Adrian Malacoda
dc0191a04a
mwparserfromhell is used for parsing wikitext but it has no support for rendering to HTML so we'll have to build it manually.
143 lines
5.2 KiB
Python
143 lines
5.2 KiB
Python
import os
|
|
import logging
|
|
import shutil
|
|
from datetime import datetime
|
|
|
|
import chevron
|
|
import bbcode
|
|
|
|
from .wiki import NAMESPACES as WIKI_NAMESPACES
|
|
import mwparserfromhell
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger("ArchiveGenerator")
|
|
|
|
def format_datetime (timestamp):
|
|
return datetime.fromtimestamp(timestamp)
|
|
|
|
def prepare_thread (thread):
|
|
thread = dict(thread)
|
|
thread['datetime'] = format_datetime(thread['poster_time'])
|
|
return thread
|
|
|
|
def prepare_post (post):
|
|
post = prepare_thread(post)
|
|
|
|
parser = bbcode.Parser()
|
|
post['body'] = parser.format(post['body']).replace("<br />", "<br />")
|
|
return post
|
|
|
|
class ArchiveGenerator():
|
|
def __init__ (self, template_dir, static_dir):
|
|
self.template_dir = template_dir
|
|
self.static_dir = static_dir
|
|
|
|
def generate_wiki (self, wiki, out_dir):
|
|
logger.info("Archiving wiki to %s", out_dir)
|
|
try:
|
|
os.makedirs(out_dir)
|
|
except FileExistsError: pass
|
|
|
|
shutil.copyfile(os.path.join(self.static_dir, "style.css"), os.path.join(out_dir, "style.css"))
|
|
renderer = TemplateRenderer(self.template_dir, out_dir)
|
|
|
|
for page in wiki.get_pages():
|
|
if page.redirect:
|
|
continue
|
|
|
|
if page.namespace != WIKI_NAMESPACES['MAIN']:
|
|
continue
|
|
|
|
page_out = "{}.html".format(page.title).replace(" ", "_")
|
|
base = ""
|
|
if "/" in page_out:
|
|
base = "../" * page_out.count("/")
|
|
try:
|
|
os.makedirs(os.path.dirname(os.path.join(out_dir, page_out)))
|
|
except FileExistsError: pass
|
|
|
|
logger.info("Archiving page %s to %s", page.title, page_out)
|
|
renderer.render_template_to_file("page", page_out, {
|
|
"title": " - {}".format(page.title),
|
|
"page": page,
|
|
"base": base,
|
|
"text": mwparserfromhell.parse(page.get_latest().text)
|
|
})
|
|
|
|
def generate_forum (self, forum, out_dir):
|
|
logger.info("Archiving forum to %s", out_dir)
|
|
try:
|
|
os.makedirs(out_dir)
|
|
except FileExistsError: pass
|
|
|
|
shutil.copyfile(os.path.join(self.static_dir, "style.css"), os.path.join(out_dir, "style.css"))
|
|
renderer = TemplateRenderer(self.template_dir, out_dir)
|
|
renderer.render_template_to_file("boards", "index.html", {
|
|
"categories": forum.get_board_tree()
|
|
})
|
|
|
|
for board in forum.get_boards():
|
|
self.generate_forum_board(forum, board, out_dir)
|
|
|
|
def generate_forum_board (self, forum, board, out_dir):
|
|
board_out_dir = os.path.join(out_dir, "board-{}".format(board['id_board']))
|
|
logger.info("Archiving board %s to %s", board['name'], board_out_dir)
|
|
try:
|
|
os.makedirs(board_out_dir)
|
|
except FileExistsError: pass
|
|
|
|
renderer = TemplateRenderer(self.template_dir, board_out_dir)
|
|
threads = [prepare_thread(thread) for thread in forum.get_threads_in_board(board)]
|
|
renderer.render_template_to_file("threads", "index.html", {
|
|
"title": " - {}".format(board['name']),
|
|
"base": "../",
|
|
"board": board,
|
|
"threads": threads
|
|
})
|
|
|
|
for thread in threads:
|
|
self.generate_forum_thread(forum, board, thread, board_out_dir)
|
|
|
|
def generate_forum_thread (self, forum, board, thread, out_dir):
|
|
thread_out_dir = os.path.join(out_dir, "thread-{}".format(thread['id_topic']))
|
|
logger.info("Archiving thread %s to %s", thread['subject'], thread_out_dir)
|
|
try:
|
|
os.makedirs(thread_out_dir)
|
|
except FileExistsError: pass
|
|
|
|
renderer = TemplateRenderer(self.template_dir, thread_out_dir)
|
|
renderer.render_template_to_file("page-0-redirect", "index.html")
|
|
|
|
page = 0
|
|
while True:
|
|
posts = [prepare_post(post) for post in forum.get_posts_in_thread(thread, page)]
|
|
if len(posts) < 1:
|
|
break
|
|
|
|
logger.info("Archiving page %s of thread %s", page, thread['subject'])
|
|
renderer.render_template_to_file("posts", "page-{}.html".format(page), {
|
|
"title": " - {} - Page {}".format(thread['subject'], page + 1),
|
|
"base": "../../",
|
|
"board": board,
|
|
"thread": thread,
|
|
"page": page,
|
|
"next": page + 1,
|
|
"prev": page - 1,
|
|
"posts": posts
|
|
})
|
|
page = page + 1
|
|
|
|
class TemplateRenderer():
|
|
def __init__ (self, template_dir, out_dir):
|
|
self.template_dir = template_dir
|
|
self.partials_dir = os.path.join(template_dir, "partials")
|
|
self.out_dir = out_dir
|
|
self.extension = "mustache"
|
|
|
|
def open_template (self, name):
|
|
return open(os.path.join(self.template_dir, "{}.{}".format(name, self.extension)))
|
|
|
|
def render_template_to_file (self, template_name, out_file, data={}):
|
|
with self.open_template(template_name) as template:
|
|
with open(os.path.join(self.out_dir, out_file), "w") as out:
|
|
out.write(chevron.render(template, data, self.partials_dir, self.extension)) |