191 lines
7.4 KiB
Python
191 lines
7.4 KiB
Python
import os
|
|
import logging
|
|
import shutil
|
|
|
|
from traceback import print_exc
|
|
|
|
import chevron
|
|
import bbcode
|
|
import html
|
|
|
|
from .wiki import Template, Renderer, reformat_page_title, NAMESPACES as WIKI_NAMESPACES
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger("ArchiveGenerator")
|
|
|
|
def prepare_thread (thread):
|
|
thread.subject = html.unescape(thread.subject)
|
|
return thread
|
|
|
|
def prepare_post (post):
|
|
post = prepare_thread(post)
|
|
parser = bbcode.Parser()
|
|
post.body = html.unescape(parser.format(post.body))
|
|
return post
|
|
|
|
class ArchiveGenerator():
|
|
def __init__ (self, template_dir, static_dir):
|
|
self.template_dir = template_dir
|
|
self.static_dir = static_dir
|
|
|
|
def generate_index (self, out_dir):
|
|
logger.info("Generating index page at %s", out_dir)
|
|
try:
|
|
os.makedirs(out_dir)
|
|
except FileExistsError: pass
|
|
|
|
shutil.copyfile(os.path.join(self.static_dir, "style.css"), os.path.join(out_dir, "style.css"))
|
|
renderer = TemplateRenderer(self.template_dir, out_dir)
|
|
renderer.render_template_to_file("index", "index.html", {})
|
|
|
|
def generate_wiki (self, wiki, out_dir):
|
|
logger.info("Archiving wiki to %s", out_dir)
|
|
try:
|
|
os.makedirs(out_dir)
|
|
except FileExistsError: pass
|
|
|
|
shutil.copyfile(os.path.join(self.static_dir, "style.css"), os.path.join(out_dir, "style.css"))
|
|
renderer = TemplateRenderer(self.template_dir, out_dir)
|
|
renderer.render_template_to_file("redirect", "index.html", {
|
|
"target": "Main_Page.html"
|
|
})
|
|
|
|
categories = {}
|
|
templates = dict([(page.title.split(":")[1], Template(page.get_latest().text)) for page in wiki.get_pages() if page.namespace == WIKI_NAMESPACES['TEMPLATE']])
|
|
wikitext_renderer = Renderer(templates)
|
|
for page in wiki.get_pages():
|
|
try:
|
|
if page.namespace != WIKI_NAMESPACES['MAIN']:
|
|
continue
|
|
|
|
page_out = f"{reformat_page_title(page.title)}.html"
|
|
base = ""
|
|
if "/" in page_out:
|
|
base = "../" * page_out.count("/")
|
|
try:
|
|
os.makedirs(os.path.dirname(os.path.join(out_dir, page_out)))
|
|
except FileExistsError: pass
|
|
|
|
if page.redirect:
|
|
logger.info("Archiving redirect page (%s -> %s) to %s", page.title, page.redirect, page_out)
|
|
renderer.render_template_to_file("redirect", page_out, {
|
|
"target": f"{base}{reformat_page_title(page.redirect)}.html"
|
|
})
|
|
else:
|
|
logger.info("Archiving page %s to %s", page.title, page_out)
|
|
(rendered, page_categories) = wikitext_renderer.render(page.get_latest().text, page=page)
|
|
|
|
for category in page_categories:
|
|
if not category in categories:
|
|
categories[category] = []
|
|
|
|
categories[category].append({
|
|
"url": page_out,
|
|
"title": page.title
|
|
})
|
|
|
|
renderer.render_template_to_file("page", page_out, {
|
|
"title": " - {}".format(page.title),
|
|
"page": page,
|
|
"base": base,
|
|
"text": rendered
|
|
})
|
|
except Exception as e:
|
|
logger.error("Error encountered when archiving %s: %s", page.title, e)
|
|
print_exc()
|
|
if isinstance(e, ValueError):
|
|
raise e
|
|
|
|
for category, pages in categories.items():
|
|
category_out = f"Category:{reformat_page_title(category)}.html"
|
|
logger.info("Archiving category %s to %s", category, category_out)
|
|
|
|
try:
|
|
renderer.render_template_to_file("category", category_out, {
|
|
"title": f" - {category}",
|
|
"category": category,
|
|
"pages": pages
|
|
})
|
|
except Exception as e:
|
|
logger.error("Error encountered when archiving %s: %s", category, e)
|
|
print_exc()
|
|
|
|
def generate_forum (self, forum, out_dir):
|
|
logger.info("Archiving forum to %s", out_dir)
|
|
try:
|
|
os.makedirs(out_dir)
|
|
except FileExistsError: pass
|
|
|
|
shutil.copyfile(os.path.join(self.static_dir, "style.css"), os.path.join(out_dir, "style.css"))
|
|
renderer = TemplateRenderer(self.template_dir, out_dir)
|
|
renderer.render_template_to_file("boards", "index.html", {
|
|
"categories": forum.get_board_tree()
|
|
})
|
|
|
|
for board in forum.get_boards():
|
|
self.generate_forum_board(forum, board, out_dir)
|
|
|
|
def generate_forum_board (self, forum, board, out_dir):
|
|
board_out_dir = os.path.join(out_dir, "board-{}".format(board.id))
|
|
logger.info("Archiving board %s to %s", board.name, board_out_dir)
|
|
try:
|
|
os.makedirs(board_out_dir)
|
|
except FileExistsError: pass
|
|
|
|
renderer = TemplateRenderer(self.template_dir, board_out_dir)
|
|
threads = [prepare_thread(thread) for thread in forum.get_threads_in_board(board)]
|
|
renderer.render_template_to_file("threads", "index.html", {
|
|
"title": " - {}".format(board.name),
|
|
"base": "../",
|
|
"board": board,
|
|
"threads": threads
|
|
})
|
|
|
|
for thread in threads:
|
|
self.generate_forum_thread(forum, board, thread, board_out_dir)
|
|
|
|
def generate_forum_thread (self, forum, board, thread, out_dir):
|
|
thread_out_dir = os.path.join(out_dir, "thread-{}".format(thread.id))
|
|
logger.info("Archiving thread %s to %s", thread.subject, thread_out_dir)
|
|
try:
|
|
os.makedirs(thread_out_dir)
|
|
except FileExistsError: pass
|
|
|
|
renderer = TemplateRenderer(self.template_dir, thread_out_dir)
|
|
renderer.render_template_to_file("redirect", "index.html", {
|
|
"target": "page-0.html"
|
|
})
|
|
|
|
page = 0
|
|
while True:
|
|
posts = [prepare_post(post) for post in forum.get_posts_in_thread(thread, page)]
|
|
if len(posts) < 1:
|
|
break
|
|
|
|
logger.info("Archiving page %s of thread %s", page, thread.subject)
|
|
renderer.render_template_to_file("posts", "page-{}.html".format(page), {
|
|
"title": " - {} - Page {}".format(thread.subject, page + 1),
|
|
"base": "../../",
|
|
"board": board,
|
|
"thread": thread,
|
|
"page": page,
|
|
"next": page + 1,
|
|
"prev": page - 1,
|
|
"posts": posts
|
|
})
|
|
page = page + 1
|
|
|
|
class TemplateRenderer():
|
|
def __init__ (self, template_dir, out_dir):
|
|
self.template_dir = template_dir
|
|
self.partials_dir = os.path.join(template_dir, "partials")
|
|
self.out_dir = out_dir
|
|
self.extension = "mustache"
|
|
|
|
def open_template (self, name):
|
|
return open(os.path.join(self.template_dir, "{}.{}".format(name, self.extension)))
|
|
|
|
def render_template_to_file (self, template_name, out_file, data={}):
|
|
with self.open_template(template_name) as template:
|
|
with open(os.path.join(self.out_dir, out_file), "w") as out:
|
|
out.write(chevron.render(template, data, self.partials_dir, self.extension)) |