diff --git a/epilogue/archive_generator.py b/epilogue/archive_generator.py index 80892fd..6f2341b 100644 --- a/epilogue/archive_generator.py +++ b/epilogue/archive_generator.py @@ -2,6 +2,8 @@ import os import logging import shutil import math +import json +import gzip from itertools import chain from traceback import print_exc @@ -24,6 +26,8 @@ DEX_TYPES = [ ] DEXES = list(chain.from_iterable([[f"{dex_type}{language}" for dex_type in DEX_TYPES] for language in DEX_LANGUAGES])) +FORUM_THREAD_INDEX = "thread_index.json.gz" + class ArchiveLinker(Linker): def __init__ (self, directory_names=[]): super().__init__() @@ -158,10 +162,15 @@ class ArchiveGenerator(): "categories": forum.get_board_tree() }) + threads = [] for board in forum.get_boards(): - self.generate_forum_board(forum, board, out_dir) + forum_threads = forum.get_threads_in_board(board) + threads = threads + forum_threads + self.generate_forum_board(forum, board, forum_threads, out_dir) - def generate_forum_board (self, forum, board, out_dir): + self.generate_thread_index(threads, os.path.join(out_dir, FORUM_THREAD_INDEX)) + + def generate_forum_board (self, forum, board, threads, out_dir): board_out_dir = os.path.join(out_dir, "board-{}".format(board.id)) logger.info("Archiving board %s to %s", board.name, board_out_dir) try: @@ -169,7 +178,7 @@ class ArchiveGenerator(): except FileExistsError: pass renderer = TemplateRenderer(self.template_dir, board_out_dir) - threads = [prepare_thread(thread) for thread in forum.get_threads_in_board(board)] + threads = [prepare_thread(thread) for thread in threads] renderer.render_template_to_file("threads", "index.html", { "title": " - {}".format(board.name), "base": "../", @@ -214,6 +223,13 @@ class ArchiveGenerator(): }) page = page + 1 + def generate_thread_index (self,threads, out_path): +# with open(out_path, "wb") as out: +# pickle.dump({thread.id: {"parent": thread.parent} for thread in threads}, out, protocol=4) + threads = {thread.id: {"parent": thread.parent} for thread in threads} + with gzip.open(out_path, "w") as out: + out.write(json.dumps(threads).encode()) + class TemplateRenderer(): def __init__ (self, template_dir, out_dir): self.template_dir = template_dir