glitch-city-laboratories/epilogue
1
0
Fork 0
Code Issues 4 Releases Wiki Activity

Implement thread index for mapping thread ids back to board ids, for use with the redirector.

Browse Source 
The archive domain (archives.glitchcity.info) will host this file and the redirector will pull and unpack it when it starts up.
This commit is contained in:
Adrian Kuschelyagi Malacoda 2020-08-30 16:50:21 -05:00
parent ef3f3dd60c
commit c37cf4fc44
1 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
epilogue/archive_generator.py
View File

@ -2,6 +2,8 @@ import os
import logging import logging
import shutil import shutil
import math import math
import json
import gzip
from itertools import chain from itertools import chain
from traceback import print_exc from traceback import print_exc
@ -24,6 +26,8 @@ DEX_TYPES = [
] ]
DEXES = list(chain.from_iterable([[f"{dex_type}{language}" for dex_type in DEX_TYPES] for language in DEX_LANGUAGES])) DEXES = list(chain.from_iterable([[f"{dex_type}{language}" for dex_type in DEX_TYPES] for language in DEX_LANGUAGES]))
FORUM_THREAD_INDEX = "thread_index.json.gz"
class ArchiveLinker(Linker): class ArchiveLinker(Linker):
def __init__ (self, directory_names=[]): def __init__ (self, directory_names=[]):
super().__init__() super().__init__()
@ -158,10 +162,15 @@ class ArchiveGenerator():
"categories": forum.get_board_tree() "categories": forum.get_board_tree()
}) })
threads = []
for board in forum.get_boards(): for board in forum.get_boards():
self.generate_forum_board(forum, board, out_dir) forum_threads = forum.get_threads_in_board(board)
threads = threads + forum_threads
self.generate_forum_board(forum, board, forum_threads, out_dir)
def generate_forum_board (self, forum, board, out_dir): self.generate_thread_index(threads, os.path.join(out_dir, FORUM_THREAD_INDEX))
def generate_forum_board (self, forum, board, threads, out_dir):
board_out_dir = os.path.join(out_dir, "board-{}".format(board.id)) board_out_dir = os.path.join(out_dir, "board-{}".format(board.id))
logger.info("Archiving board %s to %s", board.name, board_out_dir) logger.info("Archiving board %s to %s", board.name, board_out_dir)
try: try:
@ -169,7 +178,7 @@ class ArchiveGenerator():
except FileExistsError: pass except FileExistsError: pass
renderer = TemplateRenderer(self.template_dir, board_out_dir) renderer = TemplateRenderer(self.template_dir, board_out_dir)
threads = [prepare_thread(thread) for thread in forum.get_threads_in_board(board)] threads = [prepare_thread(thread) for thread in threads]
renderer.render_template_to_file("threads", "index.html", { renderer.render_template_to_file("threads", "index.html", {
"title": " - {}".format(board.name), "title": " - {}".format(board.name),
"base": "../", "base": "../",
@ -214,6 +223,13 @@ class ArchiveGenerator():
}) })
page = page + 1 page = page + 1
def generate_thread_index (self,threads, out_path):
# with open(out_path, "wb") as out:
# pickle.dump({thread.id: {"parent": thread.parent} for thread in threads}, out, protocol=4)
threads = {thread.id: {"parent": thread.parent} for thread in threads}
with gzip.open(out_path, "w") as out:
out.write(json.dumps(threads).encode())
class TemplateRenderer(): class TemplateRenderer():
def __init__ (self, template_dir, out_dir): def __init__ (self, template_dir, out_dir):
self.template_dir = template_dir self.template_dir = template_dir