Re-enable archival of wiki by default.

deploy_archives

@@ -1,8 +1,8 @@
-#!/bin/sh
+#!/bin/sh -x
 HOSTNAME=glitchcity.info
 ARCHIVE_PATH=/var/www/html/gclarchives
 
 cd archives
-tar -cvf forums.tar forums
+tar -cf forums.tar forums && gzip -f forums.tar
-scp forums.tar $HOSTNAME:$ARCHIVE_PATH
+scp forums.tar.gz $HOSTNAME:$ARCHIVE_PATH
-ssh $HOSTNAME "cd $ARCHIVE_PATH; tar -xvf forums.tar"
+ssh $HOSTNAME "cd $ARCHIVE_PATH; tar -xf forums.tar.gz"

epilogue/__init__.py

@@ -3,6 +3,8 @@ from .forum import Forum
 from .wiki import Wiki
 from .archive_generator import ArchiveGenerator
 
+import sys
+
 BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 FORUM_DATABASE = os.path.join(BASEDIR, "forum", "forum.sqlite")
 WIKI_DIRECTORY = os.path.join(BASEDIR, "wiki")
@@ -13,16 +15,31 @@ ARCHIVES_BASEDIR = "archives"
 FORUM_ARCHIVES = os.path.join(ARCHIVES_BASEDIR, "forums")
 WIKI_ARCHIVES = os.path.join(ARCHIVES_BASEDIR, "wiki")
 
+DEFAULT_ARGUMENTS = ["wiki", "forum"]
+
 def main():
+    args = sys.argv[1:]
+    if not args:
+        args = DEFAULT_ARGUMENTS
+
+    if "forum" in args or "forums" in args:
+        archive_forum()
+
+    if "wiki" in args:
+        archive_wiki()
+
+def archive_forum():
     forum = Forum(FORUM_DATABASE)
 
+    generator = ArchiveGenerator(TEMPLATES_DIR, STATIC_DIR)
+    generator.generate_forum(forum, FORUM_ARCHIVES)
+
+def archive_wiki():
     wiki = None
     for entry in os.listdir(WIKI_DIRECTORY):
         if entry.endswith(".xml"):
             wiki = Wiki(os.path.join(WIKI_DIRECTORY, entry))
 
-    generator = ArchiveGenerator(TEMPLATES_DIR, STATIC_DIR)
-    #generator.generate_forum(forum, FORUM_ARCHIVES)
-
     if wiki:
+        generator = ArchiveGenerator(TEMPLATES_DIR, STATIC_DIR)
         generator.generate_wiki(wiki, WIKI_ARCHIVES)

epilogue/archive_generator.py

@@ -1,10 +1,10 @@
 import os
 import logging
 import shutil
-from datetime import datetime
 
 import chevron
 import bbcode
+import html
 
 from .wiki import NAMESPACES as WIKI_NAMESPACES
 import mwparserfromhell
@@ -12,19 +12,14 @@ import mwparserfromhell
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("ArchiveGenerator")
 
-def format_datetime (timestamp):
-    return datetime.fromtimestamp(timestamp)
-
 def prepare_thread (thread):
-    thread = dict(thread)
+    thread.subject = html.unescape(thread.subject)
-    thread['datetime'] = format_datetime(thread['poster_time'])
     return thread
 
 def prepare_post (post):
     post = prepare_thread(post)
-
     parser = bbcode.Parser()
-    post['body'] = parser.format(post['body']).replace("&lt;br /&gt;", "<br />")
+    post.body = html.unescape(parser.format(post.body))
     return post
 
 class ArchiveGenerator():
@@ -80,8 +75,8 @@ class ArchiveGenerator():
             self.generate_forum_board(forum, board, out_dir)
 
     def generate_forum_board (self, forum, board, out_dir):
-        board_out_dir = os.path.join(out_dir, "board-{}".format(board['id_board']))
+        board_out_dir = os.path.join(out_dir, "board-{}".format(board.id))
-        logger.info("Archiving board %s to %s", board['name'], board_out_dir)
+        logger.info("Archiving board %s to %s", board.name, board_out_dir)
         try:
             os.makedirs(board_out_dir)
         except FileExistsError: pass
@@ -89,7 +84,7 @@ class ArchiveGenerator():
         renderer = TemplateRenderer(self.template_dir, board_out_dir)
         threads = [prepare_thread(thread) for thread in forum.get_threads_in_board(board)]
         renderer.render_template_to_file("threads", "index.html", {
-            "title": " - {}".format(board['name']),
+            "title": " - {}".format(board.name),
             "base": "../",
             "board": board,
             "threads": threads
@@ -99,8 +94,8 @@ class ArchiveGenerator():
             self.generate_forum_thread(forum, board, thread, board_out_dir)
 
     def generate_forum_thread (self, forum, board, thread, out_dir):
-        thread_out_dir = os.path.join(out_dir, "thread-{}".format(thread['id_topic']))
+        thread_out_dir = os.path.join(out_dir, "thread-{}".format(thread.id))
-        logger.info("Archiving thread %s to %s", thread['subject'], thread_out_dir)
+        logger.info("Archiving thread %s to %s", thread.subject, thread_out_dir)
         try:
             os.makedirs(thread_out_dir)
         except FileExistsError: pass
@@ -114,9 +109,9 @@ class ArchiveGenerator():
             if len(posts) < 1:
                 break
             
-            logger.info("Archiving page %s of thread %s", page, thread['subject'])
+            logger.info("Archiving page %s of thread %s", page, thread.subject)
             renderer.render_template_to_file("posts", "page-{}.html".format(page), {
-                "title": " - {} - Page {}".format(thread['subject'], page + 1),
+                "title": " - {} - Page {}".format(thread.subject, page + 1),
                 "base": "../../",
                 "board": board,
                 "thread": thread,

epilogue/forum.py

@@ -1,8 +1,15 @@
 import sqlite3
+from datetime import datetime
 
 PREFIX = "smf_"
-GET_BOARDS = "SELECT * FROM `{}boards`".format(PREFIX)
+GET_BOARDS = """
-GET_CATEGORIES = "SELECT * FROM `{}categories`".format(PREFIX)
+    SELECT * FROM `{}boards`
+    ORDER BY `board_order` ASC
+""".format(PREFIX)
+GET_CATEGORIES = """
+    SELECT * FROM `{}categories`
+    ORDER BY `cat_order` ASC
+    """.format(PREFIX)
 GET_THREADS = """
     SELECT * FROM `{}topics` AS `topics`, `{}messages` AS `messages`
     WHERE `topics`.`id_board`=? AND `messages`.`id_msg`=`topics`.`id_first_msg`
@@ -16,42 +23,78 @@ GET_POSTS = """
     LIMIT ? OFFSET ?
 """.format(PREFIX)
 
+def fix_encoding (string):
+    return string.encode("latin1", errors="ignore").decode(errors="ignore")
+
 class Forum():
     def __init__ (self, db_path):
         self.connection = sqlite3.connect(db_path)
         self.connection.row_factory = sqlite3.Row
 
     def get_board_tree (self):
-        categories = [dict(category) for category in self.get_categories()]
+        categories = self.get_categories()
-        boards = [dict(board) for board in self.get_boards()]
+        boards = self.get_boards()
         for category in categories:
-            category['children'] = [board for board in boards if board['id_cat'] == category['id_cat']]
+            category.children = [child for child in boards if child.category == category.id and child.child_level == 0]
         for board in boards:
-            board['children'] = [board for board in boards if board['id_parent'] == board['id_board']]
+            board.children = [child for child in boards if child.parent_board == board.id]
         return categories
 
     def get_categories (self):
         cursor = self.connection.cursor()
         cursor.execute(GET_CATEGORIES)
-        return cursor.fetchall()
+        return [Category(category) for category in cursor.fetchall()]
 
     def get_boards (self):
         cursor = self.connection.cursor()
         cursor.execute(GET_BOARDS)
-        return cursor.fetchall()
+        return [Board(board) for board in cursor.fetchall()]
 
     def get_threads_in_board (self, board, page=0, per_page=2000):
         try:
-            board = board['id_board']
+            board = board.id
         except ValueError: pass
         cursor = self.connection.cursor()
         cursor.execute(GET_THREADS, (board, per_page, page * per_page))
-        return cursor.fetchall()
+        return [Thread(thread) for thread in cursor.fetchall()]
 
     def get_posts_in_thread (self, thread, page=0, per_page=15):
         try:
-            thread = thread['id_topic']
+            thread = thread.id
         except ValueError: pass
         cursor = self.connection.cursor()
         cursor.execute(GET_POSTS, (thread, per_page, page * per_page))
-        return cursor.fetchall()
+        return [Post(post) for post in cursor.fetchall()]
+
+class Category():
+    def __init__ (self, row):
+        self.id = row['id_cat']
+        self.name = fix_encoding(row['name'])
+        self.children = []
+
+class Board():
+    def __init__ (self, row):
+        self.id = row['id_board']
+        self.category = row['id_cat']
+        self.parent_board = row['id_parent']
+        self.child_level = row['child_level']
+        self.name = fix_encoding(row['name'])
+        self.description = fix_encoding(row['description'])
+        self.children = []
+
+class Thread():
+    def __init__ (self, row):
+        self.id = row['id_topic']
+        self.parent = row['id_board']
+        self.datetime = datetime.fromtimestamp(row['poster_time'])
+        self.subject = fix_encoding(row['subject'])
+        self.poster_name = fix_encoding(row['poster_name'])
+
+class Post():
+    def __init__ (self, row):
+        self.id = row['id_msg']
+        self.parent = row['id_topic']
+        self.datetime = datetime.fromtimestamp(row['poster_time'])
+        self.subject = fix_encoding(row['subject'])
+        self.body = fix_encoding(row['body'])
+        self.poster_name = fix_encoding(row['poster_name'])

static/style.css

@@ -1,8 +1,10 @@
 body { font-family: monospace; }
 
+ul.boards { margin-left: 0; padding-left: 0; }
+.board { margin-bottom: 5px; }
 .category, .board { list-style-type: none;}
 .category .name, .board .name { font-weight: bold; }
-.board .board { margin-left: 10px; }
+.board .board { margin-left: 15px; }
 
 #threads { width: 100%; }
 

templates/partials/child_boards.mustache

@@ -1,7 +1,9 @@
-{{#children}}
+<ul class="boards">
-<li class="board">
+    {{#children}}
-    <div class="name"><a href="board-{{id_board}}">{{name}}</a></div>
+    <li class="board">
-    <div class="description">{{description}}</div>
+        <div class="name"><a href="board-{{id}}">{{name}}</a></div>
+        <div class="description">{{{description}}}</div>
         {{>child_boards}}
-</li>
+    </li>
-{{/children}}
+    {{/children}}
+</ul>

templates/posts.mustache

@@ -3,7 +3,7 @@
 <h3>{{thread.subject}} - Page {{next}}</h3>
 {{>pagination}}
 {{#posts}}
-<article id="post-{{id_msg}}">
+<article id="msg{{id}}">
     <header>
         <h4>{{subject}}</h4>
         <div><span class="label">Posted by:</span> {{poster_name}}</div>

templates/threads.mustache

@@ -8,7 +8,7 @@
     </tr>
     {{#threads}}
     <tr>
-        <td class="thread-subject"><a href="thread-{{id_topic}}">{{subject}}</a></td>
+        <td class="thread-subject"><a href="thread-{{id}}">{{subject}}</a></td>
         <td class="thread-poster">{{poster_name}}</td>
         <td class="thread-date">{{datetime}}</td>
     </tr>