initial commit for the-great-escape yuku scraper

tge/outputters/__init__.py

@@ -0,0 +1,10 @@
+from . import json
+
+outputters = [json]
+
+def get_outputter (name):
+    for outputter in outputters:
+        if outputter.__name__.endswith(".{}".format(name)):
+            return outputter
+
+    raise Exception("Unknown outputter: {}".format(name))

tge/outputters/json.py

@@ -0,0 +1,28 @@
+from ..model import User, Category, Forum, Board, Post, Thread
+
+import json
+import os
+
+def output (data, destination):
+    if isinstance(data, Forum):
+        output_forum(data, destination)
+    elif isinstance(data, Board):
+        output_board(data, destination)
+    elif isinstance(data, Thread):
+        output_thread(data, destination)
+
+def output_forum (data, destination):
+    os.makedirs(destination)
+    for category in data.categories:
+        os.makedirs(os.path.join(destination, category.title))
+        for board in category.children:
+            output_board(board, os.path.join(destination, category.title, board.title))
+
+def output_board (data, destination):
+    os.makedirs(destination)
+    for thread in data.children:
+        output_thread(thread, os.path.join(destination, thread.title))
+
+def output_thread (data, destination):
+    with open(destination, "w") as out_file:
+        out_file.write(json.dumps(data, default=vars, indent=4))