improved handling of categories and refactored Linker stuff into a class, also use ./ as default base path so Category: links will work correctly

This unfortunately means the redirector will need to have special logic to be aware of dexes too, but it could probably be as simple as "if directory name contains 'Dex' then treat it as a directory name"

epilogue/archive_generator.py

@@ -2,17 +2,26 @@ import os
 import logging
 import shutil
 
+from itertools import chain
 from traceback import print_exc
 
 import chevron
 import bbcode
 import html
 
-from .wiki import Template, Renderer, reformat_page_title, NAMESPACES as WIKI_NAMESPACES
+from .wiki import Template, Renderer, Linker, reformat_page_title, NAMESPACES as WIKI_NAMESPACES
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("ArchiveGenerator")
 
+DEX_LANGUAGES = ["", "DE", "ES", "FR", "IT", "JP", "KO"]
+DEX_TYPES = [
+    "GlitchDex", "AttackDex", "DexDex", "AreaDex", "TrainerDex", "FieldMoveDex", "ItemDex", "FamilyDex", "DecDex", "DayDex",
+    "MDIGlitchDex", "MetascriptDex", "TMHMDex", "StatDex", "PosterDex", "TypeDex", "UnownDex", "DollDex", "DefaultNameDex",
+    "BattleTypeDe", "BadgeDescriptionDex", "FacingDex"
+]
+DEXES = list(chain.from_iterable([[f"{dex_type}{language}" for dex_type in DEX_TYPES] for language in DEX_LANGUAGES]))
+
 def prepare_thread (thread):
     thread.subject = html.unescape(thread.subject)
     return thread
@@ -52,14 +61,15 @@ class ArchiveGenerator():
 
         categories = {}
         templates = dict([(page.title.split(":")[1], Template(page.get_latest().text)) for page in wiki.get_pages() if page.namespace == WIKI_NAMESPACES['TEMPLATE']])
-        wikitext_renderer = Renderer(templates)
+        linker = Linker(directory_names=DEXES)
+        wikitext_renderer = Renderer(templates, linker)
         for page in wiki.get_pages():
             try:
                 if page.namespace != WIKI_NAMESPACES['MAIN']:
                     continue
 
-                page_out = "{}.html".format(page.title).replace(" ", "_")
+                page_out = linker.translate_page_title(page.title)
-                base = ""
+                base = "./"
                 if "/" in page_out:
                     base = "../" * page_out.count("/")
                     try:
@@ -69,11 +79,11 @@ class ArchiveGenerator():
                 if page.redirect:
                     logger.info("Archiving redirect page (%s -> %s) to %s", page.title, page.redirect, page_out)
                     renderer.render_template_to_file("redirect", page_out, {
-                        "target": f"{base}{reformat_page_title(page.redirect)}.html"
+                        "target": f"{base}{linker.translate_page_title(page.redirect)}"
                     })
                 else:
                     logger.info("Archiving page %s to %s", page.title, page_out)
-                    (rendered, page_categories) = wikitext_renderer.render(page.get_latest().text, page=page)
+                    (rendered, page_categories) = wikitext_renderer.render(page.get_latest().text, base, page=page)
 
                     for category in page_categories:
                         if not category in categories:

epilogue/wiki.py

@@ -32,6 +32,7 @@ INTERWIKI_NAMESPACES = {
 
 FILE_NAMESPACES = ["File:", "Image:"]
 CATEGORY_NAMESPACE = "Category:"
+CATEGORY_LINK_NAMESPACE = f":{CATEGORY_NAMESPACE}"
 
 class Wiki():
     def __init__ (self, xml_path):
@@ -85,10 +86,11 @@ class Contributor():
                 self.username = child.text
 
 class Renderer():
-    def __init__ (self, templates={}):
+    def __init__ (self, templates={}, linker=None):
         self.templates = templates
+        self.linker = linker if linker else Linker()
 
-    def render (self, wikitext, *args, **kwargs):
+    def render (self, wikitext, base="", *args, **kwargs):
         categories = []
         wikitext = self.transclude_templates(wikitext, *args, **kwargs)
         
@@ -100,11 +102,15 @@ class Renderer():
             wikitext.remove(link)
             categories.append(link.title[len(CATEGORY_NAMESPACE):])
         
-        rendered = [render(wikitext)]
+        rendered = [render(wikitext, base, self.linker)]
         if categories:
             rendered.append('<h2>Categories</h2><ul class="categories">')
             for category in categories:
-                rendered.append('<li><a href="Category:{}.html">{}</a></li>'.format(reformat_page_title(category), category))
+                rendered.append('<li><a href="{}Category:{}.html">{}</a></li>'.format(
+                    base,
+                    reformat_page_title(category),
+                    category
+                ))
             rendered.append("</ul>")
         
         return ("".join(rendered), categories)
@@ -125,23 +131,32 @@ class Renderer():
             except ValueError: pass
         return wikitext
 
-def render (wikitext):
+def render (wikitext, base="", linker=None):
     rendered = []
+
+    if not linker:
+        linker = Linker()
+
     for node in wikitext.ifilter(False):
         # node types:
         # https://mwparserfromhell.readthedocs.io/en/latest/api/mwparserfromhell.nodes.html#module-mwparserfromhell.nodes.text
         node_type = type(node)
         if node_type is Wikilink:
-            image_name = translate_image_title(node.title)
+            image_name = linker.translate_image_title(node.title)
             if image_name:
-                rendered.append('<img src="{}" />'.format(
+                rendered.append('<img src="{}{}" />'.format(
+                    base,
                     image_name,
-                    render(mwparserfromhell.parse(node.text))
+                    render(mwparserfromhell.parse(node.text), base, linker)
                 ))
             else:
+                url = linker.translate_interwiki_title(node.title)
+                if not url:
+                    url = f"{base}{linker.translate_page_title(node.title)}"
+                
                 rendered.append('<a href="{}">{}</a>'.format(
-                    translate_page_title(node.title),
+                    url,
-                    render(node.text if node.text else node.title)
+                    render(node.text if node.text else node.title, base, linker)
                 ))
         elif node_type is ExternalLink:
             rendered.append('<a href="{}">{}</a>'.format(
@@ -151,13 +166,13 @@ def render (wikitext):
         elif node_type is Tag:
             rendered.append("<{}>{}</{}>".format(
                 render(node.tag),
-                render(node.contents),
+                render(node.contents, base, linker),
                 render(node.tag)
             ))
         elif node_type is Heading:
             rendered.append("<h{}>{}</h{}>".format(
                 node.level,
-                render(node.title),
+                render(node.title, base, linker),
                 node.level
             ))
         elif node_type is Text:
@@ -165,20 +180,40 @@ def render (wikitext):
 
     return "".join(rendered).strip().replace("\n\n", "<br /><br />")
 
-def translate_page_title (page_title):
+class Linker():
-    for namespace, url in INTERWIKI_NAMESPACES.items():
+    def __init__ (self, file_namespaces=FILE_NAMESPACES, interwiki_namespaces=INTERWIKI_NAMESPACES, directory_names=[]):
-        if page_title.startswith(namespace):
+        self.file_namespaces = file_namespaces
-            return url.format(page_title[len(namespace):])
+        self.interwiki_namespaces = interwiki_namespaces
+        self.directory_names = directory_names
 
-    return "{}.html".format(reformat_page_title(page_title))
+    def translate_interwiki_title (self, page_title):
+        for namespace, url in self.interwiki_namespaces.items():
+            if page_title.startswith(namespace):
+                return url.format(page_title[len(namespace):])
 
-def translate_image_title (page_title):
+    def translate_page_title (self, page_title):
-    for namespace in FILE_NAMESPACES:
+        if page_title.startswith(CATEGORY_LINK_NAMESPACE):
-        if page_title.startswith(namespace):
+            page_title = page_title[1:]
-            return reformat_page_title(page_title[len(namespace):])
+        
+        directory_name = ""
+        for name in self.directory_names:
+            if page_title.startswith(f"{name}/"):
+                directory_name = name
+                page_title = page_title[len(directory_name) + 1:]
+                break
+
+        return f"{reformat_page_title(directory_name)}{'/' if directory_name else ''}{reformat_page_title(page_title)}.html"
+
+    def translate_image_title (self, page_title):
+        for namespace in self.file_namespaces:
+            if page_title.startswith(namespace):
+                return reformat_page_title(page_title[len(namespace):])
 
 def reformat_page_title (page_title):
-    return "{}{}".format(page_title[0].upper(), page_title[1:].replace(" ", "_"))
+    if not page_title:
+        return ""
+    
+    return f"{page_title[0].upper()}{page_title[1:].replace(' ', '_').replace('/', '%2F')}"
 
 class Template():
     def __init__ (self, wikicode):