style and convention fixes to make pylint happy
This commit is contained in:
@@ -1,9 +1,12 @@
|
||||
"""Outputters take scraped objects and save them to a certain format."""
|
||||
|
||||
from . import json, pickle
|
||||
|
||||
outputters = [json, pickle]
|
||||
OUTPUTTERS = [json, pickle]
|
||||
|
||||
def get_outputter (name):
|
||||
for outputter in outputters:
|
||||
def get_outputter(name):
|
||||
"""Get the outputter with the specified name."""
|
||||
for outputter in OUTPUTTERS:
|
||||
if outputter.__name__.endswith(".{}".format(name)):
|
||||
return outputter
|
||||
|
||||
|
@@ -1,10 +1,13 @@
|
||||
from ..model import User, Category, Forum, Board, Post, Thread
|
||||
from ..util import sanitize_title
|
||||
"""JSON outputter."""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
def output (data, destination):
|
||||
from ..model import Forum, Board, Thread
|
||||
from ..util import sanitize_title
|
||||
|
||||
def output(data, destination):
|
||||
"""Output the given object to the specified folder."""
|
||||
if isinstance(data, Forum):
|
||||
output_forum(data, destination)
|
||||
elif isinstance(data, Board):
|
||||
@@ -12,18 +15,24 @@ def output (data, destination):
|
||||
elif isinstance(data, Thread):
|
||||
output_thread(data, destination)
|
||||
|
||||
def output_forum (data, destination):
|
||||
def output_forum(data, destination):
|
||||
"""Output the given Forum object to the specified folder."""
|
||||
os.makedirs(destination)
|
||||
|
||||
with open(os.path.join(destination, "index.json"), "w") as out_file:
|
||||
out_file.write(json.dumps({"title": data.title}, indent=4))
|
||||
|
||||
for category in data.categories:
|
||||
os.makedirs(os.path.join(destination, sanitize_title(category.title)))
|
||||
category_dir = os.path.join(destination, sanitize_title(category.title))
|
||||
os.makedirs(category_dir)
|
||||
for board in category.children:
|
||||
output_board(board, os.path.join(destination, sanitize_title(category.title), sanitize_title(board.title)))
|
||||
output_board(
|
||||
board,
|
||||
os.path.join(category_dir, sanitize_title(board.title))
|
||||
)
|
||||
|
||||
def output_board (data, destination):
|
||||
def output_board(data, destination):
|
||||
"""Output the given Board object to the specified folder."""
|
||||
os.makedirs(destination)
|
||||
os.makedirs(os.path.join(destination, "threads"))
|
||||
with open(os.path.join(destination, "index.json"), "w") as out_file:
|
||||
@@ -35,6 +44,7 @@ def output_board (data, destination):
|
||||
for thread in data.children:
|
||||
output_thread(thread, os.path.join(destination, "threads", sanitize_title(thread.title)))
|
||||
|
||||
def output_thread (data, destination):
|
||||
def output_thread(data, destination):
|
||||
"""Output the given Thread object to the specified file."""
|
||||
with open(destination, "w") as out_file:
|
||||
out_file.write(json.dumps(data, default=vars, indent=4))
|
||||
|
@@ -1,5 +1,9 @@
|
||||
"""Outputter based on Python's pickle module.
|
||||
The output of this outputter can be read with the pickle scraper."""
|
||||
|
||||
import pickle
|
||||
|
||||
def output (data, destination):
|
||||
def output(data, destination):
|
||||
"""Output the given object into the specified pickle file."""
|
||||
with open(destination, "wb") as out_file:
|
||||
pickle.dump(data, out_file)
|
||||
|
Reference in New Issue
Block a user