style and convention fixes to make pylint happy

This commit is contained in:
Adrian Malacoda
2016-12-16 00:29:59 -06:00
parent 1db0d315b8
commit 05c766011f
9 changed files with 148 additions and 68 deletions

View File

@@ -1,9 +1,12 @@
"""Outputters take scraped objects and save them to a certain format."""
from . import json, pickle
outputters = [json, pickle]
OUTPUTTERS = [json, pickle]
def get_outputter (name):
for outputter in outputters:
def get_outputter(name):
"""Get the outputter with the specified name."""
for outputter in OUTPUTTERS:
if outputter.__name__.endswith(".{}".format(name)):
return outputter

View File

@@ -1,10 +1,13 @@
from ..model import User, Category, Forum, Board, Post, Thread
from ..util import sanitize_title
"""JSON outputter."""
import json
import os
def output (data, destination):
from ..model import Forum, Board, Thread
from ..util import sanitize_title
def output(data, destination):
"""Output the given object to the specified folder."""
if isinstance(data, Forum):
output_forum(data, destination)
elif isinstance(data, Board):
@@ -12,18 +15,24 @@ def output (data, destination):
elif isinstance(data, Thread):
output_thread(data, destination)
def output_forum (data, destination):
def output_forum(data, destination):
"""Output the given Forum object to the specified folder."""
os.makedirs(destination)
with open(os.path.join(destination, "index.json"), "w") as out_file:
out_file.write(json.dumps({"title": data.title}, indent=4))
for category in data.categories:
os.makedirs(os.path.join(destination, sanitize_title(category.title)))
category_dir = os.path.join(destination, sanitize_title(category.title))
os.makedirs(category_dir)
for board in category.children:
output_board(board, os.path.join(destination, sanitize_title(category.title), sanitize_title(board.title)))
output_board(
board,
os.path.join(category_dir, sanitize_title(board.title))
)
def output_board (data, destination):
def output_board(data, destination):
"""Output the given Board object to the specified folder."""
os.makedirs(destination)
os.makedirs(os.path.join(destination, "threads"))
with open(os.path.join(destination, "index.json"), "w") as out_file:
@@ -35,6 +44,7 @@ def output_board (data, destination):
for thread in data.children:
output_thread(thread, os.path.join(destination, "threads", sanitize_title(thread.title)))
def output_thread (data, destination):
def output_thread(data, destination):
"""Output the given Thread object to the specified file."""
with open(destination, "w") as out_file:
out_file.write(json.dumps(data, default=vars, indent=4))

View File

@@ -1,5 +1,9 @@
"""Outputter based on Python's pickle module.
The output of this outputter can be read with the pickle scraper."""
import pickle
def output (data, destination):
def output(data, destination):
"""Output the given object into the specified pickle file."""
with open(destination, "wb") as out_file:
pickle.dump(data, out_file)