"""Scrapers accept an input located somewhere (at a URL or local file) and scrape them into objects, which can be dumped by an outputter.""" from . import yuku, pickle SCRAPERS = [yuku, pickle] def get_scraper(name): """Get the scraper with the specified name.""" for scraper in SCRAPERS: if scraper.__name__.endswith(".{}".format(name)): return scraper raise Exception("Unknown scraper: {}".format(name)) def guess_scraper(url): """Attempt to guess the correct scraper for handling the given path or URL.""" for scraper in SCRAPERS: if "can_scrape_url" in vars(scraper) and scraper.can_scrape_url(url): return scraper raise Exception("Unable to guess scraper for forum url: {}".format(url))