elsanctum/the-great-escape
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

grab board description from forum index (we can't get it from the board index)

Browse Source
This commit is contained in:
Adrian Malacoda 2016-11-27 01:13:45 -06:00
parent 3f4eecc238
commit 39b8bfff30
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tge/scrapers/yuku.py
View File

@ -29,6 +29,7 @@ def scrape_index (url):
forum.categories.append(category) forum.categories.append(category)
for board_link in category_element.find("a[href^='/forums/']").items(): for board_link in category_element.find("a[href^='/forums/']").items():
board = scrape_board("{}://{}{}".format(urlparts.scheme, urlparts.netloc, board_link.attr.href)) board = scrape_board("{}://{}{}".format(urlparts.scheme, urlparts.netloc, board_link.attr.href))
board.description = board_link.closest("div").find("p").eq(0).text()
category.children.append(board) category.children.append(board)
print("Finished scraping all boards in category: {}".format(category.title)) print("Finished scraping all boards in category: {}".format(category.title))