elsanctum/the-great-escape
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add exponential backoff for retrying

Browse Source
This commit is contained in:
Adrian Malacoda 2016-11-27 13:03:13 -06:00
parent f47895fd46
commit b67ab06b55
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tge/scrapers/yuku.py
View File

@ -3,6 +3,7 @@ from urllib.parse import urlparse
from time import strptime, mktime
import dateutil.parser
from pyquery import PyQuery as pq
from retrying import retry
time_format = "%b %d %y %I:%M %p"
@ -18,6 +19,7 @@ def scrape (url):
elif (not path) or path == "/":
return scrape_index(url)
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
def scrape_index (url):
print("Scraping forum index from url: {}".format(url))
urlparts = urlparse(url)
@ -35,6 +37,7 @@ def scrape_index (url):
return forum
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
def scrape_board (url):
print("Scraping board from url: {}".format(url))
urlparts = urlparse(url)
@ -57,6 +60,7 @@ def scrape_board (url):
return board
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
def scrape_thread (url):
print("Scraping thread from url: {}".format(url))
urlparts = urlparse(url)