elsanctum/the-great-escape
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add exponential backoff for retrying

Browse Source
This commit is contained in:
Adrian Malacoda 2016-11-27 13:03:13 -06:00
parent f47895fd46
commit b67ab06b55
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tge/scrapers/yuku.py
View File

@ -3,6 +3,7 @@ from urllib.parse import urlparse
from time import strptime, mktime from time import strptime, mktime
import dateutil.parser import dateutil.parser
from pyquery import PyQuery as pq from pyquery import PyQuery as pq
from retrying import retry
time_format = "%b %d %y %I:%M %p" time_format = "%b %d %y %I:%M %p"
@ -18,6 +19,7 @@ def scrape (url):
elif (not path) or path == "/": elif (not path) or path == "/":
return scrape_index(url) return scrape_index(url)
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
def scrape_index (url): def scrape_index (url):
print("Scraping forum index from url: {}".format(url)) print("Scraping forum index from url: {}".format(url))
urlparts = urlparse(url) urlparts = urlparse(url)
@ -35,6 +37,7 @@ def scrape_index (url):
return forum return forum
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
def scrape_board (url): def scrape_board (url):
print("Scraping board from url: {}".format(url)) print("Scraping board from url: {}".format(url))
urlparts = urlparse(url) urlparts = urlparse(url)
@ -57,6 +60,7 @@ def scrape_board (url):
return board return board
@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000)
def scrape_thread (url): def scrape_thread (url):
print("Scraping thread from url: {}".format(url)) print("Scraping thread from url: {}".format(url))
urlparts = urlparse(url) urlparts = urlparse(url)