cphipps/changedetection.io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re #214 - configurable wait extra seconds for webdriver requests before extracting text (#606)

Browse Source
This commit is contained in:
dgtlmoon
2022-05-17 18:35:33 +02:00
committed by GitHub
parent 31fea55ee4
commit 67c833d2bc
10 changed files with 62 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
changedetectionio/content_fetcher.py
View File

@@ -28,6 +28,9 @@ class Fetcher():
system_http_proxy = os.getenv('HTTP_PROXY')
system_https_proxy = os.getenv('HTTPS_PROXY')
# Time ONTOP of the system defined env minimum time
render_extract_delay=0
@abstractmethod
def get_error(self):
return self.error
@@ -147,7 +150,7 @@ class base_html_playwright(Fetcher):
# - `'commit'` - consider operation to be finished when network response is received and the document started loading.
# Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
# This seemed to solve nearly all 'TimeoutErrors'
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
page.wait_for_timeout(extra_wait * 1000)
except playwright._impl._api_types.TimeoutError as e:
raise EmptyReply(url=url, status_code=None)
@@ -240,7 +243,7 @@ class base_html_webdriver(Fetcher):
# raise EmptyReply(url=url, status_code=r.status_code)
# @todo - dom wait loaded?
time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
self.content = self.driver.page_source
self.headers = {}
self.screenshot = self.driver.get_screenshot_as_png()