cphipps/changedetection.io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use proxies.json instead of proxies.txt - see wiki Proxies section (#945)

Browse Source
This commit is contained in:
dgtlmoon
2022-09-15 15:25:23 +02:00
committed by GitHub
parent 4c759490da
commit 2ba55bb477
3 changed files with 43 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
changedetectionio/fetch_site_status.py
View File

@@ -21,6 +21,7 @@ class perform_site_check():
self.datastore = datastore
# If there was a proxy list enabled, figure out what proxy_args/which proxy to use
# Returns the proxy as a URL
# if watch.proxy use that
# fetcher.proxy_override = watch.proxy or main config proxy
# Allows override the proxy on a per-request basis
@@ -33,18 +34,19 @@ class perform_site_check():
# If its a valid one
if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
proxy_args = watch['proxy']
proxy_args = self.datastore.proxy_list.get(watch['proxy']).get('url')
# not valid (including None), try the system one
else:
system_proxy = self.datastore.data['settings']['requests']['proxy']
# Is not None and exists
if any([system_proxy in p for p in self.datastore.proxy_list]):
proxy_args = system_proxy
if self.datastore.proxy_list.get():
proxy_args = self.datastore.proxy_list.get(system_proxy).get('url')
# Fallback - Did not resolve anything, use the first available
if proxy_args is None:
proxy_args = self.datastore.proxy_list[0][0]
first_default = list(self.datastore.proxy_list)[0]
proxy_args = self.datastore.proxy_list.get(first_default).get('url')
return proxy_args
@@ -68,6 +70,8 @@ class perform_site_check():
stripped_text_from_html = ""
watch = self.datastore.data['watching'].get(uuid)
if not watch:
return
# Protect against file:// access
if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
@@ -90,7 +94,7 @@ class perform_site_check():
if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
timeout = self.datastore.data['settings']['requests']['timeout']
timeout = self.datastore.data['settings']['requests'].get('timeout')
url = watch.get('url')
request_body = self.datastore.data['watching'][uuid].get('body')
request_method = self.datastore.data['watching'][uuid].get('method')
@@ -110,9 +114,10 @@ class perform_site_check():
# If the klass doesnt exist, just use a default
klass = getattr(content_fetcher, "html_requests")
proxy_args = self.set_proxy_from_list(watch)
fetcher = klass(proxy_override=proxy_args)
proxy_url = self.set_proxy_from_list(watch)
if proxy_url:
print ("UUID {} Using proxy {}".format(uuid, proxy_url))
fetcher = klass(proxy_override=proxy_url)
# Configurable per-watch or global extra delay before extracting text (for webDriver types)
system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)