cphipps/changedetection.io
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fetching - Prefer to use SockPuppetBrowser (#2163)

Browse Source
This commit is contained in:
dgtlmoon
2024-02-07 20:58:21 +01:00
committed by GitHub
parent 5119efe4fb
commit c5a4e0aaa3
14 changed files with 58 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
changedetectionio/tests/custom_browser_url/test_custom_browser_url.py
View File

@@ -7,10 +7,11 @@ from ..util import live_server_setup, wait_for_all_checks
def do_test(client, live_server, make_test_use_extra_browser=False):
# Grep for this string in the logs?
test_url = f"https://changedetection.io/ci-test.html"
test_url = f"https://changedetection.io/ci-test.html?non-custom-default=true"
# "non-custom-default" should not appear in the custom browser connection
custom_browser_name = 'custom browser URL'
# needs to be set and something like 'ws://127.0.0.1:3000?stealth=1&--disable-web-security=true'
# needs to be set and something like 'ws://127.0.0.1:3000'
assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
#####################
@@ -19,9 +20,7 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
data={"application-empty_pages_are_a_change": "",
"requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_webdriver",
# browserless-custom-url is setup in .github/workflows/test-only.yml
# the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs
'requests-extra_browsers-0-browser_connection_url': 'ws://browserless-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1',
'requests-extra_browsers-0-browser_connection_url': 'ws://sockpuppetbrowser-custom-url:3000',
'requests-extra_browsers-0-browser_name': custom_browser_name
},
follow_redirects=True
@@ -51,7 +50,8 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
res = client.post(
url_for("edit_page", uuid="first"),
data={
"url": test_url,
# 'run_customer_browser_url_tests.sh' will search for this string to know if we hit the right browser container or not
"url": f"https://changedetection.io/ci-test.html?custom-browser-search-string=1",
"tags": "",
"headers": "",
'fetch_backend': f"extra_browser_{custom_browser_name}",

2
changedetectionio/tests/test_jsonpath_jq_selector.py
View File

@@ -456,7 +456,7 @@ def test_ignore_json_order(client, live_server):
def test_correct_header_detect(client, live_server):
# Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
# Specify extra html that JSON is sometimes wrapped in - when using Browserless/Puppeteer etc
# Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
with open("test-datastore/endpoint-content.txt", "w") as f:
f.write('<html><body>{"hello" : 123, "world": 123}')

8
changedetectionio/tests/test_request.py
View File

@@ -14,7 +14,7 @@ def test_headers_in_request(client, live_server):
# Add our URL to the import page
test_url = url_for('test_headers', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml
# Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'changedet')
# Add the test URL twice, we will check
@@ -89,7 +89,7 @@ def test_body_in_request(client, live_server):
# Add our URL to the import page
test_url = url_for('test_body', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml
# Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'cdio')
res = client.post(
@@ -181,7 +181,7 @@ def test_method_in_request(client, live_server):
# Add our URL to the import page
test_url = url_for('test_method', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml
# Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'cdio')
# Add the test URL twice, we will check
@@ -258,7 +258,7 @@ def test_headers_textfile_in_request(client, live_server):
# Add our URL to the import page
test_url = url_for('test_headers', _external=True)
if os.getenv('PLAYWRIGHT_DRIVER_URL'):
# Because its no longer calling back to localhost but from browserless, set in test-only.yml
# Because its no longer calling back to localhost but from the browser container, set in test-only.yml
test_url = test_url.replace('localhost', 'cdio')
print ("TEST URL IS ",test_url)