Dont use default Requests user-agent and accept headers in playwright+selenium requests, breaks sites such as united.com. (#1004)

changedetectionio/content_fetcher.py

@@ -575,6 +575,11 @@ class html_requests(Fetcher):
             ignore_status_codes=False,
             current_css_filter=None):
 
+        # Make requests use a more modern looking user-agent
+        if not 'User-Agent' in request_headers:
+            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
+                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
+
         proxies = {}
 
         # Allows override the proxy on a per-request basis