Be sure that documents returned with a application/json header are not parsed with inscriptis (#337)

* Auto-detect JSON by Content-Type header
* Add test to not parse JSON responses with inscriptis
This commit is contained in:
Unpublished
2022-01-02 22:35:33 +01:00
committed by GitHub
parent fbd9ecab62
commit 023951a10e
4 changed files with 69 additions and 3 deletions

View File

@@ -15,6 +15,7 @@ class Fetcher():
error = None
status_code = None
content = None # Should always be bytes.
headers = None
fetcher_description ="No description"
@@ -113,6 +114,7 @@ class html_webdriver(Fetcher):
# @todo - dom wait loaded?
time.sleep(5)
self.content = driver.page_source
self.headers = {}
driver.quit()
@@ -156,4 +158,5 @@ class html_requests(Fetcher):
self.status_code = r.status_code
self.content = html
self.headers = r.headers