Be sure that only valid history index lines are read

changedetectionio/__init__.py

@@ -33,7 +33,7 @@ from flask_wtf import CSRFProtect
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.39.20.3'
+__version__ = '0.39.20.4'
 
 datastore = None
 
@@ -819,8 +819,8 @@ def changedetection_app(config=None, datastore_o=None):
         # Read as binary and force decode as UTF-8
         # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
         try:
-            with open(newest_file, 'rb') as f:
-                newest_version_file_contents = f.read().decode('utf-8')
+            with open(newest_file, 'r', encoding='utf-8', errors='ignore') as f:
+                newest_version_file_contents = f.read()
         except Exception as e:
             newest_version_file_contents = "Unable to read {}.\n".format(newest_file)
 
@@ -832,8 +832,8 @@ def changedetection_app(config=None, datastore_o=None):
             previous_file = history[dates[-2]]
 
         try:
-            with open(previous_file, 'rb') as f:
-                previous_version_file_contents = f.read().decode('utf-8')
+            with open(previous_file, 'r', encoding='utf-8', errors='ignore') as f:
+                previous_version_file_contents = f.read()
         except Exception as e:
             previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
 
@@ -909,7 +909,7 @@ def changedetection_app(config=None, datastore_o=None):
         timestamp = list(watch.history.keys())[-1]
         filename = watch.history[timestamp]
         try:
-            with open(filename, 'r') as f:
+            with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
                 tmp = f.readlines()
 
                 # Get what needs to be highlighted

changedetectionio/model/Watch.py

@@ -118,7 +118,10 @@ class model(dict):
         if os.path.isfile(fname):
             logging.debug("Reading history index " + str(time.time()))
             with open(fname, "r") as f:
-                tmp_history = dict(i.strip().split(',', 2) for i in f.readlines())
+                for i in f.readlines():
+                    if ',' in i:
+                        k, v = i.strip().split(',', 2)
+                        tmp_history[k] = v
 
         if len(tmp_history):
             self.__newest_history_key = list(tmp_history.keys())[-1]
@@ -158,7 +161,8 @@ class model(dict):
 
         logging.debug("Saving history text {}".format(snapshot_fname))
 
-        # in /diff/ we are going to assume for now that it's UTF-8 when reading
+        # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
+        # most sites are utf-8 and some are even broken utf-8
         with open(snapshot_fname, 'wb') as f:
             f.write(contents)
             f.close()