Distill.io JSON export file importer (#592)

changedetectionio/tests/test_import.py

@@ -5,18 +5,17 @@ import time
 from flask import url_for
 
 from .util import live_server_setup
-
-
-def test_import(client, live_server):
-
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
+def test_import(client, live_server):
     # Give the endpoint time to spin up
     time.sleep(1)
 
     res = client.post(
         url_for("import_page"),
         data={
+            "distill-io": "",
             "urls": """https://example.com
 https://example.com tag1
 https://example.com tag1, other tag"""
@@ -26,3 +25,96 @@ https://example.com tag1, other tag"""
     assert b"3 Imported" in res.data
     assert b"tag1" in res.data
     assert b"other tag" in res.data
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+
+    # Clear flask alerts
+    res = client.get( url_for("index"))
+    res = client.get( url_for("index"))
+
+def xtest_import_skip_url(client, live_server):
+
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    res = client.post(
+        url_for("import_page"),
+        data={
+            "distill-io": "",
+            "urls": """https://example.com
+:ht000000broken
+"""
+        },
+        follow_redirects=True,
+    )
+    assert b"1 Imported" in res.data
+    assert b"ht000000broken" in res.data
+    assert b"1 Skipped" in res.data
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    # Clear flask alerts
+    res = client.get( url_for("index"))
+
+def test_import_distillio(client, live_server):
+
+    distill_data='''
+{
+    "client": {
+        "local": 1
+    },
+    "data": [
+        {
+            "name": "Unraid | News",
+            "uri": "https://unraid.net/blog",
+            "config": "{\\"selections\\":[{\\"frames\\":[{\\"index\\":0,\\"excludes\\":[],\\"includes\\":[{\\"type\\":\\"xpath\\",\\"expr\\":\\"(//div[@id='App']/div[contains(@class,'flex')]/main[contains(@class,'relative')]/section[contains(@class,'relative')]/div[@class='container']/div[contains(@class,'flex')]/div[contains(@class,'w-full')])[1]\\"}]}],\\"dynamic\\":true,\\"delay\\":2}],\\"ignoreEmptyText\\":true,\\"includeStyle\\":false,\\"dataAttr\\":\\"text\\"}",
+            "tags": ["nice stuff", "nerd-news"],
+            "content_type": 2,
+            "state": 40,
+            "schedule": "{\\"type\\":\\"INTERVAL\\",\\"params\\":{\\"interval\\":4447}}",
+            "ts": "2022-03-27T15:51:15.667Z"
+        }
+    ]
+}		   
+
+'''
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+    client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    res = client.post(
+        url_for("import_page"),
+        data={
+            "distill-io": distill_data,
+            "urls" : ''
+        },
+        follow_redirects=True,
+    )
+
+
+    assert b"Unable to read JSON file, was it broken?" not in res.data
+    assert b"1 Imported from Distill.io" in res.data
+
+    res = client.get( url_for("edit_page", uuid="first"))
+
+    assert b"https://unraid.net/blog" in res.data
+    assert b"Unraid | News" in res.data
+
+
+    # flask/wtforms should recode this, check we see it
+    # wtforms encodes it like id=&#39 ,but html.escape makes it like id=&#x27
+    # - so just check it manually :(
+    #import json
+    #import html
+    #d = json.loads(distill_data)
+    # embedded_d=json.loads(d['data'][0]['config'])
+    # x=html.escape(embedded_d['selections'][0]['frames'][0]['includes'][0]['expr']).encode('utf-8')
+    assert b"xpath:(//div[@id='App']/div[contains(@class,'flex')]/main[contains(@class,'relative')]/section[contains(@class,'relative')]/div[@class='container']/div[contains(@class,'flex')]/div[contains(@class,'w-full')])[1]" in res.data
+
+    # did the tags work?
+    res = client.get( url_for("index"))
+
+    assert b"nice stuff" in res.data
+    assert b"nerd-news" in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    # Clear flask alerts
+    res = client.get(url_for("index"))