diff --git a/README.md b/README.md index 1a6f38dc..b9c02b92 100755 --- a/README.md +++ b/README.md @@ -55,8 +55,7 @@ The system continuously scans the network for, **New devices**, **New connection - DB maintenance, Backup, Restore tools and CSV Export / Import - Help/FAQ Section - | ![Screen 1][screen1] | ![Screen 2][screen2] | - | -------------------- | -------------------- | + | ![Screen 1][screen1] | ![Screen 2][screen2] | | ![Screen 3][screen3] | ![Screen 4][screen4] | | ![Screen 5][screen5] | ![Screen 6][screen6] | | ![Report 1][report1] | ![Report 2][report2] | diff --git a/back/pialert.py b/back/pialert.py index cd9754fc..83921c69 100755 --- a/back/pialert.py +++ b/back/pialert.py @@ -51,11 +51,14 @@ fullConfPath = pialertPath + confPath fullDbPath = pialertPath + dbPath STOPARPSCAN = pialertPath + "/db/setting_stoparpscan" -time_now = datetime.datetime.now() -log_timestamp = time_now +time_started = datetime.datetime.now() +log_timestamp = time_started sql_connection = None +#------------------------------------------------------------------------------- +def timeNow(): + return datetime.datetime.now().replace(microsecond=0) #------------------------------------------------------------------------------- def file_print(*args): @@ -227,6 +230,11 @@ DDNS_USER = 'dynu_user' DDNS_PASSWORD = 'A0000000B0000000C0000000D0000000' DDNS_UPDATE_URL = 'https://api.dynu.com/nic/update?' +# Pholus settings +# ---------------------- +PHOLUS_ACTIVE = False +PHOLUS_TIMEOUT = 60 + # PIHOLE settings # ---------------------- PIHOLE_ACTIVE = False @@ -281,6 +289,14 @@ def closeDB (): #------------------------------------------------------------------------------- # Import user values +def check_config_dict(key, default, config): + if key in config: + return config[key] + else: + return default + +#------------------------------------------------------------------------------- + def importConfig (): # Specify globals so they can be overwritten with the new config @@ -302,6 +318,8 @@ def importConfig (): global DDNS_ACTIVE, DDNS_DOMAIN, DDNS_USER, DDNS_PASSWORD, DDNS_UPDATE_URL # PiHole global PIHOLE_ACTIVE, DHCP_ACTIVE + # Pholus + global PHOLUS_ACTIVE, PHOLUS_TIMEOUT # load the variables from pialert.conf config_file = Path(fullConfPath) @@ -309,68 +327,72 @@ def importConfig (): config_dict = {} exec(code, {"__builtins__": {}}, config_dict) - SCAN_SUBNETS = config_dict['SCAN_SUBNETS'] - PRINT_LOG = config_dict['PRINT_LOG'] - TIMEZONE = config_dict['TIMEZONE'] - PIALERT_WEB_PROTECTION = config_dict['PIALERT_WEB_PROTECTION'] - PIALERT_WEB_PASSWORD = config_dict['PIALERT_WEB_PASSWORD'] - INCLUDED_SECTIONS = config_dict['INCLUDED_SECTIONS'] - SCAN_CYCLE_MINUTES = config_dict['SCAN_CYCLE_MINUTES'] - DAYS_TO_KEEP_EVENTS = config_dict['DAYS_TO_KEEP_EVENTS'] - REPORT_DASHBOARD_URL = config_dict['REPORT_DASHBOARD_URL'] + SCAN_SUBNETS = check_config_dict('SCAN_SUBNETS', SCAN_SUBNETS , config_dict) + PRINT_LOG = check_config_dict('PRINT_LOG', PRINT_LOG , config_dict) + TIMEZONE = check_config_dict('TIMEZONE', TIMEZONE , config_dict) + PIALERT_WEB_PROTECTION = check_config_dict('PIALERT_WEB_PROTECTION', PIALERT_WEB_PROTECTION , config_dict) + PIALERT_WEB_PASSWORD = check_config_dict('PIALERT_WEB_PASSWORD', PIALERT_WEB_PASSWORD , config_dict) + INCLUDED_SECTIONS = check_config_dict('INCLUDED_SECTIONS', INCLUDED_SECTIONS , config_dict) + SCAN_CYCLE_MINUTES = check_config_dict('SCAN_CYCLE_MINUTES', SCAN_CYCLE_MINUTES , config_dict) + DAYS_TO_KEEP_EVENTS = check_config_dict('DAYS_TO_KEEP_EVENTS', DAYS_TO_KEEP_EVENTS , config_dict) + REPORT_DASHBOARD_URL = check_config_dict('REPORT_DASHBOARD_URL', REPORT_DASHBOARD_URL , config_dict) # Email - REPORT_MAIL = config_dict['REPORT_MAIL'] - SMTP_SERVER = config_dict['SMTP_SERVER'] - SMTP_PORT = config_dict['SMTP_PORT'] - REPORT_TO = config_dict['REPORT_TO'] - REPORT_FROM = config_dict['REPORT_FROM'] - SMTP_SKIP_LOGIN = config_dict['SMTP_SKIP_LOGIN'] - SMTP_USER = config_dict['SMTP_USER'] - SMTP_PASS = config_dict['SMTP_PASS'] - SMTP_SKIP_TLS = config_dict['SMTP_SKIP_TLS'] + REPORT_MAIL = check_config_dict('REPORT_MAIL', REPORT_MAIL , config_dict) + SMTP_SERVER = check_config_dict('SMTP_SERVER', SMTP_SERVER , config_dict) + SMTP_PORT = check_config_dict('SMTP_PORT', SMTP_PORT , config_dict) + REPORT_TO = check_config_dict('REPORT_TO', REPORT_TO , config_dict) + REPORT_FROM = check_config_dict('REPORT_FROM', REPORT_FROM , config_dict) + SMTP_SKIP_LOGIN = check_config_dict('SMTP_SKIP_LOGIN', SMTP_SKIP_LOGIN , config_dict) + SMTP_USER = check_config_dict('SMTP_USER', SMTP_USER , config_dict) + SMTP_PASS = check_config_dict('SMTP_PASS', SMTP_PASS , config_dict) + SMTP_SKIP_TLS = check_config_dict('SMTP_SKIP_TLS', SMTP_SKIP_TLS , config_dict) # Webhooks - REPORT_WEBHOOK = config_dict['REPORT_WEBHOOK'] - WEBHOOK_URL = config_dict['WEBHOOK_URL'] - WEBHOOK_PAYLOAD = config_dict['WEBHOOK_PAYLOAD'] - WEBHOOK_REQUEST_METHOD = config_dict['WEBHOOK_REQUEST_METHOD'] + REPORT_WEBHOOK = check_config_dict('REPORT_WEBHOOK', REPORT_WEBHOOK , config_dict) + WEBHOOK_URL = check_config_dict('WEBHOOK_URL', WEBHOOK_URL , config_dict) + WEBHOOK_PAYLOAD = check_config_dict('WEBHOOK_PAYLOAD', WEBHOOK_PAYLOAD , config_dict) + WEBHOOK_REQUEST_METHOD = check_config_dict('WEBHOOK_REQUEST_METHOD', WEBHOOK_REQUEST_METHOD , config_dict) # Apprise - REPORT_APPRISE = config_dict['REPORT_APPRISE'] - APPRISE_HOST = config_dict['APPRISE_HOST'] - APPRISE_URL = config_dict['APPRISE_URL'] + REPORT_APPRISE = check_config_dict('REPORT_APPRISE', REPORT_APPRISE , config_dict) + APPRISE_HOST = check_config_dict('APPRISE_HOST', APPRISE_HOST , config_dict) + APPRISE_URL = check_config_dict('APPRISE_URL', APPRISE_URL , config_dict) # NTFY - REPORT_NTFY = config_dict['REPORT_NTFY'] - NTFY_HOST = config_dict['NTFY_HOST'] - NTFY_TOPIC = config_dict['NTFY_TOPIC'] - NTFY_USER = config_dict['NTFY_USER'] - NTFY_PASSWORD = config_dict['NTFY_PASSWORD'] + REPORT_NTFY = check_config_dict('REPORT_NTFY', REPORT_NTFY , config_dict) + NTFY_HOST = check_config_dict('NTFY_HOST', NTFY_HOST , config_dict) + NTFY_TOPIC = check_config_dict('NTFY_TOPIC', NTFY_TOPIC , config_dict) + NTFY_USER = check_config_dict('NTFY_USER', NTFY_USER , config_dict) + NTFY_PASSWORD = check_config_dict('NTFY_PASSWORD', NTFY_PASSWORD , config_dict) # PUSHSAFER - REPORT_PUSHSAFER = config_dict['REPORT_PUSHSAFER'] - PUSHSAFER_TOKEN = config_dict['PUSHSAFER_TOKEN'] + REPORT_PUSHSAFER = check_config_dict('REPORT_PUSHSAFER', REPORT_PUSHSAFER , config_dict) + PUSHSAFER_TOKEN = check_config_dict('PUSHSAFER_TOKEN', PUSHSAFER_TOKEN , config_dict) # MQTT - REPORT_MQTT = config_dict['REPORT_MQTT'] - MQTT_BROKER = config_dict['MQTT_BROKER'] - MQTT_PORT = config_dict['MQTT_PORT'] - MQTT_USER = config_dict['MQTT_USER'] - MQTT_PASSWORD = config_dict['MQTT_PASSWORD'] - MQTT_QOS = config_dict['MQTT_QOS'] - MQTT_DELAY_SEC = config_dict['MQTT_DELAY_SEC'] + REPORT_MQTT = check_config_dict('REPORT_MQTT', REPORT_MQTT , config_dict) + MQTT_BROKER = check_config_dict('MQTT_BROKER', MQTT_BROKER , config_dict) + MQTT_PORT = check_config_dict('MQTT_PORT', MQTT_PORT , config_dict) + MQTT_USER = check_config_dict('MQTT_USER', MQTT_USER , config_dict) + MQTT_PASSWORD = check_config_dict('MQTT_PASSWORD', MQTT_PASSWORD , config_dict) + MQTT_QOS = check_config_dict('MQTT_QOS', MQTT_QOS , config_dict) + MQTT_DELAY_SEC = check_config_dict('MQTT_DELAY_SEC', MQTT_DELAY_SEC , config_dict) # DynDNS - DDNS_ACTIVE = config_dict['DDNS_ACTIVE'] - DDNS_DOMAIN = config_dict['DDNS_DOMAIN'] - DDNS_USER = config_dict['DDNS_USER'] - DDNS_PASSWORD = config_dict['DDNS_PASSWORD'] - DDNS_UPDATE_URL = config_dict['DDNS_UPDATE_URL'] + DDNS_ACTIVE = check_config_dict('DDNS_ACTIVE', DDNS_ACTIVE , config_dict) + DDNS_DOMAIN = check_config_dict('DDNS_DOMAIN', DDNS_DOMAIN , config_dict) + DDNS_USER = check_config_dict('DDNS_USER', DDNS_USER , config_dict) + DDNS_PASSWORD = check_config_dict('DDNS_PASSWORD', DDNS_PASSWORD , config_dict) + DDNS_UPDATE_URL = check_config_dict('DDNS_UPDATE_URL', DDNS_UPDATE_URL , config_dict) # PiHole - PIHOLE_ACTIVE = config_dict['PIHOLE_ACTIVE'] - DHCP_ACTIVE = config_dict['DHCP_ACTIVE'] + PIHOLE_ACTIVE = check_config_dict('PIHOLE_ACTIVE', PIHOLE_ACTIVE, config_dict) + DHCP_ACTIVE = check_config_dict('DHCP_ACTIVE', DHCP_ACTIVE , config_dict) + + # PHOLUS + PHOLUS_ACTIVE = check_config_dict('PHOLUS_ACTIVE', PHOLUS_ACTIVE , config_dict) + PHOLUS_TIMEOUT = check_config_dict('PHOLUS_TIMEOUT', PHOLUS_TIMEOUT , config_dict) openDB() @@ -439,8 +461,12 @@ def importConfig (): ('DDNS_UPDATE_URL', 'DynDNS update URL', '', 'text', '', '' , str(DDNS_UPDATE_URL) , 'DynDNS'), # PiHole - ('PIHOLE_ACTIVE', 'Enable PiHole mapping', 'If enabled you need to map /etc/pihole/pihole-FTL.db in your docker-compose.yml', 'boolean', '', '' , str(PIHOLE_ACTIVE) , 'PiHole'), - ('DHCP_ACTIVE', 'Enable PiHole DHCP', 'If enabled you need to map /etc/pihole/dhcp.leases in your docker-compose.yml', 'boolean', '', '' , str(DHCP_ACTIVE) , 'PiHole') + ('PIHOLE_ACTIVE', 'Enable PiHole mapping', '', 'boolean', '', '' , str(PIHOLE_ACTIVE) , 'PiHole'), + ('DHCP_ACTIVE', 'Enable PiHole DHCP', '', 'boolean', '', '' , str(DHCP_ACTIVE) , 'PiHole'), + + # Pholus + ('PHOLUS_ACTIVE', 'Enable Pholus scans', '', 'boolean', '', '' , str(PHOLUS_ACTIVE) , 'Pholus'), + ('PHOLUS_TIMEOUT', 'Pholus timeout', '', 'integer', '', '' , str(PHOLUS_TIMEOUT) , 'Pholus') ] # Insert into DB @@ -452,8 +478,6 @@ def importConfig (): closeDB() #------------------------------------------------------------------------------- -# importConfig() - #=============================================================================== # USER CONFIG VARIABLES - END #=============================================================================== @@ -466,18 +490,18 @@ check_report = [1, "internet_IP", "update_vendors_silent"] mqtt_thread_up = False # timestamps of last execution times -startTime = time_now -now_minus_24h = time_now - timedelta(hours = 24) +startTime = time_started +now_minus_24h = time_started - timedelta(hours = 24) last_network_scan = now_minus_24h last_internet_IP_scan = now_minus_24h last_run = now_minus_24h last_cleanup = now_minus_24h -last_update_vendors = time_now - timedelta(days = 6) # update vendors 24h after first run and than once a week +last_update_vendors = time_started - timedelta(days = 6) # update vendors 24h after first run and than once a week def main (): # Initialize global variables - global time_now, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors, mqtt_thread_up + global time_started, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors, mqtt_thread_up # second set of global variables global startTime, log_timestamp, sql_connection, sql @@ -498,41 +522,41 @@ def main (): while True: # update NOW time - time_now = datetime.datetime.now() + time_started = datetime.datetime.now() # re-load user configuration importConfig() # proceed if 1 minute passed - if last_run + timedelta(minutes=1) < time_now : + if last_run + timedelta(minutes=1) < time_started : # last time any scan or maintennace/Upkeep was run - last_run = time_now + last_run = time_started reporting = False # Header updateState("Process: Start") - file_print('[', time_now.replace (microsecond=0), '] Process: Start') + file_print('[', timeNow(), '] Process: Start') # Timestamp - startTime = time_now - startTime = startTime.replace (microsecond=0) + startTime = time_started + startTime = startTime.replace (microsecond=0) # determine run/scan type based on passed time - if last_internet_IP_scan + timedelta(minutes=3) < time_now: + if last_internet_IP_scan + timedelta(minutes=3) < time_started: cycle = 'internet_IP' - last_internet_IP_scan = time_now + last_internet_IP_scan = time_started reporting = check_internet_IP() # Update vendors once a week - if last_update_vendors + timedelta(days = 7) < time_now: - last_update_vendors = time_now + if last_update_vendors + timedelta(days = 7) < time_started: + last_update_vendors = time_started cycle = 'update_vendors' update_devices_MAC_vendors() - if last_network_scan + timedelta(minutes=SCAN_CYCLE_MINUTES) < time_now and os.path.exists(STOPARPSCAN) == False: - last_network_scan = time_now + if last_network_scan + timedelta(minutes=SCAN_CYCLE_MINUTES) < time_started and os.path.exists(STOPARPSCAN) == False: + last_network_scan = time_started cycle = 1 # network scan scan_network() @@ -541,8 +565,8 @@ def main (): email_reporting() # clean up the DB once a day - if last_cleanup + timedelta(hours = 24) < time_now: - last_cleanup = time_now + if last_cleanup + timedelta(hours = 24) < time_started: + last_cleanup = time_started cycle = 'cleanup' cleanup_database() @@ -554,12 +578,12 @@ def main (): action = str(cycle) if action == "1": action = "network_scan" - file_print('[', time_now.replace (microsecond=0), '] Last action: ', action) + file_print('[', timeNow(), '] Last action: ', action) cycle = "" # Footer updateState("Process: Wait") - file_print('[', time_now.replace (microsecond=0), '] Process: Wait') + file_print('[', timeNow(), '] Process: Wait') else: # do something cycle = "" @@ -755,9 +779,13 @@ def cleanup_database (): file_print(' Optimize Database') # Cleanup Events - file_print(' Upkeep Events, up to the lastest '+str(DAYS_TO_KEEP_EVENTS)+' days') + file_print(' Upkeep Events, delete all older than '+str(DAYS_TO_KEEP_EVENTS)+' days') sql.execute ("DELETE FROM Events WHERE eve_DateTime <= date('now', '-"+str(DAYS_TO_KEEP_EVENTS)+" day')") + # Cleanup Pholus_Scan + file_print(' Upkeep Pholus_Scan, delete all older than 7 days') + sql.execute ("DELETE FROM Pholus_Scan WHERE Time <= date('now', '-7 day')") + # Shrink DB file_print(' Shrink Database') sql.execute ("VACUUM;") @@ -1061,6 +1089,10 @@ def execute_arpscan_on_interface (SCAN_SUBNETS): # Retry is 6 to avoid false offline devices arpscan_args = ['sudo', 'arp-scan', '--ignoredups', '--retry=6'] + subnets + + mask = subnets[0] + interface = subnets[1].split('=')[1] + # Execute command try: # try runnning a subprocess @@ -1070,6 +1102,9 @@ def execute_arpscan_on_interface (SCAN_SUBNETS): file_print(e.output) result = "" + if PHOLUS_ACTIVE: + performPholusScan(interface, mask) + return result #------------------------------------------------------------------------------- @@ -1124,6 +1159,9 @@ def read_DHCP_leases (): #------------------------------------------------------------------------------- def save_scanned_devices (p_arpscan_devices, p_cycle_interval): cycle = 1 # always 1, only one cycle supported + + openDB() + # Delete previous scan data sql.execute ("DELETE FROM CurrentScan WHERE cur_ScanCycle = ?", (cycle,)) @@ -1541,7 +1579,7 @@ def update_devices_names (): # BUGFIX #97 - Updating name of Devices w/o IP for device in sql.execute ("SELECT * FROM Devices WHERE dev_Name IN ('(unknown)','') AND dev_LastIP <> '-'") : # Resolve device name - newName = resolve_device_name (device['dev_MAC'], device['dev_LastIP']) + newName = resolve_device_name (device['dev_MAC'], device['dev_LastIP']) #<<<< continue here TODO DEV > get >latest< Pholus scan results and match if newName == -1 : notFound += 1 @@ -1561,6 +1599,42 @@ def update_devices_names (): # DEBUG - print number of rows updated # file_print(sql.rowcount) +#------------------------------------------------------------------------------- +def performPholusScan (interface, mask): + updateState("Scan: Pholus") + file_print('[', timeNow(), '] Scan: Pholus') + + pholus_args = ['python3', '/home/pi/pialert/pholus/pholus3.py', interface, "-rdns_scanning", mask, "-stimeout", str(PHOLUS_TIMEOUT)] + + # Execute command + try: + # try runnning a subprocess + output = subprocess.check_output (pholus_args, universal_newlines=True) + except subprocess.CalledProcessError as e: + # An error occured, handle it + file_print(e.output) + file_print("Error - PholusScan - check logs") + output = "" + + if output != "": + file_print('[', timeNow(), '] Scan: Pholus SUCCESS') + write_file (logPath + '/pialert_pholus.log', output) + + params = [] + + for line in output.split("\n"): + columns = line.split("|") + if len(columns) == 4: + params.append(( interface + " " + mask, timeNow() , columns[0], columns[1], columns[2], columns[3], '')) + + if len(params) > 0: + openDB () + sql.executemany ("""INSERT INTO Pholus_Scan ("Info", "Time", "MAC", "IP_v4_or_v6", "Record_Type", "Value", "Extra") VALUES (?, ?, ?, ?, ?, ?, ?)""", params) + + else: + file_print('[', timeNow(), '] Scan: Pholus FAIL - check logs') + + #------------------------------------------------------------------------------- def resolve_device_name (pMAC, pIP): try : @@ -2582,6 +2656,33 @@ def upgradeDB (): "Group" TEXT ); """) + + # indicates, if Pholus_Scan table is available + pholusScanMissing = sql.execute(""" + SELECT name FROM sqlite_master WHERE type='table' + AND name='Pholus_Scan'; + """).fetchone() == None + + # Re-creating Pholus_Scan table + file_print("[upgradeDB] Re-creating Pholus_Scan table") + + # if pholusScanMissing == False: + # sql.execute("DROP TABLE Pholus_Scan;") + + if pholusScanMissing: + sql.execute(""" + CREATE TABLE "Pholus_Scan" ( + "Index" INTEGER, + "Info" TEXT, + "Time" TEXT, + "MAC" TEXT, + "IP_v4_or_v6" TEXT, + "Record_Type" TEXT, + "Value" TEXT, + "Extra" TEXT, + PRIMARY KEY("Index" AUTOINCREMENT) + ); + """) # don't hog DB access closeDB () diff --git a/front/css/pialert.css b/front/css/pialert.css index 8d9ab1d5..1bf797bd 100755 --- a/front/css/pialert.css +++ b/front/css/pialert.css @@ -34,6 +34,10 @@ display: grid; margin: 5px; } +.logs-size +{ + font-size: 14px; +} .log-area { padding: 3px; diff --git a/front/maintenance.php b/front/maintenance.php index c21d3b73..b394c3e2 100755 --- a/front/maintenance.php +++ b/front/maintenance.php @@ -348,12 +348,12 @@ if (submit && isset($_POST['skinselector_set'])) {
-
-
pialert.log
+
pialert.log
@@ -367,13 +367,29 @@ if (submit && isset($_POST['skinselector_set'])) {
-
pialert_front.log
+
pialert_front.log
-
+
+
+ +
+ +
+
+
+
pialert_pholus.log
+
+ +
+
+
+ +
@@ -382,7 +398,7 @@ if (submit && isset($_POST['skinselector_set'])) {
-
IP_changes.log
+
IP_changes.log
@@ -397,7 +413,7 @@ if (submit && isset($_POST['skinselector_set'])) {
-
stdout.log
+
stdout.log
@@ -412,7 +428,7 @@ if (submit && isset($_POST['skinselector_set'])) {
-
stderr.log
+
stderr.log
diff --git a/front/php/templates/language/en_us.php b/front/php/templates/language/en_us.php index a7bc685e..fde45249 100755 --- a/front/php/templates/language/en_us.php +++ b/front/php/templates/language/en_us.php @@ -547,6 +547,12 @@ the scan will take hours to complete instead of seconds. 'DHCP_ACTIVE_name' => 'Enable PiHole DHCP', 'DHCP_ACTIVE_description' => 'If enabled you need to map :/etc/pihole/dhcp.leases in your docker-compose.yml file.', +// Pholus +'PHOLUS_ACTIVE_name' => 'Enable Pholus scan', +'PHOLUS_ACTIVE_description' => 'Pholus is a sniffing tool to discover additional information about the devices on the network, including the device name. Please be aware it can spam the network with unnecessary traffic.', +'PHOLUS_TIMEOUT_name' => 'Pholus timeout', +'PHOLUS_TIMEOUT_description' => 'How long (s) should Pholus be sniffing the network. Tested with 60s on a network with 50 devices.', + ); ?> diff --git a/front/settings.php b/front/settings.php index 7a9ba080..aadf5f30 100644 --- a/front/settings.php +++ b/front/settings.php @@ -242,7 +242,7 @@ $db->close();