From 8256a84efeec40262e16029a59096bb97c4d050f Mon Sep 17 00:00:00 2001 From: Jokob-sk Date: Sun, 6 Aug 2023 11:16:43 +1000 Subject: [PATCH] ARPSCAN to plugin rewrite --- pialert/database.py | 20 ++++++++++ pialert/device.py | 55 ++++++++++++--------------- pialert/networkscan.py | 51 ++++++++----------------- pialert/scanners/arpscan.py | 74 ------------------------------------- 4 files changed, 59 insertions(+), 141 deletions(-) delete mode 100755 pialert/scanners/arpscan.py diff --git a/pialert/database.py b/pialert/database.py index b78d2e80..f33c59ef 100755 --- a/pialert/database.py +++ b/pialert/database.py @@ -385,6 +385,26 @@ class DB(): PRIMARY KEY("Index" AUTOINCREMENT) ); """) + self.commitDB() + + # indicates, if CurrentScan table is available + currentScanMissing = self.sql.execute(""" + SELECT name FROM sqlite_master WHERE type='table' + AND name='CurrentScan'; + """).fetchone() == None + + if currentScanMissing == False: + self.sql.execute("DROP TABLE CurrentScan;") + + self.sql.execute(""" CREATE TABLE CurrentScan ( + cur_ScanCycle INTEGER NOT NULL, + cur_MAC STRING(50) NOT NULL COLLATE NOCASE, + cur_IP STRING(50) NOT NULL COLLATE NOCASE, + cur_Vendor STRING(250), + cur_ScanMethod STRING(10) + ); + """) + self.commitDB() diff --git a/pialert/device.py b/pialert/device.py index 0bfb7148..a6d766e3 100755 --- a/pialert/device.py +++ b/pialert/device.py @@ -77,10 +77,9 @@ def save_scanned_devices (db): local_ip = '0.0.0.0' # Check if local mac has been detected with other methods - sql.execute ("SELECT COUNT(*) FROM CurrentScan WHERE cur_MAC = ? ", (local_mac) ) + sql.execute (f"SELECT COUNT(*) FROM CurrentScan WHERE cur_MAC = '{local_mac}'") if sql.fetchone()[0] == 0 : - sql.execute ("INSERT INTO CurrentScan (cur_ScanCycle, cur_MAC, cur_IP, cur_Vendor, cur_ScanMethod) "+ - "VALUES ( 1, ?, ?, Null, 'local_MAC') ", (local_mac, local_ip) ) + sql.execute (f"""INSERT INTO CurrentScan (cur_ScanCycle, cur_MAC, cur_IP, cur_Vendor, cur_ScanMethod) VALUES ( 1, '{local_mac}', '{local_ip}', Null, 'local_MAC') """) #------------------------------------------------------------------------------- def print_scan_stats (db): @@ -108,7 +107,7 @@ def print_scan_stats (db): WHERE dev_AlertDeviceDown = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) """) + ) """) mylog('verbose', ['[Scan Stats] Down Alerts........: ' + str (sql.fetchone()[0]) ]) # New Down Alerts @@ -117,12 +116,12 @@ def print_scan_stats (db): AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) """) + ) """) mylog('verbose', ['[Scan Stats] New Down Alerts....: ' + str (sql.fetchone()[0]) ]) # New Connections sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan - WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle + WHERE dev_MAC = cur_MAC AND dev_PresentLastScan = 0""") mylog('verbose', ['[Scan Stats] New Connections....: ' + str ( sql.fetchone()[0]) ]) @@ -131,12 +130,12 @@ def print_scan_stats (db): WHERE dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) """) + ) """) mylog('verbose', ['[Scan Stats] Disconnections.....: ' + str ( sql.fetchone()[0]) ]) # IP Changes sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan - WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle + WHERE dev_MAC = cur_MAC AND dev_LastIP <> cur_IP """) mylog('verbose', ['[Scan Stats] IP Changes.........: ' + str ( sql.fetchone()[0]) ]) @@ -149,23 +148,21 @@ def create_new_devices (db): # arpscan - Insert events for new devices mylog('debug','[New Devices] New devices - 1 Events') - sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, + sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, eve_EventType, eve_AdditionalInfo, eve_PendingAlertEmail) - SELECT cur_MAC, cur_IP, ?, 'New Device', cur_Vendor, 1 + SELECT cur_MAC, cur_IP, '{startTime}', 'New Device', cur_Vendor, 1 FROM CurrentScan WHERE NOT EXISTS (SELECT 1 FROM Devices - WHERE dev_MAC = cur_MAC) """, - (startTime) ) + WHERE dev_MAC = cur_MAC) """ ) mylog('debug','[New Devices] Insert Connection into session table') - sql.execute ("""INSERT INTO Sessions (ses_MAC, ses_IP, ses_EventTypeConnection, ses_DateTimeConnection, + sql.execute (f"""INSERT INTO Sessions (ses_MAC, ses_IP, ses_EventTypeConnection, ses_DateTimeConnection, ses_EventTypeDisconnection, ses_DateTimeDisconnection, ses_StillConnected, ses_AdditionalInfo) - SELECT cur_MAC, cur_IP,'Connected',?, NULL , NULL ,1, cur_Vendor + SELECT cur_MAC, cur_IP,'Connected','{startTime}', NULL , NULL ,1, cur_Vendor FROM CurrentScan WHERE NOT EXISTS (SELECT 1 FROM Sessions - WHERE ses_MAC = cur_MAC) """, - (startTime) ) + WHERE ses_MAC = cur_MAC) """) # arpscan - Create new devices mylog('debug','[New Devices] 2 Create devices') @@ -254,14 +251,13 @@ def create_new_devices (db): # DHCP Leases - Insert events for new devices mylog('debug','[New Devices] 5 DHCP Leases Events') - sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, + sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, eve_EventType, eve_AdditionalInfo, eve_PendingAlertEmail) - SELECT DHCP_MAC, DHCP_IP, ?, 'New Device', '(DHCP lease)',1 + SELECT DHCP_MAC, DHCP_IP, '{startTime}', 'New Device', '(DHCP lease)',1 FROM DHCP_Leases WHERE NOT EXISTS (SELECT 1 FROM Devices - WHERE dev_MAC = DHCP_MAC) """, - (startTime, ) ) + WHERE dev_MAC = DHCP_MAC) """) # DHCP Leases - Create New Devices mylog('debug','[New Devices] 6 DHCP Leases Create devices') @@ -296,33 +292,28 @@ def update_devices_data_from_scan (db): startTime = timeNow() # Update Last Connection mylog('debug','[Update Devices] 1 Last Connection') - sql.execute ("""UPDATE Devices SET dev_LastConnection = ?, + sql.execute (f"""UPDATE Devices SET dev_LastConnection = '{startTime}', dev_PresentLastScan = 1 - WHERE dev_ScanCycle = ? - AND dev_PresentLastScan = 0 + WHERE dev_PresentLastScan = 0 AND EXISTS (SELECT 1 FROM CurrentScan - WHERE dev_MAC = cur_MAC) """, - (startTime)) + WHERE dev_MAC = cur_MAC) """) # Clean no active devices mylog('debug','[Update Devices] 2 Clean no active devices') sql.execute ("""UPDATE Devices SET dev_PresentLastScan = 0 WHERE NOT EXISTS (SELECT 1 FROM CurrentScan - WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) """) + WHERE dev_MAC = cur_MAC) """) # Update IP & Vendor mylog('debug','[Update Devices] - 3 LastIP & Vendor') sql.execute ("""UPDATE Devices SET dev_LastIP = (SELECT cur_IP FROM CurrentScan - WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle), + WHERE dev_MAC = cur_MAC), dev_Vendor = (SELECT cur_Vendor FROM CurrentScan WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) + ) WHERE EXISTS (SELECT 1 FROM CurrentScan - WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) """) + WHERE dev_MAC = cur_MAC) """) # Pi-hole Network - Update (unknown) Name mylog('debug','[Update Devices] - 4 Unknown Name') diff --git a/pialert/networkscan.py b/pialert/networkscan.py index bc575c2c..d210f922 100755 --- a/pialert/networkscan.py +++ b/pialert/networkscan.py @@ -1,7 +1,7 @@ import conf -from scanners.arpscan import execute_arpscan + from scanners.pihole import copy_pihole_network, read_DHCP_leases from database import insertOnlineHistory from device import create_new_devices, print_scan_stats, save_scanned_devices, update_devices_data_from_scan, update_devices_names @@ -36,14 +36,6 @@ def scan_network (db): db.commitDB() - # Moved to the ARPSCAN Plugin - # arp-scan command - # conf.arpscan_devices = [] - # if conf.ENABLE_ARPSCAN: - # mylog('verbose','[Network Scan] arp-scan start') - # conf.arpscan_devices = execute_arpscan (conf.userSubnets) - # mylog('verbose','[Network Scan] arp-scan ends') - # Pi-hole method if conf.PIHOLE_ACTIVE : mylog('verbose','[Network Scan] Pi-hole start') @@ -213,14 +205,7 @@ def void_ghost_disconnections (db): #------------------------------------------------------------------------------- def pair_sessions_events (db): sql = db.sql #TO-DO - - # NOT NECESSARY FOR INCREMENTAL UPDATE - # print_log ('Pair session - 1 Clean') - # sql.execute ("""UPDATE Events - # SET eve_PairEventRowid = NULL - # WHERE eve_EventType IN ('New Device', 'Connected') - # """ ) - + # Pair Connection / New Device events mylog('debug','[Pair Session] - 1 Connections / New Devices') @@ -275,53 +260,49 @@ def insert_events (db): # Check device down mylog('debug','[Events] - 1 - Devices down') - sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, + sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, eve_EventType, eve_AdditionalInfo, eve_PendingAlertEmail) - SELECT dev_MAC, dev_LastIP, ?, 'Device Down', '', 1 + SELECT dev_MAC, dev_LastIP, '{startTime}', 'Device Down', '', 1 FROM Devices WHERE dev_AlertDeviceDown = 1 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) """, - (startTime) ) + ) """) # Check new connections mylog('debug','[Events] - 2 - New Connections') - sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, + sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, eve_EventType, eve_AdditionalInfo, eve_PendingAlertEmail) - SELECT cur_MAC, cur_IP, ?, 'Connected', '', dev_AlertEvents + SELECT cur_MAC, cur_IP, '{startTime}', 'Connected', '', dev_AlertEvents FROM Devices, CurrentScan - WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle - AND dev_PresentLastScan = 0 """, - (startTime) ) + WHERE dev_MAC = cur_MAC + AND dev_PresentLastScan = 0 """) # Check disconnections mylog('debug','[Events] - 3 - Disconnections') - sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, + sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, eve_EventType, eve_AdditionalInfo, eve_PendingAlertEmail) - SELECT dev_MAC, dev_LastIP, ?, 'Disconnected', '', + SELECT dev_MAC, dev_LastIP, '{startTime}', 'Disconnected', '', dev_AlertEvents FROM Devices WHERE dev_AlertDeviceDown = 0 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC - AND dev_ScanCycle = cur_ScanCycle) """, - (startTime) ) + ) """) # Check IP Changed mylog('debug','[Events] - 4 - IP Changes') - sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, + sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime, eve_EventType, eve_AdditionalInfo, eve_PendingAlertEmail) - SELECT cur_MAC, cur_IP, ?, 'IP Changed', + SELECT cur_MAC, cur_IP, '{startTime}', 'IP Changed', 'Previous IP: '|| dev_LastIP, dev_AlertEvents FROM Devices, CurrentScan - WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle - AND dev_LastIP <> cur_IP """, - (startTime) ) + WHERE dev_MAC = cur_MAC + AND dev_LastIP <> cur_IP """ ) mylog('debug','[Events] - Events end') \ No newline at end of file diff --git a/pialert/scanners/arpscan.py b/pialert/scanners/arpscan.py deleted file mode 100755 index 42a9c502..00000000 --- a/pialert/scanners/arpscan.py +++ /dev/null @@ -1,74 +0,0 @@ -import re -import subprocess -import conf - -from logger import mylog -from helper import write_file -from const import logPath - -#------------------------------------------------------------------------------- -def execute_arpscan (userSubnets): - - # output of possible multiple interfaces - arpscan_output = "" - - # scan each interface - index = 0 - for interface in userSubnets : - arpscan_output += execute_arpscan_on_interface (interface) - index += 1 - - # Search IP + MAC + Vendor as regular expresion - re_ip = r'(?P((2[0-5]|1[0-9]|[0-9])?[0-9]\.){3}((2[0-5]|1[0-9]|[0-9])?[0-9]))' - re_mac = r'(?P([0-9a-fA-F]{2}[:-]){5}([0-9a-fA-F]{2}))' - re_hw = r'(?P.*)' - re_pattern = re.compile (re_ip + '\s+' + re_mac + '\s' + re_hw) - - # Create Userdict of devices - devices_list = [device.groupdict() - for device in re.finditer (re_pattern, arpscan_output)] - - mylog('debug', ['[ARP Scan] Found: Devices including duplicates ', len(devices_list) ]) - - # Delete duplicate MAC - unique_mac = [] - unique_devices = [] - - for device in devices_list : - if device['mac'] not in unique_mac: - unique_mac.append(device['mac']) - unique_devices.append(device) - - # return list - mylog('debug', ['[ARP Scan] Found: Devices without duplicates ', len(unique_devices) ]) - - return unique_devices - -#------------------------------------------------------------------------------- -def execute_arpscan_on_interface (interface): - # Prepare command arguments - subnets = interface.strip().split() - # Retry is 6 to avoid false offline devices - mylog('debug', ['[ARP Scan] - arpscan command: sudo arp-scan --ignoredups --retry=6 ', str(subnets)]) - arpscan_args = ['sudo', 'arp-scan', '--ignoredups', '--retry=6'] + subnets - - # Execute command - if conf.LOG_LEVEL == 'debug': - # try runnning a subprocess - result = subprocess.check_output (arpscan_args, universal_newlines=True) - else: - try: - # try runnning a subprocess safely - result = subprocess.check_output (arpscan_args, universal_newlines=True) - except subprocess.CalledProcessError as e: - # An error occured, handle it - error_type = type(e).__name__ # Capture the error type - - mylog('none', [f'[ARP Scan] Error type : {error_type}']) - mylog('none', [f'[ARP Scan] Set LOG_LEVEL=debug for more details']) - mylog('none', [f'[ARP Scan] Error output: {e.output}']) - - result = "" - - mylog('debug', ['[ARP Scan] on Interface Completed with results: ', result]) - return result