ARPSCAN to plugin rewrite

pialert/database.py

@@ -387,6 +387,26 @@ class DB():
 
         self.commitDB()        
         
+        # indicates, if CurrentScan table is available
+        currentScanMissing = self.sql.execute("""
+        SELECT name FROM sqlite_master WHERE type='table'
+        AND name='CurrentScan';
+        """).fetchone() == None
+
+        if currentScanMissing == False:
+            self.sql.execute("DROP TABLE CurrentScan;")
+
+        self.sql.execute(""" CREATE TABLE CurrentScan (
+                                cur_ScanCycle INTEGER NOT NULL,
+                                cur_MAC STRING(50) NOT NULL COLLATE NOCASE,
+                                cur_IP STRING(50) NOT NULL COLLATE NOCASE,
+                                cur_Vendor STRING(250),
+                                cur_ScanMethod STRING(10)
+                            );
+                        """)
+
+        self.commitDB()
+
 
     #-------------------------------------------------------------------------------
     def get_table_as_json(self, sqlQuery):

pialert/device.py

@@ -77,10 +77,9 @@ def save_scanned_devices (db):
         local_ip = '0.0.0.0'
 
     # Check if local mac has been detected with other methods
-    sql.execute ("SELECT COUNT(*) FROM CurrentScan WHERE cur_MAC = ? ", (local_mac) )
+    sql.execute (f"SELECT COUNT(*) FROM CurrentScan WHERE cur_MAC = '{local_mac}'")
     if sql.fetchone()[0] == 0 :
-        sql.execute ("INSERT INTO CurrentScan (cur_ScanCycle, cur_MAC, cur_IP, cur_Vendor, cur_ScanMethod) "+
+        sql.execute (f"""INSERT INTO CurrentScan (cur_ScanCycle, cur_MAC, cur_IP, cur_Vendor, cur_ScanMethod) VALUES ( 1, '{local_mac}', '{local_ip}', Null, 'local_MAC') """)
-                     "VALUES ( 1, ?, ?, Null, 'local_MAC') ", (local_mac, local_ip) )
 
 #-------------------------------------------------------------------------------
 def print_scan_stats (db):
@@ -108,7 +107,7 @@ def print_scan_stats (db):
                     WHERE dev_AlertDeviceDown = 1                      
                       AND NOT EXISTS (SELECT 1 FROM CurrentScan
                                       WHERE dev_MAC = cur_MAC
-                                        AND dev_ScanCycle = cur_ScanCycle) """)
+                                         ) """)
     mylog('verbose', ['[Scan Stats]        Down Alerts........: ' + str (sql.fetchone()[0]) ])
 
     # New Down Alerts
@@ -117,12 +116,12 @@ def print_scan_stats (db):
                       AND dev_PresentLastScan = 1                      
                       AND NOT EXISTS (SELECT 1 FROM CurrentScan
                                       WHERE dev_MAC = cur_MAC
-                                        AND dev_ScanCycle = cur_ScanCycle) """)
+                                         ) """)
     mylog('verbose', ['[Scan Stats]        New Down Alerts....: ' + str (sql.fetchone()[0]) ])
 
     # New Connections
     sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan
-                    WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle
+                    WHERE dev_MAC = cur_MAC  
                       AND dev_PresentLastScan = 0""")                      
     mylog('verbose', ['[Scan Stats]        New Connections....: ' + str ( sql.fetchone()[0]) ])
 
@@ -131,12 +130,12 @@ def print_scan_stats (db):
                     WHERE dev_PresentLastScan = 1                      
                       AND NOT EXISTS (SELECT 1 FROM CurrentScan
                                       WHERE dev_MAC = cur_MAC
-                                        AND dev_ScanCycle = cur_ScanCycle) """)
+                                         ) """)
     mylog('verbose', ['[Scan Stats]        Disconnections.....: ' + str ( sql.fetchone()[0]) ])
 
     # IP Changes
     sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan
-                    WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle                      
+                    WHERE dev_MAC = cur_MAC                        
                       AND dev_LastIP <> cur_IP """)
     mylog('verbose', ['[Scan Stats]        IP Changes.........: ' + str ( sql.fetchone()[0]) ])
 
@@ -149,23 +148,21 @@ def create_new_devices (db):
 
     # arpscan - Insert events for new devices
     mylog('debug','[New Devices] New devices - 1 Events')
-    sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
                         eve_EventType, eve_AdditionalInfo,
                         eve_PendingAlertEmail)
-                    SELECT cur_MAC, cur_IP, ?, 'New Device', cur_Vendor, 1
+                    SELECT cur_MAC, cur_IP, '{startTime}', 'New Device', cur_Vendor, 1
                     FROM CurrentScan
                     WHERE NOT EXISTS (SELECT 1 FROM Devices
-                                      WHERE dev_MAC = cur_MAC) """,
+                                      WHERE dev_MAC = cur_MAC) """ ) 
-                    (startTime) ) 
 
     mylog('debug','[New Devices] Insert Connection into session table')
-    sql.execute ("""INSERT INTO Sessions (ses_MAC, ses_IP, ses_EventTypeConnection, ses_DateTimeConnection,
+    sql.execute (f"""INSERT INTO Sessions (ses_MAC, ses_IP, ses_EventTypeConnection, ses_DateTimeConnection,
                         ses_EventTypeDisconnection, ses_DateTimeDisconnection, ses_StillConnected, ses_AdditionalInfo)
-                    SELECT cur_MAC, cur_IP,'Connected',?, NULL , NULL ,1, cur_Vendor
+                    SELECT cur_MAC, cur_IP,'Connected','{startTime}', NULL , NULL ,1, cur_Vendor
                     FROM CurrentScan 
                     WHERE NOT EXISTS (SELECT 1 FROM Sessions
-                                      WHERE ses_MAC = cur_MAC) """,
+                                      WHERE ses_MAC = cur_MAC) """)
-                    (startTime) ) 
                     
     # arpscan - Create new devices
     mylog('debug','[New Devices] 2 Create devices')
@@ -254,14 +251,13 @@ def create_new_devices (db):
     # DHCP Leases - Insert events for new devices
     mylog('debug','[New Devices] 5 DHCP Leases Events')
 
-    sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
                         eve_EventType, eve_AdditionalInfo,
                         eve_PendingAlertEmail)
-                    SELECT DHCP_MAC, DHCP_IP, ?, 'New Device', '(DHCP lease)',1
+                    SELECT DHCP_MAC, DHCP_IP, '{startTime}', 'New Device', '(DHCP lease)',1
                     FROM DHCP_Leases
                     WHERE NOT EXISTS (SELECT 1 FROM Devices
-                                      WHERE dev_MAC = DHCP_MAC) """,
+                                      WHERE dev_MAC = DHCP_MAC) """) 
-                    (startTime, ) ) 
 
     # DHCP Leases - Create New Devices
     mylog('debug','[New Devices] 6 DHCP Leases Create devices')
@@ -296,33 +292,28 @@ def update_devices_data_from_scan (db):
     startTime = timeNow()
     # Update Last Connection
     mylog('debug','[Update Devices] 1 Last Connection')
-    sql.execute ("""UPDATE Devices SET dev_LastConnection = ?,
+    sql.execute (f"""UPDATE Devices SET dev_LastConnection = '{startTime}',
                         dev_PresentLastScan = 1
-                    WHERE dev_ScanCycle = ?
+                    WHERE dev_PresentLastScan = 0
-                      AND dev_PresentLastScan = 0
                       AND EXISTS (SELECT 1 FROM CurrentScan 
-                                  WHERE dev_MAC = cur_MAC) """,
+                                  WHERE dev_MAC = cur_MAC) """)
-                    (startTime))
 
     # Clean no active devices
     mylog('debug','[Update Devices] 2 Clean no active devices')
     sql.execute ("""UPDATE Devices SET dev_PresentLastScan = 0
                     WHERE NOT EXISTS (SELECT 1 FROM CurrentScan 
-                                      WHERE dev_MAC = cur_MAC
+                                      WHERE dev_MAC = cur_MAC) """)
-                                        AND dev_ScanCycle = cur_ScanCycle) """)
 
     # Update IP & Vendor
     mylog('debug','[Update Devices] - 3 LastIP & Vendor')
     sql.execute ("""UPDATE Devices
                     SET dev_LastIP = (SELECT cur_IP FROM CurrentScan
-                                      WHERE dev_MAC = cur_MAC
+                                      WHERE dev_MAC = cur_MAC),
-                                        AND dev_ScanCycle = cur_ScanCycle),
                         dev_Vendor = (SELECT cur_Vendor FROM CurrentScan
                                       WHERE dev_MAC = cur_MAC
-                                        AND dev_ScanCycle = cur_ScanCycle)
+                                        )
                     WHERE EXISTS (SELECT 1 FROM CurrentScan
-                                  WHERE dev_MAC = cur_MAC
+                                  WHERE dev_MAC = cur_MAC) """) 
-                                    AND dev_ScanCycle = cur_ScanCycle) """) 
 
     # Pi-hole Network - Update (unknown) Name
     mylog('debug','[Update Devices] - 4 Unknown Name')

pialert/networkscan.py

@@ -1,7 +1,7 @@
 
 
 import conf
-from scanners.arpscan import execute_arpscan
+
 from scanners.pihole import copy_pihole_network, read_DHCP_leases
 from database import insertOnlineHistory
 from device import create_new_devices, print_scan_stats, save_scanned_devices, update_devices_data_from_scan, update_devices_names
@@ -36,14 +36,6 @@ def scan_network (db):
 
     db.commitDB()
 
-    #  Moved to the ARPSCAN Plugin
-    # arp-scan command
-    # conf.arpscan_devices = []
-    # if conf.ENABLE_ARPSCAN:    
-    #     mylog('verbose','[Network Scan] arp-scan start')    
-    #     conf.arpscan_devices = execute_arpscan (conf.userSubnets)
-    #     mylog('verbose','[Network Scan] arp-scan ends')
-
     # Pi-hole method    
     if conf.PIHOLE_ACTIVE :       
         mylog('verbose','[Network Scan] Pi-hole start')        
@@ -214,13 +206,6 @@ def void_ghost_disconnections (db):
 def pair_sessions_events (db):
     sql = db.sql #TO-DO
    
-    # NOT NECESSARY FOR INCREMENTAL UPDATE
-    # print_log ('Pair session - 1 Clean')
-    # sql.execute ("""UPDATE Events
-    #                 SET eve_PairEventRowid = NULL
-    #                 WHERE eve_EventType IN ('New Device', 'Connected')
-    #              """ )
-    
 
     # Pair Connection / New Device events
     mylog('debug','[Pair Session] - 1 Connections / New Devices')
@@ -275,53 +260,49 @@ def insert_events (db):
     
     # Check device down
     mylog('debug','[Events] - 1 - Devices down')
-    sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
                         eve_EventType, eve_AdditionalInfo,
                         eve_PendingAlertEmail)
-                    SELECT dev_MAC, dev_LastIP, ?, 'Device Down', '', 1
+                    SELECT dev_MAC, dev_LastIP, '{startTime}', 'Device Down', '', 1
                     FROM Devices 
                     WHERE dev_AlertDeviceDown = 1
                       AND dev_PresentLastScan = 1                      
                       AND NOT EXISTS (SELECT 1 FROM CurrentScan
                                       WHERE dev_MAC = cur_MAC
-                                        AND dev_ScanCycle = cur_ScanCycle) """,
+                                         ) """)
-                    (startTime) )
 
     # Check new connections
     mylog('debug','[Events] - 2 - New Connections')
-    sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
                         eve_EventType, eve_AdditionalInfo,
                         eve_PendingAlertEmail)
-                    SELECT cur_MAC, cur_IP, ?, 'Connected', '', dev_AlertEvents
+                    SELECT cur_MAC, cur_IP, '{startTime}', 'Connected', '', dev_AlertEvents
                     FROM Devices, CurrentScan
-                    WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle
+                    WHERE dev_MAC = cur_MAC  
-                      AND dev_PresentLastScan = 0 """,
+                      AND dev_PresentLastScan = 0 """)
-                    (startTime) )
 
     # Check disconnections
     mylog('debug','[Events] - 3 - Disconnections')
-    sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
                         eve_EventType, eve_AdditionalInfo,
                         eve_PendingAlertEmail)
-                    SELECT dev_MAC, dev_LastIP, ?, 'Disconnected', '',
+                    SELECT dev_MAC, dev_LastIP, '{startTime}', 'Disconnected', '',
                         dev_AlertEvents
                     FROM Devices
                     WHERE dev_AlertDeviceDown = 0
                       AND dev_PresentLastScan = 1
                       AND NOT EXISTS (SELECT 1 FROM CurrentScan
                                       WHERE dev_MAC = cur_MAC
-                                        AND dev_ScanCycle = cur_ScanCycle) """,
+                                         ) """)
-                    (startTime) )
 
     # Check IP Changed
     mylog('debug','[Events] - 4 - IP Changes')
-    sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
                         eve_EventType, eve_AdditionalInfo,
                         eve_PendingAlertEmail)
-                    SELECT cur_MAC, cur_IP, ?, 'IP Changed',
+                    SELECT cur_MAC, cur_IP, '{startTime}', 'IP Changed',
                         'Previous IP: '|| dev_LastIP, dev_AlertEvents
                     FROM Devices, CurrentScan
-                    WHERE dev_MAC = cur_MAC AND dev_ScanCycle = cur_ScanCycle                      
+                    WHERE dev_MAC = cur_MAC                        
-                      AND dev_LastIP <> cur_IP """,
+                      AND dev_LastIP <> cur_IP """ )
-                    (startTime) )
     mylog('debug','[Events] - Events end')

pialert/scanners/arpscan.py

@@ -1,74 +0,0 @@
-import re
-import subprocess
-import conf
-
-from logger import mylog
-from helper import write_file
-from const import logPath
-
-#-------------------------------------------------------------------------------
-def execute_arpscan (userSubnets):
-
-    # output of possible multiple interfaces
-    arpscan_output = ""
-
-    # scan each interface
-    index = 0
-    for interface in userSubnets :   
-        arpscan_output += execute_arpscan_on_interface (interface)    
-        index += 1         
-    
-    # Search IP + MAC + Vendor as regular expresion
-    re_ip = r'(?P<ip>((2[0-5]|1[0-9]|[0-9])?[0-9]\.){3}((2[0-5]|1[0-9]|[0-9])?[0-9]))'
-    re_mac = r'(?P<mac>([0-9a-fA-F]{2}[:-]){5}([0-9a-fA-F]{2}))'
-    re_hw = r'(?P<hw>.*)'
-    re_pattern = re.compile (re_ip + '\s+' + re_mac + '\s' + re_hw)
-
-    # Create Userdict of devices
-    devices_list = [device.groupdict()
-        for device in re.finditer (re_pattern, arpscan_output)]
-
-    mylog('debug', ['[ARP Scan] Found: Devices including duplicates ', len(devices_list) ]) 
-    
-    # Delete duplicate MAC
-    unique_mac = [] 
-    unique_devices = [] 
-
-    for device in devices_list :
-        if device['mac'] not in unique_mac: 
-            unique_mac.append(device['mac'])
-            unique_devices.append(device)    
-
-    # return list
-    mylog('debug', ['[ARP Scan] Found: Devices without duplicates ', len(unique_devices)  ]) 
-
-    return unique_devices
-
-#-------------------------------------------------------------------------------
-def execute_arpscan_on_interface (interface):    
-    # Prepare command arguments
-    subnets = interface.strip().split()
-    # Retry is 6 to avoid false offline devices
-    mylog('debug', ['[ARP Scan] - arpscan command: sudo arp-scan --ignoredups --retry=6 ', str(subnets)])
-    arpscan_args = ['sudo', 'arp-scan', '--ignoredups', '--retry=6'] + subnets
-
-    # Execute command
-    if conf.LOG_LEVEL == 'debug':
-        # try runnning a subprocess
-        result = subprocess.check_output (arpscan_args, universal_newlines=True)
-    else:
-        try:
-            # try runnning a subprocess safely
-            result = subprocess.check_output (arpscan_args, universal_newlines=True)
-        except subprocess.CalledProcessError as e:
-            # An error occured, handle it
-            error_type = type(e).__name__  # Capture the error type            
-            
-            mylog('none', [f'[ARP Scan] Error type  : {error_type}'])
-            mylog('none', [f'[ARP Scan] Set LOG_LEVEL=debug for more details'])
-            mylog('none', [f'[ARP Scan] Error output: {e.output}'])
-
-            result = ""
-
-    mylog('debug', ['[ARP Scan] on Interface Completed with results: ', result])
-    return result