Pholus 0.1

README.md

@@ -56,7 +56,6 @@ The system continuously scans the network for, **New devices**, **New connection
   - Help/FAQ Section 
 
   | ![Screen 1][screen1] | ![Screen 2][screen2] |  
-  | -------------------- | -------------------- |
   | ![Screen 3][screen3] | ![Screen 4][screen4] |
   | ![Screen 5][screen5] | ![Screen 6][screen6] |
   | ![Report 1][report1] | ![Report 2][report2] |

back/pialert.py

@@ -51,11 +51,14 @@ fullConfPath = pialertPath + confPath
 fullDbPath   = pialertPath + dbPath
 STOPARPSCAN = pialertPath + "/db/setting_stoparpscan"
 
-time_now = datetime.datetime.now()
+time_started = datetime.datetime.now()
-log_timestamp = time_now
+log_timestamp = time_started
 sql_connection = None
 
 
+#-------------------------------------------------------------------------------
+def timeNow():
+    return datetime.datetime.now().replace(microsecond=0)
 #-------------------------------------------------------------------------------
 def file_print(*args):
 
@@ -227,6 +230,11 @@ DDNS_USER               = 'dynu_user'
 DDNS_PASSWORD           = 'A0000000B0000000C0000000D0000000'
 DDNS_UPDATE_URL         = 'https://api.dynu.com/nic/update?'
 
+# Pholus settings
+# ----------------------
+PHOLUS_ACTIVE           = False                         
+PHOLUS_TIMEOUT          = 60   
+
 # PIHOLE settings
 # ----------------------
 PIHOLE_ACTIVE           = False                         
@@ -281,6 +289,14 @@ def closeDB ():
 #-------------------------------------------------------------------------------
 # Import user values
 
+def check_config_dict(key, default, config):
+    if key in config:
+        return config[key]
+    else: 
+        return default
+
+#-------------------------------------------------------------------------------
+
 def importConfig (): 
 
     # Specify globals so they can be overwritten with the new config
@@ -302,6 +318,8 @@ def importConfig ():
     global DDNS_ACTIVE, DDNS_DOMAIN, DDNS_USER, DDNS_PASSWORD, DDNS_UPDATE_URL
     # PiHole
     global PIHOLE_ACTIVE, DHCP_ACTIVE
+    # Pholus
+    global PHOLUS_ACTIVE, PHOLUS_TIMEOUT  
 
     # load the variables from  pialert.conf
     config_file = Path(fullConfPath)
@@ -309,68 +327,72 @@ def importConfig ():
     config_dict = {}
     exec(code, {"__builtins__": {}}, config_dict)        
     
-    SCAN_SUBNETS = config_dict['SCAN_SUBNETS']
+    SCAN_SUBNETS = check_config_dict('SCAN_SUBNETS', SCAN_SUBNETS , config_dict)
-    PRINT_LOG = config_dict['PRINT_LOG']
+    PRINT_LOG = check_config_dict('PRINT_LOG', PRINT_LOG , config_dict)
-    TIMEZONE = config_dict['TIMEZONE']
+    TIMEZONE = check_config_dict('TIMEZONE', TIMEZONE , config_dict)
-    PIALERT_WEB_PROTECTION = config_dict['PIALERT_WEB_PROTECTION']
+    PIALERT_WEB_PROTECTION = check_config_dict('PIALERT_WEB_PROTECTION', PIALERT_WEB_PROTECTION , config_dict)
-    PIALERT_WEB_PASSWORD = config_dict['PIALERT_WEB_PASSWORD']
+    PIALERT_WEB_PASSWORD = check_config_dict('PIALERT_WEB_PASSWORD', PIALERT_WEB_PASSWORD , config_dict)
-    INCLUDED_SECTIONS = config_dict['INCLUDED_SECTIONS']
+    INCLUDED_SECTIONS = check_config_dict('INCLUDED_SECTIONS', INCLUDED_SECTIONS , config_dict)
-    SCAN_CYCLE_MINUTES = config_dict['SCAN_CYCLE_MINUTES']
+    SCAN_CYCLE_MINUTES = check_config_dict('SCAN_CYCLE_MINUTES', SCAN_CYCLE_MINUTES , config_dict)
-    DAYS_TO_KEEP_EVENTS = config_dict['DAYS_TO_KEEP_EVENTS']
+    DAYS_TO_KEEP_EVENTS = check_config_dict('DAYS_TO_KEEP_EVENTS', DAYS_TO_KEEP_EVENTS , config_dict)
-    REPORT_DASHBOARD_URL = config_dict['REPORT_DASHBOARD_URL']
+    REPORT_DASHBOARD_URL = check_config_dict('REPORT_DASHBOARD_URL', REPORT_DASHBOARD_URL , config_dict)
 
     # Email
-    REPORT_MAIL = config_dict['REPORT_MAIL']
+    REPORT_MAIL = check_config_dict('REPORT_MAIL', REPORT_MAIL , config_dict)
-    SMTP_SERVER = config_dict['SMTP_SERVER']
+    SMTP_SERVER = check_config_dict('SMTP_SERVER', SMTP_SERVER , config_dict)
-    SMTP_PORT = config_dict['SMTP_PORT']
+    SMTP_PORT = check_config_dict('SMTP_PORT', SMTP_PORT , config_dict)
-    REPORT_TO = config_dict['REPORT_TO']
+    REPORT_TO = check_config_dict('REPORT_TO', REPORT_TO , config_dict)
-    REPORT_FROM = config_dict['REPORT_FROM']
+    REPORT_FROM = check_config_dict('REPORT_FROM', REPORT_FROM , config_dict)
-    SMTP_SKIP_LOGIN = config_dict['SMTP_SKIP_LOGIN']
+    SMTP_SKIP_LOGIN = check_config_dict('SMTP_SKIP_LOGIN', SMTP_SKIP_LOGIN , config_dict)
-    SMTP_USER = config_dict['SMTP_USER']
+    SMTP_USER = check_config_dict('SMTP_USER', SMTP_USER , config_dict)
-    SMTP_PASS = config_dict['SMTP_PASS']
+    SMTP_PASS = check_config_dict('SMTP_PASS', SMTP_PASS , config_dict)
-    SMTP_SKIP_TLS = config_dict['SMTP_SKIP_TLS']
+    SMTP_SKIP_TLS = check_config_dict('SMTP_SKIP_TLS', SMTP_SKIP_TLS , config_dict)
 
     # Webhooks
-    REPORT_WEBHOOK = config_dict['REPORT_WEBHOOK']
+    REPORT_WEBHOOK = check_config_dict('REPORT_WEBHOOK', REPORT_WEBHOOK , config_dict)
-    WEBHOOK_URL = config_dict['WEBHOOK_URL']
+    WEBHOOK_URL = check_config_dict('WEBHOOK_URL', WEBHOOK_URL , config_dict)
-    WEBHOOK_PAYLOAD = config_dict['WEBHOOK_PAYLOAD']
+    WEBHOOK_PAYLOAD = check_config_dict('WEBHOOK_PAYLOAD', WEBHOOK_PAYLOAD , config_dict)
-    WEBHOOK_REQUEST_METHOD = config_dict['WEBHOOK_REQUEST_METHOD']
+    WEBHOOK_REQUEST_METHOD = check_config_dict('WEBHOOK_REQUEST_METHOD', WEBHOOK_REQUEST_METHOD , config_dict)
 
     # Apprise
-    REPORT_APPRISE = config_dict['REPORT_APPRISE']
+    REPORT_APPRISE = check_config_dict('REPORT_APPRISE', REPORT_APPRISE , config_dict)
-    APPRISE_HOST = config_dict['APPRISE_HOST']
+    APPRISE_HOST = check_config_dict('APPRISE_HOST', APPRISE_HOST , config_dict)
-    APPRISE_URL = config_dict['APPRISE_URL']
+    APPRISE_URL = check_config_dict('APPRISE_URL', APPRISE_URL , config_dict)
 
     # NTFY
-    REPORT_NTFY = config_dict['REPORT_NTFY']
+    REPORT_NTFY = check_config_dict('REPORT_NTFY', REPORT_NTFY , config_dict)
-    NTFY_HOST = config_dict['NTFY_HOST']
+    NTFY_HOST = check_config_dict('NTFY_HOST', NTFY_HOST , config_dict)
-    NTFY_TOPIC = config_dict['NTFY_TOPIC']
+    NTFY_TOPIC = check_config_dict('NTFY_TOPIC', NTFY_TOPIC , config_dict)
-    NTFY_USER = config_dict['NTFY_USER']
+    NTFY_USER = check_config_dict('NTFY_USER', NTFY_USER , config_dict)
-    NTFY_PASSWORD = config_dict['NTFY_PASSWORD']
+    NTFY_PASSWORD = check_config_dict('NTFY_PASSWORD', NTFY_PASSWORD , config_dict)
 
     # PUSHSAFER
-    REPORT_PUSHSAFER = config_dict['REPORT_PUSHSAFER']
+    REPORT_PUSHSAFER = check_config_dict('REPORT_PUSHSAFER', REPORT_PUSHSAFER , config_dict)
-    PUSHSAFER_TOKEN = config_dict['PUSHSAFER_TOKEN']
+    PUSHSAFER_TOKEN = check_config_dict('PUSHSAFER_TOKEN', PUSHSAFER_TOKEN , config_dict)
 
     # MQTT
-    REPORT_MQTT = config_dict['REPORT_MQTT']
+    REPORT_MQTT = check_config_dict('REPORT_MQTT', REPORT_MQTT , config_dict)
-    MQTT_BROKER = config_dict['MQTT_BROKER']
+    MQTT_BROKER = check_config_dict('MQTT_BROKER', MQTT_BROKER , config_dict)
-    MQTT_PORT = config_dict['MQTT_PORT']
+    MQTT_PORT = check_config_dict('MQTT_PORT', MQTT_PORT , config_dict)
-    MQTT_USER = config_dict['MQTT_USER']
+    MQTT_USER = check_config_dict('MQTT_USER', MQTT_USER , config_dict)
-    MQTT_PASSWORD = config_dict['MQTT_PASSWORD']
+    MQTT_PASSWORD = check_config_dict('MQTT_PASSWORD', MQTT_PASSWORD , config_dict)
-    MQTT_QOS = config_dict['MQTT_QOS']
+    MQTT_QOS = check_config_dict('MQTT_QOS', MQTT_QOS , config_dict)
-    MQTT_DELAY_SEC = config_dict['MQTT_DELAY_SEC']
+    MQTT_DELAY_SEC = check_config_dict('MQTT_DELAY_SEC', MQTT_DELAY_SEC , config_dict)
 
     # DynDNS
-    DDNS_ACTIVE = config_dict['DDNS_ACTIVE']
+    DDNS_ACTIVE = check_config_dict('DDNS_ACTIVE', DDNS_ACTIVE , config_dict)
-    DDNS_DOMAIN = config_dict['DDNS_DOMAIN']
+    DDNS_DOMAIN = check_config_dict('DDNS_DOMAIN', DDNS_DOMAIN , config_dict)
-    DDNS_USER = config_dict['DDNS_USER']
+    DDNS_USER = check_config_dict('DDNS_USER', DDNS_USER , config_dict)
-    DDNS_PASSWORD = config_dict['DDNS_PASSWORD']
+    DDNS_PASSWORD = check_config_dict('DDNS_PASSWORD', DDNS_PASSWORD , config_dict)
-    DDNS_UPDATE_URL = config_dict['DDNS_UPDATE_URL']
+    DDNS_UPDATE_URL = check_config_dict('DDNS_UPDATE_URL', DDNS_UPDATE_URL , config_dict)
 
     # PiHole
-    PIHOLE_ACTIVE = config_dict['PIHOLE_ACTIVE']
+    PIHOLE_ACTIVE = check_config_dict('PIHOLE_ACTIVE',  PIHOLE_ACTIVE, config_dict)
-    DHCP_ACTIVE = config_dict['DHCP_ACTIVE']
+    DHCP_ACTIVE = check_config_dict('DHCP_ACTIVE', DHCP_ACTIVE , config_dict)
+
+    # PHOLUS
+    PHOLUS_ACTIVE = check_config_dict('PHOLUS_ACTIVE', PHOLUS_ACTIVE , config_dict)
+    PHOLUS_TIMEOUT = check_config_dict('PHOLUS_TIMEOUT', PHOLUS_TIMEOUT , config_dict)
 
     openDB()    
 
@@ -439,8 +461,12 @@ def importConfig ():
         ('DDNS_UPDATE_URL', 'DynDNS update URL', '',  'text', '', '' , str(DDNS_UPDATE_URL) , 'DynDNS'),
 
         # PiHole
-        ('PIHOLE_ACTIVE', 'Enable PiHole mapping', 'If enabled you need to map /etc/pihole/pihole-FTL.db in your docker-compose.yml',  'boolean', '', '' , str(PIHOLE_ACTIVE) , 'PiHole'),
+        ('PIHOLE_ACTIVE', 'Enable PiHole mapping', '',  'boolean', '', '' , str(PIHOLE_ACTIVE) , 'PiHole'),
-        ('DHCP_ACTIVE', 'Enable PiHole DHCP', 'If enabled you need to map /etc/pihole/dhcp.leases in your docker-compose.yml',  'boolean', '', '' , str(DHCP_ACTIVE) , 'PiHole')
+        ('DHCP_ACTIVE', 'Enable PiHole DHCP', '',  'boolean', '', '' , str(DHCP_ACTIVE) , 'PiHole'),
+
+        # Pholus
+        ('PHOLUS_ACTIVE', 'Enable Pholus scans', '',  'boolean', '', '' , str(PHOLUS_ACTIVE) , 'Pholus'),
+        ('PHOLUS_TIMEOUT', 'Pholus timeout', '',  'integer', '', '' , str(PHOLUS_TIMEOUT) , 'Pholus')
 
     ]
     # Insert into DB    
@@ -452,8 +478,6 @@ def importConfig ():
     closeDB()
 #-------------------------------------------------------------------------------
 
-# importConfig()
-
 #===============================================================================
 # USER CONFIG VARIABLES - END
 #===============================================================================
@@ -466,18 +490,18 @@ check_report = [1, "internet_IP", "update_vendors_silent"]
 mqtt_thread_up = False
 
 # timestamps of last execution times
-startTime = time_now
+startTime = time_started
-now_minus_24h = time_now - timedelta(hours = 24)
+now_minus_24h = time_started - timedelta(hours = 24)
 
 last_network_scan = now_minus_24h
 last_internet_IP_scan = now_minus_24h
 last_run = now_minus_24h
 last_cleanup = now_minus_24h
-last_update_vendors = time_now - timedelta(days = 6) # update vendors 24h after first run and than once a week
+last_update_vendors = time_started - timedelta(days = 6) # update vendors 24h after first run and than once a week
 
 def main ():
     # Initialize global variables
-    global time_now, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors, mqtt_thread_up
+    global time_started, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors, mqtt_thread_up
     # second set of global variables
     global startTime, log_timestamp, sql_connection, sql
 
@@ -498,41 +522,41 @@ def main ():
     while True:
 
         # update NOW time
-        time_now = datetime.datetime.now()
+        time_started = datetime.datetime.now()
 
         # re-load user configuration
         importConfig() 
 
         # proceed if 1 minute passed
-        if last_run + timedelta(minutes=1) < time_now :
+        if last_run + timedelta(minutes=1) < time_started :
 
              # last time any scan or maintennace/Upkeep was run
-            last_run = time_now
+            last_run = time_started
 
             reporting = False
 
             # Header
             updateState("Process: Start")
-            file_print('[', time_now.replace (microsecond=0), '] Process: Start')                
+            file_print('[', timeNow(), '] Process: Start')                
 
             # Timestamp
-            startTime = time_now
+            startTime = time_started
             startTime = startTime.replace (microsecond=0)      
 
             # determine run/scan type based on passed time
-            if last_internet_IP_scan + timedelta(minutes=3) < time_now:
+            if last_internet_IP_scan + timedelta(minutes=3) < time_started:
                 cycle = 'internet_IP'                
-                last_internet_IP_scan = time_now
+                last_internet_IP_scan = time_started
                 reporting = check_internet_IP()
 
             # Update vendors once a week
-            if last_update_vendors + timedelta(days = 7) < time_now:
+            if last_update_vendors + timedelta(days = 7) < time_started:
-                last_update_vendors = time_now
+                last_update_vendors = time_started
                 cycle = 'update_vendors'
                 update_devices_MAC_vendors()
 
-            if last_network_scan + timedelta(minutes=SCAN_CYCLE_MINUTES) < time_now and os.path.exists(STOPARPSCAN) == False:
+            if last_network_scan + timedelta(minutes=SCAN_CYCLE_MINUTES) < time_started and os.path.exists(STOPARPSCAN) == False:
-                last_network_scan = time_now
+                last_network_scan = time_started
                 cycle = 1 # network scan
                 scan_network() 
             
@@ -541,8 +565,8 @@ def main ():
                 email_reporting()
 
             # clean up the DB once a day
-            if last_cleanup + timedelta(hours = 24) < time_now:
+            if last_cleanup + timedelta(hours = 24) < time_started:
-                last_cleanup = time_now
+                last_cleanup = time_started
                 cycle = 'cleanup'  
                 cleanup_database()   
 
@@ -554,12 +578,12 @@ def main ():
                 action = str(cycle)
                 if action == "1":
                     action = "network_scan"
-                file_print('[', time_now.replace (microsecond=0), '] Last action: ', action)
+                file_print('[', timeNow(), '] Last action: ', action)
                 cycle = ""
 
             # Footer
             updateState("Process: Wait")
-            file_print('[', time_now.replace (microsecond=0), '] Process: Wait')            
+            file_print('[', timeNow(), '] Process: Wait')            
         else:
             # do something
             cycle = ""           
@@ -755,9 +779,13 @@ def cleanup_database ():
     file_print('    Optimize Database')
 
     # Cleanup Events
-    file_print('    Upkeep Events, up to the lastest '+str(DAYS_TO_KEEP_EVENTS)+' days')
+    file_print('    Upkeep Events, delete all older than '+str(DAYS_TO_KEEP_EVENTS)+' days')
     sql.execute ("DELETE FROM Events WHERE eve_DateTime <= date('now', '-"+str(DAYS_TO_KEEP_EVENTS)+" day')")
 
+    # Cleanup Pholus_Scan
+    file_print('    Upkeep Pholus_Scan, delete all older than 7 days')
+    sql.execute ("DELETE FROM Pholus_Scan WHERE Time <= date('now', '-7 day')")
+
     # Shrink DB
     file_print('    Shrink Database')
     sql.execute ("VACUUM;")
@@ -1061,6 +1089,10 @@ def execute_arpscan_on_interface (SCAN_SUBNETS):
     # Retry is 6 to avoid false offline devices
     arpscan_args = ['sudo', 'arp-scan', '--ignoredups', '--retry=6'] + subnets
 
+
+    mask = subnets[0]
+    interface = subnets[1].split('=')[1]
+
     # Execute command
     try:
         # try runnning a subprocess
@@ -1070,6 +1102,9 @@ def execute_arpscan_on_interface (SCAN_SUBNETS):
         file_print(e.output)
         result = ""
 
+    if PHOLUS_ACTIVE:
+        performPholusScan(interface, mask)    
+
     return result
 
 #-------------------------------------------------------------------------------
@@ -1124,6 +1159,9 @@ def read_DHCP_leases ():
 #-------------------------------------------------------------------------------
 def save_scanned_devices (p_arpscan_devices, p_cycle_interval):
     cycle = 1 # always 1, only one cycle supported
+
+    openDB()    
+
     # Delete previous scan data
     sql.execute ("DELETE FROM CurrentScan WHERE cur_ScanCycle = ?",
                 (cycle,))
@@ -1541,7 +1579,7 @@ def update_devices_names ():
     # BUGFIX #97 - Updating name of Devices w/o IP
     for device in sql.execute ("SELECT * FROM Devices WHERE dev_Name IN ('(unknown)','') AND dev_LastIP <> '-'") :
         # Resolve device name
-        newName = resolve_device_name (device['dev_MAC'], device['dev_LastIP'])
+        newName = resolve_device_name (device['dev_MAC'], device['dev_LastIP']) #<<<< continue here TODO DEV > get >latest< Pholus scan results and match
        
         if newName == -1 :
             notFound += 1
@@ -1561,6 +1599,42 @@ def update_devices_names ():
     # DEBUG - print number of rows updated
         # file_print(sql.rowcount)
 
+#-------------------------------------------------------------------------------
+def performPholusScan (interface, mask):
+    updateState("Scan: Pholus")
+    file_print('[', timeNow(), '] Scan: Pholus')  
+    
+    pholus_args = ['python3', '/home/pi/pialert/pholus/pholus3.py', interface, "-rdns_scanning", mask, "-stimeout", str(PHOLUS_TIMEOUT)]
+
+    # Execute command
+    try:
+        # try runnning a subprocess
+        output = subprocess.check_output (pholus_args, universal_newlines=True)
+    except subprocess.CalledProcessError as e:
+        # An error occured, handle it
+        file_print(e.output)
+        file_print("Error - PholusScan - check logs")
+        output = ""
+
+    if output != "":
+        file_print('[', timeNow(), '] Scan: Pholus SUCCESS')
+        write_file (logPath + '/pialert_pholus.log', output)
+
+        params = []
+
+        for line in output.split("\n"):
+            columns = line.split("|")
+            if len(columns) == 4:
+                params.append(( interface + " " + mask, timeNow() , columns[0], columns[1], columns[2], columns[3], ''))
+
+        if len(params) > 0:
+            openDB ()
+            sql.executemany ("""INSERT INTO Pholus_Scan ("Info", "Time", "MAC", "IP_v4_or_v6", "Record_Type", "Value", "Extra") VALUES (?, ?, ?, ?, ?, ?, ?)""", params) 
+
+    else:
+        file_print('[', timeNow(), '] Scan: Pholus FAIL - check logs') 
+
+
 #-------------------------------------------------------------------------------
 def resolve_device_name (pMAC, pIP):
     try :
@@ -2583,6 +2657,33 @@ def upgradeDB ():
     );      
     """)
 
+    # indicates, if Pholus_Scan table is available 
+    pholusScanMissing = sql.execute("""
+    SELECT name FROM sqlite_master WHERE type='table'
+    AND name='Pholus_Scan'; 
+    """).fetchone() == None
+
+    # Re-creating Pholus_Scan table    
+    file_print("[upgradeDB] Re-creating Pholus_Scan table")
+
+    # if pholusScanMissing == False:   
+    #      sql.execute("DROP TABLE Pholus_Scan;")       
+
+    if pholusScanMissing:
+        sql.execute("""      
+        CREATE TABLE "Pholus_Scan" (        
+        "Index"	          INTEGER,
+        "Info"	          TEXT,
+        "Time"	          TEXT,
+        "MAC"	          TEXT,
+        "IP_v4_or_v6"	  TEXT,
+        "Record_Type"	  TEXT,
+        "Value"           TEXT,
+        "Extra"           TEXT,
+        PRIMARY KEY("Index" AUTOINCREMENT)
+        );      
+        """)
+      
     # don't hog DB access  
     closeDB ()
 

front/css/pialert.css

@@ -34,6 +34,10 @@
     display: grid;
     margin: 5px;
 }
+.logs-size
+{
+    font-size: 14px;
+}
 .log-area
 {
     padding: 3px;    

front/maintenance.php

@@ -348,12 +348,12 @@ if (submit && isset($_POST['skinselector_set'])) {
                     <div class="db_info_table">
                         <div class="log-area">
                             <div class="row logs-row">
-                              <textarea id="pialert_log" class="logs" cols="70" rows="10" readonly ><?php echo file_get_contents( "./log/pialert.log" ); ?>
+                              <textarea id="pialert_log" class="logs" cols="70" rows="10" wrap='off' readonly ><?php echo file_get_contents( "./log/pialert.log" ); ?>
                               </textarea>
                             </div>
                             <div class="row logs-row" >
                               <div>
-                                <div class="log-file">pialert.log</div><span class="span-padding"><a href="./log/pialert.log" target="_blank"><i class="fa fa-download"></i> </a></span>             
+                                <div class="log-file">pialert.log <div class="logs-size"><?php echo number_format((filesize("./log/pialert.log") / 1000000),2,",",".") . ' MB';?> </div></div><span class="span-padding"><a href="./log/pialert.log" target="_blank"><i class="fa fa-download"></i> </a></span>             
                                 <div class="log-purge">
                                   <button class="btn btn-primary" onclick="logManage('pialert.log','cleanLog')"><?php echo lang('Gen_Purge');?></button>
                                 </div>
@@ -367,7 +367,7 @@ if (submit && isset($_POST['skinselector_set'])) {
                             </div>
                             <div class="row logs-row" >                            
                               <div>
-                                <div class="log-file">pialert_front.log</div><span class="span-padding"><a href="./log/pialert_front.log"><i class="fa fa-download"></i> </a></span>
+                                <div class="log-file">pialert_front.log<div class="logs-size"><?php echo number_format((filesize("./log/pialert_front.log") / 1000000),2,",",".") . ' MB';?> </div></div><span class="span-padding"><a href="./log/pialert_front.log"><i class="fa fa-download"></i> </a></span>
                                 <div class="log-purge">
                                   <button class="btn btn-primary" onclick="logManage('pialert_front.log','cleanLog')"><?php echo lang('Gen_Purge');?></button>
                                 </div>
@@ -376,13 +376,29 @@ if (submit && isset($_POST['skinselector_set'])) {
                         </div>
                         <div class="log-area">
                              
+                             <div class="row logs-row">
+                               <textarea id="pialert_pholus_log" class="logs" cols="70" rows="10" wrap='off' readonly><?php echo file_get_contents( "./log/pialert_pholus.log" ); ?>
+                               </textarea>                              
+                             </div>                          
+                             <div class="row logs-row" >
+                               <div> 
+                                 <div class="log-file">pialert_pholus.log<div class="logs-size"><?php echo number_format((filesize("./log/pialert_pholus.log") / 1000000),2,",",".") . ' MB';?> </div></div><span class="span-padding"><a href="./log/pialert_pholus.log"><i class="fa fa-download"></i> </a></span>                          
+                                 <div class="log-purge">
+                                   <button class="btn btn-primary" onclick="logManage('pialert_pholus.log','cleanLog')"><?php echo lang('Gen_Purge');?></button>
+                                 </div>                            
+                               </div>                            
+                             </div>                            
+ 
+                         </div>    
+                        <div class="log-area">
+                             
                             <div class="row logs-row">
                               <textarea id="IP_changes_log" class="logs logs-small" cols="70" rows="10" readonly><?php echo file_get_contents( "./log/IP_changes.log" ); ?>
                               </textarea>                              
                             </div>                          
                             <div class="row logs-row" >
                               <div> 
-                                <div class="log-file">IP_changes.log</div><span class="span-padding"><a href="./log/IP_changes.log"><i class="fa fa-download"></i> </a></span>                          
+                                <div class="log-file">IP_changes.log<div class="logs-size"><?php echo number_format((filesize("./log/IP_changes.log") / 1000000),2,",",".") . ' MB';?> </div></div><span class="span-padding"><a href="./log/IP_changes.log"><i class="fa fa-download"></i> </a></span>                          
                                 <div class="log-purge">
                                   <button class="btn btn-primary" onclick="logManage('IP_changes.log','cleanLog')"><?php echo lang('Gen_Purge');?></button>
                                 </div>                            
@@ -397,7 +413,7 @@ if (submit && isset($_POST['skinselector_set'])) {
                             </div>
                             <div class="row logs-row" >                            
                               <div>
-                                <div class="log-file">stdout.log</div><span class="span-padding"><a href="./log/stdout.log"><i class="fa fa-download"></i> </a></span>
+                                <div class="log-file">stdout.log<div class="logs-size"><?php echo number_format((filesize("./log/stdout.log") / 1000000),2,",",".") . ' MB';?> </div></div><span class="span-padding"><a href="./log/stdout.log"><i class="fa fa-download"></i> </a></span>
                                 <div class="log-purge">
                                   <button class="btn btn-primary" onclick="logManage('stdout.log','cleanLog')"><?php echo lang('Gen_Purge');?></button>
                                 </div>
@@ -412,7 +428,7 @@ if (submit && isset($_POST['skinselector_set'])) {
                             </div>
                             <div class="row logs-row" >
                               <div>
-                              <div class="log-file">stderr.log</div><span class="span-padding"><a href="./log/stderr.log"><i class="fa fa-download"></i> </a></span>
+                              <div class="log-file">stderr.log<div class="logs-size"><?php echo number_format((filesize("./log/stderr.log") / 1000000),2,",",".") . ' MB';?> </div></div><span class="span-padding"><a href="./log/stderr.log"><i class="fa fa-download"></i> </a></span>
                               <div class="log-purge">
                                 <button class="btn btn-primary" onclick="logManage('stderr.log','cleanLog')"><?php echo lang('Gen_Purge');?></button>
                               </div>                            

front/php/templates/language/en_us.php

@@ -547,6 +547,12 @@ the scan will take hours to complete instead of seconds.
 'DHCP_ACTIVE_name' => 'Enable PiHole DHCP',
 'DHCP_ACTIVE_description' => 'If enabled you need to map <code>:/etc/pihole/dhcp.leases</code> in your <code>docker-compose.yml</code>  file.',
 
+// Pholus
+'PHOLUS_ACTIVE_name' => 'Enable Pholus scan',
+'PHOLUS_ACTIVE_description' => '<a href="https://github.com/jokob-sk/Pi.Alert/tree/main/pholus" target="_blank" >Pholus</a> is a sniffing tool to discover additional information about the devices on the network, including the device name. Please be aware it can spam the network with unnecessary traffic.',
+'PHOLUS_TIMEOUT_name' => 'Pholus timeout',
+'PHOLUS_TIMEOUT_description' => 'How long (s) should Pholus be sniffing the network. Tested with <code>60</code>s on a network with 50 devices.',
+
 );
 
 ?>

front/settings.php

@@ -242,7 +242,7 @@ $db->close();
 <script>
 
   // number of settings has to be equal to
-  var settingsNumber = 46;
+  var settingsNumber = 48;
   
   if(<?php echo count($settings)?> != settingsNumber) 
   {

pholus/README.md

@@ -1,3 +1,6 @@
+# INFO
+For anyone reading, `pholus.py` is not actively developed as this project only uses `pholus3.py`. 
+
 # Pholus
 A multicast DNS and DNS Service Discovery Security Assessment Tool
 It can perform recconnaisance, Denial of Service, Man in the Middle attacks

pholus/pholus3.py

@@ -11,9 +11,56 @@ import codecs
 import ipaddress
 from scapy.utils import PcapWriter
 
+
 sys.setrecursionlimit(30000)
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)#supress Scapy warnings`
 
+
+
+#===============================================================================
+# UTIL
+#===============================================================================
+
+
+def sanitize_string(input):
+    if isinstance(input, bytes):
+        input = input.decode('utf-8')
+    value = b_to_str(re.sub('[^a-zA-Z0-9-_\s]', '', str(input)))
+    return value
+
+#-------------------------------------------------------------------------------
+NoneType = type(None)
+
+def b_to_str(value):
+    # if value is of type bytes, convert to string
+    if value is None:  
+        print("00>>>>> ")      
+        return str("")
+    elif isinstance(value, type(None)):
+        print("01>>>>> ")
+        return str("")
+    elif isinstance(value, NoneType):
+        print("02>>>>> ")
+        return str("")
+    elif isinstance(value, str):
+        # print("11>>>>> ",type(value))
+        return str(value+"")
+    elif isinstance(value, int):
+        b_to_str(str(value))
+    elif isinstance(value, bool):
+        b_to_str(str(value))
+    elif isinstance(value, bytes):        
+        b_to_str(value.decode('utf-8'))
+    elif isinstance(value, list):
+        for one in value:            
+            b_to_str(one)
+    else:
+        print("21>>>>> ",type(value))
+        return str(value)
+    # return ">>Couldn't determine type<<"
+ 
+ #-------------------------------------------------------------------------------
+
 ######################################
 ### OBTAIN THE SYSTEM IPV6 ADDRESS ###
 ######################################
@@ -113,7 +160,7 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
         IP_src=packets.getlayer(IPv6).src
     elif packets.haslayer(IP):
         IP_src=packets.getlayer(IP).src
-    res0= Ether_src + " " + IP_src
+    res0= Ether_src + " | " + IP_src.ljust(27)
     if packets.haslayer(DNS):
         dns=packets.getlayer(DNS)
         if (conflict or dos_ttl) and dns.ancount>0:
@@ -174,7 +221,7 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
         elif auto_fake_responses or (not (dos_ttl or conflict)):
             ## IF THIS IS A QUERY ##
             if dns.opcode==0:
-                res0 = res0 + " QUERY"
+                res0 = res0 + ""
                 if dns.qdcount>0:
                     DNSBlocks = [ ]
                     DNSBlocks.append(dns.qd)
@@ -384,11 +431,11 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
                             ### END "IF WE NEED TO AUTO RESPOND WITH A FAKE RESPONSE
                             ### NEXT LINES ARE ONLY USED TO PRINT RESULTS ###
                             if dnsqr.qclass==32769:
-                                res = res0 + " Question: "+dnsqr.qname.decode("utf-8") + " " + dns_type[dnsqr.qtype] +" QU Class:IN"
+                                res = res0 + " | Question            | "+dnsqr.qname.decode("utf-8") + " " + dns_type[dnsqr.qtype] +" QU Class:IN"
                             elif dnsqr.qclass==1:
-                                res = res0 + " Question: "+dnsqr.qname.decode("utf-8") + " "+ dns_type[dnsqr.qtype] + " QM Class:IN"
+                                res = res0 + " | Question            | "+dnsqr.qname.decode("utf-8") + " "+ dns_type[dnsqr.qtype] + " QM Class:IN"
                             elif dnsqr.qclass==255:
-                                res = res0 + " Question: "+dnsqr.qname.decode("utf-8") + " "+ dns_type[dnsqr.qtype] + " QM Class:ANY"
+                                res = res0 + " | Question            | "+dnsqr.qname.decode("utf-8") + " "+ dns_type[dnsqr.qtype] + " QM Class:ANY"
                             else:
                                 print("DNSQR:")
                                 print("-----")
@@ -415,7 +462,7 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
                                     ARtype="OPT"
                                 else:
                                     ARtype=str(dnsrropt.type)
-                                res = res0 + " Additional_Record: " + rrname.decode("utf-8")  + " " + ARtype 
+                                res = res0 + " | Additional_Record   | " + rrname.decode("utf-8")  + " " + ARtype 
                                 if dnsrropt.haslayer(EDNS0TLV):
                                     edns0tlv=dnsrropt.getlayer(EDNS0TLV)
                                     if edns0tlv.optcode==4:
@@ -431,11 +478,28 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
                             while isinstance(block,DNSRR):#Somewhat equivalent: while not isinstance(an, NoPayload):
                                 dnsrr=block.getlayer(DNSRR)
                                 if dnsrr.rclass==32769:
-                                    res = res0 + " DNS Resource Record: "+ dnsrr.rrname + " " + dns_type[dnsrr.type] +" QU Class:IN "+dnsrr.rdata
+
+                                    str_res0   = str(b_to_str(res0)) + ""
+                                    str_rrname = str(b_to_str(dnsrr.rrname)) + ""
+                                    str_type   = str(b_to_str(dns_type[dnsrr.type])) + ""
+                                    str_rdata  = str(b_to_str(dnsrr.rdata)) + ""
+
+                                    res = str_res0       + " | DNS Resource Record | " + str_rrname + " " + str_type + " QU Class:IN " + str_rdata
                                 elif dnsrr.rclass==1:
-                                    res = res0 + " DNS Resource Record: "+dnsrr.rrname + " "+ dns_type[dnsrr.type] + " QM Class:IN "+dnsrr.rdata
+
+                                    str_res0   = str(b_to_str(res0)) + ""
+                                    str_rrname = str(b_to_str(dnsrr.rrname)) + ""
+                                    str_type   = str(b_to_str(dns_type[dnsrr.type])) + ""
+                                    str_rdata  = str(b_to_str(dnsrr.rdata)) + ""
+
+                                    res = str_res0 + " | DNS Resource Record | " + str_rrname + " " + str_type + " QM Class:IN " + str_rdata
                                 elif dnsrr.qclass==255:
-                                    res = res0 + " Question: "+dnsrr.qname + " "+ dns_type[dnsrr.qtype] + " QM Class:ANY"
+
+                                    str_res0   = str(b_to_str(res0)) + ""
+                                    str_qname  = str(b_to_str(dnsrr.qname)) + ""
+                                    str_qtype  = str(b_to_str(dns_type[dnsrr.qtype])) + ""
+
+                                    res = str_res0 + " | Question            | " + str_qname + " " + str_qtype + " QM Class:ANY"
                                 else:
                                     print("DNSRR:")
                                     print("-----")
@@ -443,10 +507,17 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
                                     print("DEBUGGING IS NEEDED HERE")
                                     exit(0)
                                 if dnsrr.type==33:#SRV Record
+
+                                    str_res0   = str(b_to_str(res0)) + ""
+                                    str_rrname = str(b_to_str(dnsrr.rrname)) + ""
+                                    str_type   = str(b_to_str(dns_type[dnsrr.type])) + ""                                    
+                                    str_rclass = str(b_to_str(dnsrr.rclass)) + ""
+
                                     priority=str(dnsrr.rdata)[0].encode("HEX")+str(dnsrr.rdata)[1].encode("HEX")
                                     weight=str(dnsrr.rdata)[2].encode("HEX")+str(dnsrr.rdata)[3].encode("HEX")
                                     port_number=str(dnsrr.rdata)[4].encode("HEX")+str(dnsrr.rdata)[5].encode("HEX")
-                                    res = res0 + " Additional_Record: "+dnsrr.rrname + " " + dns_type[dnsrr.type]+" " + str(dnsrr.rclass) + " priority="+str(int(priority,16))+" weight="+str(int(weight,16))+" port="+str(int(port_number,16))+" target="+str(dnsrr.rdata)[6::]
+
+                                    res = str_res0 + " | Additional_Record   | "+ str_rrname + " " + str_type+" " + str_rclass + " priority="+str(int(priority,16))+" weight="+str(int(weight,16))+" port="+str(int(port_number,16))+" target="+str(dnsrr.rdata)[6::]
                                 else:
                                     rdata=dnsrr.rdata
                                     if isinstance(rdata,bytes):
@@ -454,7 +525,14 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
                                     if "._tcp." not in rdata and "._udp." not in rdata:
                                         if rdata == "_dhnap.":
                                             rdata=rdata+"_tcp."
-                                    res = res0 + " Additional_Record: "+dnsrr.rrname + " " + dns_type[dnsrr.type]+" " + str(dnsrr.rclass) + ' "' +rdata+'"'
+
+                                    str_res0   = str(b_to_str(res0)) + ""
+                                    str_rrname = str(b_to_str(dnsrr.rrname)) + ""
+                                    str_type   = str(b_to_str(dns_type[dnsrr.type])) + ""
+                                    str_rdata  = str(b_to_str(dnsrr.rdata)) + ""
+                                    str_rclass = str(b_to_str(dnsrr.rclass)) + ""
+
+                                    res = str_res0 + " | Additional_Record   | "+str_rrname + " " + str_type+" " + str_rclass + ' "' +str_rdata+'"'
                                 if show_ttl:
                                     res = res + " TTL:"+str(dnsrr.ttl)
                                 if print_res==1:
@@ -478,14 +556,14 @@ def ext_handler(packets,queue,unidns,show_ttl,print_res,dos_ttl,conflict,ttl,int
                                 priority=str(dnsrr.rdata)[0].encode("HEX")+str(dnsrr.rdata)[1].encode("HEX")
                                 weight=str(dnsrr.rdata)[2].encode("HEX")+str(dnsrr.rdata)[3].encode("HEX")
                                 port_number=str(dnsrr.rdata)[4].encode("HEX")+str(dnsrr.rdata)[5].encode("HEX")
-                                res = res0 + " Answer: "+dnsrr.rrname + " " + dns_type[dnsrr.type]+" " + rclass + " priority="+str(int(priority,16))+" weight="+str(int(weight,16))+" port="+str(int(port_number,16))+" target="+str(dnsrr.rdata)[6::]
+                                res = res0 + " | Answer              | "+dnsrr.rrname + " " + dns_type[dnsrr.type]+" " + rclass + " priority="+str(int(priority,16))+" weight="+str(int(weight,16))+" port="+str(int(port_number,16))+" target="+str(dnsrr.rdata)[6::]
                             else:
                                 if "._tcp." not in rdata and "._udp." not in rdata:
                                     if rdata  == "_dhnap.":
                                         rdata=dnsrr.rdata+"_tcp."
                                 if isinstance(rdata,list):
                                     rdata = b" ".join(rdata).decode("utf-8")
-                                res = res0 + " Answer: "+dnsrr.rrname.decode("utf-8") + " " + dns_type[dnsrr.type]+" " + rclass + ' "' +rdata+'"'
+                                res = res0 + " | Answer              | "+dnsrr.rrname.decode("utf-8") + " " + dns_type[dnsrr.type]+" " + rclass + ' "' +rdata+'"'
                             if show_ttl:
                                 res = res + " TTL:"+str(dnsrr.ttl)
                             if print_res==1: