From 445b4de69e01921afb3ad21bafd9cb5151c0440b Mon Sep 17 00:00:00 2001 From: Jokob-sk Date: Thu, 24 Aug 2023 15:54:31 +1000 Subject: [PATCH] convert pholus to plugin v0.2 --- front/php/templates/language/en_us.json | 20 +- front/php/templates/language/es_es.json | 18 -- .../README.md | 0 .../config.json | 92 ++++++--- .../script.py | 94 +-------- front/plugins/snmp_discovery/config.json | 2 +- front/settings.php | 2 +- pialert/__main__.py | 26 +-- pialert/conf.py | 9 - pialert/device.py | 8 +- pialert/helper.py | 141 ++++++++++++- pialert/initialise.py | 14 -- pialert/plugin_utils.py | 2 +- pialert/scanners/pholusscan.py | 193 ------------------ 14 files changed, 220 insertions(+), 401 deletions(-) rename front/plugins/{pholus_scan__ignore => pholus_scan}/README.md (100%) rename front/plugins/{pholus_scan__ignore => pholus_scan}/config.json (70%) rename front/plugins/{pholus_scan__ignore => pholus_scan}/script.py (58%) diff --git a/front/php/templates/language/en_us.json b/front/php/templates/language/en_us.json index 6be6c1e7..7164d886 100755 --- a/front/php/templates/language/en_us.json +++ b/front/php/templates/language/en_us.json @@ -579,25 +579,7 @@ "DDNS_PASSWORD_name" : "DynDNS password", "DDNS_PASSWORD_description" : "", "DDNS_UPDATE_URL_name" : "DynDNS update URL", - "DDNS_UPDATE_URL_description" : "Update URL starting with http:// or https://.", - "PiHole_display_name" : "PiHole", - "PiHole_icon" : "", - "Pholus_display_name" : "Pholus", - "Pholus_icon" : "", - "PHOLUS_ACTIVE_name" : "Cycle run", - "PHOLUS_ACTIVE_description" : "Pholus is a sniffing tool to discover additional information about the devices on the network, including the device name. If enabled this will execute the scan before every network scan cycle until there are no (unknown) or (name not found) devices. Please be aware it can spam the network with unnecessary traffic. Depends on the SCAN_SUBNETS setting. For a scheduled or one-off scan, check the PHOLUS_RUN setting.", - "PHOLUS_TIMEOUT_name" : "Cycle run timeout", - "PHOLUS_TIMEOUT_description" : "How long in seconds should Pholus be sniffing on each interface if above condition is fulfilled. The longer you leave it on, the more likely devices would broadcast more info. This timeout adds to the time it takes to perform an arp-scan on your network.", - "PHOLUS_FORCE_name" : "Cycle force scan", - "PHOLUS_FORCE_description" : "Force scan every network scan, even if there are no (unknown) or (name not found) devices. Be careful enabling this as the sniffing can easily flood your network.", - "PHOLUS_RUN_name" : "Scheduled run", - "PHOLUS_RUN_description" : "Enable a regular Pholus scan / sniff on your network. The scheduling settings can be found below. If you select once Pholus is run only once on start for the time specified in PHOLUS_RUN_TIMEOUT setting.", - "PHOLUS_RUN_TIMEOUT_name" : "Scheduled run timeout", - "PHOLUS_RUN_TIMEOUT_description" : "The timeout in seconds for the scheduled Pholus scan. Same notes regarding the duration apply as on the PHOLUS_TIMEOUT setting. A scheduled scan doesn not check if there are (unknown) or (name not found) devices, the scan is executed either way.", - "PHOLUS_RUN_SCHD_name" : "Schedule", - "PHOLUS_RUN_SCHD_description" : "Only enabled if you select schedule in the PHOLUS_RUN setting. Make sure you enter the schedule in the correct cron-like format (e.g. validate at crontab.guru). For example entering 0 4 * * * will run the scan after 4 am in the TIMEZONE you set above. Will be run NEXT time the time passes.", - "PHOLUS_DAYS_DATA_name" : "Data retention", - "PHOLUS_DAYS_DATA_description" : "How many days of Pholus scan entries should be kept (globally, not device specific!) Enter 0 to disable.", + "DDNS_UPDATE_URL_description" : "Update URL starting with http:// or https://.", "Nmap_display_name" : "Nmap", "Nmap_icon" : "", "NMAP_ACTIVE_name" : "Cycle run", diff --git a/front/php/templates/language/es_es.json b/front/php/templates/language/es_es.json index f38126a8..b8aac324 100755 --- a/front/php/templates/language/es_es.json +++ b/front/php/templates/language/es_es.json @@ -573,24 +573,6 @@ "DDNS_PASSWORD_description" : "", "DDNS_UPDATE_URL_name" : "URL de actualización de DynDNS", "DDNS_UPDATE_URL_description" : "Actualice la URL que comienza con http:// o https://.", - "PiHole_display_name" : "PiHole", - "PiHole_icon" : "", - "Pholus_display_name" : "Pholus", - "Pholus_icon" : "", - "PHOLUS_ACTIVE_name" : "Ejecución del ciclo", - "PHOLUS_ACTIVE_description" : "Pholus es una herramienta de rastreo para descubrir información adicional sobre los dispositivos en la red, incluido el nombre del dispositivo. Si está habilitado, ejecutará el escaneo antes de cada ciclo de escaneo de red hasta que no haya dispositivos (unknown) o (name not found). Tenga en cuenta que puede enviar spam a la red con tráfico innecesario. Depende de la configuración de SCAN_SUBNETS. Para un análisis programado o único, verifique la configuración de PHOLUS_RUN.", - "PHOLUS_TIMEOUT_name" : "Tiempo de espera de ciclo", - "PHOLUS_TIMEOUT_description" : "¿Cuánto tiempo en segundos debe rastrear Pholus en cada interfaz si se cumple la condición anterior? Cuanto más tiempo lo deje encendido, es más probable que los dispositivos transmitan más información. Este tiempo de espera se suma al tiempo que lleva realizar un escaneo arp en su red.", - "PHOLUS_FORCE_name" : "Escaneo de fuerza de ciclo", - "PHOLUS_FORCE_description" : "Fuerce el escaneo de cada escaneo de red, incluso si no hay dispositivos (unknown) o (name not found). Tenga cuidado al habilitar esto, ya que la detección puede inundar fácilmente su red.", - "PHOLUS_RUN_name" : "Ejecución programada", - "PHOLUS_RUN_description" : "Habilite un escaneo regular de Pholus en su red. Los ajustes de programación se pueden encontrar a continuación. Si selecciona una vez, Pholus se ejecuta solo una vez al inicio durante el tiempo especificado en la configuración de PHOLUS_RUN_TIMEOUT.", - "PHOLUS_RUN_TIMEOUT_name" : "Tiempo de espera de ejecución programado", - "PHOLUS_RUN_TIMEOUT_description" : "El tiempo de espera en segundos para el escaneo Pholus programado. Se aplican las mismas notas con respecto a la duración que en la configuración de PHOLUS_TIMEOUT. Un escaneo programado no verifica si hay dispositivos (unknown) o (name not found), el escaneo se ejecuta de cualquier manera.", - "PHOLUS_RUN_SCHD_name" : "Programar", - "PHOLUS_RUN_SCHD_description" : "Solo está habilitado si selecciona programar en la configuración de PHOLUS_RUN. Asegúrese de ingresar el horario en el formato similar a cron correcto(por ejemplo, validar en crontab.guru). Por ejemplo, ingresar 0 4 * * * ejecutará el escaneo después de las 4 am en el TIMEZONE que configuró arriba. Se ejecutará la PRÓXIMA vez que pase el tiempo.", - "PHOLUS_DAYS_DATA_name" : "Retención de datos", - "PHOLUS_DAYS_DATA_description" : "Cuántos días de entradas de escaneo de Pholus deben conservarse (globalmente, ¡no específico del dispositivo!). El archivo pialert_pholus.log no se modifica. Introduzca 0 para desactivar.", "Nmap_display_name" : "Nmap", "Nmap_icon" : "", "NMAP_ACTIVE_name" : "Ejecución del ciclo", diff --git a/front/plugins/pholus_scan__ignore/README.md b/front/plugins/pholus_scan/README.md similarity index 100% rename from front/plugins/pholus_scan__ignore/README.md rename to front/plugins/pholus_scan/README.md diff --git a/front/plugins/pholus_scan__ignore/config.json b/front/plugins/pholus_scan/config.json similarity index 70% rename from front/plugins/pholus_scan__ignore/config.json rename to front/plugins/pholus_scan/config.json index 53aa9e0e..7e17da58 100755 --- a/front/plugins/pholus_scan__ignore/config.json +++ b/front/plugins/pholus_scan/config.json @@ -47,7 +47,7 @@ { "name" : "timeout", "type" : "setting", - "value" : "RUN_TIMEOUT" + "value" : "PHOLUS_RUN_TIMEOUT" } ], @@ -68,10 +68,16 @@ "language_code":"es_es", "string" : "Cuando ejecutar" }], - "description": [{ + "description": [ + { "language_code":"en_us", - "string" : "Specify when your Name-discovery scan will run. Typical setting would be on_new_device or schedule and then you specify a cron-like schedule in the PHOLUS_RUN_SCHDsetting." - }] + "string" : "Pholus is a sniffing tool to discover additional information about the devices on the network, including the device name. If enabled this will execute the scan before every network scan cycle until there are no (unknown) or (name not found) devices. Please be aware it can spam the network with unnecessary traffic. Depends on the SCAN_SUBNETS setting. For a scheduled or one-off scan, check the PHOLUS_RUN setting.Specify when your Name-discovery scan will run. Typical setting would be on_new_device or schedule and then you specify a cron-like schedule in the PHOLUS_RUN_SCHDsetting." + }, + { + "language_code":"es_es", + "string" : "Pholus es una herramienta de rastreo para descubrir información adicional sobre los dispositivos en la red, incluido el nombre del dispositivo. Si está habilitado, ejecutará el escaneo antes de cada ciclo de escaneo de red hasta que no haya dispositivos (unknown) o (name not found). Tenga en cuenta que puede enviar spam a la red con tráfico innecesario. Depende de la configuración de SCAN_SUBNETS. Para un análisis programado o único, verifique la configuración de PHOLUS_RUN." + } + ] }, { "function": "CMD", @@ -134,7 +140,7 @@ "language_code":"en_us", "string" : "Schedule" }, - { + { "language_code":"es_es", "string" : "Schedule" }], @@ -143,6 +149,31 @@ "string" : "Only enabled if you select schedule in the PHOLUS_RUN setting. Make sure you enter the schedule in the correct cron-like format (e.g. validate at crontab.guru). For example entering 30 3 * * * will run the scan at 3:30 am. Will be run NEXT time the time passes.
" }] }, + { + "function": "DAYS_DATA", + "type": "integer", + "default_value":30, + "options": [], + "localized": ["name", "description"], + "name" : [{ + "language_code":"en_us", + "string" : "Schedule" + }, + { + "language_code":"es_es", + "string" : "Retención de datos" + }], + "description": [ + { + "language_code":"en_us", + "string" : "How many days of Pholus scan entries should be kept (globally, not device specific!) Enter 0 to disable." + }, + { + "language_code":"es_es", + "string" : "Cuántos días de entradas de escaneo de Pholus deben conservarse (globalmente, ¡no específico del dispositivo!). El archivo pialert_pholus.log no se modifica. Introduzca 0 para desactivar." + } + ] + }, { "function": "WATCH", "type": "text.multiselect", @@ -153,13 +184,13 @@ "language_code":"en_us", "string" : "Watched" }, - { + { "language_code":"es_es", "string" : "Watched" }] , "description":[{ "language_code":"en_us", - "string" : "Send a notification if selected values change. Use CTRL + Click to select/deselect. " + "string" : "Send a notification if selected values change. Use CTRL + Click to select/deselect. " }] }, { @@ -195,7 +226,7 @@ [ { "column": "Object_PrimaryID", - "mapped_to_column": "cur_MAC", + "mapped_to_column": "MAC", "css_classes": "col-sm-2", "show": true, "type": "device_name_mac", @@ -212,11 +243,11 @@ }] }, { - "column": "Watched_Value1", - "mapped_to_column": "cur_IP", + "column": "Object_SecondaryID", + "mapped_to_column": "IP_v4_or_v6", "css_classes": "col-sm-2", "show": true, - "type": "device_ip", + "type": "label", "default_value":"", "options": [], "localized": ["name"], @@ -224,14 +255,14 @@ "language_code":"en_us", "string" : "IP" }, - { + { "language_code":"es_es", "string" : "IP" }] }, { - "column": "Watched_Value2", - "mapped_to_column": "cur_Vendor", + "column": "Watched_Value1", + "mapped_to_column": "Info", "css_classes": "col-sm-2", "show": true, "type": "label", @@ -240,16 +271,30 @@ "localized": ["name"], "name":[{ "language_code":"en_us", - "string" : "Vendor" + "string" : "Info" }, { "language_code":"es_es", - "string" : "Proveedor" + "string" : "Info" + }] + } , + { + "column": "Watched_Value2", + "mapped_to_column": "Record_Type", + "css_classes": "col-sm-2", + "show": true, + "type": "label", + "default_value":"", + "options": [], + "localized": ["name"], + "name":[{ + "language_code":"en_us", + "string" : "Type" }] } , { - "column": "Extra", - "mapped_to_column": "cur_ScanMethod", + "column": "Watched_Value3", + "mapped_to_column": "Value", "css_classes": "col-sm-2", "show": true, "type": "label", @@ -258,15 +303,16 @@ "localized": ["name"], "name":[{ "language_code":"en_us", - "string" : "Scan method" + "string" : "Info" }, - { + { "language_code":"es_es", - "string" : "Método de escaneado" + "string" : "Info" }] - } , + }, { "column": "DateTimeCreated", + "mapped_to_column": "Time", "css_classes": "col-sm-2", "show": true, "type": "label", @@ -330,7 +376,7 @@ "language_code":"en_us", "string" : "Status" }, - { + { "language_code":"es_es", "string" : "Estado" }] diff --git a/front/plugins/pholus_scan__ignore/script.py b/front/plugins/pholus_scan/script.py similarity index 58% rename from front/plugins/pholus_scan__ignore/script.py rename to front/plugins/pholus_scan/script.py index c821dd19..1989f906 100755 --- a/front/plugins/pholus_scan__ignore/script.py +++ b/front/plugins/pholus_scan/script.py @@ -15,6 +15,7 @@ sys.path.append('/home/pi/pialert/pialert') from logger import mylog from plugin_helper import Plugin_Object, Plugin_Objects +from helper import timeNowTZ from const import fullPholusPath, logPath CUR_PATH = str(pathlib.Path(__file__).parent.resolve()) @@ -159,7 +160,7 @@ def execute_pholus_on_interface(interface, timeoutSec, mask): mylog('verbose', ['[PholusScan] Scan: Pholus SUCCESS']) # check the last run output - f = open(CUR_PATH + '/pialert_pholus_lastrun.log', 'r+') + f = open(logPath + '/pialert_pholus_lastrun.log', 'r+') newLines = f.read().split('\n') f.close() @@ -173,102 +174,13 @@ def execute_pholus_on_interface(interface, timeoutSec, mask): columns = line.split("|") if len(columns) == 4: # "Info", "Time", "MAC", "IP_v4_or_v6", "Record_Type", "Value" - params.append( interface + " " + mask, timeNowTZ() , columns[0].replace(" ", ""), columns[1].replace(" ", ""), columns[2].replace(" ", ""), columns[3]) + params.append( [interface + " " + mask, timeNowTZ() , columns[0].replace(" ", ""), columns[1].replace(" ", ""), columns[2].replace(" ", ""), columns[3]]) return params -#------------------------------------------------------------------------------- -def cleanResult(str): - # alternative str.split('.')[0] - str = str.replace("._airplay", "") - str = str.replace("._tcp", "") - str = str.replace(".local", "") - str = str.replace("._esphomelib", "") - str = str.replace("._googlecast", "") - str = str.replace(".lan", "") - str = str.replace(".home", "") - str = re.sub(r'-[a-fA-F0-9]{32}', '', str) # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 - str = re.sub(r'#.*', '', str) # Remove everything after '#' including the '#' - # remove trailing dots - if str.endswith('.'): - str = str[:-1] - - return str -# Disclaimer - I'm interfacing with a script I didn't write (pholus3.py) so it's possible I'm missing types of answers -# it's also possible the pholus3.py script can be adjusted to provide a better output to interface with it -# Hit me with a PR if you know how! :) -def resolve_device_name_pholus (pMAC, pIP, allRes): - - pholusMatchesIndexes = [] - index = 0 - for result in allRes: - # limiting entries used for name resolution to the ones containing the current IP (v4 only) - if result["MAC"] == pMAC and result["Record_Type"] == "Answer" and result["IP_v4_or_v6"] == pIP and '._googlezone' not in result["Value"]: - # found entries with a matching MAC address, let's collect indexes - pholusMatchesIndexes.append(index) - - index += 1 - - # return if nothing found - if len(pholusMatchesIndexes) == 0: - return -1 - - # we have some entries let's try to select the most useful one - - # airplay matches contain a lot of information - # Matches for example: - # Brand Tv (50)._airplay._tcp.local. TXT Class:32769 "acl=0 deviceid=66:66:66:66:66:66 features=0x77777,0x38BCB46 rsf=0x3 fv=p20.T-FFFFFF-03.1 flags=0x204 model=XXXX manufacturer=Brand serialNumber=XXXXXXXXXXX protovers=1.1 srcvers=777.77.77 pi=FF:FF:FF:FF:FF:FF psi=00000000-0000-0000-0000-FFFFFFFFFF gid=00000000-0000-0000-0000-FFFFFFFFFF gcgl=0 pk=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and '._airplay._tcp.local. TXT Class:32769' in str(allRes[i]["Value"]) : - return allRes[i]["Value"].split('._airplay._tcp.local. TXT Class:32769')[0] - - # second best - contains airplay - # Matches for example: - # _airplay._tcp.local. PTR Class:IN "Brand Tv (50)._airplay._tcp.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_airplay._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('._googlecast') not in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - # Contains PTR Class:32769 - # Matches for example: - # 3.1.168.192.in-addr.arpa. PTR Class:32769 "MyPc.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:32769' in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - # Contains AAAA Class:IN - # Matches for example: - # DESKTOP-SOMEID.local. AAAA Class:IN "fe80::fe80:fe80:fe80:fe80" - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'AAAA Class:IN' in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('.local.')[0]) - - # Contains _googlecast._tcp.local. PTR Class:IN - # Matches for example: - # _googlecast._tcp.local. PTR Class:IN "Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77._googlecast._tcp.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_googlecast._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('Google-Cast-Group') not in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - # Contains A Class:32769 - # Matches for example: - # Android.local. A Class:32769 "192.168.1.6" - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and ' A Class:32769' in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split(' A Class:32769')[0]) - - # # Contains PTR Class:IN - # Matches for example: - # _esphomelib._tcp.local. PTR Class:IN "ceiling-light-1._esphomelib._tcp.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:IN' in allRes[i]["Value"]: - if allRes[i]["Value"] and len(allRes[i]["Value"].split('"')) > 1: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - return -1 #=============================================================================== diff --git a/front/plugins/snmp_discovery/config.json b/front/plugins/snmp_discovery/config.json index da234cdb..53f8ad7f 100755 --- a/front/plugins/snmp_discovery/config.json +++ b/front/plugins/snmp_discovery/config.json @@ -41,7 +41,7 @@ }], "description": [{ "language_code":"en_us", - "string" : "Este complemento se utiliza para descubrir dispositivos a través de las tablas arp de un enrutador o conmutador compatible con RFC1213.." + "string" : "This plugin is used to discover devices via the arp table(s) of a RFC1213 compliant router or switch." }, { "language_code":"es_es", diff --git a/front/settings.php b/front/settings.php index b0ebb282..e6e6668b 100755 --- a/front/settings.php +++ b/front/settings.php @@ -32,7 +32,7 @@ $result = $db->query("SELECT * FROM Settings"); // array $settingKeyOfLists = array(); -$settingCoreGroups = array('General', 'NewDeviceDefaults', 'Email', 'Webhooks', 'Apprise', 'NTFY', 'PUSHSAFER', 'MQTT', 'DynDNS', 'PiHole', 'Pholus', 'Nmap', 'API'); +$settingCoreGroups = array('General', 'NewDeviceDefaults', 'Email', 'Webhooks', 'Apprise', 'NTFY', 'PUSHSAFER', 'MQTT', 'DynDNS', 'PiHole', 'Nmap', 'API'); $settings = array(); while ($row = $result -> fetchArray (SQLITE3_ASSOC)) { // Push row data diff --git a/pialert/__main__.py b/pialert/__main__.py index d5041237..6c7551e3 100755 --- a/pialert/__main__.py +++ b/pialert/__main__.py @@ -24,7 +24,7 @@ import multiprocessing import conf from const import * from logger import mylog -from helper import filePermissions, isNewVersion, timeNowTZ, updateState +from helper import filePermissions, isNewVersion, timeNowTZ, updateState, get_setting_value from api import update_api from networkscan import process_scan from initialise import importConfigs @@ -34,7 +34,6 @@ from reporting import check_and_run_event, send_notifications from plugin import run_plugin_scripts # different scanners -from scanners.pholusscan import performPholusScan from scanners.nmapscan import performNmapScan from scanners.internet import check_internet_IP @@ -60,7 +59,6 @@ main structure of Pi Alert run plugins (scheduled) check internet IP check vendor - run PHOLUS run NMAP run "scan_network()" processing scan results @@ -161,25 +159,7 @@ def main (): conf.last_update_vendors = loop_start_time conf.cycle = 'update_vendors' mylog('verbose', ['[MAIN] cycle:',conf.cycle]) - update_devices_MAC_vendors(db) - - # Execute scheduled or one-off Pholus scan if enabled and run conditions fulfilled - if conf.PHOLUS_RUN == "schedule" or conf.PHOLUS_RUN == "once": - - pholusSchedule = [sch for sch in conf.mySchedules if sch.service == "pholus"][0] - run = False - - # run once after application starts - if conf.PHOLUS_RUN == "once" and pholusSchedule.last_run == 0: - run = True - - # run if overdue scheduled time - if conf.PHOLUS_RUN == "schedule": - run = pholusSchedule.runScheduleCheck() - - if run: - pholusSchedule.last_run = datetime.datetime.now(conf.tz).replace(microsecond=0) - performPholusScan(db, conf.PHOLUS_RUN_TIMEOUT, conf.userSubnets) + update_devices_MAC_vendors(db) # Execute scheduled or one-off Nmap scan if enabled and run conditions fulfilled if conf.NMAP_RUN == "schedule" or conf.NMAP_RUN == "once": @@ -236,7 +216,7 @@ def main (): conf.last_cleanup = loop_start_time conf.cycle = 'cleanup' mylog('verbose', ['[MAIN] cycle:',conf.cycle]) - db.cleanup_database(startTime, conf.DAYS_TO_KEEP_EVENTS, conf.PHOLUS_DAYS_DATA, conf.HRS_TO_KEEP_NEWDEV, conf.PLUGINS_KEEP_HIST) + db.cleanup_database(startTime, conf.DAYS_TO_KEEP_EVENTS, get_setting_value('PHOLUS_DAYS_DATA'), conf.HRS_TO_KEEP_NEWDEV, conf.PLUGINS_KEEP_HIST) # Commit SQL db.commitDB() diff --git a/pialert/conf.py b/pialert/conf.py index e327c093..a139db6e 100755 --- a/pialert/conf.py +++ b/pialert/conf.py @@ -103,15 +103,6 @@ DDNS_USER = 'dynu_user' DDNS_PASSWORD = 'A0000000B0000000C0000000D0000000' DDNS_UPDATE_URL = 'https://api.dynu.com/nic/update?' -# PHOLUS -PHOLUS_ACTIVE = False -PHOLUS_TIMEOUT = 20 -PHOLUS_FORCE = False -PHOLUS_RUN = 'once' -PHOLUS_RUN_TIMEOUT = 600 -PHOLUS_RUN_SCHD = '0 4 * * *' -PHOLUS_DAYS_DATA = 0 - # Nmap NMAP_ACTIVE = True NMAP_TIMEOUT = 150 diff --git a/pialert/device.py b/pialert/device.py index 8f896a4b..b6ce2119 100755 --- a/pialert/device.py +++ b/pialert/device.py @@ -3,11 +3,11 @@ import subprocess import conf import re -from helper import timeNowTZ, get_setting, get_setting_value +from helper import timeNowTZ, get_setting, get_setting_value,resolve_device_name_dig, resolve_device_name_pholus from scanners.internet import check_IP_format, get_internet_IP from logger import mylog, print_log from mac_vendor import query_MAC_vendor -from scanners.pholusscan import performPholusScan, resolve_device_name_dig, resolve_device_name_pholus + #------------------------------------------------------------------------------- @@ -363,10 +363,6 @@ def update_devices_names (db): unknownDevices = sql.fetchall() db.commitDB() - # perform Pholus scan if (unknown) devices found - if conf.PHOLUS_ACTIVE and (len(unknownDevices) > 0 or conf.PHOLUS_FORCE): - performPholusScan(db, conf.PHOLUS_TIMEOUT, conf.userSubnets) - # skip checks if no unknown devices if len(unknownDevices) == 0 and conf.PHOLUS_FORCE == False: return diff --git a/pialert/helper.py b/pialert/helper.py index 1dd061ee..e790f3a2 100755 --- a/pialert/helper.py +++ b/pialert/helper.py @@ -6,6 +6,7 @@ import datetime import os import re import subprocess +import pytz from pytz import timezone from datetime import timedelta import json @@ -20,7 +21,12 @@ from logger import mylog, logResult #------------------------------------------------------------------------------- def timeNowTZ(): - return datetime.datetime.now(conf.tz).replace(microsecond=0) + if isinstance(conf.TIMEZONE, str): + tz = pytz.timezone(conf.TIMEZONE) + else: + tz = conf.TIMEZONE + + return datetime.datetime.now(tz).replace(microsecond=0) def timeNow(): return datetime.datetime.now().replace(microsecond=0) @@ -370,4 +376,135 @@ def get_setting_value(key): return setVal - return '' \ No newline at end of file + return '' + +#------------------------------------------------------------------------------- +# Disclaimer - I'm interfacing with a script I didn't write (pholus3.py) so it's possible I'm missing types of answers +# it's also possible the pholus3.py script can be adjusted to provide a better output to interface with it +# Hit me with a PR if you know how! :) +def resolve_device_name_pholus (pMAC, pIP, allRes): + + pholusMatchesIndexes = [] + + index = 0 + for result in allRes: + # limiting entries used for name resolution to the ones containing the current IP (v4 only) + if result["MAC"] == pMAC and result["Record_Type"] == "Answer" and result["IP_v4_or_v6"] == pIP and '._googlezone' not in result["Value"]: + # found entries with a matching MAC address, let's collect indexes + pholusMatchesIndexes.append(index) + + index += 1 + + # return if nothing found + if len(pholusMatchesIndexes) == 0: + return -1 + + # we have some entries let's try to select the most useful one + + # airplay matches contain a lot of information + # Matches for example: + # Brand Tv (50)._airplay._tcp.local. TXT Class:32769 "acl=0 deviceid=66:66:66:66:66:66 features=0x77777,0x38BCB46 rsf=0x3 fv=p20.T-FFFFFF-03.1 flags=0x204 model=XXXX manufacturer=Brand serialNumber=XXXXXXXXXXX protovers=1.1 srcvers=777.77.77 pi=FF:FF:FF:FF:FF:FF psi=00000000-0000-0000-0000-FFFFFFFFFF gid=00000000-0000-0000-0000-FFFFFFFFFF gcgl=0 pk=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + for i in pholusMatchesIndexes: + if checkIPV4(allRes[i]['IP_v4_or_v6']) and '._airplay._tcp.local. TXT Class:32769' in str(allRes[i]["Value"]) : + return allRes[i]["Value"].split('._airplay._tcp.local. TXT Class:32769')[0] + + # second best - contains airplay + # Matches for example: + # _airplay._tcp.local. PTR Class:IN "Brand Tv (50)._airplay._tcp.local." + for i in pholusMatchesIndexes: + if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_airplay._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('._googlecast') not in allRes[i]["Value"]: + return cleanResult(allRes[i]["Value"].split('"')[1]) + + # Contains PTR Class:32769 + # Matches for example: + # 3.1.168.192.in-addr.arpa. PTR Class:32769 "MyPc.local." + for i in pholusMatchesIndexes: + if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:32769' in allRes[i]["Value"]: + return cleanResult(allRes[i]["Value"].split('"')[1]) + + # Contains AAAA Class:IN + # Matches for example: + # DESKTOP-SOMEID.local. AAAA Class:IN "fe80::fe80:fe80:fe80:fe80" + for i in pholusMatchesIndexes: + if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'AAAA Class:IN' in allRes[i]["Value"]: + return cleanResult(allRes[i]["Value"].split('.local.')[0]) + + # Contains _googlecast._tcp.local. PTR Class:IN + # Matches for example: + # _googlecast._tcp.local. PTR Class:IN "Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77._googlecast._tcp.local." + for i in pholusMatchesIndexes: + if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_googlecast._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('Google-Cast-Group') not in allRes[i]["Value"]: + return cleanResult(allRes[i]["Value"].split('"')[1]) + + # Contains A Class:32769 + # Matches for example: + # Android.local. A Class:32769 "192.168.1.6" + for i in pholusMatchesIndexes: + if checkIPV4(allRes[i]['IP_v4_or_v6']) and ' A Class:32769' in allRes[i]["Value"]: + return cleanResult(allRes[i]["Value"].split(' A Class:32769')[0]) + + # # Contains PTR Class:IN + # Matches for example: + # _esphomelib._tcp.local. PTR Class:IN "ceiling-light-1._esphomelib._tcp.local." + for i in pholusMatchesIndexes: + if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:IN' in allRes[i]["Value"]: + if allRes[i]["Value"] and len(allRes[i]["Value"].split('"')) > 1: + return cleanResult(allRes[i]["Value"].split('"')[1]) + + return -1 + +#------------------------------------------------------------------------------- + +def resolve_device_name_dig (pMAC, pIP): + + newName = "" + + try : + dig_args = ['dig', '+short', '-x', pIP] + + # Execute command + try: + # try runnning a subprocess + newName = subprocess.check_output (dig_args, universal_newlines=True) + except subprocess.CalledProcessError as e: + # An error occured, handle it + mylog('none', ['[device_name_dig] ', e.output]) + # newName = "Error - check logs" + return -1 + + # Check returns + newName = newName.strip() + + if len(newName) == 0 : + return -1 + + # Cleanup + newName = cleanResult(newName) + + if newName == "" or len(newName) == 0: + return -1 + + # Return newName + return newName + + # not Found + except subprocess.CalledProcessError : + return -1 + +#------------------------------------------------------------------------------- +def cleanResult(str): + # alternative str.split('.')[0] + str = str.replace("._airplay", "") + str = str.replace("._tcp", "") + str = str.replace(".local", "") + str = str.replace("._esphomelib", "") + str = str.replace("._googlecast", "") + str = str.replace(".lan", "") + str = str.replace(".home", "") + str = re.sub(r'-[a-fA-F0-9]{32}', '', str) # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 + str = re.sub(r'#.*', '', str) # Remove everything after '#' including the '#' + # remove trailing dots + if str.endswith('.'): + str = str[:-1] + + return str \ No newline at end of file diff --git a/pialert/initialise.py b/pialert/initialise.py index f8f6a3a3..9d726839 100755 --- a/pialert/initialise.py +++ b/pialert/initialise.py @@ -154,15 +154,6 @@ def importConfigs (db): conf.DDNS_PASSWORD = ccd('DDNS_PASSWORD', 'A0000000B0000000C0000000D0000000' , c_d, 'DynDNS password', 'password', '', 'DynDNS') conf.DDNS_UPDATE_URL = ccd('DDNS_UPDATE_URL', 'https://api.dynu.com/nic/update?' , c_d, 'DynDNS update URL', 'text', '', 'DynDNS') - # PHOLUS - conf.PHOLUS_ACTIVE = ccd('PHOLUS_ACTIVE', False , c_d, 'Enable Pholus scans', 'boolean', '', 'Pholus') - conf.PHOLUS_TIMEOUT = ccd('PHOLUS_TIMEOUT', 20 , c_d, 'Pholus timeout', 'integer', '', 'Pholus') - conf.PHOLUS_FORCE = ccd('PHOLUS_FORCE', False , c_d, 'Pholus force check', 'boolean', '', 'Pholus') - conf.PHOLUS_RUN = ccd('PHOLUS_RUN', 'once' , c_d, 'Pholus enable schedule', 'text.select', "['disabled', 'once', 'schedule']", 'Pholus') - conf.PHOLUS_RUN_TIMEOUT = ccd('PHOLUS_RUN_TIMEOUT', 600 , c_d, 'Pholus timeout schedule', 'integer', '', 'Pholus') - conf.PHOLUS_RUN_SCHD = ccd('PHOLUS_RUN_SCHD', '0 4 * * *' , c_d, 'Pholus schedule', 'text', '', 'Pholus') - conf.PHOLUS_DAYS_DATA = ccd('PHOLUS_DAYS_DATA', 0 , c_d, 'Pholus keep days', 'integer', '', 'Pholus') - # Nmap conf.NMAP_ACTIVE = ccd('NMAP_ACTIVE', True , c_d, 'Enable Nmap scans', 'boolean', '', 'Nmap') conf.NMAP_TIMEOUT = ccd('NMAP_TIMEOUT', 150 , c_d, 'Nmap timeout', 'integer', '', 'Nmap') @@ -197,11 +188,6 @@ def importConfigs (db): # reset schedules conf.mySchedules = [] - # init pholus schedule - pholusSchedule = Cron(conf.PHOLUS_RUN_SCHD).schedule(start_date=datetime.datetime.now(conf.tz)) - - conf.mySchedules.append(schedule_class("pholus", pholusSchedule, pholusSchedule.next(), False)) - # init nmap schedule nmapSchedule = Cron(conf.NMAP_RUN_SCHD).schedule(start_date=datetime.datetime.now(conf.tz)) conf.mySchedules.append(schedule_class("nmap", nmapSchedule, nmapSchedule.next(), False)) diff --git a/pialert/plugin_utils.py b/pialert/plugin_utils.py index 5ebc67fa..4ac687a9 100755 --- a/pialert/plugin_utils.py +++ b/pialert/plugin_utils.py @@ -135,7 +135,7 @@ def resolve_wildcards_arr(commandArr, params): for comPart in commandArr: - commandArr[i] = comPart.replace('{' + param[0] + '}', param[1]).replace('{s-quote}',"'") + commandArr[i] = comPart.replace('{' + str(param[0]) + '}', str(param[1])).replace('{s-quote}',"'") i += 1 diff --git a/pialert/scanners/pholusscan.py b/pialert/scanners/pholusscan.py index 837d116f..412fb018 100755 --- a/pialert/scanners/pholusscan.py +++ b/pialert/scanners/pholusscan.py @@ -7,196 +7,3 @@ from logger import mylog #------------------------------------------------------------------------------- -def performPholusScan (db, timeoutSec, userSubnets): - sql = db.sql # TO-DO - # scan every interface - for subnet in userSubnets: - - temp = subnet.split("--interface=") - - if len(temp) != 2: - mylog('none', ["[PholusScan] Skip scan (need subnet in format '192.168.1.0/24 --inteface=eth0'), got: ", subnet]) - return - - mask = temp[0].strip() - interface = temp[1].strip() - - # logging & updating app state - updateState(db,"Scan: Pholus") - mylog('none', ['[PholusScan] Scan: Pholus for ', str(timeoutSec), 's ('+ str(round(int(timeoutSec) / 60, 1)) +'min)']) - mylog('verbose', ["[PholusScan] Pholus scan on [interface] ", interface, " [mask] " , mask]) - - # the scan always lasts 2x as long, so the desired user time from settings needs to be halved - adjustedTimeout = str(round(int(timeoutSec) / 2, 0)) - - # python3 -m trace --trace /home/pi/pialert/pholus/pholus3.py eth1 -rdns_scanning 192.168.1.0/24 -stimeout 600 - pholus_args = ['python3', fullPholusPath, interface, "-rdns_scanning", mask, "-stimeout", adjustedTimeout] - - # Execute command - output = "" - - try: - # try runnning a subprocess with a forced (timeout + 30 seconds) in case the subprocess hangs - output = subprocess.check_output (pholus_args, universal_newlines=True, stderr=subprocess.STDOUT, timeout=(timeoutSec + 30)) - except subprocess.CalledProcessError as e: - # An error occured, handle it - mylog('none', ['[PholusScan]', e.output]) - mylog('none', ["[PholusScan] Error - Pholus Scan - check logs"]) - except subprocess.TimeoutExpired as timeErr: - mylog('none', ['[PholusScan] Pholus TIMEOUT - the process forcefully terminated as timeout reached']) - - if output == "": # check if the subprocess failed - mylog('none', ['[PholusScan] Scan: Pholus FAIL - check logs']) - else: - mylog('verbose', ['[PholusScan] Scan: Pholus SUCCESS']) - - # check the last run output - f = open(logPath + '/pialert_pholus_lastrun.log', 'r+') - newLines = f.read().split('\n') - f.close() - - # cleanup - select only lines containing a separator to filter out unnecessary data - newLines = list(filter(lambda x: '|' in x, newLines)) - - # build SQL query parameters to insert into the DB - params = [] - - for line in newLines: - columns = line.split("|") - if len(columns) == 4: - params.append(( interface + " " + mask, timeNowTZ() , columns[0].replace(" ", ""), columns[1].replace(" ", ""), columns[2].replace(" ", ""), columns[3], '')) - - if len(params) > 0: - sql.executemany ("""INSERT INTO Pholus_Scan ("Info", "Time", "MAC", "IP_v4_or_v6", "Record_Type", "Value", "Extra") VALUES (?, ?, ?, ?, ?, ?, ?)""", params) - db.commitDB() - -#------------------------------------------------------------------------------- -def cleanResult(str): - # alternative str.split('.')[0] - str = str.replace("._airplay", "") - str = str.replace("._tcp", "") - str = str.replace(".local", "") - str = str.replace("._esphomelib", "") - str = str.replace("._googlecast", "") - str = str.replace(".lan", "") - str = str.replace(".home", "") - str = re.sub(r'-[a-fA-F0-9]{32}', '', str) # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77 - str = re.sub(r'#.*', '', str) # Remove everything after '#' including the '#' - # remove trailing dots - if str.endswith('.'): - str = str[:-1] - - return str - - -# Disclaimer - I'm interfacing with a script I didn't write (pholus3.py) so it's possible I'm missing types of answers -# it's also possible the pholus3.py script can be adjusted to provide a better output to interface with it -# Hit me with a PR if you know how! :) -def resolve_device_name_pholus (pMAC, pIP, allRes): - - pholusMatchesIndexes = [] - - index = 0 - for result in allRes: - # limiting entries used for name resolution to the ones containing the current IP (v4 only) - if result["MAC"] == pMAC and result["Record_Type"] == "Answer" and result["IP_v4_or_v6"] == pIP and '._googlezone' not in result["Value"]: - # found entries with a matching MAC address, let's collect indexes - pholusMatchesIndexes.append(index) - - index += 1 - - # return if nothing found - if len(pholusMatchesIndexes) == 0: - return -1 - - # we have some entries let's try to select the most useful one - - # airplay matches contain a lot of information - # Matches for example: - # Brand Tv (50)._airplay._tcp.local. TXT Class:32769 "acl=0 deviceid=66:66:66:66:66:66 features=0x77777,0x38BCB46 rsf=0x3 fv=p20.T-FFFFFF-03.1 flags=0x204 model=XXXX manufacturer=Brand serialNumber=XXXXXXXXXXX protovers=1.1 srcvers=777.77.77 pi=FF:FF:FF:FF:FF:FF psi=00000000-0000-0000-0000-FFFFFFFFFF gid=00000000-0000-0000-0000-FFFFFFFFFF gcgl=0 pk=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and '._airplay._tcp.local. TXT Class:32769' in str(allRes[i]["Value"]) : - return allRes[i]["Value"].split('._airplay._tcp.local. TXT Class:32769')[0] - - # second best - contains airplay - # Matches for example: - # _airplay._tcp.local. PTR Class:IN "Brand Tv (50)._airplay._tcp.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_airplay._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('._googlecast') not in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - # Contains PTR Class:32769 - # Matches for example: - # 3.1.168.192.in-addr.arpa. PTR Class:32769 "MyPc.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:32769' in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - # Contains AAAA Class:IN - # Matches for example: - # DESKTOP-SOMEID.local. AAAA Class:IN "fe80::fe80:fe80:fe80:fe80" - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'AAAA Class:IN' in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('.local.')[0]) - - # Contains _googlecast._tcp.local. PTR Class:IN - # Matches for example: - # _googlecast._tcp.local. PTR Class:IN "Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77._googlecast._tcp.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_googlecast._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('Google-Cast-Group') not in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - # Contains A Class:32769 - # Matches for example: - # Android.local. A Class:32769 "192.168.1.6" - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and ' A Class:32769' in allRes[i]["Value"]: - return cleanResult(allRes[i]["Value"].split(' A Class:32769')[0]) - - # # Contains PTR Class:IN - # Matches for example: - # _esphomelib._tcp.local. PTR Class:IN "ceiling-light-1._esphomelib._tcp.local." - for i in pholusMatchesIndexes: - if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:IN' in allRes[i]["Value"]: - if allRes[i]["Value"] and len(allRes[i]["Value"].split('"')) > 1: - return cleanResult(allRes[i]["Value"].split('"')[1]) - - return -1 - -#------------------------------------------------------------------------------- - -def resolve_device_name_dig (pMAC, pIP): - - newName = "" - - try : - dig_args = ['dig', '+short', '-x', pIP] - - # Execute command - try: - # try runnning a subprocess - newName = subprocess.check_output (dig_args, universal_newlines=True) - except subprocess.CalledProcessError as e: - # An error occured, handle it - mylog('none', ['[device_name_dig] ', e.output]) - # newName = "Error - check logs" - return -1 - - # Check returns - newName = newName.strip() - - if len(newName) == 0 : - return -1 - - # Cleanup - newName = cleanResult(newName) - - if newName == "" or len(newName) == 0: - return -1 - - # Return newName - return newName - - # not Found - except subprocess.CalledProcessError : - return -1