diff --git a/README.md b/README.md index a31be8df..ebfeb1e3 100755 --- a/README.md +++ b/README.md @@ -31,11 +31,12 @@ The system continuously scans the network for, **New devices**, **New connection that were not discovered by the other methods. -## 🧩 Notification Integrations +## 🧩 Integrations - [Apprise](https://hub.docker.com/r/caronc/apprise), [Pushsafer](https://www.pushsafer.com/), [NTFY](https://ntfy.sh/) - [Webhooks](https://github.com/jokob-sk/Pi.Alert/blob/main/docs/WEBHOOK_N8N.md) ([sample JSON](docs/webhook_json_sample.json)) - Home Assistant via [MQTT](https://www.home-assistant.io/integrations/mqtt/) - discovery ~10s per device, deleting not supported, use [MQTT Explorer](https://mqtt-explorer.com/) for now + - A simple [API endpoint](https://github.com/jokob-sk/Pi.Alert/blob/main/docs/API.md) ## 🔐 Security diff --git a/back/pialert.py b/back/pialert.py index 9e983a84..e0f8fb04 100755 --- a/back/pialert.py +++ b/back/pialert.py @@ -288,6 +288,8 @@ def importConfig (): global PHOLUS_ACTIVE, PHOLUS_TIMEOUT, PHOLUS_FORCE, PHOLUS_DAYS_DATA, PHOLUS_RUN, PHOLUS_RUN_SCHD, PHOLUS_RUN_TIMEOUT # Nmap global NMAP_ACTIVE, NMAP_TIMEOUT, NMAP_RUN, NMAP_RUN_SCHD, NMAP_ARGS + # API + global ENABLE_API, API_RUN, API_RUN_SCHD, API_RUN_INTERVAL mySettings = [] # reset settings # get config file @@ -385,8 +387,14 @@ def importConfig (): NMAP_RUN = ccd('NMAP_RUN', 'none' , c_d, 'Nmap enable schedule', 'selecttext', "['none', 'once', 'schedule']", 'Nmap') NMAP_RUN_SCHD = ccd('NMAP_RUN_SCHD', '0 2 * * *' , c_d, 'Nmap schedule', 'text', '', 'Nmap') NMAP_ARGS = ccd('NMAP_ARGS', '-p -10000' , c_d, 'Nmap custom arguments', 'text', '', 'Nmap') + + # API + ENABLE_API = ccd('ENABLE_API', True , c_d, 'Enable API', 'boolean', '', 'API') + API_RUN = ccd('API_RUN', 'schedule' , c_d, 'API execution', 'selecttext', "['none', 'interval', 'schedule']", 'API') + API_RUN_SCHD = ccd('API_RUN_SCHD', '*/3 * * * *' , c_d, 'API schedule', 'text', '', 'API') + API_RUN_INTERVAL = ccd('API_RUN_INTERVAL', 10 , c_d, 'API update interval', 'integer', '', 'API') - # Insert into DB + # Insert settings into the DB sql.execute ("DELETE FROM Settings") sql.executemany ("""INSERT INTO Settings ("Code_Name", "Display_Name", "Description", "Type", "Options", "RegEx", "Value", "Group", "Events" ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", mySettings) @@ -416,6 +424,10 @@ def importConfig (): nmapSchedule = Cron(NMAP_RUN_SCHD).schedule(start_date=datetime.datetime.now(tz)) mySchedules.append(serviceSchedule("nmap", nmapSchedule, nmapSchedule.next(), False)) + # init API schedule + apiSchedule = Cron(API_RUN_SCHD).schedule(start_date=datetime.datetime.now(tz)) + mySchedules.append(serviceSchedule("api", apiSchedule, apiSchedule.next(), False)) + # Format and prepare the list of subnets updateSubnets() @@ -433,6 +445,7 @@ now_minus_24h = time_started - datetime.timedelta(hours = 24) last_network_scan = now_minus_24h last_internet_IP_scan = now_minus_24h +last_API_update = now_minus_24h last_run = now_minus_24h last_cleanup = now_minus_24h last_update_vendors = time_started - datetime.timedelta(days = 6) # update vendors 24h after first run and than once a week @@ -442,7 +455,7 @@ newVersionAvailable = False def main (): # Initialize global variables - global time_started, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors + global time_started, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors, last_API_update # second set of global variables global startTime, log_timestamp, sql_connection, sql @@ -468,6 +481,12 @@ def main (): # check if there is a front end initiated event which needs to be executed check_and_run_event() + # Execute API update if enabled via the interval schedule settings and if enough time passed + if API_RUN == "interval" and last_API_update + datetime.timedelta(seconds = API_RUN_INTERVAL) < time_started: + + last_API_update = time_started + update_api() + # proceed if 1 minute passed if last_run + datetime.timedelta(minutes=1) < time_started : @@ -496,7 +515,7 @@ def main (): cycle = 'update_vendors' update_devices_MAC_vendors() - # Execute Pholus scheduled or one-off scan if enabled and run conditions fulfilled + # Execute scheduled or one-off Pholus scan if enabled and run conditions fulfilled if PHOLUS_RUN == "schedule" or PHOLUS_RUN == "once": pholusSchedule = [sch for sch in mySchedules if sch.service == "pholus"][0] @@ -514,7 +533,7 @@ def main (): pholusSchedule.last_run = datetime.datetime.now(tz).replace(microsecond=0) performPholusScan(PHOLUS_RUN_TIMEOUT) - # Execute Nmap scheduled or one-off scan if enabled and run conditions fulfilled + # Execute scheduled or one-off Nmap scan if enabled and run conditions fulfilled if NMAP_RUN == "schedule" or NMAP_RUN == "once": nmapSchedule = [sch for sch in mySchedules if sch.service == "nmap"][0] @@ -532,6 +551,19 @@ def main (): nmapSchedule.last_run = datetime.datetime.now(tz).replace(microsecond=0) performNmapScan(get_all_devices()) + # Execute scheduled API update if enabled + if API_RUN == "schedule": + + apiSchedule = [sch for sch in mySchedules if sch.service == "api"][0] + run = False + + # run if overdue scheduled time + run = apiSchedule.runScheduleCheck() + + if run: + apiSchedule.last_run = datetime.datetime.now(tz).replace(microsecond=0) + update_api() + # Perform a network scan via arp-scan or pihole if last_network_scan + datetime.timedelta(minutes=SCAN_CYCLE_MINUTES) < time_started: last_network_scan = time_started @@ -2189,7 +2221,7 @@ def send_notifications (): # Send Mail if json_internet != [] or json_new_devices != [] or json_down_devices != [] or json_events != [] or debug_force_notification: - update_api() + update_api(True) file_print(' Changes detected, sending reports') @@ -2913,19 +2945,25 @@ def to_binary_sensor(input): return result - #=============================================================================== # API #=============================================================================== -def update_api(): +def update_api(isNotification = False): + + # Proceed only if enabled in settings + if ENABLE_API == False: + return + file_print(' [API] Updating files in /front/api') folder = pialertPath + '/front/api/' - write_file(folder + 'notification_text.txt' , mail_text) - write_file(folder + 'notification_text.html' , mail_html) - write_file(folder + 'notification_json_final.json' , json.dumps(json_final)) - + if isNotification: + # Update last notification alert in all formats + write_file(folder + 'notification_text.txt' , mail_text) + write_file(folder + 'notification_text.html' , mail_html) + write_file(folder + 'notification_json_final.json' , json.dumps(json_final)) + # prepare databse tables we want to expose dataSourcesSQLs = [ ["devices", sql_devices_all], ["nmap_scan", sql_nmap_scan_all], diff --git a/dockerfiles/README.md b/dockerfiles/README.md index f6523736..1cf2bb5a 100755 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -48,6 +48,7 @@ docker run -d --rm --network=host \ |Optional| `:/home/pi/pialert/front/log` | Logs folder useful for debugging if you have issues setting up the container | |Optional| `:/etc/pihole/pihole-FTL.db` | PiHole's `pihole-FTL.db` database file. Required if you want to use PiHole | |Optional| `:/etc/pihole/dhcp.leases` | PiHole's `dhcp.leases` file. Required if you want to use PiHole | +|Optional| `:/home/pi/pialert/front/api` | A simple [API endpoint](https://github.com/jokob-sk/Pi.Alert/blob/main/docs/API.md) containing static (but regularly updated) json and other files. | ### Config (`pialert.conf`) @@ -56,7 +57,7 @@ docker run -d --rm --network=host \ - ❗ Set the `SCAN_SUBNETS` variable. * The adapter will probably be `eth0` or `eth1`. (Run `iwconfig` to find your interface name(s)) * Specify the network filter (which **significantly** speeds up the scan process). For example, the filter `192.168.1.0/24` covers IP ranges 192.168.1.0 to 192.168.1.255. - * Examples for one and two subnets (❗ Note the `['...', '...']` format for two or more subnets): + * Examples for one and two subnets (❗ Note the `['...', '...']` format): * One subnet: `SCAN_SUBNETS = ['192.168.1.0/24 --interface=eth0']` * Two subnets: `SCAN_SUBNETS = ['192.168.1.0/24 --interface=eth0', '192.168.1.0/24 --interface=eth1']` diff --git a/docs/API.md b/docs/API.md index b9aa8396..41125fbb 100755 --- a/docs/API.md +++ b/docs/API.md @@ -1,12 +1,23 @@ -## Where are API endpoints located +## API endpoints -PiAlert comes with a simple API. These API endpoints are static files, which are updated during: +PiAlert comes with a simple API. These API endpoints are static files, that are periodically updated based on your settings. -1) A notification event -2) TBD + +### When are the endpoints updated + +Once you enable the API (`ENABLE_API` setting), the endpoints are updated during these events: + +1) Always during a notification event. +2) (optional) If `API_RUN` is set to `schedule` on a specified cron-like schedule specified by the `API_RUN_SCHD` setting. +3) (optional) If `API_RUN` is set to `interval` every N seconds specified by the `API_RUN_INTERVAL` setting (minimum 5). + + +### Location of the endpoints In the container, these files are located under the `/home/pi/pialert/front/api/` folder and thus on the `/api/` url. +### Available endpoints + You can access the following files: | File name | Description | @@ -16,8 +27,72 @@ You can access the following files: | `notification_json_final.json` | The json version of the last notification (e.g. used for webhooks - [sample JSON](https://github.com/jokob-sk/Pi.Alert/blob/main/back/webhook_json_sample.json)). | | `table_devices.json` | The current (at the time of the last update as mentioned above on this page) state of all of the available Devices detected by the app. | | `table_nmap_scan.json` | The current state of the discovered ports by the regular NMAP scans. | - | `pholus_scan.json` | The latest state of the [pholus](https://github.com/jokob-sk/Pi.Alert/tree/main/pholus) (A multicast DNS and DNS Service Discovery Security Assessment Tool) scan results. | + | `table_pholus_scan.json` | The latest state of the [pholus](https://github.com/jokob-sk/Pi.Alert/tree/main/pholus) (A multicast DNS and DNS Service Discovery Security Assessment Tool) scan results. | | `table_events_pending_alert.json` | The list of the unprocessed (pending) notification events. | Current/latest state of the aforementioned files depends on your settings. +### JSON Data format + +The endpoints starting with the `table_` prefix contain most, if not all, data contained in the corresponding database table. The common format for those is: + +```JSON +{ + "data": [ + { + "db_column_name": "data", + "db_column_name2": "data2" + }, + { + "db_column_name": "data3", + "db_column_name2": "data4" + } + ] +} + +``` + +Example JSON of the `table_devices.json` endpoint with two Devices (database rows): + +```JSON +{ + "data": [ + { + "dev_MAC": "Internet", + "dev_Name": "Net - Huawei", + "dev_DeviceType": "Router", + "dev_Vendor": null, + "dev_Group": "Always on", + "dev_FirstConnection": "2021-01-01 00:00:00", + "dev_LastConnection": "2021-01-28 22:22:11", + "dev_LastIP": "192.168.1.24", + "dev_StaticIP": 0, + "dev_PresentLastScan": 1, + "dev_LastNotification": "2023-01-28 22:22:28.998715", + "dev_NewDevice": 0, + "dev_Network_Node_MAC_ADDR": "", + "dev_Network_Node_port": "", + "dev_Icon": "globe" + }, + { + "dev_MAC": "a4:8f:ff:aa:ba:1f", + "dev_Name": "Net - USG", + "dev_DeviceType": "Firewall", + "dev_Vendor": "Ubiquiti Inc", + "dev_Group": "", + "dev_FirstConnection": "2021-02-12 22:05:00", + "dev_LastConnection": "2021-07-17 15:40:00", + "dev_LastIP": "192.168.1.1", + "dev_StaticIP": 1, + "dev_PresentLastScan": 1, + "dev_LastNotification": "2021-07-17 15:40:10.667717", + "dev_NewDevice": 0, + "dev_Network_Node_MAC_ADDR": "Internet", + "dev_Network_Node_port": 1, + "dev_Icon": "shield-halved" + } + ] +} + +``` + diff --git a/front/php/templates/language/en_us.php b/front/php/templates/language/en_us.php index 3a3a20e2..b58cb24c 100755 --- a/front/php/templates/language/en_us.php +++ b/front/php/templates/language/en_us.php @@ -646,6 +646,16 @@ the arp-scan will take hours to complete instead of seconds. 'NMAP_ARGS_name' => 'Arguments', 'NMAP_ARGS_description' => 'Arguments used to run the Nmap scan. Be careful to specify the arguments correctly. For example -p -10000 scans ports from 1 to 10000.', +// Nmap +'API_settings_group' => ' API', +'ENABLE_API_name' => 'Enable API', +'ENABLE_API_description' => 'If enabled the app will start publishing and updating simple API endpoints under the /home/pi/pialert/front/api/ folder and thus on the pialert_url/api/File_name` url.', +'API_RUN_name' => 'Scheduling updates', +'API_RUN_description' => 'Scheduling settings to specify when the API endpoints should be updated. If set to schedule then endpoints will be updated on a specified cron-like schedule specified by the API_RUN_SCHD setting. Otherwise if set to interval endpoints will be updated every N seconds specified by the API_RUN_INTERVAL setting.', +'API_RUN_SCHD_name' => 'Schedule', +'API_RUN_SCHD_description' => 'Depends on the API_RUN settings to be set to schedule. Make sure you enter the schedule in the correct cron-like format (e.g. validate at crontab.guru).', +'API_RUN_INTERVAL_name' => 'API update interval', +'API_RUN_INTERVAL_description' => 'Depends on the API_RUN settings to be set to interval. The minimum cycle is 5 seconds.', ); ?> diff --git a/front/settings.php b/front/settings.php index 2dad3ac4..5af3938b 100755 --- a/front/settings.php +++ b/front/settings.php @@ -285,7 +285,7 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {