🔃 Sync Hub v0.8

front/php/templates/header.php

@@ -165,7 +165,7 @@ if ($ENABLED_DARKMODE === True) {
           <!-- Full Screen -->		 
           <li>
             <a id="notifications-button" href='userNotifications.php' role="button" span class='fa-solid fa-bell'></a>
-            <span  id="unread-notifications-bell-count" title="" class="badge bg-yellow unread-notifications-bell" >0</span>
+            <span  id="unread-notifications-bell-count" title="" class="badge bg-red unread-notifications-bell" >0</span>
           </li>	                
           <!-- Server Status -->
           <li>

front/plugins/sync/README.md

@@ -1,6 +1,52 @@
 ## Overview
 
-Synchronization plugin to synchronize multiple app instances. The plugin sends encrypted `last_result.log` files for individual plugins. 
+Synchronization plugin to synchronize multiple app instances. The Plugin can sychronize 2 types of data:
+
+1. 💻 Devices: The plugin sends an encrypted `table_devices.json` file to synchronize the whole Devices DB table.
+1. 🔌 Plugin data:  The plugin sends encrypted `last_result.log` files for individual plugins. 
+
+### Synchronizing 💻 Devices data
+
+This is probably what most of the setups will use. Required settings follow.
+
+#### Node (Source) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+- Node name `SYNC_node_name`
+- Hub URL `SYNC_hub_url`
+- Send Devices `SYNC_devices` 👈
+
+#### Hub (Target) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+
+### Synchronizing 🔌 Plugins data
+
+This mechanism will be probably used in special use cases. Required settings follow.
+
+#### Node (Source) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+- Node name `SYNC_node_name`
+- Hub URL `SYNC_hub_url`
+- Send Plugins `SYNC_plugins` 👈
+
+#### Hub (Target) Settings
+
+- When to run `SYNC_RUN`
+- Schedule `SYNC_RUN_SCHD`
+- API token `SYNC_api_token`
+- Encryption Key `SYNC_encryption_key`
+
 
 ### Usage
 

front/plugins/sync/config.json

@@ -81,6 +81,82 @@
         "string" : "When the node sync should run. Data might be lost if you run the sync less frequently. Good options are <code>always_after_scan</code>, <code>on_new_device</code>, <code>on_notification</code>"
     }]
   },
+  {
+    "function": "RUN_SCHD",
+    "type": "text",
+    "display_condition": {
+      "type" : "setting",
+      "name" : "SYNC_instance_type",
+      "value": "hub"
+    },   
+    "default_value":"*/5 * * * *",
+    "options": [],
+    "localized": ["name", "description"],
+    "name" : [{
+        "language_code":"en_us",
+        "string" : "Schedule"
+    },
+    {
+        "language_code":"es_es",
+        "string" : "Schedule"
+    },
+    {
+        "language_code":"de_de",
+        "string" : "Schedule"
+    }],
+    "description": [{
+        "language_code":"en_us",
+        "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
+    },
+    {
+        "language_code":"es_es",
+        "string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
+    },
+    {
+        "language_code":"de_de",
+        "string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
+    }]
+  },    
+  {
+    "function": "api_token",
+    "type": "text",
+    "maxLength": 50,
+    "default_value": "",
+    "options": [],
+    "localized": ["name", "description"],
+    "name": [
+      {
+        "language_code": "en_us",
+        "string": "API token"
+      }
+    ],
+    "description": [
+      {
+        "language_code": "en_us",
+        "string": "API token to secure communication. The API token needs to be the same on the hub and on the nodes."
+      } 
+    ]
+  },
+  {
+    "function": "encryption_key",
+    "type": "text",
+    "maxLength": 50,
+    "default_value": "",
+    "options": [],
+    "localized": ["name", "description"],
+    "name": [
+      {
+        "language_code": "en_us",
+        "string": "Encryption Key"
+      }
+    ],
+    "description": [
+      {
+        "language_code": "en_us",
+        "string": "Encryption key used to encrypt the sent data. The key needs to be the same on the hub and on the nodes."
+      } 
+    ]
+  },
   {
     "function": "hub_url",
     "type": "text",
@@ -183,46 +259,6 @@
       } 
     ]
   },
-  {
-    "function": "api_token",
-    "type": "text",
-    "maxLength": 50,
-    "default_value": "",
-    "options": [],
-    "localized": ["name", "description"],
-    "name": [
-      {
-        "language_code": "en_us",
-        "string": "API token"
-      }
-    ],
-    "description": [
-      {
-        "language_code": "en_us",
-        "string": "API token to secure communication. The API token needs to be the same on the hub and on the nodes."
-      } 
-    ]
-  },
-  {
-    "function": "encryption_key",
-    "type": "text",
-    "maxLength": 50,
-    "default_value": "",
-    "options": [],
-    "localized": ["name", "description"],
-    "name": [
-      {
-        "language_code": "en_us",
-        "string": "Encryption Key"
-      }
-    ],
-    "description": [
-      {
-        "language_code": "en_us",
-        "string": "Encryption key used to encrypt the sent data. The key needs to be the same on the hub and on the nodes."
-      } 
-    ]
-  },
   {
     "function": "CMD",
     "type": "readonly",
@@ -258,42 +294,6 @@
       }	        
     ]
   },
-  {
-    "function": "RUN_SCHD",
-    "type": "text",
-    "display_condition": {
-      "type" : "setting",
-      "name" : "SYNC_instance_type",
-      "value": "hub"
-    },   
-    "default_value":"*/5 * * * *",
-    "options": [],
-    "localized": ["name", "description"],
-    "name" : [{
-        "language_code":"en_us",
-        "string" : "Schedule"
-    },
-    {
-        "language_code":"es_es",
-        "string" : "Schedule"
-    },
-    {
-        "language_code":"de_de",
-        "string" : "Schedule"
-    }],
-    "description": [{
-        "language_code":"en_us",
-        "string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
-    },
-    {
-        "language_code":"es_es",
-        "string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
-    },
-    {
-        "language_code":"de_de",
-        "string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#SYNC_RUN\"><code>SYNC_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
-    }]
-  },    
   {
     "function": "RUN_TIMEOUT",
     "type": "integer",

front/plugins/sync/hub.php

@@ -35,8 +35,12 @@ if ($_SERVER['REQUEST_METHOD'] === 'POST') {
     }
 
     // Generate a unique file path to avoid overwriting existing files
-    $files = glob("{$storage_path}/last_result.{encoded,decoded}.{$node_name}.*.log", GLOB_BRACE);
+    $encoded_files = glob("{$storage_path}/last_result.encoded.{$node_name}.*.log");
+    $decoded_files = glob("{$storage_path}/last_result.decoded.{$node_name}.*.log");
+
+    $files = array_merge($encoded_files, $decoded_files);
     $file_count = count($files) + 1;
+
     $file_path = "{$storage_path}/last_result.encoded.{$node_name}.{$file_count}.log";
 
 

front/plugins/sync/sync.py

@@ -128,24 +128,18 @@ def main():
                         unique_mac_addresses.add(device['dev_MAC'])
                         device_data.append(device)
 
+    
+
     if len(device_data) > 0:
         # Retrieve existing dev_MAC values from the Devices table
         placeholders = ', '.join('?' for _ in unique_mac_addresses)
         cursor.execute(f'SELECT dev_MAC FROM Devices WHERE dev_MAC IN ({placeholders})', tuple(unique_mac_addresses))
         existing_mac_addresses = set(row[0] for row in cursor.fetchall())
         
-        # Filter out existing devices
-        new_devices = [device for device in device_data if device['dev_MAC'] not in existing_mac_addresses]
 
-        #  Remove 'rowid' key if it exists 
+        # insert devices into the lats_result.log to manage state
-        for device in new_devices:
+        for device in device_data:
-            device.pop('rowid', None)
+            if device['dev_PresentLastScan'] == 1:
-
-
-        # Prepare the insert statement
-        if new_devices:
-            # insert devices into the lats_result.log to manage state
-            for device in new_devices:
                 plugin_objects.add_object(
                     primaryId   = device['dev_MAC'],
                     secondaryId = device['dev_LastIP'],
@@ -156,6 +150,19 @@ def main():
                     extra       = '',
                     foreignKey  = device['dev_GUID'])
 
+        # Filter out existing devices
+        new_devices = [device for device in device_data if device['dev_MAC'] not in existing_mac_addresses]
+
+        #  Remove 'rowid' key if it exists 
+        for device in new_devices:
+            device.pop('rowid', None)
+
+        mylog('verbose', [f'[{pluginName}] All devices: "{len(device_data)}"'])
+        mylog('verbose', [f'[{pluginName}] New devices: "{len(new_devices)}"'])
+
+        # Prepare the insert statement
+        if new_devices:
+
             columns = ', '.join(k for k in new_devices[0].keys() if k != 'rowid')
             placeholders = ', '.join('?' for k in new_devices[0] if k != 'rowid')
             sql = f'INSERT INTO Devices ({columns}) VALUES ({placeholders})'

server/plugin.py

@@ -225,13 +225,15 @@ def execute_plugin(db, all_plugins, plugin, pluginsState = plugins_state() ):
         file_dir = os.path.join(pluginsPath, plugin["code_name"])
         file_prefix = 'last_result'
 
-
         # Decode files, rename them, and get the list of files
         files_to_process = decode_and_rename_files(file_dir, file_prefix)
 
         for filename in files_to_process:
+
+            full_path = os.path.join(file_dir, filename)
+
             # Open the decrypted file and process its contents
-            with open(os.path.join(file_dir, filename), 'r') as f:
+            with open(full_path, 'r') as f:
                 newLines = f.read().split('\n')
 
                 # if the script produced some output, clean it up to ensure it's the correct format        
@@ -272,8 +274,10 @@ def execute_plugin(db, all_plugins, plugin, pluginsState = plugins_state() ):
                     else:
                         mylog('none', ['[Plugins] Skipped invalid line in the output: ', line])            
             
-            # TODO: delete processed files 
+            # keep current instance log file, delete all from other nodes
-            # os.rename(file_path, os.path.join(file_dir, new_filename))
+            if filename != 'last_result.log' and os.path.exists(full_path):
+                os.remove(full_path)
+                mylog('verbose', [f'[Plugins] Processed and deleted file: {full_path} '])   
 
     
     # app-db-query