Loading plugins v 0.3 🔌

front/php/templates/language/en_us.json

@@ -331,7 +331,7 @@
     "HelpFAQ_Cat_Presence_401_head": "A device is displayed as present although it is \"Offline\".",
     "HelpFAQ_Cat_Presence_401_text": "If this happens, you have the possibility to delete the events for the device in question (details view). Another possibility would be to switch on the device and wait until NetAlertX recognizes the device as \"online\" with the next scan and then simply switch the device off again. Now NetAlertX should properly note the state of the device in the database with the next scan.",
     "HelpFAQ_Title": "Help / FAQ",
-    "LOADED_PLUGINS_description": "Which Plugins to load. Adding plugins might slow the application. Read more about which plugins need to be enabled, types, or scanning options in the <a target=\"_blank\" href=\"https://github.com/jokob-sk/NetAlertX/tree/main/front/plugins\">plugins docs</a>",
+    "LOADED_PLUGINS_description": "Which Plugins to load. Adding plugins might slow the application. Read more about which plugins need to be enabled, types, or scanning options in the <a target=\"_blank\" href=\"https://github.com/jokob-sk/NetAlertX/tree/main/front/plugins\">plugins docs</a>. Unloaded plugins will lose your settings. Only <code>disabled</code> plugins can be unloaded.",
     "LOADED_PLUGINS_name": "Loaded plugins",
     "LOG_LEVEL_description": "This setting will enable more verbose logging. Useful for debugging events writing into the database.",
     "LOG_LEVEL_name": "Print additional logging",

server/__main__.py

@@ -84,10 +84,12 @@ def main ():
     # Header + init app state
     updateState("Initializing")    
 
+    all_plugins = None
+
     while True:
 
         # re-load user configuration and plugins   
-        importConfigs(db)
+        all_plugins = importConfigs(db, all_plugins)
 
         # update time started
         conf.loop_start_time = timeNowTZ()       
@@ -96,14 +98,14 @@ def main ():
 
         # Handle plugins executed ONCE
         if conf.plugins_once_run == False:
-            pluginsState = run_plugin_scripts(db, 'once')  
+            pluginsState = run_plugin_scripts(db, all_plugins, 'once')  
             conf.plugins_once_run = True
 
         # check if there is a front end initiated event which needs to be executed
-        pluginsState = check_and_run_user_event(db, pluginsState)
+        pluginsState = check_and_run_user_event(db, all_plugins, pluginsState)
 
         # Update API endpoints              
-        update_api(db)
+        update_api(db, all_plugins)
         
         # proceed if 1 minute passed
         if conf.last_scan_run + datetime.timedelta(minutes=1) < conf.loop_start_time :
@@ -119,13 +121,13 @@ def main ():
             startTime = startTime.replace (microsecond=0) 
 
             # Check if any plugins need to run on schedule
-            pluginsState = run_plugin_scripts(db,'schedule', pluginsState) 
+            pluginsState = run_plugin_scripts(db, all_plugins, 'schedule', pluginsState) 
 
             # determine run/scan type based on passed time
             # --------------------------------------------
            
             # Runs plugin scripts which are set to run every timne after a scans finished            
-            pluginsState = run_plugin_scripts(db,'always_after_scan', pluginsState)
+            pluginsState = run_plugin_scripts(db, all_plugins, 'always_after_scan', pluginsState)
 
             
             # process all the scanned data into new devices
@@ -139,7 +141,7 @@ def main ():
             # --------
             # Reporting   
             # run plugins before notification processing (e.g. Plugins to discover device names)
-            pluginsState = run_plugin_scripts(db, 'before_name_updates', pluginsState)
+            pluginsState = run_plugin_scripts(db, all_plugins, 'before_name_updates', pluginsState)
 
             # Resolve devices names
             mylog('debug','[Main] Resolve devices names')
@@ -153,7 +155,7 @@ def main ():
             #  new devices were found
             if len(newDevices) > 0:
                 #  run all plugins registered to be run when new devices are found                    
-                pluginsState = run_plugin_scripts(db, 'on_new_device', pluginsState)                
+                pluginsState = run_plugin_scripts(db, all_plugins, 'on_new_device', pluginsState)                
 
             # Notification handling
             # ----------------------------------------
@@ -167,7 +169,7 @@ def main ():
 
             # run all enabled publisher gateways 
             if notificationObj.HasNotifications:
-                pluginsState = run_plugin_scripts(db, 'on_notification', pluginsState) 
+                pluginsState = run_plugin_scripts(db, all_plugins, 'on_notification', pluginsState) 
                 notification.setAllProcessed()
                 notification.clearPendingEmailFlag()
 

server/api.py

@@ -12,14 +12,14 @@ apiEndpoints = []
 #===============================================================================
 # API
 #===============================================================================
-def update_api(db, isNotification = False, updateOnlyDataSources = []):
+def update_api(db, all_plugins, isNotification = False, updateOnlyDataSources = []):
     mylog('debug', ['[API] Update API starting'])
     # return
 
     folder = apiPath 
 
     # Save plugins    
-    write_file(folder + 'plugins.json'  , json.dumps({"data" : conf.plugins}))  
+    write_file(folder + 'plugins.json'  , json.dumps({"data" : all_plugins}))  
 
     #  prepare database tables we want to expose 
     dataSourcesSQLs = [

server/conf.py

@@ -9,7 +9,6 @@ mySettingsSQLsafe = []
 cycle = 1
 userSubnets = []
 mySchedules = [] # bad solution for global - TO-DO
-plugins = []  # bad solution for global - TO-DO
 tz = ''
 
 # modified time of the most recently imported config file

server/initialise.py

@@ -55,7 +55,7 @@ def ccd(key, default, config_dir, name, inputtype, options, group, events=[], de
     return result
 #-------------------------------------------------------------------------------
 
-def importConfigs (db): 
+def importConfigs (db, all_plugins): 
 
     sql = db.sql
 
@@ -75,9 +75,9 @@ def importConfigs (db):
     mylog('debug', ['[Import Config] fileModifiedTime         :', fileModifiedTime])
     
 
-    if (fileModifiedTime == conf.lastImportedConfFile) :
+    if (fileModifiedTime == conf.lastImportedConfFile) and all_plugins is not None:
         mylog('debug', ['[Import Config] skipping config file import'])
-        return
+        return all_plugins
 
     # Header
     updateState("Import config", showSpinner = True)  
@@ -146,16 +146,18 @@ def importConfigs (db):
 
     # Plugins START
     # -----------------
-    conf.plugins = get_plugins_configs()
+    all_plugins = get_plugins_configs()
 
-    mylog('none', ['[Config] Plugins: Number of available plugins (including not loaded): ', len(conf.plugins)])
+    mylog('none', ['[Config] Plugins: Number of all plugins (including not loaded): ', len(all_plugins)])
+
+    plugin_indexes_to_remove = []
 
     #  handle plugins
     index = 0
-    for plugin in conf.plugins:
+    for plugin in all_plugins:
 
         # Header on the frontend and the app_state.json
-        updateState(f"Import plugin {index} of {len(conf.plugins)}") 
+        updateState(f"Import plugin {index} of {len(all_plugins)}") 
 
         index +=1
 
@@ -168,8 +170,8 @@ def importConfigs (db):
         plugin_run = get_plugin_setting(plugin, "RUN")
 
         #  get user-defined run value if available
-        if key in c_d:
+        if pref + "_RUN" in c_d:
-            plugin_run =  c_d[key]
+            plugin_run =  c_d[pref + "_RUN" ]
 
 
         # only include loaded plugins, and the ones that are enabled
@@ -222,10 +224,25 @@ def importConfigs (db):
             sql.executemany ("""INSERT INTO Plugins_Language_Strings ("Language_Code", "String_Key", "String_Value", "Extra") VALUES (?, ?, ?, ?)""", stringSqlParams )
 
         else:
-            mylog('none', [f'[Config] Skipping plugin {pref} because not in the LOADED_PLUGINS setting'])
+            # log which plugins to remove 
+            index_to_remove = 0
+            for plugin in all_plugins:
+                if plugin["unique_prefix"] == pref:
+                    break
+                index_to_remove +=1
+
+            plugin_indexes_to_remove.append(index_to_remove)
 
             
 
+    # remove plugin at index_to_remove from list
+    # Sort the list of indexes in descending order to avoid index shifting issues
+    plugin_indexes_to_remove.sort(reverse=True)
+    for indx in plugin_indexes_to_remove:
+        pref = all_plugins[indx]["unique_prefix"]  
+        mylog('none', [f'[Config] ⛔ Unloading plugin {pref} because not in the LOADED_PLUGINS setting or disabled by default'])
+        all_plugins.pop(indx)
+
 
     conf.plugins_once_run = False
     # -----------------
@@ -240,10 +257,10 @@ def importConfigs (db):
     db.commitDB()
 
     #  update only the settings datasource
-    update_api(db, False, ["settings"])  
+    update_api(db, all_plugins, False, ["settings"])  
     
     # run plugins that are modifying the config   
-    run_plugin_scripts(db, 'before_config_save' )
+    run_plugin_scripts(db, all_plugins, 'before_config_save' )
 
     # Used to determine the next import
     conf.lastImportedConfFile = os.path.getmtime(config_file)   
@@ -252,6 +269,8 @@ def importConfigs (db):
 
     mylog('minimal', '[Config] Imported new config')
 
+    return all_plugins
+
 
 
 #-------------------------------------------------------------------------------

server/plugin.py

@@ -101,14 +101,14 @@ class plugins_state:
         self.processScan        = processScan
 
 #-------------------------------------------------------------------------------
-def run_plugin_scripts(db, runType, pluginsState = plugins_state()):
+def run_plugin_scripts(db, all_plugins, runType, pluginsState = plugins_state()):
 
     # Header
     updateState("Run: Plugins")
 
     mylog('debug', ['[Plugins] Check if any plugins need to be executed on run type: ', runType])
 
-    for plugin in conf.plugins:
+    for plugin in all_plugins:
 
         shouldRun = False        
         prefix = plugin["unique_prefix"]
@@ -131,7 +131,7 @@ def run_plugin_scripts(db, runType, pluginsState = plugins_state()):
                         
             print_plugin_info(plugin, ['display_name'])
             mylog('debug', ['[Plugins] CMD: ', get_plugin_setting(plugin, "CMD")["value"]])
-            pluginsState = execute_plugin(db, plugin, pluginsState) 
+            pluginsState = execute_plugin(db, all_plugins, plugin, pluginsState) 
             #  update last run time
             if runType == "schedule":
                 for schd in conf.mySchedules:
@@ -146,7 +146,7 @@ def run_plugin_scripts(db, runType, pluginsState = plugins_state()):
 
 #-------------------------------------------------------------------------------
 # Executes the plugin command specified in the setting with the function specified as CMD 
-def execute_plugin(db, plugin, pluginsState = plugins_state() ):
+def execute_plugin(db, all_plugins, plugin, pluginsState = plugins_state() ):
     sql = db.sql  
 
 
@@ -374,7 +374,7 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
         pluginsState = process_plugin_events(db, plugin, pluginsState, sqlParams)
 
         # update API endpoints
-        update_api(db, False, ["plugins_events","plugins_objects", "plugins_history", "appevents"])  
+        update_api(db, all_plugins, False, ["plugins_events","plugins_objects", "plugins_history", "appevents"])  
     
     return pluginsState
 
@@ -729,7 +729,7 @@ class plugin_object_class:
 #===============================================================================
 # Handling of  user initialized front-end events
 #===============================================================================
-def check_and_run_user_event(db, pluginsState):
+def check_and_run_user_event(db, all_plugins, pluginsState):
     # Check if the log file exists
     logFile = os.path.join(logPath, "execution_queue.log")
     
@@ -749,12 +749,12 @@ def check_and_run_user_event(db, pluginsState):
         event, param = columns
 
         if event == 'test':
-            pluginsState = handle_test(param, db, pluginsState)
+            pluginsState = handle_test(param, db, all_plugins, pluginsState)
         if event == 'run':
-            pluginsState = handle_run(param, db, pluginsState)
+            pluginsState = handle_run(param, db, all_plugins, pluginsState)
         if event == 'update_api':
             # update API endpoints
-            update_api(db, False, param.split(','))  
+            update_api(db, all_plugins, False, param.split(','))  
 
     # Clear the log file
     open(logFile, "w").close()
@@ -763,14 +763,14 @@ def check_and_run_user_event(db, pluginsState):
 
 
 #-------------------------------------------------------------------------------
-def handle_run(runType, db, pluginsState):
+def handle_run(runType, db, all_plugins, pluginsState):
     
     mylog('minimal', ['[', timeNowTZ(), '] START Run: ', runType])
     
     # run the plugin to run
-    for plugin in conf.plugins:
+    for plugin in all_plugins:
         if plugin["unique_prefix"] == runType:                
-            pluginsState = execute_plugin(db, plugin, pluginsState) 
+            pluginsState = execute_plugin(db, all_plugins, plugin, pluginsState) 
 
     mylog('minimal', ['[', timeNowTZ(), '] END Run: ', runType])
     return pluginsState
@@ -778,7 +778,7 @@ def handle_run(runType, db, pluginsState):
 
 
 #-------------------------------------------------------------------------------
-def handle_test(runType, db, pluginsState):
+def handle_test(runType, db, all_plugins, pluginsState):
 
     mylog('minimal', ['[', timeNowTZ(), '] [Test] START Test: ', runType])
     
@@ -792,7 +792,7 @@ def handle_test(runType, db, pluginsState):
     notificationObj = notification.create(sample_json, "")
 
     # Run test
-    pluginsState = handle_run(runType, db, pluginsState)
+    pluginsState = handle_run(runType, db, all_plugins, pluginsState)
 
     # Remove sample notification
     notificationObj.remove(notificationObj.GUID)