Updated flow-refresh.py to install node modules from sidecar extraEnv…

… = NODE_MODULES (#253)

charts/node-red/Chart.yaml

@@ -9,7 +9,7 @@ icon: https://nodered.org/about/resources/media/node-red-icon-2.png
 
 type: application
 
-version: 0.24.1
+version: 0.24.2
 appVersion: 3.0.2
 
 keywords:

charts/node-red/README.md.gotmpl

@@ -95,6 +95,18 @@ Default values are: `node-red-settings:1`.
 The `k8s-sidecar` will then call the `node-red` api to reload the flows. This will be done via a script. To run this script successfully you need to provide the `username` and `password`
 of your admin user. The admin user needs to have the right to use the `node-red` API.
 
+The `k8s-sidecar` can also call the `node-red` api to install additional node modules (npm packages) before refreshing or importing the flow.json.
+You need to list your flows requiert 'NODE_MODULES' in the `sidecar.extraNodeModules`: e.g.
+
+```yaml
+sidecar:
+ extraNodeModules:
+    - node-red-contrib-xkeys_setunitid
+    - node-red-contrib-microsoft-teams-tasks
+    - node-red-contrib-json
+```
+To install the node modules successfully, the node red pod needs access to the `npmrc.registry` to download the declaired modules/packages.
+
 ## Contributing 🤝
 
 ### Contributing via GitHub

charts/node-red/scripts/flow_refresh.py

@@ -3,43 +3,147 @@
 import time
 import os
 import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 import json
 import sys
 import re
 
 # SET VARIABLES FROM CONTAINER ENVIRONMENT
-SLEEP_TIME_SIDECAR = 5 if os.getenv("SLEEP_TIME_SIDECAR") is None else int(re.sub('[A-z]', "",os.getenv("SLEEP_TIME_SIDECAR")))
+SLEEP_TIME_SIDECAR = 5 if os.getenv("SLEEP_TIME_SIDECAR") is None else int(re.sub("[A-z]", "", os.getenv("SLEEP_TIME_SIDECAR")))
 USERNAME = os.getenv("USERNAME")
 PASSWORD = os.getenv("PASSWORD")
 URL = os.getenv("URL")
+# REQUEST CONNECTION PARAMETERS
+FLOW_REQ_RETRY_TOTAL = 20 if os.getenv("REQ_RETRY_TOTAL") is None else int(os.getenv("REQ_RETRY_TOTAL"))
+FLOW_REQ_RETRY_CONNECT = 30 if os.getenv("REQ_RETRY_CONNECT") is None else int(os.getenv("REQ_RETRY_CONNECT"))
+FLOW_REQ_RETRY_READ = 15 if os.getenv("REQ_RETRY_READ") is None else int(os.getenv("REQ_RETRY_READ"))
+FLOW_REQ_RETRY_BACKOFF_FACTOR = 1.1 if os.getenv("REQ_RETRY_BACKOFF_FACTOR") is None else float(os.getenv("REQ_RETRY_BACKOFF_FACTOR"))
+FLOW_REQ_TIMEOUT = 60 if os.getenv("REQ_TIMEOUT") is None else float(os.getenv("REQ_TIMEOUT"))
+FLOW_REQ_TLS_VERIFY = False if os.getenv("REQ_SKIP_TLS_VERIFY") == "true" else None
 
-print('node-red flow refresh api call via k8s-sidecar')
-print('sidecar sleeping for', SLEEP_TIME_SIDECAR, 'seconds...')
-time.sleep(SLEEP_TIME_SIDECAR)
-
-# GET NODE RED BEARER TOKEN
-PAYLOAD_TOKEN = {"client_id": "node-red-admin", "grant_type": "password", "scope": "*", "username": USERNAME, "password": PASSWORD}
-r_token = requests.post(URL + '/auth/token', data=PAYLOAD_TOKEN, timeout=30)
-if r_token.status_code == requests.codes.ok:
-    print('node-red bearer token successfully created -', r_token.status_code)  
-    token = (json.loads(r_token.text)["access_token"])
-else:
-    print('could not create bearer token....', r_token.status_code) 
-    sys.exit('Error-Code -', r_token.status_code)
-
-# FLOW REFRESH/RELOAD FLOWS FROM SECRET/CONFIGMAP
-PAYLOAD_FLOW_REFRESH = "{\"flows\": [{\"type\": \"tab\"}]}"
-HEADERS_FLOW_REFRESH={
-  'Authorization': 'Bearer' + ' ' + token,
-  'content-type': 'application/json; charset=utf-8',
-  'Node-RED-Deployment-Type': 'reload',
-  'Node-RED-API-Version': 'v2'
-}
-
-r_flow_refresh = requests.post(URL + '/flows', headers=HEADERS_FLOW_REFRESH, data=PAYLOAD_FLOW_REFRESH, timeout=30) 
-
-if r_flow_refresh.status_code == requests.codes.ok:
-    print('node-red flows successfully reloaded -', r_flow_refresh.status_code)
-    sys.exit(0)   
-else:
-    sys.exit('Error-Code', r_flow_refresh.status_code)
+EXTRA_NODE_MODULES = None if os.path.isfile('/data/extra-node-modules.json') is False else json.load((open('/data/extra-node-modules.json', "r")))
+script_errors = {}
+
+def main():
+    print("----START PYTHON SIDECAR SCRIPT----")
+    print("node-red node module install and flow refresh api call via k8s-sidecar")
+    print(f"sidecar sleeping for {SLEEP_TIME_SIDECAR} seconds...")
+    time.sleep(SLEEP_TIME_SIDECAR)
+
+    # REQUESTS INIT SESSION
+    r = requests.Session()
+    retries = Retry(
+        total=FLOW_REQ_RETRY_TOTAL,
+        connect=FLOW_REQ_RETRY_CONNECT,
+        read=FLOW_REQ_RETRY_READ,
+        backoff_factor=FLOW_REQ_RETRY_BACKOFF_FACTOR,
+    )
+    r.mount("http://", HTTPAdapter(max_retries=retries))
+    r.mount("https://", HTTPAdapter(max_retries=retries))
+
+    # GET NODE RED BEARER TOKEN
+    print("----TOKEN----")
+    payload_token = {
+        "client_id": "node-red-admin",
+        "grant_type": "password",
+        "scope": "*",
+        "username": USERNAME,
+        "password": PASSWORD,
+    }
+    r_token = r.post(
+        "%s" % URL + "/auth/token",
+        data=payload_token,
+        timeout=FLOW_REQ_TIMEOUT,
+        verify=FLOW_REQ_TLS_VERIFY,
+    )
+    if r_token.status_code == requests.codes.ok:
+        print(f"node-red bearer token successfully created - {r_token.status_code}")
+        token = json.loads(r_token.text)["access_token"]
+    else:
+        print(f"could not create bearer token.... {r_token.status_code}")
+        sys.exit(r_token.status_code)
+
+    # NODE MODULE INSTALL VIA HELM SIDECAR EXTRA NODE MODULES CONFIG MAP
+    print("----INSTALL EXTRA NODE MODULES----")
+    if EXTRA_NODE_MODULES is not None:
+        print(f"found extra node modules in configmap - {EXTRA_NODE_MODULES}")
+        # GET ISNTALLED NODE MODULES
+        headers_node_module = {
+            "Authorization": "Bearer" + " " + token,
+            "Accept": "application/json",
+        }
+        r_node_modules = r.get(
+            "%s" % URL + "/nodes",
+            headers=headers_node_module,
+            timeout=FLOW_REQ_TIMEOUT,
+            verify=FLOW_REQ_TLS_VERIFY,
+        )
+        node_modules = json.loads(r_node_modules.text)
+        # PARSE ALL INSTALLED MODULES FROM EVERY ARRAY AN SET AS UNIQUE LIST FROM AKTIVE NODE RED DEPLOYEMNT
+        modules_installed = list(set([item.get("module") for item in node_modules]))
+        print(f"currently installed node modules - {modules_installed}")
+
+        for module in EXTRA_NODE_MODULES:
+            if module not in modules_installed:
+                payload_node_module = '{"module": "' + module + '"}'
+                headers_node_module = {
+                    "Authorization": "Bearer" + " " + token,
+                    "Content-type": "application/json",
+                }
+                # INSTALL NODE MODULES FROM ITERATION
+                r_node_modules = r.post(
+                    "%s" % URL + "/nodes",
+                    headers=headers_node_module,
+                    data=payload_node_module,
+                    timeout=FLOW_REQ_TIMEOUT,
+                    verify=FLOW_REQ_TLS_VERIFY,
+                )
+                if r_node_modules.status_code == requests.codes.ok:
+                    print(f"node module {module} successfully installed -  {r_node_modules.status_code}")
+                else:
+                    print(f"error-status-code = {r_node_modules.status_code} while installing {module}")
+                    script_errors.update({module: r_node_modules.status_code})
+                    # sys.exit at the end! Non hard exit.
+            else:
+                print(f"node module {module} already installed...")
+    else:
+        print("no extra node-modules found from configmap ...")
+        print("skipping extra node modules installation...")
+
+    # FLOW REFRESH/RELOAD FLOWS FROM SECRET/CONFIGMAP
+    print("----RELOAD FLOWS----")
+    payload_flow_refresh = '{"flows": [{"type": "tab"}]}'
+    headers_flow_refresh = {
+        "Authorization": "Bearer" + " " + token,
+        "content-type": "application/json; charset=utf-8",
+        "Node-RED-Deployment-Type": "reload",
+        "Node-RED-API-Version": "v2",
+    }
+
+    r_flow_refresh = r.post(
+        "%s" % URL + "/flows",
+        headers=headers_flow_refresh,
+        data=payload_flow_refresh,
+        timeout=FLOW_REQ_TIMEOUT,
+        verify=FLOW_REQ_TLS_VERIFY,
+    )
+
+    if r_flow_refresh.status_code == requests.codes.ok:
+        print(f"node-red flows successfully reloaded - {r_flow_refresh.status_code}")
+    else:
+        print(f"could not refresh flows - {r_flow_refresh.status_code}")
+        script_errors.update({"flow_refresh": r_flow_refresh.status_code})
+        # sys.exit(r_flow_refresh.status_code)
+
+    # NODE MODULE INSTALL VIA HELM SIDECAR EXTRA ENVIROMENT
+    print("----SCRIPT EXIT----")
+    if script_errors:
+        print(json.dumps(script_errors, indent=4))
+        sys.exit("script error")
+    else:
+        print("yeah right")
+        sys.exit(0)
+
+if __name__ == "__main__":
+    main()

charts/node-red/templates/_helpers.tpl

@@ -75,7 +75,6 @@ Create the name of the sidecar-configmap
 {{ printf "%s-flow-refresh-cm" (include "node-red.fullname" $) | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
-
 {{/*
 Create the name of the certificate
 */}}

charts/node-red/templates/sidecar-cm.yaml

@@ -3,9 +3,13 @@ apiVersion: v1
 data:
   flow_refresh.py: | {{ range $.Files.Lines "scripts/flow_refresh.py" }}
     {{ . }}{{ end }}
+  {{- if .Values.sidecar.extraNodeModules }}
+  extra-node-modules.json: |-
+    {{ toJson .Values.sidecar.extraNodeModules }}
+  {{- end }}
 kind: ConfigMap
 metadata:
   name: {{ include "node-red.sidecarConfigMapName" . }}
   labels:
     {{ .Values.sidecar.env.label}}: {{ .Values.sidecar.env.label_value}}
-{{- end }}
+{{- end }}

charts/node-red/values.yaml

@@ -258,6 +258,8 @@ sidecar:
     #  name: node-red-api-admin-password
     # -- Key of the secret that contains the password
     #  key: password
+  # -- Extra Node-Modules that will be installed  from the sidecar script
+  extraNodeModules: []
   # -- Extra Environments for the sidecar
   extraEnv: []
   # -- Resources for the sidecar