chore: merge develop into master

.github/dependabot.yml

@@ -8,7 +8,7 @@ updates:
       - "/anonymizer_module"
       - "/collector_module"
       - "/corrector_module"
-      #- "/opendata_collector_module"
+      - "/opendata_collector_module"
       - "/opendata_module"
       - "/reports_module"
     schedule:
@@ -24,6 +24,52 @@ updates:
         applies-to: version-updates
         update-types:
           - "major"
+    ignore: # ignoring the following packages as they require python >= 3.9
+      - dependency-name: "matplotlib"
+        versions:
+          - ">= 3.8.0"
+      - dependency-name: "pandas"
+        versions:
+          - ">= 2.1.0"
+      - dependency-name: "weasyprint"
+        versions:
+          - ">= 62.0"
+      - dependency-name: "numpy"
+        versions:
+          - ">= 1.25.0"
+      - dependency-name: "contourpy"
+        versions:
+          - ">= 1.2.0"
+      - dependency-name: "markupsafe"
+        versions:
+          - ">= 3.0.0"
+      - dependency-name: "Pillow"
+        versions:
+          - ">= 11.0"
+      - dependency-name: "django"
+        versions:
+          - ">= 5.0"
+      - dependency-name: "pyparsing"
+        versions:
+          - ">= 3.2.0"
+      - dependency-name: "pydyf"
+        versions:
+          - ">= 0.11.0"
+      - dependency-name: "rpds-py"
+        versions:
+          - ">= 0.21.0"
+      - dependency-name: "vcrpy"
+        versions:
+          - ">= 7.0.0"
+      - dependency-name: "setuptools"
+        versions:
+          - ">= 75.4.0"
+      - dependency-name: "urllib3"
+        versions:
+          - ">= 2.3.0"
+      - dependency-name: "pymongo"
+        versions:
+          - ">= 4.11.0"
 
   - package-ecosystem: "github-actions"
     directory: "/.github/workflows"

.github/workflows/run-tests.yml

@@ -1,5 +1,5 @@
 name: X-Road Metrics Tests
-on: [pull_request]
+on: [workflow_dispatch, pull_request]
 jobs:
   Test-Anonymizer:
     name: Anonymizer Tests
@@ -81,26 +81,26 @@ jobs:
         run: pip install tox
       - name: Run tox tests
         run: tox
-  # Test-Opendata-Collector:
-  #   name: OpenData Collector Tests
-  #   runs-on: ubuntu-22.04
-  #   defaults:
-  #     run:
-  #       working-directory: ./opendata_collector_module
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Set up Python 3.8
-  #       uses: actions/setup-python@v5
-  #       with:
-  #         python-version: '3.8'
-  #         cache: 'pip'
-  #         cache-dependency-path: ./opendata_collector_module/setup.py
-  #     - name: Install dependencies
-  #       run: pip install -r test_requirements.txt
-  #     - name: Install tox
-  #       run: pip install tox
-  #     - name: Run tox tests
-  #       run: tox
+  Test-Opendata-Collector:
+    name: OpenData Collector Tests
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: ./opendata_collector_module
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.8'
+          cache: 'pip'
+          cache-dependency-path: ./opendata_collector_module/setup.py
+      - name: Install dependencies
+        run: pip install -r test_requirements.txt
+      - name: Install tox
+        run: pip install tox
+      - name: Run tox tests
+        run: tox
   Test-Reports:
     name: Reports Tests
     runs-on: ubuntu-22.04

.ort.yml

@@ -7,39 +7,12 @@ excludes:
   - pattern: "3RD-PARTY-NOTICES.txt"
     reason: "DOCUMENTATION_OF"
     comment: "3rd party notices."
-
-  scopes:
-  - pattern: "checkstyle"
-    reason: "BUILD_DEPENDENCY_OF"
-    comment: "Packages for code styling checks (testing) only."
-
-  - pattern: "devDependencies"
-    reason: "DEV_DEPENDENCY_OF"
-    comment: "Packages for development only."
-
-  - pattern: "jacocoAgent"
-    reason: "TEST_DEPENDENCY_OF"
-    comment: "Packages for code coverage (testing) only."
-
-  - pattern: "jacocoAnt"
-    reason: "TEST_DEPENDENCY_OF"
-    comment: "Packages for code coverage (testing) only."
-
-  - pattern: "test.*"
-    reason: "TEST_DEPENDENCY_OF"
-    comment: "Packages for testing only."
-
-  - pattern: "annotationProcessor"
-    reason: "DEV_DEPENDENCY_OF"
-    comment: "Packages for development only."
-
-  - pattern: "compileClasspath"
-    reason: "DEV_DEPENDENCY_OF"
-    comment: "Packages for development only."
-
-  - pattern: "compileOnly"
-    reason: "DEV_DEPENDENCY_OF"
-    comment: "Packages for development only."
+  - pattern: "**/tests/**"
+    reason: "TEST_OF"
+    comment: "This directory contains tests."
+  - pattern: "**/test_results.html"
+    reason: "TEST_OF"
+    comment: "This file contains contains test results."
 
 resolutions:
   rule_violations:

README.md

@@ -57,6 +57,8 @@ Networking module is written in [**R**](https://www.r-project.org/).
 get access to the
 [X-Road Metrics Backlog](https://nordic-institute.atlassian.net/jira/software/c/projects/OPMONDEV/boards/2/backlog).
 
+Follow the [developer's guide](./docs/dev_guide/developer_guide.md) to setup your local development environment
+
 Submit a pull request to
 [X-Road Metrics source code Github repository](https://github.com/nordic-institute/X-Road-Metrics)
 or an enhancement request to the

analysis_module/Makefile

@@ -1,10 +1,24 @@
 clean:
 	@echo "Cleaning project..."
 	@find . -type d -name "__pycache__" -exec rm -rf {} +
+	@rm -f ./test_results.xml
 	@rm -rf .coverage
 	@rm -rf .tox/
 	@rm -rf build/
 	@rm -rf dist/
 	@rm -rf *.egg-info/
-	@rm -rf opmon_collector/tests/responses/Test/
-	@echo "Project cleaned."
+	@rm -rf ../htmlcov/analysis_module/
+	@echo "Project cleaned."
+
+license:
+	@echo "Checking if licenseheaders library is installed..."
+	@if pip show licenseheaders > /dev/null 2>&1; then \
+		echo "licenseheaders is installed."; \
+	else \
+		echo "licenseheaders is not installed. Installing it..."; \
+		pip install licenseheaders --no-input; \
+	fi
+	@echo "Adding license to opmon_analyzer module..."
+	@licenseheaders -t ../LICENSE.MD -E py -d ./opmon_analyzer
+	@echo "Note: bin directory is not updated by the tool. Make sure to update it manually."
+	@echo "License added."

analysis_module/opmon_analyzer/AnalyzerDatabaseManager.py

@@ -1,24 +1,26 @@
-#  The MIT License
-#  Copyright (c) 2021- Nordic Institute for Interoperability Solutions (NIIS)
-#  Copyright (c) 2017-2020 Estonian Information System Authority (RIA)
 #
-#  Permission is hereby granted, free of charge, to any person obtaining a copy
-#  of this software and associated documentation files (the "Software"), to deal
-#  in the Software without restriction, including without limitation the rights
-#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-#  copies of the Software, and to permit persons to whom the Software is
-#  furnished to do so, subject to the following conditions:
+# The MIT License 
+# Copyright (c) 2021- Nordic Institute for Interoperability Solutions (NIIS)
+# Copyright (c) 2017-2020 Estonian Information System Authority (RIA)
+#  
+# Permission is hereby granted, free of charge, to any person obtaining a copy 
+# of this software and associated documentation files (the "Software"), to deal 
+# in the Software without restriction, including without limitation the rights 
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
+# copies of the Software, and to permit persons to whom the Software is 
+# furnished to do so, subject to the following conditions: 
+#  
+# The above copyright notice and this permission notice shall be included in 
+# all copies or substantial portions of the Software. 
+#  
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+# THE SOFTWARE.
 #
-#  The above copyright notice and this permission notice shall be included in
-#  all copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-#  THE SOFTWARE.
 
 import urllib.parse
 

analysis_module/opmon_analyzer/__init__.py

@@ -1,23 +1,25 @@
-#  The MIT License
-#  Copyright (c) 2021- Nordic Institute for Interoperability Solutions (NIIS)
-#  Copyright (c) 2017-2020 Estonian Information System Authority (RIA)
 #
-#  Permission is hereby granted, free of charge, to any person obtaining a copy
-#  of this software and associated documentation files (the "Software"), to deal
-#  in the Software without restriction, including without limitation the rights
-#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-#  copies of the Software, and to permit persons to whom the Software is
-#  furnished to do so, subject to the following conditions:
+# The MIT License 
+# Copyright (c) 2021- Nordic Institute for Interoperability Solutions (NIIS)
+# Copyright (c) 2017-2020 Estonian Information System Authority (RIA)
+#  
+# Permission is hereby granted, free of charge, to any person obtaining a copy 
+# of this software and associated documentation files (the "Software"), to deal 
+# in the Software without restriction, including without limitation the rights 
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
+# copies of the Software, and to permit persons to whom the Software is 
+# furnished to do so, subject to the following conditions: 
+#  
+# The above copyright notice and this permission notice shall be included in 
+# all copies or substantial portions of the Software. 
+#  
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+# THE SOFTWARE.
 #
-#  The above copyright notice and this permission notice shall be included in
-#  all copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-#  THE SOFTWARE.
 
 __version__ = '0.1.0'

analysis_module/opmon_analyzer/analyzer_conf.py

@@ -1,24 +1,26 @@
-#  The MIT License
-#  Copyright (c) 2021- Nordic Institute for Interoperability Solutions (NIIS)
-#  Copyright (c) 2017-2020 Estonian Information System Authority (RIA)
 #
-#  Permission is hereby granted, free of charge, to any person obtaining a copy
-#  of this software and associated documentation files (the "Software"), to deal
-#  in the Software without restriction, including without limitation the rights
-#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-#  copies of the Software, and to permit persons to whom the Software is
-#  furnished to do so, subject to the following conditions:
+# The MIT License 
+# Copyright (c) 2021- Nordic Institute for Interoperability Solutions (NIIS)
+# Copyright (c) 2017-2020 Estonian Information System Authority (RIA)
+#  
+# Permission is hereby granted, free of charge, to any person obtaining a copy 
+# of this software and associated documentation files (the "Software"), to deal 
+# in the Software without restriction, including without limitation the rights 
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
+# copies of the Software, and to permit persons to whom the Software is 
+# furnished to do so, subject to the following conditions: 
+#  
+# The above copyright notice and this permission notice shall be included in 
+# all copies or substantial portions of the Software. 
+#  
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+# THE SOFTWARE.
 #
-#  The above copyright notice and this permission notice shall be included in
-#  all copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-#  THE SOFTWARE.
 
 class DataModelConfiguration:
     """Create data model time windows and thresholds based on settings.yaml"""

analysis_module/opmon_analyzer/constants.py

@@ -1,3 +1,26 @@
+#
+# The MIT License 
+# Copyright (c) 2021- Nordic Institute for Interoperability Solutions (NIIS)
+# Copyright (c) 2017-2020 Estonian Information System Authority (RIA)
+#  
+# Permission is hereby granted, free of charge, to any person obtaining a copy 
+# of this software and associated documentation files (the "Software"), to deal 
+# in the Software without restriction, including without limitation the rights 
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
+# copies of the Software, and to permit persons to whom the Software is 
+# furnished to do so, subject to the following conditions: 
+#  
+# The above copyright notice and this permission notice shall be included in 
+# all copies or substantial portions of the Software. 
+#  
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+# THE SOFTWARE.
+#
 timestamp_field = 'requestInTs'
 
 #  The MIT License