Live intersection map

.vscode/settings.json

@@ -26,37 +26,50 @@
   "cSpell.words": [
     "bbox",
     "BOOTSTRAPSERVERS",
+    "Bsms",
     "cdot",
     "cimms",
     "collstats",
     "conflictmonitor",
     "conflictvisualizer",
+    "customdb",
     "cviz",
     "cvmanager",
     "CVPEP",
     "formik",
     "geojsonconverter",
+    "hamcrest",
     "healthcheck",
+    "hookform",
     "inet",
     "INITDB",
     "inprog",
+    "JDBC",
     "keyfile",
+    "ksession",
     "LOGLEVEL",
     "MESSAGETYPE",
     "millis",
     "mongosh",
+    "multidict",
+    "Multivalued",
+    "OIDC",
+    "OIDCID",
     "pgdb",
     "postgis",
     "pytz",
     "reduxjs",
     "rsus",
     "SASL",
     "SNMP",
+    "sqlalchemy",
+    "subquery",
     "svcs",
     "TABLENAME",
     "usdotjpoode",
     "utctimestamp",
     "wydot",
     "Zabbix"
-  ]
+  ],
+  "java.configuration.updateBuildConfiguration": "automatic"
 }

docker-compose-addons.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 include:
   - docker-compose.yml
 
@@ -123,11 +121,11 @@ services:
         max-size: '10m'
         max-file: '5'
 
-  firmware_manager:
+  firmware_manager_upgrade_scheduler:
     build:
       context: services
-      dockerfile: Dockerfile.firmware_manager
-    image: jpo_firmware_manager:latest
+      dockerfile: Dockerfile.fmus
+    image: jpo_firmware_manager_upgrade_scheduler:latest
     restart: on-failure:3
 
     ports:
@@ -138,9 +136,32 @@ services:
       PG_DB_USER: ${PG_DB_USER}
       PG_DB_PASS: ${PG_DB_PASS}
 
+      UPGRADE_RUNNER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT}
+
+      LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL}
+    volumes:
+      - ${GOOGLE_APPLICATION_CREDENTIALS}:/google/gcp_credentials.json
+      - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage
+    logging:
+      options:
+        max-size: '10m'
+        max-file: '5'
+
+  firmware_manager_upgrade_runner:
+    build:
+      context: services
+      dockerfile: Dockerfile.fmur
+    image: jpo_firmware_manager_upgrade_runner:latest
+    restart: on-failure:3
+
+    ports:
+      - '8090:8080'
+    environment:
       BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER}
       BLOB_STORAGE_BUCKET: ${BLOB_STORAGE_BUCKET}
 
+      FW_UPGRADE_MAX_RETRY_LIMIT: ${FW_UPGRADE_MAX_RETRY_LIMIT}
+
       GCP_PROJECT: ${GCP_PROJECT_ID}
       GOOGLE_APPLICATION_CREDENTIALS: '/google/gcp_credentials.json'
 

docker-compose-full-cm.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   cvmanager_api:
     build:
@@ -117,6 +116,7 @@ services:
     environment:
       KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
       KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
+      KEYSTORE_PASSWORD: ${KEYCLOAK_KEYSTORE_PASSWORD}
       WEBAPP_ORIGIN: http://${WEBAPP_DOMAIN}
       WEBAPP_CM_ORIGIN: http://${WEBAPP_CM_DOMAIN}
       KC_HEALTH_ENABLED: true

docker-compose-mongo.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 include:
   - docker-compose.yml
 

docker-compose-no-cm.yml

@@ -1,4 +1,3 @@
-version: '3.9'
 services:
   cvmanager_api:
     build:
@@ -117,6 +116,7 @@ services:
     environment:
       KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
       KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
+      KEYSTORE_PASSWORD: ${KEYCLOAK_KEYSTORE_PASSWORD}
       WEBAPP_ORIGIN: http://${WEBAPP_DOMAIN}
       WEBAPP_CM_ORIGIN: http://${WEBAPP_CM_DOMAIN}
       KC_HEALTH_ENABLED: true

docker-compose-obu-ota-server.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   # OBU OTA Server and Nginx proxy services
   jpo_ota_backend:
@@ -11,7 +10,7 @@ services:
       - 8085:8085
     environment:
       SERVER_HOST: ${OBU_OTA_SERVER_HOST}
-      LOGGING_LEVEL: ${OBU_OTA_SERVER_LOGGING_LEVEL}
+      LOGGING_LEVEL: ${OBU_OTA_LOGGING_LEVEL}
       BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER}
       BLOB_STORAGE_BUCKET: ${OBU_OTA_BLOB_STORAGE_BUCKET}
       BLOB_STORAGE_PATH: ${OBU_OTA_BLOB_STORAGE_PATH}
@@ -25,6 +24,7 @@ services:
       PG_DB_PASS: ${PG_DB_PASS}
 
       MAX_COUNT: ${MAX_COUNT}
+      NGINX_ENCRYPTION: ${NGINX_ENCRYPTION}
     volumes:
       - ./resources/ota/firmwares:/firmwares
     logging:

docker-compose-webapp-deployment.yml

@@ -1,7 +1,6 @@
 # This file is used to build the webapp image for deployment.
 # The COUNTS_MSG_TYPES and DOT_NAME variables must be set in .env before building to populate
 # correctly in the deployed webapp as they are build-time variables.
-version: '3'
 services:
   cvmanager_webapp:
     build:

docker-compose.yml

@@ -118,6 +118,7 @@ services:
     environment:
       KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
       KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
+      KEYSTORE_PASSWORD: ${KEYCLOAK_KEYSTORE_PASSWORD}
       WEBAPP_ORIGIN: http://${WEBAPP_DOMAIN}
       WEBAPP_CM_ORIGIN: http://${WEBAPP_CM_DOMAIN}
       KC_HEALTH_ENABLED: true
@@ -138,6 +139,12 @@ services:
     logging:
       options:
         max-size: '10m'
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:8080/health']
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
 
   kafka:
     image: bitnami/kafka:latest
@@ -252,7 +259,7 @@ services:
     healthcheck:
       test: |
         test $$(mongosh --username ${MONGO_INITDB_ROOT_USERNAME} --password ${MONGO_INITDB_ROOT_PASSWORD}  --quiet --eval "try { rs.initiate({ _id: 'rs0', members: [{ _id: 0, host: '${DB_HOST_IP}' }] }).ok } catch (_) { rs.status().ok }") -eq 1
-      interval: 10s
+      interval: 20s
       start_period: 60s
     entrypoint:
       - bash

resources/keycloak/.gitignore

@@ -1,2 +1,3 @@
 *.jar
-!sample_theme.jar
+!sample_theme.jar
+ssl_cert/**

resources/keycloak/Dockerfile

@@ -4,12 +4,47 @@ RUN dnf install --installroot /mnt/rootfs curl jq \
     --releasever 9 --setopt install_weak_deps=false --nodocs -y; \
     dnf --installroot /mnt/rootfs clean all
 
+FROM maven:3.8-eclipse-temurin-21-alpine as jarbuilder
+
+WORKDIR /home
+
+COPY ./custom-user-provider/pom.xml ./custom-user-provider/
+COPY ./custom-user-provider/src ./custom-user-provider/src
+
+WORKDIR /home/custom-user-provider
+
+RUN mvn clean package -DskipTests
+
+FROM quay.io/keycloak/keycloak:21.1 as builder
+
+# Configure a database vendor
+ENV KC_DB=postgres
+
+WORKDIR /opt/keycloak
+
+COPY --from=jarbuilder /home/custom-user-provider/target/keycloak-custom-providers.jar /opt/keycloak/providers/
+
+# Use environment variables for the keystore password
+ARG KEYSTORE_PASSWORD
+ENV KEYSTORE_PASSWORD=${KEYSTORE_PASSWORD}
+
+# Copy the external certificate and key from keycloak resources directory
+COPY ./ssl_cert/cert.pem /opt/keycloak/conf/server.crt
+COPY ./ssl_cert/key.pem /opt/keycloak/conf/server.key
+
+# Import the certificate into a Java keystore
+RUN keytool -import -trustcacerts -file /opt/keycloak/conf/server.crt -alias server -keystore /opt/keycloak/conf/server.keystore -storepass ${KEYSTORE_PASSWORD} -noprompt
+
+# Build Keycloak
+RUN /opt/keycloak/bin/kc.sh build
+
 FROM quay.io/keycloak/keycloak:21.1
 ARG KEYCLOAK_LOGIN_THEME_NAME
-
 COPY --from=ubi-micro-build /mnt/rootfs /
+COPY --from=builder /opt/keycloak/ /opt/keycloak/
 COPY custom-welcome /opt/keycloak/themes/custom-welcome
 COPY realm.json /opt/keycloak/data/import/realm.json
 COPY ${KEYCLOAK_LOGIN_THEME_NAME} /opt/keycloak/providers/theme.jar
-HEALTHCHECK --interval=5s --timeout=10s --retries=30 \
+
+HEALTHCHECK --interval=5s --timeout=10s --start-period=30s --retries=30 \
 CMD curl --fail http://localhost:8080/health || exit 1

resources/keycloak/README.md

@@ -10,3 +10,26 @@ The `realm.json` file included in this project initializes Keycloak with a sampl
 ## Keycloak Theme
 
 A sample keycloak theme is provided in the `sample_theme.jar` file. This is a sample theme generated using [Keycloakify](https://github.com/CDOT-CV/keycloakify-starter), to use a custom theme put a generated .jar file in this directory and then update the `KEYCLOAK_LOGIN_THEME_NAME` with the name of the new .jar file.
+
+## TLS Configuration
+
+Due to the addition of the Keycloak custom user provider, a Java keystore containing an SSL certificate is now required to build the Keycloak image.
+
+### Development
+
+For development, you can use the create_local_cert.sh script to generate a self-signed certificate:
+
+```sh
+./create_local_cert.sh
+```
+
+This script generates two files, ./ssl_cert/cert.pem and ./ssl_cert/key.pem. These are picked up by the Dockerfile on build.
+
+### Production
+
+For production deployments, a custom certificate should be generated and loaded into the image as a volume before being built. This process is as follows:
+
+1. Create a certificate to be used by Keycloak. This should ideally be signed by a CA. The Dockerfile requires the following files to exist: ./ssl_cert/cert.pem and ./ssl_cert/key.pem
+2. Create a random password to be used for the java keystore. Set this in the docker image as en env variable "KEYSTORE_PASSWORD"
+3. Load the certificate.crt and private.key files into the docker build as a volume, mounted under the /cert directory
+4. Build the docker image!

resources/keycloak/create_local_cert.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+########## ONLY USE THIS SCRIPT FOR DEVELOPMENT PURPOSES ##########
+########## FOR PRODUCTION, PLEASE USE A VALID SSL CERTIFICATE ##########
+
+mkdir -p ./ssl_cert
+
+# generate self-signed ssl key and cert
+openssl req -newkey rsa:2048 -nodes -keyout ./ssl_cert/key.pem -x509 -days 3650 -out ./ssl_cert/cert.pem 
+
+# If you would prefer, you can use the auto-generated information:
+# -subj "/C=US/ST=Colorado/L=Denver/O=My Org/OU=Developers/CN=localhost/emailAddress=test@gmail.com"

resources/keycloak/custom-user-provider/.gitignore

@@ -0,0 +1 @@
+target/**

resources/keycloak/custom-user-provider/README.md

@@ -0,0 +1,98 @@
+# Custom Postgres Keycloak User provider
+
+This is a java project which builds a custom user provider to be integrated into keycloak. This utilizes environment variables set in the realm.json to authenticate to the postgres database, and allows keycloak to query the postgres database to retrieve user information.
+
+## Running the project
+
+This is a maven Spring-boot java project. This project generated a .jar file in ./target/keycloak-custom-providers-0.1.0-SNAPSHOT.jar. Please see the commands below for installing the project and generating the .jar file:
+
+### Installation
+
+```sh
+mvn clean install
+```
+
+### Regenerating the .jar file
+
+The .jar file generated in the target folder is used by the Dockerfile to build the custom user provider into the keycloak image.
+
+```sh
+mvn package
+```
+
+## User Data
+
+This custom user provider primarily uses the public.users postgres table, while joining this data with the public.user_organization, public.organizations, and public.roles tables to incorporate organization and role data.
+
+Data stored in public.users table:
+
+| Field             | Type      | Description                                                                        |
+| ----------------- | --------- | ---------------------------------------------------------------------------------- |
+| user_id           | integer   | Unique user identifier, defaulted by postgres, used to link users to organizations |
+| keycloak_id       | UUID      | Unique user identifier, set by keycloak custom user provider                       |
+| email             | String?   | Email of user, also used as username                                               |
+| first_name        | String?   | First name                                                                         |
+| last_name         | String?   | Last name                                                                          |
+| created_timestamp | Long      | Time of user creation, in milliseconds since epoch                                 |
+| super_user        | int (0-1) | 1 if user has super-user admin privileges                                          |
+
+Data pulled into custom user config joined from public.user_organization, public.organizations, and public.roles:
+
+| Field         | Type   | Description                                             |
+| ------------- | ------ | ------------------------------------------------------- |
+| organizations | List   | List of all organization-specific roles granted by user |
+| - org         | String | Organization name                                       |
+| - role        | String | Role within organization                                |
+
+## User Federation
+
+This custom user provider is configured as a user federation provider in Keycloak, for the cvmanager realm. This is configured through the realm.json, like so:
+
+```json
+"org.keycloak.storage.UserStorageProvider": [
+    {
+        "id": "a6d0ac0a-279e-40f6-a226-81cc7d5be501",
+        "name": "postgres-user-provider",
+        "providerId": "custom-user-provider",
+        "subComponents": {},
+        "config": {
+        "JDBC_URL": ["${KC_DB_URL}"],
+        "VALIDATION_QUERY": ["select 1"],
+        "cachePolicy": ["NO_CACHE"],
+        "DB_USERNAME": ["${KC_DB_USERNAME}"],
+        "DB_PASSWORD": ["${KC_DB_PASSWORD}"],
+        "enabled": ["true"],
+        "JDBC_DRIVER": ["org.postgresql.Driver"]
+        }
+    }
+],
+```
+
+This federated user provider handles the user information described above, but does not handle passwords, credentials, or keycloak-specific fields as "EMAIL_VERIFIED" and "REQUIRED_USER_ACTIONS". Those fields are managed by keycloak, which stores them in it's own Postgres tables.
+
+Keycloak custom user provider walkthrough: [Using Custom User Providers with Keycloak](https://www.baeldung.com/java-keycloak-custom-user-providers)
+
+## Token Mapping
+
+A custom token mapper is configured to include the above user information in generated JWT tokens, and is configured on the cvmanager-gui client in keycloak. The returned JWT token looks as follows:
+
+```json
+...
+"cvmanager_data": {
+    "super_user": "1",
+    "organizations": [
+        {
+            "org": "Test Org",
+            "role": "admin"
+        },
+        {
+            "org": "Test Org 2",
+            "role": "user"
+        }
+    ],
+    "user_created_timestamp": 1719549350683
+},
+...
+```
+
+Keycloak custom toke mapper walkthrough: [Custom Protocol Mapper with Keycloak](https://www.baeldung.com/keycloak-custom-protocol-mapper)