Merge pull request #6 from usdot-jpo-ode/config/jpo-mec-deposit

Adding MEC Deposit Resources

Author: Michael7371

docker-compose-connect.yml

@@ -24,8 +24,10 @@ services:
     depends_on:
       mongo:
         condition: service_healthy
+        required: false
       kafka:
         condition: service_healthy
+        required: false
     environment:
       CONNECT_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
       CONNECT_REST_ADVERTISED_HOST_NAME: connect
@@ -54,6 +56,7 @@ services:
       - all
       - kafka_connect
       - kafka_connect_standalone
+      - kafka_connect_setup
     image: jpo-jikkou
     build:
       context: jikkou
@@ -68,13 +71,15 @@ services:
     depends_on:
       kafka-connect:
         condition: service_healthy
+        required: false
     environment:
       CONNECT_URL: ${CONNECT_URL}
       CONNECT_TASKS_MAX: ${CONNECT_TASKS_MAX}
       CONNECT_CREATE_ODE: ${CONNECT_CREATE_ODE}
       CONNECT_CREATE_GEOJSONCONVERTER: ${CONNECT_CREATE_GEOJSONCONVERTER}
       CONNECT_CREATE_CONFLICTMONITOR: ${CONNECT_CREATE_CONFLICTMONITOR}
       CONNECT_CREATE_DEDUPLICATOR: ${CONNECT_CREATE_DEDUPLICATOR}
+      CONNECT_CREATE_MECDEPOSIT: ${CONNECT_CREATE_MECDEPOSIT}
       MONGO_CONNECTOR_USERNAME: ${MONGO_ADMIN_DB_USER}
       MONGO_CONNECTOR_PASSWORD: ${MONGO_ADMIN_DB_PASS:?}
       MONGO_DB_IP: ${MONGO_IP}

docker-compose-kafka.yml

@@ -19,7 +19,7 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '1'
+          cpus: "1"
           memory: 4G
     volumes:
       - kafka:/bitnami
@@ -40,10 +40,9 @@ services:
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
     logging:
       options:
-        max-size: "10m"  
+        max-size: "10m"
         max-file: "5"
 
-
   kafka-setup:
     profiles:
       - all
@@ -58,11 +57,12 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '0.5'
+          cpus: "0.5"
           memory: 1G
     depends_on:
       kafka:
         condition: service_healthy
+        required: false
     environment:
       KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
       KAFKA_TOPIC_PARTITIONS: ${KAFKA_TOPIC_PARTITIONS}
@@ -74,12 +74,13 @@ services:
       KAFKA_TOPIC_CREATE_GEOJSONCONVERTER: ${KAFKA_TOPIC_CREATE_GEOJSONCONVERTER}
       KAFKA_TOPIC_CREATE_CONFLICTMONITOR: ${KAFKA_TOPIC_CREATE_CONFLICTMONITOR}
       KAFKA_TOPIC_CREATE_DEDUPLICATOR: ${KAFKA_TOPIC_CREATE_DEDUPLICATOR}
-      MONGO_CONNECTOR_USERNAME: ${MONGO_ADMIN_DB_USER}
-      MONGO_CONNECTOR_PASSWORD: ${MONGO_ADMIN_DB_PASS}
-      MONGO_DB_IP: ${MONGO_IP}
-      MONGO_DB_NAME: ${MONGO_DB_NAME}
+      KAFKA_TOPIC_CREATE_MECDEPOSIT: ${KAFKA_TOPIC_CREATE_MECDEPOSIT}
     volumes:
       - ${KAFKA_TOPIC_CONFIG_RELATIVE_PATH:-./jikkou/kafka-topics-values.yaml}:/app/kafka-topics-values.yaml
+    logging:
+      options:
+        max-size: "10m"
+        max-file: "5"
 
   kafka-schema-registry:
     profiles:
@@ -91,11 +92,12 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '0.5'
+          cpus: "0.5"
           memory: 1G
     depends_on:
       kafka:
         condition: service_healthy
+        required: false
     ports:
       - "8081:8081"
     environment:
@@ -108,6 +110,10 @@ services:
       interval: 30s
       timeout: 10s
       retries: 4
+    logging:
+      options:
+        max-size: "10m"
+        max-file: "5"
 
   kafka-ui:
     profiles:
@@ -120,19 +126,24 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '0.5'
+          cpus: "0.5"
           memory: 1G
     ports:
       - 8001:8080
     depends_on:
       kafka:
         condition: service_healthy
+        required: false
     environment:
-      DYNAMIC_CONFIG_ENABLED: 'true'
+      DYNAMIC_CONFIG_ENABLED: "true"
       KAFKA_CLUSTERS_0_NAME: local
       KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: ${KAFKA_BOOTSTRAP_SERVERS}
       KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: kafka-connect
       KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: ${CONNECT_URL}
+    logging:
+      options:
+        max-size: "10m"
+        max-file: "5"
 
 volumes:
-  kafka:
+  kafka:

docker-compose-mongo.yml

@@ -52,7 +52,8 @@ services:
       - ./mongo/manage-volume-cron:/data/manage-volume-cron
       - ./mongo/manage_volume.js:/data/manage_volume.js
     healthcheck:
-      test: "echo 'db.runCommand(\"ping\").ok' | mongosh --quiet --username ${MONGO_ADMIN_DB_USER} --password ${MONGO_ADMIN_DB_PASS} --authenticationDatabase admin --eval \"rs.status().ok\""
+      # Removal of replica set status check as the mongo-setup container is what actually configures the replica set
+      test: mongosh --quiet --username ${MONGO_ADMIN_DB_USER} --password ${MONGO_ADMIN_DB_PASS} --authenticationDatabase admin --eval "db.adminCommand('ping').ok"
       interval: 10s
       timeout: 10s
       retries: 10
@@ -66,7 +67,9 @@ services:
     image: mongo:8
     hostname: mongo_setup
     depends_on:
-      - mongo
+      mongo:
+        condition: service_healthy
+        required: false
     restart: on-failure
     deploy:
       resources:
@@ -88,7 +91,7 @@ services:
       CONNECT_CREATE_GEOJSONCONVERTER: ${CONNECT_CREATE_GEOJSONCONVERTER}
       CONNECT_CREATE_CONFLICTMONITOR: ${CONNECT_CREATE_CONFLICTMONITOR}
       CONNECT_CREATE_DEDUPLICATOR: ${CONNECT_CREATE_DEDUPLICATOR}
-      
+      CONNECT_CREATE_MECDEPOSIT: ${CONNECT_CREATE_MECDEPOSIT}
     entrypoint: ["/bin/bash", "setup_mongo.sh"]
     volumes:
       - ${MONGO_SETUP_SCRIPT_RELATIVE_PATH}:/setup_mongo.sh
@@ -111,7 +114,9 @@ services:
     ports:
       - "8002:8081"
     depends_on:
-      - mongo
+      mongo:
+        condition: service_healthy
+        required: false
     environment:
       ME_CONFIG_MONGODB_ENABLE_ADMIN: "true"
       ME_CONFIG_BASICAUTH_USERNAME: ${MONGO_EXPRESS_USER}

jikkou/kafka-connectors-template.jinja

@@ -69,4 +69,8 @@ spec:
 {% else %}
 {{ create_connector(values.apps.ode_duplicated) }}
 {{ create_connector(values.apps.geojsonconverter_duplicated) }}
-{% endif %}
+{% endif %}
+
+{% if system.env.CONNECT_CREATE_MECDEPOSIT %}
+{{ create_connector(values.apps.mecdeposit) }}
+{% endif %}

jikkou/kafka-connectors-values.yaml

@@ -350,4 +350,14 @@ apps:
         collectionName: CmEventStateProgressionNotificationAggregation
         generateTimestamp: true
         timestampField: notificationGeneratedAt
-
+  mecdeposit:
+    name: mecdeposit
+    connectors:
+      - topicName: topic.MecEtxMqttDepositMetrics
+        collectionName: MecEtxMqttDepositMetrics
+        useTimestamp: true
+        timestampField: odeReceivedAt
+      - topicName: topic.MecEtxApiDepositMetrics
+        collectionName: MecEtxApiDepositMetrics
+        useTimestamp: true
+        timestampField: odeReceivedAt

jikkou/kafka-topics-template.jinja

@@ -74,4 +74,8 @@ spec:
 
 {% if system.env.KAFKA_TOPIC_CREATE_DEDUPLICATOR %}
 {{ create_topics(values.apps.deduplicator) }}
-{% endif %}
+{% endif %}
+
+{% if system.env.KAFKA_TOPIC_CREATE_MECDEPOSIT %}
+{{ create_topics(values.apps.mecdeposit) }}
+{% endif %}

jikkou/kafka-topics-values.yaml

@@ -182,4 +182,11 @@ apps:
         cleanUpPolicy: compact
       - topicName: topic.KafkaConnectStatus
         partitions: 10
-        cleanUpPolicy: compact
+        cleanUpPolicy: compact
+  mecdeposit:
+    name: jpo-mecdeposit
+    streamTopics:
+      - topic.MecEtxMqttDepositMetrics
+      - topic.MecEtxApiDepositMetrics
+    tableTopics: {}
+    customTopics: {}

sample.env

@@ -21,10 +21,10 @@ RESTART_POLICY="on-failure:3"
 #       - kafka
 #       - mongo
 #       - kafka_connect
-#       - kafka-connect-setup
+#       - kafka_connect_setup
 #  - kafka_connect
 #       - kafka_connect
-#       - kafka-connect-setup
+#       - kafka_connect_setup
 #  - deduplicator
 #       - deduplicator
 # EXAMPLE: COMPOSE_PROFILES=kafka_connect_standalone,kafka_ui,mongo_express
@@ -50,6 +50,7 @@ KAFKA_TOPIC_CREATE_ODE=true                 # Create topics for ODE
 KAFKA_TOPIC_CREATE_GEOJSONCONVERTER=true    # Create topics for GeoJSON Converter
 KAFKA_TOPIC_CREATE_CONFLICTMONITOR=true     # Create topics for Conflict Monitor
 KAFKA_TOPIC_CREATE_DEDUPLICATOR=true        # Create topics for Deduplicator
+KAFKA_TOPIC_CREATE_MECDEPOSIT=true         # Create topics for MecDeposit
 # Relative path to the Kafka topic yaml configuration script, upper level directories are supported
 # NOTE: This script is used to create kafka topics
 KAFKA_TOPIC_CONFIG_RELATIVE_PATH="./jikkou/kafka-topics-values.yaml"
@@ -117,6 +118,7 @@ CONNECT_CREATE_ODE=true                 # Create kafka connectors to MongoDB for
 CONNECT_CREATE_GEOJSONCONVERTER=true    # Create kafka connectors to MongoDB for GeoJSON Converter
 CONNECT_CREATE_CONFLICTMONITOR=true     # Create kafka connectors to MongoDB for Conflict Monitor
 CONNECT_CREATE_DEDUPLICATOR=true        # Create kafka connectors to MongoDB for Deduplicator
+CONNECT_CREATE_MECDEPOSIT=true          # Create kafka connectors to MongoDB for MecDeposit
 # Relative path to the Kafka Connector yaml configuration script, upper level directories are supported
 # NOTE: This script is used to create kafka connectors
 CONNECT_CONFIG_RELATIVE_PATH="./jikkou/kafka-connectors-values.yaml"