Merge branch 'develop' into monitoring-stack

Author: Michael7371

README.md

@@ -105,6 +105,8 @@ The following enviroment variables can be used to configure Kafka Topic creation
 | `KAFKA_TOPIC_CREATE_GEOJSONCONVERTER` | Whether to create topics for the GeoJSON Converter |
 | `KAFKA_TOPIC_CREATE_CONFLICTMONITOR` | Whether to create topics for the Conflict Monitor |
 | `KAFKA_TOPIC_CREATE_DEDUPLICATOR` | Whether to create topics for the Deduplicator |
+| `KAFKA_TOPIC_CREATE_OTHER` | Whether to create topics for other applications, this is only useful when you attach a custom `kafka-topics-values.yaml` file with other topics |
+| `KAFKA_TOPICS_VALUES_FILE` | Path to a custom `kafka-topics-values.yaml` file|
 | `KAFKA_TOPIC_PARTITIONS` | Number of partitions |
 | `KAFKA_TOPIC_REPLICAS` | Number of replicas |
 | `KAFKA_TOPIC_MIN_INSYNC_REPLICAS` | Minumum number of in-sync replicas (for use with ack=all) |
@@ -178,7 +180,9 @@ The following environment variables can be used to configure Kafka Connectors:
 | `CONNECT_CREATE_ODE` | Whether to create kafka connectors for the ODE |
 | `CONNECT_CREATE_GEOJSONCONVERTER` | Whether to create topics for the GeojsonConverter |
 | `CONNECT_CREATE_CONFLICTMONITOR` | Whether to create kafka connectors for the Conflict Monitor |
-| `CONNECT_CREATE_DEDUPLICATOR` | Whether to create topics for the Deduplicator |
+| `CONNECT_CREATE_DEDUPLICATOR` | Whether to create kafka connectors for the Deduplicator |
+| `CONNECT_KAFKA_CONNECTORS_VALUES_FILE` | Path to a custom `kafka-connectors-values.yaml` file |
+| `CONNECT_CREATE_OTHER` | Whether to create kafka connectors for other applications, this is only useful when you attach a custom `kafka-connectors-values.yaml` file with other connectors |
 
 ### Quick Run
 

docker-compose-connect.yml

@@ -82,5 +82,8 @@ services:
       CONNECT_CREATE_MECDEPOSIT: ${CONNECT_CREATE_MECDEPOSIT:-false}
       MONGO_CONNECTOR_USERNAME: ${MONGO_ADMIN_DB_USER:-admin}
       MONGO_CONNECTOR_PASSWORD: ${MONGO_ADMIN_DB_PASS:-replace_me}
+      
       MONGO_DB_IP: ${MONGO_IP:-mongo}
-      MONGO_DB_NAME: ${MONGO_DB_NAME:-CV}
+      MONGO_DB_NAME: ${MONGO_DB_NAME:-CV}
+    volumes:
+      - ${CONNECT_KAFKA_CONNECTORS_VALUES_FILE:-./jikkou/kafka-connectors-values.yaml}:/kafka-connectors-values.yaml

docker-compose-kafka.yml

@@ -64,21 +64,36 @@ services:
         condition: service_healthy
         required: false
     environment:
+      KAFKA_TOPIC_CREATE_ODE: ${KAFKA_TOPIC_CREATE_ODE:-true}
+      KAFKA_TOPIC_CREATE_GEOJSONCONVERTER: ${KAFKA_TOPIC_CREATE_GEOJSONCONVERTER:-true}
+      KAFKA_TOPIC_CREATE_CONFLICTMONITOR: ${KAFKA_TOPIC_CREATE_CONFLICTMONITOR:-true}
+      KAFKA_TOPIC_CREATE_DEDUPLICATOR: ${KAFKA_TOPIC_CREATE_DEDUPLICATOR:-false}
+      KAFKA_TOPIC_CREATE_MECDEPOSIT: ${KAFKA_TOPIC_CREATE_MECDEPOSIT:-false}
+
       KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
       KAFKA_TOPIC_PARTITIONS: ${KAFKA_TOPIC_PARTITIONS:-1}
       KAFKA_TOPIC_REPLICAS: ${KAFKA_TOPIC_REPLICAS:-1}
       KAFKA_TOPIC_MIN_INSYNC_REPLICAS: ${KAFKA_TOPIC_MIN_INSYNC_REPLICAS:-1}
       KAFKA_TOPIC_RETENTION_MS: ${KAFKA_TOPIC_RETENTION_MS:-300000}
       KAFKA_TOPIC_DELETE_RETENTION_MS: ${KAFKA_TOPIC_DELETE_RETENTION_MS:-3600000}
+
       KAFKA_TOPIC_CREATE_ODE: ${KAFKA_TOPIC_CREATE_ODE:-true}
       KAFKA_TOPIC_CREATE_GEOJSONCONVERTER: ${KAFKA_TOPIC_CREATE_GEOJSONCONVERTER:-true}
       KAFKA_TOPIC_CREATE_CONFLICTMONITOR: ${KAFKA_TOPIC_CREATE_CONFLICTMONITOR:-true}
       KAFKA_TOPIC_CREATE_DEDUPLICATOR: ${KAFKA_TOPIC_CREATE_DEDUPLICATOR:-false}
       KAFKA_TOPIC_CREATE_MECDEPOSIT: ${KAFKA_TOPIC_CREATE_MECDEPOSIT:-false}
+      KAFKA_TOPIC_CREATE_OTHER: ${KAFKA_TOPIC_CREATE_OTHER:-false}
+      
+      KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
+      KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM}
+      KAFKA_SASL_JAAS_CONFIG: ${KAFKA_SASL_JAAS_CONFIG}
+      KAFKA_SSL_ENDPOINT_ALGORITHM: ${KAFKA_SSL_ENDPOINT_ALGORITHM}
     logging:
       options:
         max-size: "10m"
         max-file: "5"
+    volumes:
+      - ${KAFKA_TOPICS_VALUES_FILE:-./jikkou/kafka-topics-values.yaml}:/kafka-topics-values.yaml
 
   kafka-schema-registry:
     profiles:

jikkou/kafka-connectors-template.jinja

@@ -70,3 +70,7 @@ spec:
 {% if system.env.CONNECT_CREATE_MECDEPOSIT %}
 {{ create_connector(values.apps.mecdeposit) }}
 {% endif %}
+
+{% if system.env.CONNECT_CREATE_OTHER %}
+{{ create_connector(values.apps.other) }}
+{% endif %}

jikkou/kafka-connectors-values.yaml

@@ -370,3 +370,7 @@ apps:
       - topicName: topic.MecDepositMetrics
         collectionName: MecDepositMetrics
         generateTimestamp: true
+  # Allow for custom connectors to be added - users can override this file and add other kafka connectors here
+  other:
+    name: other
+    connectors:

jikkou/kafka-topics-template.jinja

@@ -79,3 +79,7 @@ spec:
 {% if system.env.KAFKA_TOPIC_CREATE_MECDEPOSIT %}
 {{ create_topics(values.apps.mecdeposit) }}
 {% endif %}
+
+{% if system.env.KAFKA_TOPIC_CREATE_OTHER %}
+{{ create_topics(values.apps.other) }}
+{% endif %}

jikkou/kafka-topics-values.yaml

@@ -197,3 +197,8 @@ apps:
       - topic.MecDepositMetrics
     tableTopics: {}
     customTopics: {}
+  other:
+    name: other-topics
+    streamTopics: {}
+    tableTopics: {}
+    customTopics: {}

sample.env

@@ -55,6 +55,10 @@ KAFKA_TOPIC_CREATE_GEOJSONCONVERTER=true    # Create topics for GeoJSON Converte
 KAFKA_TOPIC_CREATE_CONFLICTMONITOR=true     # Create topics for Conflict Monitor
 KAFKA_TOPIC_CREATE_DEDUPLICATOR=false       # Create topics for Deduplicator
 KAFKA_TOPIC_CREATE_MECDEPOSIT=false         # Create topics for MecDeposit
+KAFKA_TOPIC_CREATE_OTHER=false              # Create topics for other applications
+
+# Relative path to the Kafka topics values file, upper level directories are supported
+KAFKA_TOPICS_VALUES_FILE="./jikkou/kafka-topics-values.yaml"
 
 # Confluent Cloud Support
 KAFKA_SECURITY_PROTOCOL=PLAINTEXT
@@ -132,6 +136,11 @@ CONNECT_CREATE_GEOJSONCONVERTER=true    # Create kafka connectors to MongoDB for
 CONNECT_CREATE_CONFLICTMONITOR=true     # Create kafka connectors to MongoDB for Conflict Monitor
 CONNECT_CREATE_DEDUPLICATOR=false       # Create kafka connectors to MongoDB for Deduplicator
 CONNECT_CREATE_MECDEPOSIT=false         # Create kafka connectors to MongoDB for MecDeposit
+CONNECT_CREATE_OTHER=false              # Create kafka connectors to MongoDB for other applications
+
+# Relative path to the Kafka connect connectors values file, upper level directories are supported
+CONNECT_KAFKA_CONNECTORS_VALUES_FILE="./jikkou/kafka-connectors-values.yaml"
+
 
 ### Kafka connect variables - END ###