Merge pull request #19 from CDOT-CV/Feature/custom-kafka-connectors

Custom Kafka topics / Connectors

Author: Michael7371

README.md

@@ -101,6 +101,8 @@ The following enviroment variables can be used to configure Kafka Topic creation
 | `KAFKA_TOPIC_CREATE_GEOJSONCONVERTER` | Whether to create topics for the GeoJSON Converter |
 | `KAFKA_TOPIC_CREATE_CONFLICTMONITOR` | Whether to create topics for the Conflict Monitor |
 | `KAFKA_TOPIC_CREATE_DEDUPLICATOR` | Whether to create topics for the Deduplicator |
+| `KAFKA_TOPIC_CREATE_OTHER` | Whether to create topics for other applications, this is only useful when you attach a custom `kafka-topics-values.yaml` file with other topics |
+| `KAFKA_TOPICS_VALUES_FILE` | Path to a custom `kafka-topics-values.yaml` file|
 | `KAFKA_TOPIC_PARTITIONS` | Number of partitions |
 | `KAFKA_TOPIC_REPLICAS` | Number of replicas |
 | `KAFKA_TOPIC_MIN_INSYNC_REPLICAS` | Minumum number of in-sync replicas (for use with ack=all) |
@@ -174,7 +176,9 @@ The following environment variables can be used to configure Kafka Connectors:
 | `CONNECT_CREATE_ODE` | Whether to create kafka connectors for the ODE |
 | `CONNECT_CREATE_GEOJSONCONVERTER` | Whether to create topics for the GeojsonConverter |
 | `CONNECT_CREATE_CONFLICTMONITOR` | Whether to create kafka connectors for the Conflict Monitor |
-| `CONNECT_CREATE_DEDUPLICATOR` | Whether to create topics for the Deduplicator |
+| `CONNECT_CREATE_DEDUPLICATOR` | Whether to create kafka connectors for the Deduplicator |
+| `CONNECT_KAFKA_CONNECTORS_VALUES_FILE` | Path to a custom `kafka-connectors-values.yaml` file |
+| `CONNECT_CREATE_OTHER` | Whether to create kafka connectors for other applications, this is only useful when you attach a custom `kafka-connectors-values.yaml` file with other connectors |
 
 ### Quick Run
 

docker-compose-connect.yml

@@ -80,7 +80,11 @@ services:
       CONNECT_CREATE_CONFLICTMONITOR: ${CONNECT_CREATE_CONFLICTMONITOR}
       CONNECT_CREATE_DEDUPLICATOR: ${CONNECT_CREATE_DEDUPLICATOR}
       CONNECT_CREATE_MECDEPOSIT: ${CONNECT_CREATE_MECDEPOSIT}
+      CONNECT_CREATE_OTHER: ${CONNECT_CREATE_OTHER}
+
       MONGO_CONNECTOR_USERNAME: ${MONGO_ADMIN_DB_USER}
       MONGO_CONNECTOR_PASSWORD: ${MONGO_ADMIN_DB_PASS:?}
       MONGO_DB_IP: ${MONGO_IP}
-      MONGO_DB_NAME: ${MONGO_DB_NAME}
+      MONGO_DB_NAME: ${MONGO_DB_NAME}
+    volumes:
+      - ${CONNECT_KAFKA_CONNECTORS_VALUES_FILE:-./jikkou/kafka-connectors-values.yaml}:/kafka-connectors-values.yaml

docker-compose-kafka.yml

@@ -70,11 +70,13 @@ services:
       KAFKA_TOPIC_MIN_INSYNC_REPLICAS: ${KAFKA_TOPIC_MIN_INSYNC_REPLICAS}
       KAFKA_TOPIC_RETENTION_MS: ${KAFKA_TOPIC_RETENTION_MS}
       KAFKA_TOPIC_DELETE_RETENTION_MS: ${KAFKA_TOPIC_DELETE_RETENTION_MS}
+
       KAFKA_TOPIC_CREATE_ODE: ${KAFKA_TOPIC_CREATE_ODE}
       KAFKA_TOPIC_CREATE_GEOJSONCONVERTER: ${KAFKA_TOPIC_CREATE_GEOJSONCONVERTER}
       KAFKA_TOPIC_CREATE_CONFLICTMONITOR: ${KAFKA_TOPIC_CREATE_CONFLICTMONITOR}
       KAFKA_TOPIC_CREATE_DEDUPLICATOR: ${KAFKA_TOPIC_CREATE_DEDUPLICATOR}
       KAFKA_TOPIC_CREATE_MECDEPOSIT: ${KAFKA_TOPIC_CREATE_MECDEPOSIT}
+      KAFKA_TOPIC_CREATE_OTHER: ${KAFKA_TOPIC_CREATE_OTHER}
 
       KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
       KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM}
@@ -84,6 +86,8 @@ services:
       options:
         max-size: "10m"
         max-file: "5"
+    volumes:
+      - ${KAFKA_TOPICS_VALUES_FILE:-./jikkou/kafka-topics-values.yaml}:/kafka-topics-values.yaml
 
   kafka-schema-registry:
     profiles:

jikkou/kafka-connectors-template.jinja

@@ -70,3 +70,7 @@ spec:
 {% if system.env.CONNECT_CREATE_MECDEPOSIT %}
 {{ create_connector(values.apps.mecdeposit) }}
 {% endif %}
+
+{% if system.env.CONNECT_CREATE_OTHER %}
+{{ create_connector(values.apps.other) }}
+{% endif %}

jikkou/kafka-connectors-values.yaml

@@ -370,3 +370,7 @@ apps:
       - topicName: topic.MecDepositMetrics
         collectionName: MecDepositMetrics
         generateTimestamp: true
+  # Allow for custom connectors to be added - users can override this file and add other kafka connectors here
+  other:
+    name: other
+    connectors:

jikkou/kafka-topics-template.jinja

@@ -79,3 +79,7 @@ spec:
 {% if system.env.KAFKA_TOPIC_CREATE_MECDEPOSIT %}
 {{ create_topics(values.apps.mecdeposit) }}
 {% endif %}
+
+{% if system.env.KAFKA_TOPIC_CREATE_OTHER %}
+{{ create_topics(values.apps.other) }}
+{% endif %}

jikkou/kafka-topics-values.yaml

@@ -197,3 +197,8 @@ apps:
       - topic.MecDepositMetrics
     tableTopics: {}
     customTopics: {}
+  other:
+    name: other-topics
+    streamTopics: {}
+    tableTopics: {}
+    customTopics: {}

sample.env

@@ -49,6 +49,10 @@ KAFKA_TOPIC_CREATE_GEOJSONCONVERTER=true    # Create topics for GeoJSON Converte
 KAFKA_TOPIC_CREATE_CONFLICTMONITOR=true     # Create topics for Conflict Monitor
 KAFKA_TOPIC_CREATE_DEDUPLICATOR=false       # Create topics for Deduplicator
 KAFKA_TOPIC_CREATE_MECDEPOSIT=false         # Create topics for MecDeposit
+KAFKA_TOPIC_CREATE_OTHER=false              # Create topics for other applications
+
+# Relative path to the Kafka topics values file, upper level directories are supported
+KAFKA_TOPICS_VALUES_FILE="./jikkou/kafka-topics-values.yaml"
 
 # Confluent Cloud Support
 KAFKA_SECURITY_PROTOCOL=PLAINTEXT
@@ -121,5 +125,10 @@ CONNECT_CREATE_GEOJSONCONVERTER=true    # Create kafka connectors to MongoDB for
 CONNECT_CREATE_CONFLICTMONITOR=true     # Create kafka connectors to MongoDB for Conflict Monitor
 CONNECT_CREATE_DEDUPLICATOR=false       # Create kafka connectors to MongoDB for Deduplicator
 CONNECT_CREATE_MECDEPOSIT=false         # Create kafka connectors to MongoDB for MecDeposit
+CONNECT_CREATE_OTHER=false              # Create kafka connectors to MongoDB for other applications
+
+# Relative path to the Kafka connect connectors values file, upper level directories are supported
+CONNECT_KAFKA_CONNECTORS_VALUES_FILE="./jikkou/kafka-connectors-values.yaml"
+
 
 ### Kafka connect variables - END ###