Merge branch 'develop' of github.com:CDOT-CV/jpo-utils into mongo-ttl-updates

Author: John-Wiens

azure-pipelines.yml

@@ -0,0 +1,30 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+  branches:
+    include:
+      - develop
+  paths:
+    include:
+      - 'jikkou/*'
+      - 'kafka-connect/*'
+
+pool:
+  vmImage: ubuntu-latest
+
+steps:
+  - task: CopyFiles@2
+    inputs:
+      SourceFolder: '.'
+      Contents: '**'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)'
+
+  # Publish the artifacts directory for consumption in publish pipeline
+  - task: PublishBuildArtifacts@1
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)'
+      ArtifactName: 'jpo-utils-services'
+      publishLocation: 'Container'

jikkou/kafka-connectors-template.jinja

@@ -12,7 +12,7 @@ metadata:
     kafka.jikkou.io/connect-cluster: {{ values.clusterName }}
 spec:
   connectorClass: {{ system.env.KAFKA_CONNECT_CONNECTOR_CLASS | default(values.connectorClass) }}
-  tasksMax: {{ system.env.KAFKA_TOPIC_PARTITIONS | default(values.tasksMax) }}
+  tasksMax: {{ system.env.CONNECT_TASKS_MAX | default(values.tasksMax) }}
   config:
     collection: {{ connector.collectionName }}
     connection.uri: mongodb://{{ system.env.MONGO_CONNECTOR_USERNAME }}:{{ system.env.MONGO_CONNECTOR_PASSWORD }}@{{ system.env.MONGO_DB_IP }}:27017/

jikkou/kafka-topics-values.yaml

@@ -174,6 +174,7 @@ apps:
       - topic.DeduplicatedOdeRawEncodedTIMJson
       - topic.DeduplicatedOdeBsmJson 
       - topic.DeduplicatedProcessedBsm
+      - topic.DeduplicatedProcessedSpat
     tableTopics: {}
     customTopics: {}
   kafkaconnect:
@@ -185,10 +186,10 @@ apps:
         partitions: 1
         cleanUpPolicy: compact
       - topicName: topic.KafkaConnectOffsets
-        partitions: 20
+        partitions: 3
         cleanUpPolicy: compact
       - topicName: topic.KafkaConnectStatus
-        partitions: 10
+        partitions: 3
         cleanUpPolicy: compact
   mecdeposit:
     name: jpo-mecdeposit

kafka-connect/Dockerfile

@@ -1,6 +1,9 @@
 FROM confluentinc/cp-kafka-connect:7.7.0
 
+# MongoDB Kafka Connector
 # Docs: https://www.mongodb.com/docs/kafka-connector/current/
 RUN confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.13.0
+
+# Transforms
 # Docs: https://docs.confluent.io/platform/current/connect/transforms/overview.html
 RUN confluent-hub install --no-prompt confluentinc/connect-transforms:1.4.7

mongo/create_indexes.js

@@ -88,7 +88,7 @@ const odeCollections = [
 const geoJsonConverterCollections = [
     {name: "ProcessedMap", ttlField: "recordGeneratedAt", timeField: "properties.timeStamp", intersectionField: "properties.intersectionId", expireTime: expireSeconds},
     {name: "ProcessedSpat", ttlField: "recordGeneratedAt", timeField: "utcTimeStamp", intersectionField: "intersectionId", expireTime: expireSeconds},
-    {name: "ProcessedBsm", ttlField: "recordGeneratedAt", timeField: "timeStamp", geoSpatialField: "features.geometry.coordinates", expireTime: expireSeconds},
+    {name: "ProcessedBsm", ttlField: "recordGeneratedAt", timeField: "timeStamp", geoSpatialField: "geometry.coordinates", expireTime: expireSeconds},
 ];