Merge pull request #190 from vrk-kpa/REKDAT-98_container-restart-zuli…

…p-notifications

REKDAT-98: Add MonitoringStack to send zulip notifications on container restarts

cdk/bin/cdk.ts

@@ -18,6 +18,7 @@ import {SolrStack} from "../lib/solr-stack";
 import {FileSystemStack} from "../lib/filesystem-stack";
 import {CkanStack} from "../lib/ckan-stack";
 import {ShieldStack} from "../lib/shield-stack";
+import {MonitoringStack} from '../lib/monitoring-stack';
 
 const app = new cdk.App();
 
@@ -280,6 +281,16 @@ const ShieldStackDev = new ShieldStack(app, 'ShieldStack-dev', {
   vpc: VpcStackDev.vpc
 })
 
+const MonitoringStackDev = new MonitoringStack(app, 'MonitoringStack-dev', {
+  sendToZulipLambda: LambdaStackDev.sendToZulipLambda,
+  envProps: envProps,
+  env: {
+    account: devStackProps.account,
+    region: devStackProps.region,
+  },
+  environment: devStackProps.environment,
+  vpc: VpcStackDev.vpc
+});
 
 // Production
 
@@ -481,3 +492,14 @@ const ShieldStackProd = new ShieldStack(app, 'ShieldStack-prod', {
   requestSampleAllTrafficEnabled: false,
   vpc: VpcStackProd.vpc
 })
+
+const MonitoringStackProd = new MonitoringStack(app, 'MonitoringStack-prod', {
+  sendToZulipLambda: LambdaStackProd.sendToZulipLambda,
+  envProps: envProps,
+  env: {
+    account: prodStackProps.account,
+    region: prodStackProps.region,
+  },
+  environment: prodStackProps.environment,
+  vpc: VpcStackProd.vpc
+});

cdk/lib/lambda-stack.ts

@@ -5,9 +5,12 @@ import {Construct} from "constructs";
 import {LambdaStackProps} from "./lambda-stack-props";
 import {Credentials} from "aws-cdk-lib/aws-rds";
 import {Key} from "aws-cdk-lib/aws-kms";
+import { SendToZulip } from "./lambdas/send-to-zulip";
+import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs";
 
 export class LambdaStack extends Stack {
   readonly ckanCredentials: Credentials;
+  readonly sendToZulipLambda: NodejsFunction;
   constructor(scope: Construct, id: string, props: LambdaStackProps ) {
     super(scope, id, props);
 
@@ -26,5 +29,17 @@ export class LambdaStack extends Stack {
     })
 
     this.ckanCredentials = Credentials.fromSecret(createDatabases.ckanSecret);
+
+    const sendToZulip = new SendToZulip(this, 'send-to-zulip', {
+      zulipApiUser: 'avoindata-bot@turina.dvv.fi',
+      zulipApiUrl: 'turina.dvv.fi',
+      zulipStream: 'DGA',
+      zulipTopic: 'Container restarts',
+      envProps: props.envProps,
+      env: props.env,
+      environment: props.environment,
+      vpc: props.vpc
+    });
+    this.sendToZulipLambda = sendToZulip.lambda;
   }
 }

cdk/lib/lambdas/send-to-zulip-props.ts

@@ -0,0 +1,9 @@
+import {CommonStackProps} from "../common-stack-props";
+
+export interface SendToZulipProps extends CommonStackProps {
+  zulipApiUser: string,
+  zulipApiUrl: string,
+  zulipStream: string,
+  zulipTopic: string
+}
+

cdk/lib/lambdas/send-to-zulip.function.ts

@@ -0,0 +1,79 @@
+import {Handler} from 'aws-lambda';
+import {GetSecretValueCommand, SecretsManagerClient} from "@aws-sdk/client-secrets-manager";
+import * as https from 'https';
+import FormData = require('form-data');
+
+const { ZULIP_API_URL, ZULIP_API_USER, ZULIP_API_KEY_SECRET, ZULIP_STREAM, ZULIP_TOPIC } = process.env;
+
+function eventMessage(event: any) {
+  const {detail} = event;
+  if(detail?.eventName) {
+    // Generic event
+    const {resources} = event;
+    return `${detail?.eventName}: ${resources?.join(', ')}`;
+  } else if(detail?.stoppedReason) {
+    // Container stopped event
+    const {taskArn, group, stoppedReason} = detail;
+    return `${taskArn} (${group}): ${stoppedReason}`;
+  } else {
+    return 'Unknown message type';
+  }
+}
+export const handler: Handler = async (event: any) => {
+  if(!ZULIP_API_URL || !ZULIP_API_USER || !ZULIP_API_KEY_SECRET ||
+     !ZULIP_STREAM || !ZULIP_TOPIC) {
+    return {
+      statusCode: 500,
+      body: 'Missing configuration values',
+    }
+  }
+
+  const secretsManagerClient = new SecretsManagerClient({region: "eu-west-1"});
+  const command = new GetSecretValueCommand({
+    SecretId: ZULIP_API_KEY_SECRET
+  });
+  const response = await secretsManagerClient.send(command);
+  const zulipApiKey = response.SecretString;
+
+  const message = eventMessage(event);
+
+  const data = new FormData();
+  data.append('type', 'stream');
+  data.append('to', ZULIP_STREAM);
+  data.append('topic', ZULIP_TOPIC);
+  data.append('content', message);
+
+  const options: https.RequestOptions = {
+    hostname: ZULIP_API_URL,
+    port: 443,
+    path: '/api/v1/messages',
+    method: 'POST',
+    auth: `${ZULIP_API_USER}:${zulipApiKey}`,
+    headers: data.getHeaders(),
+  };
+
+  await new Promise((resolve, reject) => {
+    const req = https.request(options, (res: any) => {
+      if(res.statusCode != 200) {
+        console.log('Response from Zulip API:', res.statusCode);
+        res.on('data', (chunk: any) => {
+          console.log(chunk.toString());
+        }).on('end', () => {
+          resolve(res);
+        });
+      } else {
+        resolve(res);
+      }
+    }).on('error', (error: any) => {
+      console.error('Error sending message to Zulip:', error);
+      reject(error);
+    });
+    data.pipe(req);
+    req.end();
+  });
+
+  return {
+    statusCode: 200,
+    body: 'Message sent to Zulip',
+  };
+};

cdk/lib/lambdas/send-to-zulip.ts

@@ -0,0 +1,25 @@
+import {Construct} from "constructs";
+import {NodejsFunction} from "aws-cdk-lib/aws-lambda-nodejs";
+import {SendToZulipProps} from "./send-to-zulip-props";
+import * as sm from 'aws-cdk-lib/aws-secretsmanager';
+
+export class SendToZulip extends Construct {
+  readonly lambda: NodejsFunction;
+  constructor(scope: Construct, id: string, props: SendToZulipProps) {
+    super(scope, id);
+
+    // Task restart zulip reporting
+    const zulipSecret = sm.Secret.fromSecretNameV2(this, 'sZulipSecret', `/${props.environment}/zulip_api_key`);
+
+    this.lambda = new NodejsFunction(this, 'function', {
+      environment: {
+        ZULIP_API_USER: props.zulipApiUser,
+        ZULIP_API_KEY_SECRET: zulipSecret.secretName,
+        ZULIP_API_URL: props.zulipApiUrl,
+        ZULIP_STREAM: props.zulipStream,
+        ZULIP_TOPIC: props.zulipTopic
+      }
+    });
+    zulipSecret.grantRead(this.lambda);
+  }
+}

cdk/lib/monitoring-stack-props.ts

@@ -0,0 +1,7 @@
+import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs';
+import { CommonStackProps } from './common-stack-props';
+
+export interface MonitoringStackProps extends CommonStackProps {
+  sendToZulipLambda: NodejsFunction
+}
+

cdk/lib/monitoring-stack.ts

@@ -0,0 +1,37 @@
+import { Stack } from 'aws-cdk-lib';
+import { MonitoringStackProps } from './monitoring-stack-props';
+
+import * as events from 'aws-cdk-lib/aws-events';
+import * as eventsTargets from 'aws-cdk-lib/aws-events-targets';
+import * as logs from 'aws-cdk-lib/aws-logs';
+
+import { Construct } from 'constructs';
+
+export class MonitoringStack extends Stack {
+  constructor(scope: Construct, id: string, props: MonitoringStackProps) {
+    super(scope, id, props);
+
+    // Task health check failure log group
+    const taskHealthCheckFailLogGroup = new logs.LogGroup(this, 'taskHealthCheckFailedGroup', {
+      logGroupName: 'taskHealthCheckFailedGroup'
+    });
+
+    // Eventbridge rule to send 
+    const sendToDeveloperZulipTarget = new eventsTargets.LambdaFunction(props.sendToZulipLambda, {});
+    const taskHealthCheckFailLogGroupTarget = new eventsTargets.CloudWatchLogGroup(taskHealthCheckFailLogGroup);
+
+    new events.Rule(this, 'taskHealthCheckFailedRule', {
+      description: 'Rule for forwarding container health check failures to zulip',
+      eventPattern: {
+        source: ['aws.ecs'],
+        detail: {
+          desiredStatus: ['STOPPED'],
+          stoppedReason: [{wildcard: '*health*'}]
+        }
+      },
+      targets: [sendToDeveloperZulipTarget, taskHealthCheckFailLogGroupTarget],
+    })
+
+  }
+}
+