Update grpc and mqtt dependencies (#7)

* Update grpc and mqtt packages

* Update grpc server start

* update mosquitto and fix tests

* Use beta release of osem api models

* Set options close to old grpc.load

* Add ghcr workflows

* Update build workflow

* Add tests for mqtts and ws

* Add config parameter for mqtts test url

* Add lint and test jobs

* Build image only if lint and build was successful

* Fix workflow error

* Set env variable for mqtts test client

* Update on push expression

.github/workflows/registry-build-publish.yaml

@@ -0,0 +1,117 @@
+name: Build and publish to Github Container Registry
+
+on:
+  push:
+    branches: [ master ]
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches:
+      - master
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+
+  lint:
+    name: Lint code
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Node.js 14
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: |
+          echo "##[set-output name=dir;]$(yarn cache dir)"
+      - name: Restore yarn cache
+        uses: actions/cache@v2.1.6
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+      - name: Install project dependencies
+        run: |
+          yarn --prefer-offline --pure-lockfile
+      - name: Run eslint
+        run: |
+          yarn run lint:ci
+
+  test:
+    name: Run tests
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Node.js 14
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: |
+          echo "##[set-output name=dir;]$(yarn cache dir)"
+      - name: Restore yarn cache
+        uses: actions/cache@v2.1.6
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+      - name: Install project dependencies
+        run: |
+          yarn --prefer-offline --pure-lockfile
+
+      - name: Execute tests
+        run: |
+          export MQTTS_TEST_URL=${{ secrets.MQTTS_URL }}
+          yarn test
+
+  build-and-push-images:
+    runs-on: ubuntu-latest
+    # Depends on lint and test jobs
+    needs: [lint, test]
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/registry-pr-purge.yaml

@@ -0,0 +1,20 @@
+name: Purge Pull Request Image
+
+# https://docs.github.com/en/actions/reference/events-that-trigger-workflows#registry_package
+# Run cleanup job if a new package was published or updated
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  purge_pr_image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Purge Pull Request Image
+        uses: vlaurin/action-ghcr-prune@v0.1.0
+        with:
+          token: ${{ secrets.GHCR_TOKEN}}
+          organization: ${{ github.repository_owner}}
+          container: ${{ github.event.repository.name }}
+          tag-regex: /^pr-${{github.event.pull_request.number}}$/
+          dry-run: true

.github/workflows/registry-purge.yaml

@@ -0,0 +1,19 @@
+name: Docker Registry Cleanup
+
+# https://docs.github.com/en/actions/reference/events-that-trigger-workflows#registry_package
+# Run cleanup job if a new package was published or updated
+on:
+  registry_package:
+
+jobs:
+  purge_untagged_images:
+    runs-on: ubuntu-latest
+    steps:
+      - name: clean packages
+        uses: vlaurin/action-ghcr-prune@v0.1.0
+        with:
+          token: ${{ secrets.GHCR_TOKEN}}
+          organization: ${{ github.repository_owner}}
+          container: ${{ github.event.repository.name }}
+          untagged: true
+          dry-run: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:8-alpine as build
+FROM node:14-alpine as build
 
 RUN apk --no-cache --virtual .build add build-base python
 
@@ -12,13 +12,15 @@ COPY yarn.lock /usr/src/app/
 
 # npm rebuild is required because the prebuilt binaries are not compatible with musl
 # remove when https://github.com/kelektiv/node.bcrypt.js/issues/528 is resolved
-RUN yarn install --pure-lockfile --production \
-  && npm rebuild bcrypt --build-from-source
+# RUN yarn install --pure-lockfile --production \
+#   && npm rebuild bcrypt --build-from-source
+
+RUN yarn install --pure-lockfile --production
 
 COPY . /usr/src/app
 
 # Final stage
-FROM node:8-alpine
+FROM node:14-alpine
 
 WORKDIR /usr/src/app
 COPY --from=build /usr/src/app /usr/src/app

config/custom-environment-variables.json

@@ -10,5 +10,8 @@
       "server_cert": "OSEM_MQTT_GRPC_SERVER_CERT",
       "server_key": "OSEM_MQTT_GRPC_SERVER_KEY"
     }
+  },
+  "mqtts_test_client": {
+    "url": "MQTTS_TEST_URL"
   }
 }

config/default.json

@@ -10,5 +10,8 @@
       "server_cert": "",
       "server_key": ""
     }
+  },
+  "mqtts_test_client:": {
+    "url": ""
   }
 }

grpc-test-client.js

@@ -1,11 +1,13 @@
 'use strict';
 
-const grpc = require('grpc'),
+const grpcLibrary = require('@grpc/grpc-js'),
+  protoLoader = require('@grpc/proto-loader'),
   fs = require('fs'),
   path = require('path'),
   { mqttProto } = require('@sensebox/osem-protos');
 
-const { MqttService } = grpc.load(mqttProto);
+const packageDefinition = protoLoader.loadSync(mqttProto);
+const MqttService = grpcLibrary.loadPackageDefinition(packageDefinition).MqttService;
 
 const readCertFile = function readCertFile (filename) {
   return fs.readFileSync(path.join(process.cwd(), 'out', filename));
@@ -15,7 +17,7 @@ const ca = readCertFile('openSenseMap_CA.crt'),
   client_key = readCertFile('client.key'),
   client_crt = readCertFile('client.crt');
 
-const credentials = grpc.credentials.createSsl(ca, client_key, client_crt);
+const credentials = grpcLibrary.credentials.createSsl(ca, client_key, client_crt);
 
 const client = new MqttService('localhost:3925', credentials);
 

package.json

@@ -8,7 +8,8 @@
   "license": "MIT",
   "scripts": {
     "start": "node src/index.js",
-    "lint": "eslint --fix \"{src,test,.scripts}/**/*.js\"",
+    "lint": "eslint --ignore-pattern node_modules --fix \"{src,test,.scripts}/**/*.js\"",
+    "lint:ci": "eslint --ignore-pattern node_modules \"{tests,packages}/**/*.js\"",
     "pretest": "docker-compose -f test/docker-compose.yml up -d",
     "test": "mocha test/index.js",
     "posttest": "docker-compose -f test/docker-compose.yml down -v"
@@ -20,12 +21,11 @@
     "mocha": "^3.4.2"
   },
   "dependencies": {
-    "@sensebox/opensensemap-api-models": "^0.0.12",
+    "@grpc/grpc-js": "^1.3.7",
+    "@grpc/proto-loader": "^0.6.4",
+    "@sensebox/opensensemap-api-models": "0.0.32-beta.0",
     "@sensebox/osem-protos": "^1.1.0",
-    "bunyan": "^1.8.12",
-    "config": "^1.28.1",
-    "grpc": "^1.9.0",
-    "mqtt": "2.15.1"
+    "mqtt": "^4.2.8"
   },
   "publishConfig": {
     "access": "public"

src/grpc-server.js

@@ -1,14 +1,24 @@
 'use strict';
 
-const grpc = require('grpc'),
+const grpcLibrary = require('@grpc/grpc-js'),
+  protoLoader = require('@grpc/proto-loader'),
   { Box } = require('@sensebox/opensensemap-api-models'),
   MQTTClient = require('./client'),
   fs = require('fs'),
   log = require('./logger'),
   config = require('config').get('grpc'),
   { mqttProto } = require('@sensebox/osem-protos');
 
-const { MqttService } = grpc.load(mqttProto);
+// const { MqttService } = grpc.load(mqttProto);
+const packageDefinition = protoLoader.loadSync(mqttProto, {
+  keepCase: true,
+  longs: String,
+  enums: String,
+  defaults: true,
+  oneofs: true,
+});
+const MqttService = grpcLibrary.loadPackageDefinition(packageDefinition).MqttService;
+
 
 const connectBox = async function connectBox (call, callback) {
   const { box_id } = call.request;
@@ -54,7 +64,7 @@ const prepareCredentials = function prepareCredentials () {
     return fs.readFileSync(config_key);
   });
 
-  return grpc.ServerCredentials.createSsl(
+  return grpcLibrary.ServerCredentials.createSsl(
     certs[0],
     [
       {
@@ -72,11 +82,12 @@ const init = function init () {
 
   const credentials = prepareCredentials();
 
-  const server = new grpc.Server();
+  const server = new grpcLibrary.Server();
   server.addService(MqttService.service, { connectBox, disconnectBox });
-  server.bind(`0.0.0.0:${port}`, credentials);
-  server.start();
-  log.info({ 'grpc-server': 'MQTT Integration GRPC server started' });
+  server.bindAsync(`0.0.0.0:${port}`, credentials, () => {
+    server.start();
+    log.info({ 'grpc-server': 'MQTT Integration GRPC server started' });
+  });
 };
 
 module.exports = { init };

test/docker-compose.yml

@@ -1,10 +1,13 @@
-version: "2"
+version: "3.9"
 
 services:
   mosquitto:
-    image: eclipse-mosquitto:1.4.12
+    image: eclipse-mosquitto:2.0.12
     ports:
-      - "1884:1883"
+      - "8883:8883"
+      - "8884:8884"
+    volumes:
+      - ./mosquitto:/mosquitto/config
   db:
     image: mongo:3.2
     ports:

test/helpers/mqtts.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const config = require('config').get('mqtts_test_client');
+
+module.exports = function mqttsBox (
+  // parameter destructuring
+  { enabled, url, topic, connectionOptions, decodeOptions, messageFormat } = {
+    // defaults
+    enabled: true,
+    url: config.get('url'),
+    topic: 'generalTestTopic',
+    messageFormat: 'csv',
+  }
+) {
+  return {
+    name: 'mqttsTestSenseBox',
+    location: [7.62, 51.96],
+    exposure: 'outdoor',
+    model: 'homeEthernet',
+    mqtt: {
+      enabled,
+      url,
+      topic,
+      connectionOptions,
+      decodeOptions,
+      messageFormat,
+    },
+  };
+};

test/helpers/senseBox.js

@@ -5,7 +5,7 @@ module.exports = function senseBox (
   { enabled, url, topic, connectionOptions, decodeOptions, messageFormat } = {
     // defaults
     enabled: true,
-    url: 'mqtt://localhost:1884',
+    url: 'mqtt://localhost:8883',
     topic: 'generalTestTopic',
     messageFormat: 'csv'
   }
@@ -21,7 +21,7 @@ module.exports = function senseBox (
       topic,
       connectionOptions,
       decodeOptions,
-      messageFormat
-    }
+      messageFormat,
+    },
   };
 };