diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3eff2f9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +version: '3.1' +services: + + redis: + image: 'redis/redis-stack:7.2.0-v9' + networks: + - hmpps + container_name: redis + environment: + - ALLOW_EMPTY_PASSWORD=yes + ports: + - '6379:6379' + healthcheck: + test: [ "CMD-SHELL", "redis-cli ping | grep PONG" ] + interval: 5s + timeout: 3s + retries: 5 + +networks: + hmpps: diff --git a/helm_deploy/hmpps-component-dependencies/templates/_envs.tpl b/helm_deploy/hmpps-component-dependencies/templates/_envs.tpl index 95022c8..733da1b 100644 --- a/helm_deploy/hmpps-component-dependencies/templates/_envs.tpl +++ b/helm_deploy/hmpps-component-dependencies/templates/_envs.tpl @@ -40,4 +40,8 @@ env: - name: REDIS_TLS_ENABLED value: "true" + + - name: REDIS_TLS_VERIFICATION + value: "true" + {{end -}} diff --git a/package-lock.json b/package-lock.json index e99ca8d..2116e7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,13 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@types/bunyan": "^1.8.11", + "@types/bunyan-format": "^0.2.9", "applicationinsights": "^2.9.5", + "bunyan": "^1.8.15", + "bunyan-format": "^0.2.1", + "connect-redis": "^7.1.1", + "redis": "^4.6.13", "superagent": "^8.1.2", "ts-node": "^10.9.1" }, @@ -1431,6 +1437,64 @@ "url": "https://opencollective.com/unts" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.5.14", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.14.tgz", + "integrity": "sha512-YGn0GqsRBFUQxklhY7v562VMOP0DcmlrHHs3IV1mFE3cbxe31IITUkqhBcIhVSI/2JqtWAJXg5mjV4aU+zD0HA==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/client/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz", + "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz", + "integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz", + "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.6", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz", @@ -1519,6 +1583,22 @@ "@babel/types": "^7.20.7" } }, + "node_modules/@types/bunyan": { + "version": "1.8.11", + "resolved": "https://registry.npmjs.org/@types/bunyan/-/bunyan-1.8.11.tgz", + "integrity": "sha512-758fRH7umIMk5qt5ELmRMff4mLDlN+xyYzC+dkPTdKwbSkJFvz6xwyScrytPU0QIBbRRwbiE8/BIg8bpajerNQ==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/bunyan-format": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@types/bunyan-format/-/bunyan-format-0.2.9.tgz", + "integrity": "sha512-WuHXkdlVOzNejhWMO7mSu+i81rpp3b1pGB8HDCv+HtOGkz0gpoTSKfaQfDgYDUKI0BfZM4XRNGsVSI11akwrEQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/cookiejar": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz", @@ -2095,6 +2175,16 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ansicolors": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.2.1.tgz", + "integrity": "sha512-tOIuy1/SK/dr94ZA0ckDohKXNeBNqZ4us6PjMVLs5h1w2GBB6uPtOknp2+VF4F/zcy9LI70W+Z+pE2Soajky1w==" + }, + "node_modules/ansistyles": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/ansistyles/-/ansistyles-0.1.3.tgz", + "integrity": "sha512-6QWEyvMgIXX0eO972y7YPBLSBsq7UWKFAoNNTLGaOJ9bstcEL9sCbcjf96dVfNDdUsRoGOK82vWFJlKApXds7g==" + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -2553,13 +2643,13 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "devOptional": true }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, + "devOptional": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2642,6 +2732,33 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/bunyan": { + "version": "1.8.15", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.15.tgz", + "integrity": "sha512-0tECWShh6wUysgucJcBAoYegf3JJoZWibxdqhTm7OHPeT42qdjkZ29QCMcKwbgU1kiH+auSIasNRXMLWXafXig==", + "engines": [ + "node >=0.10.0" + ], + "bin": { + "bunyan": "bin/bunyan" + }, + "optionalDependencies": { + "dtrace-provider": "~0.8", + "moment": "^2.19.3", + "mv": "~2", + "safe-json-stringify": "~1" + } + }, + "node_modules/bunyan-format": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/bunyan-format/-/bunyan-format-0.2.1.tgz", + "integrity": "sha512-xQs2LwWskjQdv7bVkMNwvMi7HnvDQoX4587H90nDGQGPPwHrmxsihBOIYHMVwjLMMOokITKPyFcbFneblvMEjQ==", + "dependencies": { + "ansicolors": "~0.2.1", + "ansistyles": "~0.1.1", + "xtend": "~2.1.1" + } + }, "node_modules/call-bind": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", @@ -2873,6 +2990,14 @@ "semver": "bin/semver" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -2945,7 +3070,7 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "devOptional": true }, "node_modules/confusing-browser-globals": { "version": "1.0.11", @@ -2953,6 +3078,17 @@ "integrity": "sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==", "dev": true }, + "node_modules/connect-redis": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/connect-redis/-/connect-redis-7.1.1.tgz", + "integrity": "sha512-M+z7alnCJiuzKa8/1qAYdGUXHYfDnLolOGAUjOioB07pP39qxjG+X9ibsud7qUBc4jMV5Mcy3ugGv8eFcgamJQ==", + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "express-session": ">=1" + } + }, "node_modules/continuation-local-storage": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", @@ -2968,6 +3104,21 @@ "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", "dev": true }, + "node_modules/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "peer": true + }, "node_modules/cookiejar": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz", @@ -3165,6 +3316,15 @@ "node": ">=0.4.0" } }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -3304,6 +3464,19 @@ "url": "https://dotenvx.com" } }, + "node_modules/dtrace-provider": { + "version": "0.8.8", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", + "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "nan": "^2.14.0" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -4068,6 +4241,40 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/express-session": { + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.18.0.tgz", + "integrity": "sha512-m93QLWr0ju+rOwApSsyso838LQwgfs44QtOP/WBiwtAgPIo/SAh1a5c6nn2BR6mFNZehTpqKDESzP+fRHVbxwQ==", + "peer": true, + "dependencies": { + "cookie": "0.6.0", + "cookie-signature": "1.0.7", + "debug": "2.6.9", + "depd": "~2.0.0", + "on-headers": "~1.0.2", + "parseurl": "~1.3.3", + "safe-buffer": "5.2.1", + "uid-safe": "~2.1.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/express-session/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "peer": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express-session/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "peer": true + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4298,6 +4505,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -4737,7 +4952,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, + "devOptional": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -4747,7 +4962,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "devOptional": true }, "node_modules/internal-slot": { "version": "1.0.7", @@ -6462,7 +6677,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, + "devOptional": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -6474,27 +6689,105 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, + "devOptional": true, "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "optional": true, + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/module-details-from-path": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" }, + "node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "optional": true, + "engines": { + "node": "*" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/mv": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", + "optional": true, + "dependencies": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/mv/node_modules/glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", + "optional": true, + "dependencies": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mv/node_modules/rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", + "optional": true, + "dependencies": { + "glob": "^6.0.1" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/nan": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.19.0.tgz", + "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==", + "optional": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/ncp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", + "optional": true, + "bin": { + "ncp": "bin/ncp" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -6632,6 +6925,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -6744,6 +7046,15 @@ "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", "dev": true }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -6757,7 +7068,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -6984,6 +7295,15 @@ } ] }, + "node_modules/random-bytes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", + "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -6999,6 +7319,22 @@ "node": ">=6" } }, + "node_modules/redis": { + "version": "4.6.13", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.13.tgz", + "integrity": "sha512-MHgkS4B+sPjCXpf+HfdetBwbRz6vCtsceTmw1pHNYJAsYxrfpOP6dz+piJWGos8wqG7qb3vj/Rrc5qOlmInUuA==", + "workspaces": [ + "./packages/*" + ], + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.14", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.6", + "@redis/search": "1.1.6", + "@redis/time-series": "1.0.5" + } + }, "node_modules/regexp.prototype.flags": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz", @@ -7188,6 +7524,32 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "peer": true + }, + "node_modules/safe-json-stringify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", + "optional": true + }, "node_modules/safe-regex-test": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz", @@ -8102,6 +8464,18 @@ "node": ">=4.2.0" } }, + "node_modules/uid-safe": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", + "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", + "peer": true, + "dependencies": { + "random-bytes": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -8464,6 +8838,22 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "node_modules/xtend": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.1.2.tgz", + "integrity": "sha512-vMNKzr2rHP9Dp/e1NQFnLQlwlhp9L/LfvnsVdHxN1f+uggyVI3i08uD14GPvCToPkdsRfyPqIyYGmIk58V98ZQ==", + "dependencies": { + "object-keys": "~0.4.0" + }, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/xtend/node_modules/object-keys": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.4.0.tgz", + "integrity": "sha512-ncrLw+X55z7bkl5PnUvHwFK9FcGuFYo9gtjws2XtSzL+aZ8tm830P60WJ0dSmFVaSalWieW5MD7kEdnXda9yJw==" + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index cd85014..1db2159 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,13 @@ "typescript": "^4.9.5" }, "dependencies": { + "@types/bunyan": "^1.8.11", + "@types/bunyan-format": "^0.2.9", "applicationinsights": "^2.9.5", + "bunyan": "^1.8.15", + "bunyan-format": "^0.2.1", + "connect-redis": "^7.1.1", + "redis": "^4.6.13", "superagent": "^8.1.2", "ts-node": "^10.9.1" } diff --git a/src/config.ts b/src/config.ts index 22e0a23..299e25d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -17,11 +17,19 @@ function get(name: string, fallback: T = undefined): T | string { } const config = { + production, buildNumber: get('BUILD_NUMBER', '1_0_0'), productId: get('PRODUCT_ID', 'MISSING'), gitRef: get('GIT_REF', 'xxxxxxxxxxxxxxxxxxx'), branchName: get('GIT_BRANCH', 'xxxxxxxxxxxxxxxxxxx'), serviceCatalogueUrl: get('SERVICE_CATALOGUE_URL'), + redis: { + host: get('REDIS_HOST', 'localhost'), + port: parseInt(process.env.REDIS_PORT, 10) || 6379, + password: process.env.REDIS_AUTH_TOKEN, + tlsEnabled: get('REDIS_TLS_ENABLED', 'false'), + tlsVerification: get('REDIS_TLS_VERIFICATION', 'true'), + }, environments: { dev: { appInsightsCreds: { diff --git a/src/data/ComponentInfo.ts b/src/data/ComponentInfo.ts index d2634c2..1d33c76 100644 --- a/src/data/ComponentInfo.ts +++ b/src/data/ComponentInfo.ts @@ -1,7 +1,7 @@ import ComponentNode from '../component-node' // A component name to the hostname it relies on. -export type Dependency = [componentName: string, dependencyHostname: string, type: string] +export type Dependency = { componentName: string; dependencyHostname: string; type: string } export type Component = { name: string @@ -24,7 +24,7 @@ export class ComponentInfo { } getKnownComponents(): Dependency[] { - return this.components.map(check => [check.name, undefined, 'component']) + return this.components.map(check => ({ componentName: check.name, dependencyHostname: undefined, type: 'component' })) } getComponentForHostname(hostName: string): Component { @@ -34,12 +34,12 @@ export class ComponentInfo { getComponentMap(dependencies: Dependency[]): Components { const componentMap = dependencies.reduce( (acc, dependency) => { - const [componentName, url] = dependency + const { componentName, dependencyHostname } = dependency const component = acc[componentName] || new ComponentNode(componentName) - const dependentComponent = this.getComponentForHostname(url) + const dependentComponent = this.getComponentForHostname(dependencyHostname) if (dependentComponent) { component.addDependency(dependentComponent.name) - } else if (url) component.addUnknownDependency(dependency) + } else if (dependencyHostname) component.addUnknownDependency(dependency) acc[componentName] = component return acc }, diff --git a/src/data/appInsights/index.ts b/src/data/appInsights/index.ts index 39baaee..a7a40a4 100644 --- a/src/data/appInsights/index.ts +++ b/src/data/appInsights/index.ts @@ -1,10 +1,13 @@ import { type Environment } from '../../config' +import { Dependency } from '../ComponentInfo' import AppInsights from './Client' import Queries from './queries' -const getDependencies = async (env: Environment) => { +const getDependencies = async (env: Environment): Promise => { const appInsights = new AppInsights(env.appInsightsCreds) - return appInsights.query(Queries.DEPENDENCIES()) + const results = await appInsights.query(Queries.DEPENDENCIES()) + + return results.rows.map(row => ({ componentName: row[0], dependencyHostname: row[1], type: row[2] })) } export default getDependencies diff --git a/src/data/redis/redisClient.ts b/src/data/redis/redisClient.ts new file mode 100644 index 0000000..84fdb15 --- /dev/null +++ b/src/data/redis/redisClient.ts @@ -0,0 +1,30 @@ +import { createClient } from 'redis' +import config from '../../config' +import logger from '../../utils/logger' + +export type RedisClient = ReturnType + +const url = + config.redis.tlsEnabled === 'true' ? `rediss://${config.redis.host}:${config.redis.port}` : `redis://${config.redis.host}:${config.redis.port}` + +const tlsVerification = config.redis.tlsVerification === 'true' + +export const createRedisClient = (): RedisClient => { + const client = createClient({ + url, + password: config.redis.password, + socket: { + rejectUnauthorized: tlsVerification, + reconnectStrategy: (attempts: number) => { + // Exponential back off: 20ms, 40ms, 80ms..., capped to retry every 30 seconds + const nextDelay = Math.min(2 ** attempts * 20, 30000) + logger.info(`Retry Redis connection attempt: ${attempts}, next attempt in: ${nextDelay}ms`) + return nextDelay + }, + }, + }) + + client.on('error', (e: Error) => logger.error('Redis client error', e)) + + return client +} diff --git a/src/data/redis/redisService.ts b/src/data/redis/redisService.ts new file mode 100644 index 0000000..b218bc6 --- /dev/null +++ b/src/data/redis/redisService.ts @@ -0,0 +1,22 @@ +import { ClientClosedError } from 'redis' +import logger from '../../utils/logger' +import { RedisClient } from './redisClient' + +type Data = Parameters[2] + +export default class RedisService { + constructor(private readonly redisClient: RedisClient) {} + + async write(data: Data): Promise { + try { + await this.redisClient.json.set('dependency:info', '$', data) + } catch (error) { + if (error instanceof ClientClosedError) { + logger.error(`${error.message} ...RECONNECTING`) + await this.redisClient.connect + return + } + logger.error(`Failed to json.set: ${error.message}`, error) + } + } +} diff --git a/src/dependency-categoriser.ts b/src/dependency-categoriser.ts index ea3698c..f0ba8a9 100644 --- a/src/dependency-categoriser.ts +++ b/src/dependency-categoriser.ts @@ -1,24 +1,24 @@ import type { Dependency } from './data/ComponentInfo' -const targetIncludes = (...values: string[]) => { +const hostnameIncludes = (...values: string[]) => { const valuesToCheck = values.map(value => value.toLowerCase()) - return ([, target]: Dependency) => { - const targetToCheck = target?.toLowerCase() + return ({ dependencyHostname }: Dependency) => { + const targetToCheck = dependencyHostname.toLowerCase() return valuesToCheck.some(value => targetToCheck?.includes(value)) } } const typeIncludes = (...values: string[]) => { const valuesToCheck = values.map(value => value.toLowerCase()) - return ([, , type]: Dependency) => { + return ({ type }: Dependency) => { const typeToCheck = type.toLowerCase() return valuesToCheck.some(value => typeToCheck.includes(value)) } } const awsResource = (...values: string[]) => { - const isAws = targetIncludes('amazonaws.com') - const isType = targetIncludes(...values) + const isAws = hostnameIncludes('amazonaws.com') + const isType = hostnameIncludes(...values) return (dependency: Dependency) => { return isAws(dependency) && isType(dependency) } @@ -35,20 +35,20 @@ const categories: Categorisation[] = [ ['STS', awsResource('sts')], ['S3', awsResource('s3')], ['DYNOMODB', awsResource('dynamodb')], - ['OPENSEARCH', targetIncludes('opensearch')], - ['GOTENBERG', targetIncludes('gotenberg')], - ['SENTRY', targetIncludes('sentry.io')], - ['HAZELCAST', targetIncludes('hazelcast.com')], - ['GOOGLE_ANALYTICS', targetIncludes('www.google-analytics.com')], - ['GOOGLE_APIS', targetIncludes('googleapis.com')], - ['CONTENTFUL', targetIncludes('contentful.com')], - ['ORDINANCE_SURVEY', targetIncludes('api.os.uk')], - ['GOV_NOTIFY', targetIncludes('api.notifications.service.gov.uk')], - ['MICROSOFT_LOGIN', targetIncludes('login.microsoftonline.com')], - ['MICROSOFT_GRAPH', targetIncludes('graph.microsoft.com')], + ['OPENSEARCH', hostnameIncludes('opensearch')], + ['GOTENBERG', hostnameIncludes('gotenberg')], + ['SENTRY', hostnameIncludes('sentry.io')], + ['HAZELCAST', hostnameIncludes('hazelcast.com')], + ['GOOGLE_ANALYTICS', hostnameIncludes('www.google-analytics.com')], + ['GOOGLE_APIS', hostnameIncludes('googleapis.com')], + ['CONTENTFUL', hostnameIncludes('contentful.com')], + ['ORDINANCE_SURVEY', hostnameIncludes('api.os.uk')], + ['GOV_NOTIFY', hostnameIncludes('api.notifications.service.gov.uk')], + ['MICROSOFT_LOGIN', hostnameIncludes('login.microsoftonline.com')], + ['MICROSOFT_GRAPH', hostnameIncludes('graph.microsoft.com')], ['HTTP', typeIncludes('http', 'component')], ] -const categorise = (dependency: Dependency) => categories.find(category => category[1](dependency))?.[0] || dependency[2] +const categorise = (dependency: Dependency) => categories.find(category => category[1](dependency))?.[0] || dependency.type export default categorise diff --git a/src/dependency-info-gatherer.ts b/src/dependency-info-gatherer.ts index 189f3fc..9e14cae 100644 --- a/src/dependency-info-gatherer.ts +++ b/src/dependency-info-gatherer.ts @@ -1,4 +1,5 @@ import type { Dependency, Components } from './data/ComponentInfo' +import logger from './utils/logger' type DependencyReference = { name: string; type: string } type DependencyInfo = { @@ -10,16 +11,17 @@ type DependencyInfo = { dependents: string[] } -const buildUnknownDependencies = (components: Components) => { +const buildMissingComponents = (components: Components) => { const seen: Record = {} Object.values(components) .flatMap(s => s.unknownDependencies) .forEach(i => { - seen[`${i[1]}$${i[2]}`] = i + seen[`${i.dependencyHostname}$${i.type}`] = i }) return Object.values(seen) - .sort((a, b) => a[1].localeCompare(b[1])) - .map(([, target, type]) => ({ target, type })) + .sort((a, b) => a.dependencyHostname.localeCompare(b.dependencyHostname)) + .map(({ dependencyHostname, type }) => ({ dependencyHostname, type })) + .filter(dependency => dependency.dependencyHostname.includes('service.justice.gov.uk')) } const buildCategoryToComponent = (components: Components): Record => { @@ -40,8 +42,8 @@ const buildComponentInfo = (componentsMap: Components): Record { const components = Object.keys(component.knownDependencies).map(name => name) - const other = component.unknownDependencies.map(([, target, type]) => ({ - name: target, + const other = component.unknownDependencies.map(({ dependencyHostname, type }) => ({ + name: dependencyHostname, type, })) const resources = component.dependencyCategories.map(name => name) @@ -56,12 +58,14 @@ const buildComponentInfo = (componentsMap: Components): Record { const categoryToComponent = buildCategoryToComponent(components) - const unknownDependencies = buildUnknownDependencies(components) + const missingComponents = buildMissingComponents(components) const componentDependencyInfo = buildComponentInfo(components) + const missingServices = missingComponents.map(c => c.dependencyHostname).sort() + logger.info(`Services missing from service catalogue: \n\t${missingServices.join('\n\t')}`) + return { categoryToComponent, - unknownDependencies, componentDependencyInfo, } } diff --git a/src/run.ts b/src/run.ts index 94a9836..8baa1fd 100644 --- a/src/run.ts +++ b/src/run.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-console */ import initialiseAppInsights from './utils/appInsights' import applicationInfo from './utils/applicationInfo' @@ -7,29 +6,44 @@ import type { ComponentInfo, Dependency } from './data/ComponentInfo' import gatherDependencyInfo from './dependency-info-gatherer' import getComponents from './data/serviceCatalogue' import getDependencies from './data/appInsights' +import { createRedisClient } from './data/redis/redisClient' +import RedisService from './data/redis/redisService' +import logger from './utils/logger' initialiseAppInsights(applicationInfo()) const gatherComponentDependencies = async (env: Environment, components: ComponentInfo) => { - const dependencies: Dependency[] = (await getDependencies(env)).rows.map(row => [row[0], row[1], row[2]]) + const dependencies: Dependency[] = await getDependencies(env) return components.getKnownComponents().concat(dependencies) } const run = async () => { + const redisClient = createRedisClient() + await redisClient.connect() + + const redisService = new RedisService(redisClient) + + logger.info(`Starting to gather dependency info`) + const components = await getComponents(config.serviceCatalogueUrl) const dependencies = await gatherComponentDependencies(config.environments.dev, components) const componentMap = components.getComponentMap(dependencies) - const { categoryToComponent, componentDependencyInfo, unknownDependencies } = await gatherDependencyInfo(componentMap) + const { categoryToComponent, componentDependencyInfo } = await gatherDependencyInfo(componentMap) + + const categoryCounts = Object.entries(categoryToComponent).map(([category, comps]) => `${category} => ${comps.length}`) + logger.info(`Category freqs: \n${categoryCounts.join('\n')}`) - unknownDependencies.forEach(dependency => console.log(dependency)) + logger.info(`Starting to publish dependency info`) - Object.entries(categoryToComponent).forEach(([category, comps]) => console.log(`${category} => ${comps}\n`)) + const all = { componentDependencyInfo, categoryToComponent } + await redisService.write(all) - console.log(JSON.stringify(componentDependencyInfo['create-and-vary-a-licence-api'], null, 4)) + logger.info(`Finished publishing dependency info for ${Object.keys(componentMap).length} components`) + await redisClient.quit() } run().catch(e => { - console.error(e) + logger.error(e) process.exit(1) }) diff --git a/src/utils/logger.ts b/src/utils/logger.ts new file mode 100755 index 0000000..04df6a9 --- /dev/null +++ b/src/utils/logger.ts @@ -0,0 +1,9 @@ +import bunyan from 'bunyan' +import bunyanFormat from 'bunyan-format' +import config from '../config' + +const formatOut = bunyanFormat({ outputMode: 'short', color: !config.production }) + +const logger = bunyan.createLogger({ name: 'Hmpps Component Dependencies', stream: formatOut, level: 'debug' }) + +export default logger