From d86f768c0521475bf88d73612129787d458175ff Mon Sep 17 00:00:00 2001 From: 35C4n0r Date: Wed, 7 Aug 2024 00:59:27 +0530 Subject: [PATCH 01/11] feat: add models and ingestion scripts Signed-off-by: 35C4n0r --- .gitignore | 3 + package-lock.json | 265 ++++++++++++++++++ package.json | 2 + prisma/docker-compose.yaml | 12 + .../migration.sql | 101 +++++++ prisma/migrations/migration_lock.toml | 3 + prisma/schema.prisma | 93 ++++++ src/scripts/ingestors/district.geojson.ts | 70 +++++ src/scripts/ingestors/service.geojson.ts | 161 +++++++++++ src/scripts/ingestors/state.geojson.ts | 41 +++ src/scripts/ingestors/subdistrict.geojson.ts | 118 ++++++++ src/scripts/ingestors/village.geojson.ts | 155 ++++++++++ 12 files changed, 1024 insertions(+) create mode 100644 prisma/docker-compose.yaml create mode 100644 prisma/migrations/20240806185521_add_geo_tables/migration.sql create mode 100644 prisma/migrations/migration_lock.toml create mode 100644 prisma/schema.prisma create mode 100644 src/scripts/ingestors/district.geojson.ts create mode 100644 src/scripts/ingestors/service.geojson.ts create mode 100644 src/scripts/ingestors/state.geojson.ts create mode 100644 src/scripts/ingestors/subdistrict.geojson.ts create mode 100644 src/scripts/ingestors/village.geojson.ts diff --git a/.gitignore b/.gitignore index ee0dc93..0ae4a00 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ src/.DS_Store /src/geojson-data/ /src/geoquery.in.data/ db.mmdb + + +.env diff --git a/package-lock.json b/package-lock.json index a7f2252..5b67aee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@nestjs/platform-express": "^10.0.0", "@nestjs/platform-fastify": "^10.3.0", "@nestjs/swagger": "^7.3.1", + "@prisma/client": "^5.17.0", "@samagra-x/stencil": "^0.0.6", "@turf/turf": "^6.5.0", "@types/multer": "^1.4.11", @@ -24,7 +25,9 @@ "fastify": "^4.25.2", "fastify-multer": "^2.0.3", "fastify-multipart": "^5.4.0", + "fgdb": "^1.0.0", "fuse.js": "^7.0.0", + "geojson": "^0.5.0", "multer": "^1.4.5-lts.1", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1" @@ -45,6 +48,7 @@ "husky": "8.0.3", "jest": "^29.5.0", "prettier": "^3.0.0", + "prisma": "^5.17.0", "source-map-support": "^0.5.21", "supertest": "^6.3.3", "ts-jest": "^29.1.0", @@ -2147,6 +2151,68 @@ "url": "https://opencollective.com/unts" } }, + "node_modules/@prisma/client": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.17.0.tgz", + "integrity": "sha512-N2tnyKayT0Zf7mHjwEyE8iG7FwTmXDHFZ1GnNhQp0pJUObsuel4ZZ1XwfuAYkq5mRIiC/Kot0kt0tGCfLJ70Jw==", + "hasInstallScript": true, + "engines": { + "node": ">=16.13" + }, + "peerDependencies": { + "prisma": "*" + }, + "peerDependenciesMeta": { + "prisma": { + "optional": true + } + } + }, + "node_modules/@prisma/debug": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.17.0.tgz", + "integrity": "sha512-l7+AteR3P8FXiYyo496zkuoiJ5r9jLQEdUuxIxNCN1ud8rdbH3GTxm+f+dCyaSv9l9WY+29L9czaVRXz9mULfg==", + "devOptional": true + }, + "node_modules/@prisma/engines": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.17.0.tgz", + "integrity": "sha512-+r+Nf+JP210Jur+/X8SIPLtz+uW9YA4QO5IXA+KcSOBe/shT47bCcRMTYCbOESw3FFYFTwe7vU6KTWHKPiwvtg==", + "devOptional": true, + "hasInstallScript": true, + "dependencies": { + "@prisma/debug": "5.17.0", + "@prisma/engines-version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", + "@prisma/fetch-engine": "5.17.0", + "@prisma/get-platform": "5.17.0" + } + }, + "node_modules/@prisma/engines-version": { + "version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053.tgz", + "integrity": "sha512-tUuxZZysZDcrk5oaNOdrBnnkoTtmNQPkzINFDjz7eG6vcs9AVDmA/F6K5Plsb2aQc/l5M2EnFqn3htng9FA4hg==", + "devOptional": true + }, + "node_modules/@prisma/fetch-engine": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.17.0.tgz", + "integrity": "sha512-ESxiOaHuC488ilLPnrv/tM2KrPhQB5TRris/IeIV4ZvUuKeaicCl4Xj/JCQeG9IlxqOgf1cCg5h5vAzlewN91Q==", + "devOptional": true, + "dependencies": { + "@prisma/debug": "5.17.0", + "@prisma/engines-version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", + "@prisma/get-platform": "5.17.0" + } + }, + "node_modules/@prisma/get-platform": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.17.0.tgz", + "integrity": "sha512-UlDgbRozCP1rfJ5Tlkf3Cnftb6srGrEQ4Nm3og+1Se2gWmCZ0hmPIi+tQikGDUVLlvOWx3Gyi9LzgRP+HTXV9w==", + "devOptional": true, + "dependencies": { + "@prisma/debug": "5.17.0" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -7189,6 +7255,25 @@ "bser": "2.1.1" } }, + "node_modules/fgdb": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fgdb/-/fgdb-1.0.0.tgz", + "integrity": "sha512-2ZaznM1bhXk9e5xBdLzAR8H9yGFB2N51QTfMsHnynyJkqqmPMXSvSH5U6qzsyt73aTLQCFeViSQz5an4PKg4XA==", + "dependencies": { + "jszip": "~0.2.1", + "lie": "^3.0.0", + "long": "~1.1.2", + "proj4": "^2.3.6" + } + }, + "node_modules/fgdb/node_modules/long": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/long/-/long-1.1.5.tgz", + "integrity": "sha512-TU6nAF5SdasnTr28c7e74P4Crbn9o3/zwo1pM22Wvg2i2vlZ4Eelxwu4QT7j21z0sDBlJDEnEZjXTZg2J8WJrg==", + "engines": { + "node": ">=0.6" + } + }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -7564,6 +7649,14 @@ "node": ">=6.9.0" } }, + "node_modules/geojson": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/geojson/-/geojson-0.5.0.tgz", + "integrity": "sha512-/Bx5lEn+qRF4TfQ5aLu6NH+UKtvIv7Lhc487y/c8BdludrCTpiWf9wyI0RTyqg49MFefIAvFDuEi5Dfd/zgNxQ==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/geojson-equality": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/geojson-equality/-/geojson-equality-0.1.6.tgz", @@ -7935,6 +8028,11 @@ "node": ">= 4" } }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" + }, "node_modules/import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -9050,6 +9148,11 @@ "verror": "1.10.0" } }, + "node_modules/jszip": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-0.2.1.tgz", + "integrity": "sha512-Djh0bVj/EiqNTlwKC10xsOf+HtdD6mVq4m7DWdRoUvChB0aj2BThnGl+Kl4uDlRuxlp+EvjZ2ZOo0niTJlh+LQ==" + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -9095,6 +9198,14 @@ "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.59.tgz", "integrity": "sha512-HeTsOrDF/hWhEiKqZVwg9Cqlep5x2T+IYDENvT2VRj3iX8JQ7Y+omENv+AIn0vC8m6GYhivfCed5Cgfw27r5SA==" }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/light-my-request": { "version": "5.12.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.12.0.tgz", @@ -9307,6 +9418,11 @@ "node": ">= 0.6" } }, + "node_modules/mgrs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", + "integrity": "sha512-awNbTOqCxK1DBGjalK3xqWIstBZgN6fxsMSiXLs9/spqWkF2pAhb2rrYCFSsr1/tT7PhcDGjZndG8SWYn0byYA==" + }, "node_modules/micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -10189,6 +10305,22 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/prisma": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.17.0.tgz", + "integrity": "sha512-m4UWkN5lBE6yevqeOxEvmepnL5cNPEjzMw2IqDB59AcEV6w7D8vGljDLd1gPFH+W6gUxw9x7/RmN5dCS/WTPxA==", + "devOptional": true, + "hasInstallScript": true, + "dependencies": { + "@prisma/engines": "5.17.0" + }, + "bin": { + "prisma": "build/index.js" + }, + "engines": { + "node": ">=16.13" + } + }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -10207,6 +10339,15 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, + "node_modules/proj4": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.11.0.tgz", + "integrity": "sha512-SasuTkAx8HnWQHfIyhkdUNJorSJqINHAN3EyMWYiQRVorftz9DHz650YraFgczwgtHOxqnfuDxSNv3C8MUnHeg==", + "dependencies": { + "mgrs": "1.0.0", + "wkt-parser": "^1.3.3" + } + }, "node_modules/prom-client": { "version": "15.1.1", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.1.tgz", @@ -12343,6 +12484,11 @@ "node": ">=8.12.0" } }, + "node_modules/wkt-parser": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.3.tgz", + "integrity": "sha512-ZnV3yH8/k58ZPACOXeiHaMuXIiaTk1t0hSUVisbO0t4RjA5wPpUytcxeyiN2h+LZRrmuHIh/1UlrR9e7DHDvTw==" + }, "node_modules/wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", @@ -14034,6 +14180,56 @@ "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", "dev": true }, + "@prisma/client": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.17.0.tgz", + "integrity": "sha512-N2tnyKayT0Zf7mHjwEyE8iG7FwTmXDHFZ1GnNhQp0pJUObsuel4ZZ1XwfuAYkq5mRIiC/Kot0kt0tGCfLJ70Jw==", + "requires": {} + }, + "@prisma/debug": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.17.0.tgz", + "integrity": "sha512-l7+AteR3P8FXiYyo496zkuoiJ5r9jLQEdUuxIxNCN1ud8rdbH3GTxm+f+dCyaSv9l9WY+29L9czaVRXz9mULfg==", + "devOptional": true + }, + "@prisma/engines": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.17.0.tgz", + "integrity": "sha512-+r+Nf+JP210Jur+/X8SIPLtz+uW9YA4QO5IXA+KcSOBe/shT47bCcRMTYCbOESw3FFYFTwe7vU6KTWHKPiwvtg==", + "devOptional": true, + "requires": { + "@prisma/debug": "5.17.0", + "@prisma/engines-version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", + "@prisma/fetch-engine": "5.17.0", + "@prisma/get-platform": "5.17.0" + } + }, + "@prisma/engines-version": { + "version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053.tgz", + "integrity": "sha512-tUuxZZysZDcrk5oaNOdrBnnkoTtmNQPkzINFDjz7eG6vcs9AVDmA/F6K5Plsb2aQc/l5M2EnFqn3htng9FA4hg==", + "devOptional": true + }, + "@prisma/fetch-engine": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.17.0.tgz", + "integrity": "sha512-ESxiOaHuC488ilLPnrv/tM2KrPhQB5TRris/IeIV4ZvUuKeaicCl4Xj/JCQeG9IlxqOgf1cCg5h5vAzlewN91Q==", + "devOptional": true, + "requires": { + "@prisma/debug": "5.17.0", + "@prisma/engines-version": "5.17.0-31.393aa359c9ad4a4bb28630fb5613f9c281cde053", + "@prisma/get-platform": "5.17.0" + } + }, + "@prisma/get-platform": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.17.0.tgz", + "integrity": "sha512-UlDgbRozCP1rfJ5Tlkf3Cnftb6srGrEQ4Nm3og+1Se2gWmCZ0hmPIi+tQikGDUVLlvOWx3Gyi9LzgRP+HTXV9w==", + "devOptional": true, + "requires": { + "@prisma/debug": "5.17.0" + } + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -17923,6 +18119,24 @@ "bser": "2.1.1" } }, + "fgdb": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fgdb/-/fgdb-1.0.0.tgz", + "integrity": "sha512-2ZaznM1bhXk9e5xBdLzAR8H9yGFB2N51QTfMsHnynyJkqqmPMXSvSH5U6qzsyt73aTLQCFeViSQz5an4PKg4XA==", + "requires": { + "jszip": "~0.2.1", + "lie": "^3.0.0", + "long": "~1.1.2", + "proj4": "^2.3.6" + }, + "dependencies": { + "long": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/long/-/long-1.1.5.tgz", + "integrity": "sha512-TU6nAF5SdasnTr28c7e74P4Crbn9o3/zwo1pM22Wvg2i2vlZ4Eelxwu4QT7j21z0sDBlJDEnEZjXTZg2J8WJrg==" + } + } + }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -18206,6 +18420,11 @@ "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", "dev": true }, + "geojson": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/geojson/-/geojson-0.5.0.tgz", + "integrity": "sha512-/Bx5lEn+qRF4TfQ5aLu6NH+UKtvIv7Lhc487y/c8BdludrCTpiWf9wyI0RTyqg49MFefIAvFDuEi5Dfd/zgNxQ==" + }, "geojson-equality": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/geojson-equality/-/geojson-equality-0.1.6.tgz", @@ -18465,6 +18684,11 @@ "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", "dev": true }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" + }, "import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -19298,6 +19522,11 @@ "verror": "1.10.0" } }, + "jszip": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-0.2.1.tgz", + "integrity": "sha512-Djh0bVj/EiqNTlwKC10xsOf+HtdD6mVq4m7DWdRoUvChB0aj2BThnGl+Kl4uDlRuxlp+EvjZ2ZOo0niTJlh+LQ==" + }, "keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -19334,6 +19563,14 @@ "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.59.tgz", "integrity": "sha512-HeTsOrDF/hWhEiKqZVwg9Cqlep5x2T+IYDENvT2VRj3iX8JQ7Y+omENv+AIn0vC8m6GYhivfCed5Cgfw27r5SA==" }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "requires": { + "immediate": "~3.0.5" + } + }, "light-my-request": { "version": "5.12.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.12.0.tgz", @@ -19493,6 +19730,11 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, + "mgrs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", + "integrity": "sha512-awNbTOqCxK1DBGjalK3xqWIstBZgN6fxsMSiXLs9/spqWkF2pAhb2rrYCFSsr1/tT7PhcDGjZndG8SWYn0byYA==" + }, "micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -20131,6 +20373,15 @@ } } }, + "prisma": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.17.0.tgz", + "integrity": "sha512-m4UWkN5lBE6yevqeOxEvmepnL5cNPEjzMw2IqDB59AcEV6w7D8vGljDLd1gPFH+W6gUxw9x7/RmN5dCS/WTPxA==", + "devOptional": true, + "requires": { + "@prisma/engines": "5.17.0" + } + }, "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -20146,6 +20397,15 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, + "proj4": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.11.0.tgz", + "integrity": "sha512-SasuTkAx8HnWQHfIyhkdUNJorSJqINHAN3EyMWYiQRVorftz9DHz650YraFgczwgtHOxqnfuDxSNv3C8MUnHeg==", + "requires": { + "mgrs": "1.0.0", + "wkt-parser": "^1.3.3" + } + }, "prom-client": { "version": "15.1.1", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.1.tgz", @@ -21695,6 +21955,11 @@ } } }, + "wkt-parser": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.3.tgz", + "integrity": "sha512-ZnV3yH8/k58ZPACOXeiHaMuXIiaTk1t0hSUVisbO0t4RjA5wPpUytcxeyiN2h+LZRrmuHIh/1UlrR9e7DHDvTw==" + }, "wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", diff --git a/package.json b/package.json index 728c3a2..886416a 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@nestjs/platform-express": "^10.0.0", "@nestjs/platform-fastify": "^10.3.0", "@nestjs/swagger": "^7.3.1", + "@prisma/client": "^5.17.0", "@samagra-x/stencil": "^0.0.6", "@turf/turf": "^6.5.0", "@types/multer": "^1.4.11", @@ -57,6 +58,7 @@ "husky": "8.0.3", "jest": "^29.5.0", "prettier": "^3.0.0", + "prisma": "^5.17.0", "source-map-support": "^0.5.21", "supertest": "^6.3.3", "ts-jest": "^29.1.0", diff --git a/prisma/docker-compose.yaml b/prisma/docker-compose.yaml new file mode 100644 index 0000000..9bcac96 --- /dev/null +++ b/prisma/docker-compose.yaml @@ -0,0 +1,12 @@ +version: "3.9" + +services: + postgis: + container_name: geopostgis + image: postgis/postgis:16-3.4-alpine + ports: + - "5432:5432" + environment: + - POSTGRES_PASSWORD=password + - POSTGRES_USER=admin + - POSTGRES_DB=gis diff --git a/prisma/migrations/20240806185521_add_geo_tables/migration.sql b/prisma/migrations/20240806185521_add_geo_tables/migration.sql new file mode 100644 index 0000000..52461f4 --- /dev/null +++ b/prisma/migrations/20240806185521_add_geo_tables/migration.sql @@ -0,0 +1,101 @@ +-- CreateExtension +CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch"; + +-- CreateExtension +CREATE EXTENSION IF NOT EXISTS "postgis"; + +-- CreateTable +CREATE TABLE "State" ( + "id" SERIAL NOT NULL, + "stcode11" INTEGER NOT NULL, + "stname" TEXT NOT NULL, + "levelLocationName" TEXT NOT NULL, + "stname_sh" TEXT NOT NULL, + "shape_length" DOUBLE PRECISION NOT NULL, + "shape_area" DOUBLE PRECISION NOT NULL, + "state_lgd" INTEGER NOT NULL, + "max_simp_tol" INTEGER NOT NULL, + "metadata" JSONB, + "min_simp_tol" INTEGER NOT NULL, + "geometry" geometry NOT NULL, + + CONSTRAINT "State_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "District" ( + "id" SERIAL NOT NULL, + "dtcode11" INTEGER NOT NULL, + "dtname" TEXT NOT NULL, + "levelLocationName" TEXT NOT NULL, + "year_stat" TEXT NOT NULL, + "shape_length" DOUBLE PRECISION NOT NULL, + "shape_area" DOUBLE PRECISION NOT NULL, + "dist_lgd" INTEGER NOT NULL, + "geometry" geometry NOT NULL, + "metadata" JSONB, + "stateId" INTEGER, + + CONSTRAINT "District_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "SubDistrict" ( + "id" SERIAL NOT NULL, + "sdtcode11" INTEGER NOT NULL, + "sdtname" TEXT NOT NULL, + "levelLocationName" TEXT NOT NULL, + "Shape_Length" DOUBLE PRECISION NOT NULL, + "Shape_Area" DOUBLE PRECISION NOT NULL, + "Subdt_LGD" INTEGER NOT NULL, + "geometry" geometry NOT NULL, + "metadata" JSONB, + "districtId" INTEGER, + "stateId" INTEGER, + + CONSTRAINT "SubDistrict_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Village" ( + "id" SERIAL NOT NULL, + "vgcode" SERIAL NOT NULL, + "geometry" geometry NOT NULL, + "vgname" TEXT NOT NULL, + "metadata" JSONB, + "subDistrictId" INTEGER, + "districtId" INTEGER, + "stateId" INTEGER, + + CONSTRAINT "Village_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "State_stcode11_key" ON "State"("stcode11"); + +-- CreateIndex +CREATE UNIQUE INDEX "State_stname_key" ON "State"("stname"); + +-- CreateIndex +CREATE UNIQUE INDEX "District_dtcode11_key" ON "District"("dtcode11"); + +-- CreateIndex +CREATE UNIQUE INDEX "SubDistrict_sdtcode11_key" ON "SubDistrict"("sdtcode11"); + +-- AddForeignKey +ALTER TABLE "District" ADD CONSTRAINT "District_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("stcode11") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("dtcode11") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("stcode11") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Village" ADD CONSTRAINT "Village_subDistrictId_fkey" FOREIGN KEY ("subDistrictId") REFERENCES "SubDistrict"("sdtcode11") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Village" ADD CONSTRAINT "Village_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("dtcode11") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Village" ADD CONSTRAINT "Village_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("stcode11") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..fbffa92 --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "postgresql" \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma new file mode 100644 index 0000000..e4e4537 --- /dev/null +++ b/prisma/schema.prisma @@ -0,0 +1,93 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions? +// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init + +// Command for migration: npx prisma migrate dev --name init + +generator client { + provider = "prisma-client-js" + previewFeatures = ["postgresqlExtensions"] +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") + + extensions = [postgis(), fuzzystrmatch()] +} + +model State { + id Int @id @default(autoincrement()) + stcode11 Int @unique + + stname String @unique + levelLocationName String + stname_sh String + shape_length Float + shape_area Float + state_lgd Int + max_simp_tol Int + metadata Json? + min_simp_tol Int + geometry Unsupported("geometry") + District District[] + SubDistrict SubDistrict[] + Village Village[] +} + +model District { + id Int @id @default(autoincrement()) + dtcode11 Int @unique + dtname String + levelLocationName String + year_stat String + shape_length Float + shape_area Float + dist_lgd Int + geometry Unsupported("geometry") + metadata Json? + + stateId Int? + state State? @relation(fields: [stateId], references: [stcode11]) + subDistricts SubDistrict[] + Village Village[] +} + +model SubDistrict { + id Int @id @default(autoincrement()) + sdtcode11 Int @unique + sdtname String + levelLocationName String + Shape_Length Float + Shape_Area Float + Subdt_LGD Int + geometry Unsupported("geometry") + metadata Json? + + districtId Int? + district District? @relation(fields: [districtId], references: [dtcode11]) + + stateId Int? + state State? @relation(fields: [stateId], references: [stcode11]) + Village Village[] +} + +model Village { + id Int @id @default(autoincrement()) + vgcode Int @default(autoincrement()) + + geometry Unsupported("geometry") + vgname String + metadata Json? + + subDistrictId Int? + subDistrict SubDistrict? @relation(fields: [subDistrictId], references: [sdtcode11]) + + districtId Int? + district District? @relation(fields: [districtId], references: [dtcode11]) + + stateId Int? + state State? @relation(fields: [stateId], references: [stcode11]) +} diff --git a/src/scripts/ingestors/district.geojson.ts b/src/scripts/ingestors/district.geojson.ts new file mode 100644 index 0000000..584c7a5 --- /dev/null +++ b/src/scripts/ingestors/district.geojson.ts @@ -0,0 +1,70 @@ +import { PrismaClient } from '@prisma/client'; +import * as fs from 'fs'; +import { executeDistrictCreateQuery, executeStateCreateQuery, findState } from './service.geojson'; + +const prisma = new PrismaClient(); +const districtGeoJSONLocation = `${__dirname}/../../geojson-data/INDIA_DISTRICT.geojson`; + +const insertDistrictData = async () => { + const rawData = fs.readFileSync(districtGeoJSONLocation); + const geojson = JSON.parse(rawData.toString()); + + for (const feature of geojson.features) { + const properties = feature.properties; + const geoJsonData = JSON.stringify(feature.geometry); + let state; + + // Find the stateId based on the fuzzy state name (stname) + state = await findState(properties.stname); + state = state[0]; + if (!state) { + console.error(`State not found for district: ${properties.dtname}`); + + const newState = { + STCODE11: properties.stcode11, + STNAME: properties.stname, + levelLocationName: properties.stname, + STNAME_SH: properties.stname, + Shape_Length: 0, + Shape_Area: 0, + State_LGD: properties.State_LGD, + MaxSimpTol: 0, + MinSimpTol: 0, + metadata: JSON.stringify({ createdBy: 'insertDistrictData script' }), + }; + + await executeStateCreateQuery(newState, `{ + "type": "GeometryCollection", + "geometries": [] + }`, + ); + state = await findState(properties.stname); + state = state[0]; + console.log(`Created new state: ${properties.stname}`); + } + + try { + console.log(`INGESTING: ${properties.dtname}`, state); + await executeDistrictCreateQuery(properties, geoJsonData, state); + console.log(`Ingested: ${properties.dtname}`); + } catch (error) { + if (error.meta && error.meta.code === '23505') { + console.log(error); + console.log(`District already exists: ${properties.dtname}. Skipping...`); + } else { + console.log(state); + console.error(`Error inserting District data for ${properties.dtname}:`, error); + } + } + } + console.log('District data ingestion completed!'); +}; + +insertDistrictData() + .then(async () => { + await prisma.$disconnect(); + }) + .catch(async (e) => { + console.error('Error during district data ingestion:', e); + await prisma.$disconnect(); + }); diff --git a/src/scripts/ingestors/service.geojson.ts b/src/scripts/ingestors/service.geojson.ts new file mode 100644 index 0000000..be3cafb --- /dev/null +++ b/src/scripts/ingestors/service.geojson.ts @@ -0,0 +1,161 @@ +import { PrismaClient } from '@prisma/client'; + +const prisma = new PrismaClient(); + +export async function executeStateCreateQuery(properties, geoJsonData) { + const query = ` + INSERT INTO "State" ("stcode11", + "stname", + "levelLocationName", + "stname_sh", + "shape_length", + "shape_area", + "state_lgd", + "max_simp_tol", + "min_simp_tol", + "geometry") + VALUES ('${properties.STCODE11}', + '${properties.STNAME}', + '${properties.levelLocationName}', + '${properties.STNAME_SH}', + ${properties.Shape_Length}, + ${properties.Shape_Area}, + ${properties.State_LGD}, + ${properties.MaxSimpTol}, + ${properties.MinSimpTol}, + ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326)); + `; + + await prisma.$executeRawUnsafe(query); +} + + +export async function findState(stname) { + return prisma.$queryRawUnsafe(` + WITH input AS (SELECT '${stname}'::VARCHAR AS input_name) + SELECT st.stname, st.stcode11, levenshtein(i.input_name, st.stname) as levenshtein + FROM "State" st, + input i + WHERE levenshtein(i.input_name, st.stname) <= 5 + ORDER BY levenshtein LIMIT 1; + `); + // return prisma.state.findUnique({ + // where: { + // // stcode11_stname: { + // // stcode11: stcode11, + // stname: stname, + // // }, + // }, + // }); +} + + +export async function executeDistrictCreateQuery(properties, geoJsonData, state) { + const query = ` + INSERT INTO "District" ("dtcode11", + "dtname", + "levelLocationName", + "year_stat", + "shape_length", + "shape_area", + "dist_lgd", + "geometry", + "stateId") + VALUES ('${properties.dtcode11}', + '${properties.dtname}', + '${properties.levelLocationName}', + '${properties.year_stat}', + ${properties.SHAPE_Length}, + ${properties.SHAPE_Area}, + ${properties.Dist_LGD}, + ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326), + ${state.stcode11}); + `; + + // console.log(query); + + // try { + await prisma.$executeRawUnsafe(query); + // console.log(`Ingested: ${properties.dtname}`); + // } catch (error) { + // console.error(`Error inserting ${properties.dtname}:`, error); + // } +} + + +export async function findDistrict(dtname) { + return prisma.$queryRawUnsafe(` + WITH input AS (SELECT '${dtname}'::VARCHAR AS input_name) + SELECT dt.dtname, dt.dtcode11, levenshtein(i.input_name, dt.dtname) as levenshtein + FROM "District" dt, + input i + WHERE levenshtein(i.input_name, dt.dtname) <= 5 + OR i.input_name ILIKE '%' || dt.dtname || '%' + OR dt.dtname ILIKE '%' || i.input_name || '%' + ORDER BY levenshtein LIMIT 1; + `); + // return prisma.district.findUnique({ + // where: { + // // stcode11_stname: { + // // stcode11: stcode11, + // dtname: dtname, + // // }, + // }, + // }); +} + + +export async function executeSubDistrictCreateQuery(properties, geoJsonData, state, district) { + const query = ` + INSERT INTO "SubDistrict" ("sdtcode11", + "sdtname", + "levelLocationName", + "Shape_Length", + "Shape_Area", + "Subdt_LGD", + "geometry", + "stateId", + "districtId") + VALUES ('${properties.sdtcode11}', + '${properties.sdtname}', + '${properties.levelLocationName}', + ${properties.Shape_Length}, + ${properties.Shape_Area}, + ${properties.Subdt_LGD}, + ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326), + ${state.stcode11}, + ${district.dtcode11}); + `; + await prisma.$executeRawUnsafe(query); +} + +export async function findSubDistrict(dtname) { + return prisma.$queryRawUnsafe(` + WITH input AS (SELECT '${dtname}'::VARCHAR AS input_name) + SELECT dt.sdtname, dt.sdtcode11, levenshtein(i.input_name, dt.sdtname) as levenshtein + FROM "SubDistrict" dt, + input i + WHERE levenshtein(i.input_name, dt.sdtname) <= 5 + ORDER BY levenshtein LIMIT 1; + `); +} + + +export async function executeVillageCreateQuery(properties, geoJsonData, state, district, subDistrict) { + const query = ` + INSERT INTO "Village" ( + "vgname", + "geometry", + "stateId", + "districtId", + "subDistrictId") + VALUES ( + '${properties.NAME}', + ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326), + ${state.stcode11}, + ${district.dtcode11}, + ${subDistrict.sdtcode11}); + `; + // console.log(query); + await prisma.$executeRawUnsafe(query); +} diff --git a/src/scripts/ingestors/state.geojson.ts b/src/scripts/ingestors/state.geojson.ts new file mode 100644 index 0000000..af9dc8c --- /dev/null +++ b/src/scripts/ingestors/state.geojson.ts @@ -0,0 +1,41 @@ +import { PrismaClient } from '@prisma/client'; +import * as fs from 'fs'; +import { executeStateCreateQuery } from './service.geojson'; + +const prisma = new PrismaClient(); +const subDistrictGeoJSONLocation = `${__dirname}/../../geojson-data/INDIA_STATE.geojson`; + + +const insertStateData = async () => { + const rawData = fs.readFileSync(subDistrictGeoJSONLocation); + const geojson = JSON.parse(rawData.toString()); + + for (const feature of geojson.features) { + const properties = feature.properties; + const geoJsonData = JSON.stringify(feature.geometry); + console.log(`Ingesting: ${properties.stname}`); + try { + await executeStateCreateQuery(properties, geoJsonData); + console.log(`Ingested: ${properties.stname} !`); + } catch (error) { + if (error.meta && error.meta.code === '23505') { + console.log(`State already exists: ${properties.stname}. Skipping...`); + } else { + console.error(`Error inserting state data for ${properties.stname}:`, error); + } + } + console.log(`Ingeted: ${properties.stname} !`); + } + + console.log('State data added successfully!'); +}; + +insertStateData() + .then(async () => { + + await prisma.$disconnect(); + }) + .catch(async (e) => { + console.error('Error inserting state data:', e); + await prisma.$disconnect(); + }); diff --git a/src/scripts/ingestors/subdistrict.geojson.ts b/src/scripts/ingestors/subdistrict.geojson.ts new file mode 100644 index 0000000..5512f4d --- /dev/null +++ b/src/scripts/ingestors/subdistrict.geojson.ts @@ -0,0 +1,118 @@ +import { PrismaClient } from '@prisma/client'; +import * as fs from 'fs'; +import { + executeDistrictCreateQuery, + executeStateCreateQuery, + findState, + findDistrict, + executeSubDistrictCreateQuery, +} from './service.geojson'; + +const prisma = new PrismaClient(); +const subDistrictGeoJSONLocation = `${__dirname}/../../geojson-data/INDIA_SUBDISTRICT.geojson`; + + + +const insertSubDistrictData = async () => { + const rawData = fs.readFileSync(subDistrictGeoJSONLocation); + const geojson = JSON.parse(rawData.toString()); + + for (const feature of geojson.features) { + const properties = feature.properties; + const geoJsonData = JSON.stringify(feature.geometry); + console.log(`Ingesting: ${properties.sdtname}`, properties.stname, properties.dtname); + + // Find or create the stateId based on the state code (stcode11) and state name (stname) + let state = await findState(properties.stname); + state = state[0]; + + if (!state) { + // Create the state if it does not exist + const newState = { + STCODE11: properties.stcode11, + STNAME: properties.stname, + levelLocationName: properties.stname, + STNAME_SH: properties.stname, + Shape_Length: 0, + Shape_Area: 0, + State_LGD: 0, + MaxSimpTol: 0, + MinSimpTol: 0, + metadata: JSON.stringify({ createdBy: 'insertSubDistrictData script' }), + }; + + try { + await executeStateCreateQuery(newState, `{ + "type": "GeometryCollection", + "geometries": [] + }`); + } catch (e) { + continue; + } + state = await findState(properties.stname); + state = state[0]; + console.log(`Created new state: ${properties.stname}`); + } + + // Find or create the districtId based on the district code (dtcode11) and district name (dtname) + let district = await findDistrict(properties.dtname); + district = district[0]; + + if (!district) { + console.log(properties.dtname); + // Create the district if it does not exist + // @ts-ignore + // @ts-ignore + const newDistrict = { + dtcode11: properties.dtcode11, + dtname: properties.dtname, + levelLocationName: properties.dtname, + SHAPE_Length: 0, + SHAPE_Area: 0, + Dist_LGD: 0, + metadata: JSON.stringify({ createdBy: 'insertSubDistrictData script' }), + + // @ts-ignore + stateId: state.stcode11, + }; + + try { + await executeDistrictCreateQuery(newDistrict, `{ + "type": "GeometryCollection", + "geometries": [] + }`, state); + } catch (e) { + continue; + } + district = await findDistrict(properties.dtname); + district = district[0]; + console.log(`Created new district: ${properties.dtname}`); + } + + + + // console.log(query); + + try { + await executeSubDistrictCreateQuery(properties, geoJsonData, state, district); + console.log(`Ingested: ${properties.sdtname}`); + } catch (error) { + if (error.meta && error.meta.code === '23505') { + console.log(`SubDistrict already exists: ${properties.dtname}. Skipping...`); + } else { + console.error(`Error inserting SubDistrict data for ${properties.dtname}:`, error); + } + } + } + + console.log('Subdistrict data ingestion completed!'); +}; + +insertSubDistrictData() + .then(async () => { + await prisma.$disconnect(); + }) + .catch(async (e) => { + console.error('Error during subdistrict data ingestion:', e); + await prisma.$disconnect(); + }); diff --git a/src/scripts/ingestors/village.geojson.ts b/src/scripts/ingestors/village.geojson.ts new file mode 100644 index 0000000..6b4b10e --- /dev/null +++ b/src/scripts/ingestors/village.geojson.ts @@ -0,0 +1,155 @@ +import { PrismaClient } from '@prisma/client'; +import * as fs from 'fs'; +import { + executeDistrictCreateQuery, + executeStateCreateQuery, + findState, + findDistrict, + executeVillageCreateQuery, + findSubDistrict, executeSubDistrictCreateQuery, +} from './service.geojson'; +import * as path from 'path'; + +const prisma = new PrismaClient(); +const villageMasterLocation = `${__dirname}/../../geojson-data/indian_village_boundaries`; + +const processGeojsonFile = async (filePath) => { + const rawData = fs.readFileSync(filePath); + const geojson = JSON.parse(rawData.toString()); + + for (const feature of geojson.features) { + const properties = feature.properties; + const geoJsonData = JSON.stringify(feature.geometry); + console.log(`Ingesting: ${properties.NAME}`, properties.STATE, properties.DISTRICT, properties.SUB_DIST); + + // Find or create the stateId based on the state code (stcode11) and state name (STATE) + let state = await findState(properties.STATE); + state = state[0]; + + if (!state) { + const newState = { + STNAME: properties.STATE, + levelLocationName: properties.STATE, + STNAME_SH: properties.STATE, + Shape_Length: 0, + Shape_Area: 0, + State_LGD: 0, + MaxSimpTol: 0, + MinSimpTol: 0, + metadata: JSON.stringify({ createdBy: 'insertVillageData script' }), + }; + + try { + await executeStateCreateQuery(newState, `{ + "type": "GeometryCollection", + "geometries": [] + }`); + } catch (e) { + continue; + } + state = await findState(properties.STATE); + state = state[0]; + console.log(`Created new state: ${properties.STATE}`); + } + + // Find or create the districtId based on the district code (dtcode11) and district name (DISTRICT) + let district = await findDistrict(properties.DISTRICT); + district = district[0]; + + if (!district) { + const newDistrict = { + DISTRICT: properties.DISTRICT, + levelLocationName: properties.DISTRICT, + SHAPE_Length: 0, + SHAPE_Area: 0, + Dist_LGD: 0, + metadata: JSON.stringify({ createdBy: 'insertVillageData script' }), + // @ts-ignore + stateId: state.stcode11, + }; + + try { + await executeDistrictCreateQuery(newDistrict, `{ + "type": "GeometryCollection", + "geometries": [] + }`, state); + } catch (e) { + continue; + } + district = await findDistrict(properties.DISTRICT); + district = district[0]; + console.log(`Created new district: ${properties.DISTRICT}`); + } + + // Find or create the subDistrictId based on the subdistrict code (sdtcode11) and subdistrict name (SUB_DIST) + let subDistrict = await findSubDistrict(properties.SUB_DIST); + subDistrict = subDistrict[0]; + + if (!subDistrict) { + const newSubDistrict = { + sdtname: properties.SUB_DIST, + levelLocationName: properties.SUB_DIST, + Shape_Length: 0, + Shape_Area: 0, + Subdt_LGD: 0, + + // @ts-ignore + stateId: state.stcode11, + // @ts-ignore + districtId: district.dtcode11, + }; + + try { + await executeSubDistrictCreateQuery(newSubDistrict, `{ + "type": "GeometryCollection", + "geometries": [] + }`, state, district); + } catch (e) { + continue; + } + subDistrict = await findSubDistrict(properties.SUB_DIST); + subDistrict = subDistrict[0]; + console.log(`Created new subdistrict: ${properties.SUB_DIST}`); + } + + try { + await executeVillageCreateQuery(properties, geoJsonData, state, district, subDistrict); + console.log(`Ingested: ${properties.NAME}`); + } catch (error) { + if (error.meta && error.meta.code === '23505') { + console.log(`Village already exists: ${properties.NAME}. Skipping...`); + } else { + console.error(`Error inserting Village data for ${properties.NAME}:`, error); + } + } + } + + console.log('Village data ingestion completed!'); +}; + +const insertVillageData = async (directoryPath) => { + const walk = async (dir) => { + const files = fs.readdirSync(dir); + + for (const file of files) { + const fullPath = path.join(dir, file); + + if (fs.statSync(fullPath).isDirectory()) { + await walk(fullPath); + } else if (path.extname(fullPath) === '.geojson') { + await processGeojsonFile(fullPath); + } + } + }; + + await walk(directoryPath); +}; + +insertVillageData(villageMasterLocation) + .then(async () => { + await prisma.$disconnect(); + }) + .catch(async (e) => { + console.error('Error during village data ingestion:', e); + await prisma.$disconnect(); + }); From fd9d15ded774a9a791a314ecc862df90d74bab5a Mon Sep 17 00:00:00 2001 From: 35C4n0r Date: Tue, 3 Sep 2024 17:01:04 +0530 Subject: [PATCH 02/11] feat: improve table schema Signed-off-by: 35C4n0r --- .../migration.sql | 101 ------------- .../migration.sql | 85 +++++++++++ prisma/schema.prisma | 75 ++++------ src/scripts/ingestors/service.geojson.ts | 139 +++++++----------- 4 files changed, 169 insertions(+), 231 deletions(-) delete mode 100644 prisma/migrations/20240806185521_add_geo_tables/migration.sql create mode 100644 prisma/migrations/20240903112126_add_geo_tabless/migration.sql diff --git a/prisma/migrations/20240806185521_add_geo_tables/migration.sql b/prisma/migrations/20240806185521_add_geo_tables/migration.sql deleted file mode 100644 index 52461f4..0000000 --- a/prisma/migrations/20240806185521_add_geo_tables/migration.sql +++ /dev/null @@ -1,101 +0,0 @@ --- CreateExtension -CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch"; - --- CreateExtension -CREATE EXTENSION IF NOT EXISTS "postgis"; - --- CreateTable -CREATE TABLE "State" ( - "id" SERIAL NOT NULL, - "stcode11" INTEGER NOT NULL, - "stname" TEXT NOT NULL, - "levelLocationName" TEXT NOT NULL, - "stname_sh" TEXT NOT NULL, - "shape_length" DOUBLE PRECISION NOT NULL, - "shape_area" DOUBLE PRECISION NOT NULL, - "state_lgd" INTEGER NOT NULL, - "max_simp_tol" INTEGER NOT NULL, - "metadata" JSONB, - "min_simp_tol" INTEGER NOT NULL, - "geometry" geometry NOT NULL, - - CONSTRAINT "State_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "District" ( - "id" SERIAL NOT NULL, - "dtcode11" INTEGER NOT NULL, - "dtname" TEXT NOT NULL, - "levelLocationName" TEXT NOT NULL, - "year_stat" TEXT NOT NULL, - "shape_length" DOUBLE PRECISION NOT NULL, - "shape_area" DOUBLE PRECISION NOT NULL, - "dist_lgd" INTEGER NOT NULL, - "geometry" geometry NOT NULL, - "metadata" JSONB, - "stateId" INTEGER, - - CONSTRAINT "District_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "SubDistrict" ( - "id" SERIAL NOT NULL, - "sdtcode11" INTEGER NOT NULL, - "sdtname" TEXT NOT NULL, - "levelLocationName" TEXT NOT NULL, - "Shape_Length" DOUBLE PRECISION NOT NULL, - "Shape_Area" DOUBLE PRECISION NOT NULL, - "Subdt_LGD" INTEGER NOT NULL, - "geometry" geometry NOT NULL, - "metadata" JSONB, - "districtId" INTEGER, - "stateId" INTEGER, - - CONSTRAINT "SubDistrict_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Village" ( - "id" SERIAL NOT NULL, - "vgcode" SERIAL NOT NULL, - "geometry" geometry NOT NULL, - "vgname" TEXT NOT NULL, - "metadata" JSONB, - "subDistrictId" INTEGER, - "districtId" INTEGER, - "stateId" INTEGER, - - CONSTRAINT "Village_pkey" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "State_stcode11_key" ON "State"("stcode11"); - --- CreateIndex -CREATE UNIQUE INDEX "State_stname_key" ON "State"("stname"); - --- CreateIndex -CREATE UNIQUE INDEX "District_dtcode11_key" ON "District"("dtcode11"); - --- CreateIndex -CREATE UNIQUE INDEX "SubDistrict_sdtcode11_key" ON "SubDistrict"("sdtcode11"); - --- AddForeignKey -ALTER TABLE "District" ADD CONSTRAINT "District_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("stcode11") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("dtcode11") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("stcode11") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Village" ADD CONSTRAINT "Village_subDistrictId_fkey" FOREIGN KEY ("subDistrictId") REFERENCES "SubDistrict"("sdtcode11") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Village" ADD CONSTRAINT "Village_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("dtcode11") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Village" ADD CONSTRAINT "Village_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("stcode11") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/migrations/20240903112126_add_geo_tabless/migration.sql b/prisma/migrations/20240903112126_add_geo_tabless/migration.sql new file mode 100644 index 0000000..5ee4191 --- /dev/null +++ b/prisma/migrations/20240903112126_add_geo_tabless/migration.sql @@ -0,0 +1,85 @@ +-- CreateExtension +CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch"; + +-- CreateExtension +CREATE EXTENSION IF NOT EXISTS "postgis"; + +-- CreateTable +CREATE TABLE "State" ( + "id" SERIAL NOT NULL, + "state_code" INTEGER NOT NULL, + "state_name" TEXT NOT NULL, + "metadata" JSONB, + "geometry" geometry NOT NULL, + + CONSTRAINT "State_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "District" ( + "id" SERIAL NOT NULL, + "district_code" INTEGER NOT NULL, + "district_name" TEXT NOT NULL, + "geometry" geometry NOT NULL, + "metadata" JSONB, + "stateId" INTEGER, + + CONSTRAINT "District_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "SubDistrict" ( + "id" SERIAL NOT NULL, + "subdistrict_code" INTEGER NOT NULL, + "subdistrict_name" TEXT NOT NULL, + "geometry" geometry NOT NULL, + "metadata" JSONB, + "districtId" INTEGER, + "stateId" INTEGER, + + CONSTRAINT "SubDistrict_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Village" ( + "id" SERIAL NOT NULL, + "village_code" SERIAL NOT NULL, + "geometry" geometry NOT NULL, + "village_name" TEXT NOT NULL, + "metadata" JSONB, + "subDistrictId" INTEGER, + "districtId" INTEGER, + "stateId" INTEGER, + + CONSTRAINT "Village_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "State_state_code_key" ON "State"("state_code"); + +-- CreateIndex +CREATE UNIQUE INDEX "State_state_name_key" ON "State"("state_name"); + +-- CreateIndex +CREATE UNIQUE INDEX "District_district_code_key" ON "District"("district_code"); + +-- CreateIndex +CREATE UNIQUE INDEX "SubDistrict_subdistrict_code_key" ON "SubDistrict"("subdistrict_code"); + +-- AddForeignKey +ALTER TABLE "District" ADD CONSTRAINT "District_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("district_code") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Village" ADD CONSTRAINT "Village_subDistrictId_fkey" FOREIGN KEY ("subDistrictId") REFERENCES "SubDistrict"("subdistrict_code") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Village" ADD CONSTRAINT "Village_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("district_code") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Village" ADD CONSTRAINT "Village_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e4e4537..1cd3206 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -19,75 +19,58 @@ datasource db { } model State { - id Int @id @default(autoincrement()) - stcode11 Int @unique - - stname String @unique - levelLocationName String - stname_sh String - shape_length Float - shape_area Float - state_lgd Int - max_simp_tol Int - metadata Json? - min_simp_tol Int - geometry Unsupported("geometry") - District District[] - SubDistrict SubDistrict[] - Village Village[] + id Int @id @default(autoincrement()) + state_code Int @unique + state_name String @unique + metadata Json? + geometry Unsupported("geometry") + District District[] + SubDistrict SubDistrict[] + Village Village[] } model District { - id Int @id @default(autoincrement()) - dtcode11 Int @unique - dtname String - levelLocationName String - year_stat String - shape_length Float - shape_area Float - dist_lgd Int - geometry Unsupported("geometry") - metadata Json? + id Int @id @default(autoincrement()) + district_code Int @unique + district_name String + geometry Unsupported("geometry") + metadata Json? stateId Int? - state State? @relation(fields: [stateId], references: [stcode11]) + state State? @relation(fields: [stateId], references: [state_code]) subDistricts SubDistrict[] Village Village[] } model SubDistrict { - id Int @id @default(autoincrement()) - sdtcode11 Int @unique - sdtname String - levelLocationName String - Shape_Length Float - Shape_Area Float - Subdt_LGD Int - geometry Unsupported("geometry") - metadata Json? + id Int @id @default(autoincrement()) + subdistrict_code Int @unique + subdistrict_name String + geometry Unsupported("geometry") + metadata Json? districtId Int? - district District? @relation(fields: [districtId], references: [dtcode11]) + district District? @relation(fields: [districtId], references: [district_code]) stateId Int? - state State? @relation(fields: [stateId], references: [stcode11]) + state State? @relation(fields: [stateId], references: [state_code]) Village Village[] } model Village { - id Int @id @default(autoincrement()) - vgcode Int @default(autoincrement()) + id Int @id @default(autoincrement()) + village_code Int @default(autoincrement()) - geometry Unsupported("geometry") - vgname String - metadata Json? + geometry Unsupported("geometry") + village_name String + metadata Json? subDistrictId Int? - subDistrict SubDistrict? @relation(fields: [subDistrictId], references: [sdtcode11]) + subDistrict SubDistrict? @relation(fields: [subDistrictId], references: [subdistrict_code]) districtId Int? - district District? @relation(fields: [districtId], references: [dtcode11]) + district District? @relation(fields: [districtId], references: [district_code]) stateId Int? - state State? @relation(fields: [stateId], references: [stcode11]) + state State? @relation(fields: [stateId], references: [state_code]) } diff --git a/src/scripts/ingestors/service.geojson.ts b/src/scripts/ingestors/service.geojson.ts index be3cafb..0c61636 100644 --- a/src/scripts/ingestors/service.geojson.ts +++ b/src/scripts/ingestors/service.geojson.ts @@ -4,25 +4,21 @@ const prisma = new PrismaClient(); export async function executeStateCreateQuery(properties, geoJsonData) { const query = ` - INSERT INTO "State" ("stcode11", - "stname", - "levelLocationName", - "stname_sh", - "shape_length", - "shape_area", - "state_lgd", - "max_simp_tol", - "min_simp_tol", + INSERT INTO "State" ("state_code", + "state_name", + "metadata", "geometry") VALUES ('${properties.STCODE11}', '${properties.STNAME}', - '${properties.levelLocationName}', - '${properties.STNAME_SH}', - ${properties.Shape_Length}, - ${properties.Shape_Area}, - ${properties.State_LGD}, - ${properties.MaxSimpTol}, - ${properties.MinSimpTol}, + jsonb_build_object( + 'levelLocationName', '${properties.levelLocationName}', + 'stname_sh', '${properties.STNAME_SH}', + 'shape_length', ${properties.Shape_Length}, + 'shape_area', ${properties.Shape_Area}, + 'state_lgd', ${properties.State_LGD}, + 'max_simp_tol', ${properties.MaxSimpTol}, + 'min_simp_tol', ${properties.MinSimpTol} + ), ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326)); `; @@ -30,112 +26,86 @@ export async function executeStateCreateQuery(properties, geoJsonData) { } -export async function findState(stname) { +export async function findState(state_name) { return prisma.$queryRawUnsafe(` - WITH input AS (SELECT '${stname}'::VARCHAR AS input_name) - SELECT st.stname, st.stcode11, levenshtein(i.input_name, st.stname) as levenshtein + WITH input AS (SELECT '${state_name}'::VARCHAR AS input_name) + SELECT st.state_name, st.state_code, levenshtein(i.input_name, st.state_name) as levenshtein FROM "State" st, input i - WHERE levenshtein(i.input_name, st.stname) <= 5 + WHERE levenshtein(i.input_name, st.state_name) <= 5 ORDER BY levenshtein LIMIT 1; `); - // return prisma.state.findUnique({ - // where: { - // // stcode11_stname: { - // // stcode11: stcode11, - // stname: stname, - // // }, - // }, - // }); } export async function executeDistrictCreateQuery(properties, geoJsonData, state) { const query = ` - INSERT INTO "District" ("dtcode11", - "dtname", - "levelLocationName", - "year_stat", - "shape_length", - "shape_area", - "dist_lgd", + INSERT INTO "District" ("district_code", + "district_name", + "metadata", "geometry", "stateId") VALUES ('${properties.dtcode11}', '${properties.dtname}', - '${properties.levelLocationName}', - '${properties.year_stat}', - ${properties.SHAPE_Length}, - ${properties.SHAPE_Area}, - ${properties.Dist_LGD}, + jsonb_build_object( + 'levelLocationName', '${properties.levelLocationName}', + 'year_stat', '${properties.year_stat}', + 'shape_length', ${properties.SHAPE_Length}, + 'shape_area', ${properties.SHAPE_Area}, + 'dist_lgd', ${properties.Dist_LGD} + ), ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326), - ${state.stcode11}); + ${state.state_code}); `; - - // console.log(query); - - // try { await prisma.$executeRawUnsafe(query); - // console.log(`Ingested: ${properties.dtname}`); - // } catch (error) { - // console.error(`Error inserting ${properties.dtname}:`, error); - // } } -export async function findDistrict(dtname) { +export async function findDistrict(district_name) { return prisma.$queryRawUnsafe(` - WITH input AS (SELECT '${dtname}'::VARCHAR AS input_name) - SELECT dt.dtname, dt.dtcode11, levenshtein(i.input_name, dt.dtname) as levenshtein + WITH input AS (SELECT '${district_name}'::VARCHAR AS input_name) + SELECT dt.district_name, dt.district_code, levenshtein(i.input_name, dt.district_name) as levenshtein FROM "District" dt, input i - WHERE levenshtein(i.input_name, dt.dtname) <= 5 - OR i.input_name ILIKE '%' || dt.dtname || '%' - OR dt.dtname ILIKE '%' || i.input_name || '%' + WHERE levenshtein(i.input_name, dt.district_name) <= 5 + OR i.input_name ILIKE '%' || dt.district_name || '%' + OR dt.district_name ILIKE '%' || i.input_name || '%' ORDER BY levenshtein LIMIT 1; `); - // return prisma.district.findUnique({ - // where: { - // // stcode11_stname: { - // // stcode11: stcode11, - // dtname: dtname, - // // }, - // }, - // }); } export async function executeSubDistrictCreateQuery(properties, geoJsonData, state, district) { const query = ` - INSERT INTO "SubDistrict" ("sdtcode11", - "sdtname", - "levelLocationName", - "Shape_Length", - "Shape_Area", - "Subdt_LGD", + INSERT INTO "SubDistrict" ("subdistrict_code", + "subdistrict_name", + "metadata", "geometry", "stateId", "districtId") VALUES ('${properties.sdtcode11}', '${properties.sdtname}', - '${properties.levelLocationName}', - ${properties.Shape_Length}, - ${properties.Shape_Area}, - ${properties.Subdt_LGD}, + jsonb_build_object( + 'levelLocationName', '${properties.levelLocationName}', + 'Shape_Length', ${properties.Shape_Length}, + 'Shape_Area', ${properties.Shape_Area}, + 'Subdt_LGD', ${properties.Subdt_LGD} + ), ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326), - ${state.stcode11}, - ${district.dtcode11}); + ${state.state_code}, + ${district.district_code}); `; await prisma.$executeRawUnsafe(query); } -export async function findSubDistrict(dtname) { + +export async function findSubDistrict(subdistrict_name) { return prisma.$queryRawUnsafe(` - WITH input AS (SELECT '${dtname}'::VARCHAR AS input_name) - SELECT dt.sdtname, dt.sdtcode11, levenshtein(i.input_name, dt.sdtname) as levenshtein + WITH input AS (SELECT '${subdistrict_name}'::VARCHAR AS input_name) + SELECT dt.subdistrict_name, dt.subdistrict_code, levenshtein(i.input_name, dt.subdistrict_name) as levenshtein FROM "SubDistrict" dt, input i - WHERE levenshtein(i.input_name, dt.sdtname) <= 5 + WHERE levenshtein(i.input_name, dt.subdistrict_name) <= 5 ORDER BY levenshtein LIMIT 1; `); } @@ -144,18 +114,19 @@ export async function findSubDistrict(dtname) { export async function executeVillageCreateQuery(properties, geoJsonData, state, district, subDistrict) { const query = ` INSERT INTO "Village" ( - "vgname", + "village_name", + "metadata", "geometry", "stateId", "districtId", "subDistrictId") VALUES ( '${properties.NAME}', + jsonb_build_object(), ST_SetSRID(ST_GeomFromGeoJSON('${geoJsonData}'), 4326), - ${state.stcode11}, - ${district.dtcode11}, - ${subDistrict.sdtcode11}); + ${state.state_code}, + ${district.district_code}, + ${subDistrict.subdistrict_code}); `; - // console.log(query); await prisma.$executeRawUnsafe(query); } From b3b6dc0d6d3d830a8baa1c2dc014a9b97183933d Mon Sep 17 00:00:00 2001 From: 35C4n0r Date: Tue, 3 Sep 2024 17:02:23 +0530 Subject: [PATCH 03/11] fix: fix migration typo Signed-off-by: 35C4n0r --- .../migration.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename prisma/migrations/{20240903112126_add_geo_tabless => 20240903112126_add_geo_tables}/migration.sql (100%) diff --git a/prisma/migrations/20240903112126_add_geo_tabless/migration.sql b/prisma/migrations/20240903112126_add_geo_tables/migration.sql similarity index 100% rename from prisma/migrations/20240903112126_add_geo_tabless/migration.sql rename to prisma/migrations/20240903112126_add_geo_tables/migration.sql From 06b6bb0f4f79e33a679e1a047d1afd0a117974a1 Mon Sep 17 00:00:00 2001 From: 35C4n0r Date: Thu, 5 Sep 2024 22:09:51 +0530 Subject: [PATCH 04/11] ref: refactor db schema Signed-off-by: 35C4n0r --- .../migration.sql | 24 +++++----- prisma/schema.prisma | 46 +++++++++---------- src/scripts/ingestors/service.geojson.ts | 12 ++--- 3 files changed, 41 insertions(+), 41 deletions(-) rename prisma/migrations/{20240903112126_add_geo_tables => 20240905162901_add_geo_tables}/migration.sql (59%) diff --git a/prisma/migrations/20240903112126_add_geo_tables/migration.sql b/prisma/migrations/20240905162901_add_geo_tables/migration.sql similarity index 59% rename from prisma/migrations/20240903112126_add_geo_tables/migration.sql rename to prisma/migrations/20240905162901_add_geo_tables/migration.sql index 5ee4191..afd9cd0 100644 --- a/prisma/migrations/20240903112126_add_geo_tables/migration.sql +++ b/prisma/migrations/20240905162901_add_geo_tables/migration.sql @@ -22,7 +22,7 @@ CREATE TABLE "District" ( "district_name" TEXT NOT NULL, "geometry" geometry NOT NULL, "metadata" JSONB, - "stateId" INTEGER, + "state_id" INTEGER, CONSTRAINT "District_pkey" PRIMARY KEY ("id") ); @@ -34,8 +34,8 @@ CREATE TABLE "SubDistrict" ( "subdistrict_name" TEXT NOT NULL, "geometry" geometry NOT NULL, "metadata" JSONB, - "districtId" INTEGER, - "stateId" INTEGER, + "district_id" INTEGER, + "state_id" INTEGER, CONSTRAINT "SubDistrict_pkey" PRIMARY KEY ("id") ); @@ -47,9 +47,9 @@ CREATE TABLE "Village" ( "geometry" geometry NOT NULL, "village_name" TEXT NOT NULL, "metadata" JSONB, - "subDistrictId" INTEGER, - "districtId" INTEGER, - "stateId" INTEGER, + "subdistrict_id" INTEGER, + "district_id" INTEGER, + "state_id" INTEGER, CONSTRAINT "Village_pkey" PRIMARY KEY ("id") ); @@ -67,19 +67,19 @@ CREATE UNIQUE INDEX "District_district_code_key" ON "District"("district_code"); CREATE UNIQUE INDEX "SubDistrict_subdistrict_code_key" ON "SubDistrict"("subdistrict_code"); -- AddForeignKey -ALTER TABLE "District" ADD CONSTRAINT "District_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "District" ADD CONSTRAINT "District_state_id_fkey" FOREIGN KEY ("state_id") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; -- AddForeignKey -ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("district_code") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_district_id_fkey" FOREIGN KEY ("district_id") REFERENCES "District"("district_code") ON DELETE SET NULL ON UPDATE CASCADE; -- AddForeignKey -ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "SubDistrict" ADD CONSTRAINT "SubDistrict_state_id_fkey" FOREIGN KEY ("state_id") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; -- AddForeignKey -ALTER TABLE "Village" ADD CONSTRAINT "Village_subDistrictId_fkey" FOREIGN KEY ("subDistrictId") REFERENCES "SubDistrict"("subdistrict_code") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "Village" ADD CONSTRAINT "Village_subdistrict_id_fkey" FOREIGN KEY ("subdistrict_id") REFERENCES "SubDistrict"("subdistrict_code") ON DELETE SET NULL ON UPDATE CASCADE; -- AddForeignKey -ALTER TABLE "Village" ADD CONSTRAINT "Village_districtId_fkey" FOREIGN KEY ("districtId") REFERENCES "District"("district_code") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "Village" ADD CONSTRAINT "Village_district_id_fkey" FOREIGN KEY ("district_id") REFERENCES "District"("district_code") ON DELETE SET NULL ON UPDATE CASCADE; -- AddForeignKey -ALTER TABLE "Village" ADD CONSTRAINT "Village_stateId_fkey" FOREIGN KEY ("stateId") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "Village" ADD CONSTRAINT "Village_state_id_fkey" FOREIGN KEY ("state_id") REFERENCES "State"("state_code") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 1cd3206..0861029 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -19,14 +19,14 @@ datasource db { } model State { - id Int @id @default(autoincrement()) - state_code Int @unique - state_name String @unique - metadata Json? - geometry Unsupported("geometry") - District District[] - SubDistrict SubDistrict[] - Village Village[] + id Int @id @default(autoincrement()) + state_code Int @unique + state_name String @unique + metadata Json? + geometry Unsupported("geometry") + district District[] + Sub_district SubDistrict[] + village Village[] } model District { @@ -36,10 +36,10 @@ model District { geometry Unsupported("geometry") metadata Json? - stateId Int? - state State? @relation(fields: [stateId], references: [state_code]) - subDistricts SubDistrict[] - Village Village[] + state_id Int? + state State? @relation(fields: [state_id], references: [state_code]) + sub_districts SubDistrict[] + village Village[] } model SubDistrict { @@ -49,12 +49,12 @@ model SubDistrict { geometry Unsupported("geometry") metadata Json? - districtId Int? - district District? @relation(fields: [districtId], references: [district_code]) + district_id Int? + district District? @relation(fields: [district_id], references: [district_code]) - stateId Int? - state State? @relation(fields: [stateId], references: [state_code]) - Village Village[] + state_id Int? + state State? @relation(fields: [state_id], references: [state_code]) + village Village[] } model Village { @@ -65,12 +65,12 @@ model Village { village_name String metadata Json? - subDistrictId Int? - subDistrict SubDistrict? @relation(fields: [subDistrictId], references: [subdistrict_code]) + subdistrict_id Int? + subdistrict SubDistrict? @relation(fields: [subdistrict_id], references: [subdistrict_code]) - districtId Int? - district District? @relation(fields: [districtId], references: [district_code]) + district_id Int? + district District? @relation(fields: [district_id], references: [district_code]) - stateId Int? - state State? @relation(fields: [stateId], references: [state_code]) + state_id Int? + state State? @relation(fields: [state_id], references: [state_code]) } diff --git a/src/scripts/ingestors/service.geojson.ts b/src/scripts/ingestors/service.geojson.ts index 0c61636..4092f47 100644 --- a/src/scripts/ingestors/service.geojson.ts +++ b/src/scripts/ingestors/service.geojson.ts @@ -44,7 +44,7 @@ export async function executeDistrictCreateQuery(properties, geoJsonData, state) "district_name", "metadata", "geometry", - "stateId") + "state_id") VALUES ('${properties.dtcode11}', '${properties.dtname}', jsonb_build_object( @@ -81,8 +81,8 @@ export async function executeSubDistrictCreateQuery(properties, geoJsonData, sta "subdistrict_name", "metadata", "geometry", - "stateId", - "districtId") + "state_id", + "district_id") VALUES ('${properties.sdtcode11}', '${properties.sdtname}', jsonb_build_object( @@ -117,9 +117,9 @@ export async function executeVillageCreateQuery(properties, geoJsonData, state, "village_name", "metadata", "geometry", - "stateId", - "districtId", - "subDistrictId") + "state_id", + "district_id", + "subdistrict_id") VALUES ( '${properties.NAME}', jsonb_build_object(), From 11c91ee9f65f6ba25decada5de2333c84dba379d Mon Sep 17 00:00:00 2001 From: 35C4n0r Date: Thu, 5 Sep 2024 22:26:32 +0530 Subject: [PATCH 05/11] chore: replace schema Signed-off-by: 35C4n0r --- .../migration.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename prisma/migrations/{20240905162901_add_geo_tables => 20240905164841_add_geo_tables}/migration.sql (100%) diff --git a/prisma/migrations/20240905162901_add_geo_tables/migration.sql b/prisma/migrations/20240905164841_add_geo_tables/migration.sql similarity index 100% rename from prisma/migrations/20240905162901_add_geo_tables/migration.sql rename to prisma/migrations/20240905164841_add_geo_tables/migration.sql From 6d6054be590069d46b2ad79a924cbcfb9d6a7d20 Mon Sep 17 00:00:00 2001 From: 35C4n0r Date: Thu, 5 Sep 2024 22:34:44 +0530 Subject: [PATCH 06/11] chore: replace schema Signed-off-by: 35C4n0r --- prisma/schema.prisma | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 0861029..7e2e3ed 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -19,14 +19,14 @@ datasource db { } model State { - id Int @id @default(autoincrement()) - state_code Int @unique - state_name String @unique - metadata Json? - geometry Unsupported("geometry") - district District[] - Sub_district SubDistrict[] - village Village[] + id Int @id @default(autoincrement()) + state_code Int @unique + state_name String @unique + metadata Json? + geometry Unsupported("geometry") + district District[] + subdistrict SubDistrict[] + village Village[] } model District { @@ -36,10 +36,10 @@ model District { geometry Unsupported("geometry") metadata Json? - state_id Int? - state State? @relation(fields: [state_id], references: [state_code]) - sub_districts SubDistrict[] - village Village[] + state_id Int? + state State? @relation(fields: [state_id], references: [state_code]) + subdistrict SubDistrict[] + village Village[] } model SubDistrict { From deb13bcfb8f41ec33c45323226a71b54b06a599e Mon Sep 17 00:00:00 2001 From: 35C4n0r Date: Sat, 7 Sep 2024 20:54:59 +0530 Subject: [PATCH 07/11] feat: migrate codebase Signed-off-by: 35C4n0r --- package-lock.json | 126 ----------- .../migration.sql | 0 src/app.module.ts | 2 + src/config/config.ts | 43 ++-- src/modules/georev/georev.controller.spec.ts | 44 +++- src/modules/georev/georev.controller.ts | 22 +- src/modules/georev/georev.module.ts | 3 +- src/modules/georev/georev.service.spec.ts | 52 ++--- src/modules/georev/georev.service.ts | 25 +-- .../location/location.controller.spec.ts | 162 +++++++------- src/modules/location/location.controller.ts | 73 ++----- src/modules/location/location.module.ts | 15 +- .../location/location.search-service.spec.ts | 124 ----------- .../location/location.search-service.ts | 154 +++---------- src/modules/location/location.service.spec.ts | 115 +++++----- src/modules/location/location.service.ts | 46 +--- src/modules/prisma/prisma.module.ts | 9 + src/modules/prisma/prisma.service.ts | 9 + src/services/geojson/geojson.service.spec.ts | 36 ---- src/services/geojson/geojson.service.ts | 52 ----- .../geoquery/geoquery.service.spec.ts | 18 -- src/services/geoquery/geoquery.service.ts | 202 +++++++++++++++--- src/utils/serializer/success.ts | 14 +- test/app.e2e-spec.ts | 10 +- test/jest-e2e.json | 3 +- 25 files changed, 497 insertions(+), 862 deletions(-) rename prisma/migrations/{20240905164841_add_geo_tables => 20240906152553_init}/migration.sql (100%) delete mode 100644 src/modules/location/location.search-service.spec.ts create mode 100644 src/modules/prisma/prisma.module.ts create mode 100644 src/modules/prisma/prisma.service.ts delete mode 100644 src/services/geojson/geojson.service.spec.ts delete mode 100644 src/services/geojson/geojson.service.ts delete mode 100644 src/services/geoquery/geoquery.service.spec.ts diff --git a/package-lock.json b/package-lock.json index 5b67aee..384ac7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,9 +25,7 @@ "fastify": "^4.25.2", "fastify-multer": "^2.0.3", "fastify-multipart": "^5.4.0", - "fgdb": "^1.0.0", "fuse.js": "^7.0.0", - "geojson": "^0.5.0", "multer": "^1.4.5-lts.1", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1" @@ -7255,25 +7253,6 @@ "bser": "2.1.1" } }, - "node_modules/fgdb": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fgdb/-/fgdb-1.0.0.tgz", - "integrity": "sha512-2ZaznM1bhXk9e5xBdLzAR8H9yGFB2N51QTfMsHnynyJkqqmPMXSvSH5U6qzsyt73aTLQCFeViSQz5an4PKg4XA==", - "dependencies": { - "jszip": "~0.2.1", - "lie": "^3.0.0", - "long": "~1.1.2", - "proj4": "^2.3.6" - } - }, - "node_modules/fgdb/node_modules/long": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/long/-/long-1.1.5.tgz", - "integrity": "sha512-TU6nAF5SdasnTr28c7e74P4Crbn9o3/zwo1pM22Wvg2i2vlZ4Eelxwu4QT7j21z0sDBlJDEnEZjXTZg2J8WJrg==", - "engines": { - "node": ">=0.6" - } - }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -7649,14 +7628,6 @@ "node": ">=6.9.0" } }, - "node_modules/geojson": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/geojson/-/geojson-0.5.0.tgz", - "integrity": "sha512-/Bx5lEn+qRF4TfQ5aLu6NH+UKtvIv7Lhc487y/c8BdludrCTpiWf9wyI0RTyqg49MFefIAvFDuEi5Dfd/zgNxQ==", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/geojson-equality": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/geojson-equality/-/geojson-equality-0.1.6.tgz", @@ -8028,11 +7999,6 @@ "node": ">= 4" } }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" - }, "node_modules/import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -9148,11 +9114,6 @@ "verror": "1.10.0" } }, - "node_modules/jszip": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-0.2.1.tgz", - "integrity": "sha512-Djh0bVj/EiqNTlwKC10xsOf+HtdD6mVq4m7DWdRoUvChB0aj2BThnGl+Kl4uDlRuxlp+EvjZ2ZOo0niTJlh+LQ==" - }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -9198,14 +9159,6 @@ "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.59.tgz", "integrity": "sha512-HeTsOrDF/hWhEiKqZVwg9Cqlep5x2T+IYDENvT2VRj3iX8JQ7Y+omENv+AIn0vC8m6GYhivfCed5Cgfw27r5SA==" }, - "node_modules/lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "dependencies": { - "immediate": "~3.0.5" - } - }, "node_modules/light-my-request": { "version": "5.12.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.12.0.tgz", @@ -9418,11 +9371,6 @@ "node": ">= 0.6" } }, - "node_modules/mgrs": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", - "integrity": "sha512-awNbTOqCxK1DBGjalK3xqWIstBZgN6fxsMSiXLs9/spqWkF2pAhb2rrYCFSsr1/tT7PhcDGjZndG8SWYn0byYA==" - }, "node_modules/micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -10339,15 +10287,6 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, - "node_modules/proj4": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.11.0.tgz", - "integrity": "sha512-SasuTkAx8HnWQHfIyhkdUNJorSJqINHAN3EyMWYiQRVorftz9DHz650YraFgczwgtHOxqnfuDxSNv3C8MUnHeg==", - "dependencies": { - "mgrs": "1.0.0", - "wkt-parser": "^1.3.3" - } - }, "node_modules/prom-client": { "version": "15.1.1", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.1.tgz", @@ -12484,11 +12423,6 @@ "node": ">=8.12.0" } }, - "node_modules/wkt-parser": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.3.tgz", - "integrity": "sha512-ZnV3yH8/k58ZPACOXeiHaMuXIiaTk1t0hSUVisbO0t4RjA5wPpUytcxeyiN2h+LZRrmuHIh/1UlrR9e7DHDvTw==" - }, "node_modules/wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", @@ -18119,24 +18053,6 @@ "bser": "2.1.1" } }, - "fgdb": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fgdb/-/fgdb-1.0.0.tgz", - "integrity": "sha512-2ZaznM1bhXk9e5xBdLzAR8H9yGFB2N51QTfMsHnynyJkqqmPMXSvSH5U6qzsyt73aTLQCFeViSQz5an4PKg4XA==", - "requires": { - "jszip": "~0.2.1", - "lie": "^3.0.0", - "long": "~1.1.2", - "proj4": "^2.3.6" - }, - "dependencies": { - "long": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/long/-/long-1.1.5.tgz", - "integrity": "sha512-TU6nAF5SdasnTr28c7e74P4Crbn9o3/zwo1pM22Wvg2i2vlZ4Eelxwu4QT7j21z0sDBlJDEnEZjXTZg2J8WJrg==" - } - } - }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -18420,11 +18336,6 @@ "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", "dev": true }, - "geojson": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/geojson/-/geojson-0.5.0.tgz", - "integrity": "sha512-/Bx5lEn+qRF4TfQ5aLu6NH+UKtvIv7Lhc487y/c8BdludrCTpiWf9wyI0RTyqg49MFefIAvFDuEi5Dfd/zgNxQ==" - }, "geojson-equality": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/geojson-equality/-/geojson-equality-0.1.6.tgz", @@ -18684,11 +18595,6 @@ "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", "dev": true }, - "immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" - }, "import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -19522,11 +19428,6 @@ "verror": "1.10.0" } }, - "jszip": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-0.2.1.tgz", - "integrity": "sha512-Djh0bVj/EiqNTlwKC10xsOf+HtdD6mVq4m7DWdRoUvChB0aj2BThnGl+Kl4uDlRuxlp+EvjZ2ZOo0niTJlh+LQ==" - }, "keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -19563,14 +19464,6 @@ "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.59.tgz", "integrity": "sha512-HeTsOrDF/hWhEiKqZVwg9Cqlep5x2T+IYDENvT2VRj3iX8JQ7Y+omENv+AIn0vC8m6GYhivfCed5Cgfw27r5SA==" }, - "lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "requires": { - "immediate": "~3.0.5" - } - }, "light-my-request": { "version": "5.12.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.12.0.tgz", @@ -19730,11 +19623,6 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, - "mgrs": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz", - "integrity": "sha512-awNbTOqCxK1DBGjalK3xqWIstBZgN6fxsMSiXLs9/spqWkF2pAhb2rrYCFSsr1/tT7PhcDGjZndG8SWYn0byYA==" - }, "micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -20397,15 +20285,6 @@ "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, - "proj4": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/proj4/-/proj4-2.11.0.tgz", - "integrity": "sha512-SasuTkAx8HnWQHfIyhkdUNJorSJqINHAN3EyMWYiQRVorftz9DHz650YraFgczwgtHOxqnfuDxSNv3C8MUnHeg==", - "requires": { - "mgrs": "1.0.0", - "wkt-parser": "^1.3.3" - } - }, "prom-client": { "version": "15.1.1", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.1.tgz", @@ -21955,11 +21834,6 @@ } } }, - "wkt-parser": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.3.3.tgz", - "integrity": "sha512-ZnV3yH8/k58ZPACOXeiHaMuXIiaTk1t0hSUVisbO0t4RjA5wPpUytcxeyiN2h+LZRrmuHIh/1UlrR9e7DHDvTw==" - }, "wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", diff --git a/prisma/migrations/20240905164841_add_geo_tables/migration.sql b/prisma/migrations/20240906152553_init/migration.sql similarity index 100% rename from prisma/migrations/20240905164841_add_geo_tables/migration.sql rename to prisma/migrations/20240906152553_init/migration.sql diff --git a/src/app.module.ts b/src/app.module.ts index 026884d..af663eb 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { CityModule } from './modules/city/city.module'; import { GeorevModule } from './modules/georev/georev.module'; import { LocationModule } from './modules/location/location.module'; +import { PrismaModule } from './modules/prisma/prisma.module'; import { ConfigModule } from '@nestjs/config'; import { config } from './config/config'; import { AppController } from './app.controller'; @@ -12,6 +13,7 @@ import { AppService } from './app.service'; CityModule, GeorevModule, LocationModule, + PrismaModule, ConfigModule.forRoot({ envFilePath: `${process.cwd()}/config/env/${process.env.NODE_ENV || 'default'}.env`, load: [config], diff --git a/src/config/config.ts b/src/config/config.ts index 17d1088..469ff8e 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -1,34 +1,23 @@ export const config = () => ({ NODE_ENV: process.env.NODE_ENV || 'default', port: parseInt(process.env.PORT) || 3000, - requiredGeoLocationLevels: ['SUBDISTRICT', 'DISTRICT', 'STATE'], - geoLocationLevels: { - VILLAGE: 'VILLAGE', - SUBDISTRICT: 'SUBDISTRICT', - DISTRICT: 'DISTRICT', - STATE: 'STATE', - }, - levelsMapping: { - STATE: { - name: 'state', - path: 'state', - depth: 0, + tableLevels: [], + tableMeta: { + "STATE": { + tname: "State", + fname: "state_name" }, - DISTRICT: { - name: 'district', - path: 'state->district', - depth: 1, + "DISTRICT": { + tname: "District", + fname: "district_name", }, - SUBDISTRICT: { - name: 'subDistrict', - path: 'state->district->subDistrict', - depth: 2, + "SUBDISTRICT": { + tname: "SubDistrict", + fname: "subdistrict_name", }, - VILLAGE: { - name: 'village', - path: 'state->district->subDistrict->village', - depth: 3, - }, - }, - country: 'INDIA', + "VILLAGE": { + tname: "Village", + fname: "village_name", + } + } }); diff --git a/src/modules/georev/georev.controller.spec.ts b/src/modules/georev/georev.controller.spec.ts index ef32cf0..ef8bf2e 100644 --- a/src/modules/georev/georev.controller.spec.ts +++ b/src/modules/georev/georev.controller.spec.ts @@ -3,8 +3,9 @@ import { ConfigService } from '@nestjs/config'; import { GeorevController } from './georev.controller'; import { GeorevService } from './georev.service'; import { GeoqueryService } from '../../services/geoquery/geoquery.service'; -import { GeojsonService } from '../../services/geojson/geojson.service'; import { HttpException, HttpStatus } from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +// import { PrismaService } from '../../services/prisma/prisma.service'; // Add PrismaService import describe('GeorevController', () => { let controller: GeorevController; @@ -16,7 +17,19 @@ describe('GeorevController', () => { providers: [ GeorevService, GeoqueryService, - GeojsonService, + { + provide: PrismaService, // Mock PrismaService here + useValue: { + // Mocked methods of PrismaService, if needed + $queryRawUnsafe: jest.fn((query: string) => { + return [{ + state_name: 'DELHI', + district_name: 'North West', + subdistrict_name: 'Saraswati Vihar', + }] + }), + }, + }, { provide: ConfigService, useValue: { @@ -63,13 +76,17 @@ describe('GeorevController', () => { it('should handle missing lat lon query parameters', async () => { const lat = ''; const lon = ''; - + try{ const result = await controller.getGeoRev(lat, lon); - - expect(result).toEqual({ + } catch (error) { + expect(error).toBeInstanceOf(HttpException); + expect(error.getStatus()).toBe(HttpStatus.INTERNAL_SERVER_ERROR); + expect(error.getResponse()).toEqual({ status: 'fail', error: 'lat lon query missing', }); + } + }); it('should handle error when processing lat lon', async () => { @@ -78,12 +95,13 @@ describe('GeorevController', () => { try { await controller.getGeoRev(lat, lon); - } catch (error) { + } + catch (error) { expect(error).toBeInstanceOf(HttpException); expect(error.getStatus()).toBe(HttpStatus.INTERNAL_SERVER_ERROR); expect(error.getResponse()).toEqual({ status: 'fail', - error: 'coordinates must contain numbers', + error: 'Invalid latitude or longitude', }); } }); @@ -92,6 +110,16 @@ describe('GeorevController', () => { const lat = '1.2345'; // valid latitude const lon = '2.3456'; // valid longitude + jest.spyOn(service, 'getGeoRev').mockRejectedValue( + new HttpException( + { + status: 'fail', + error: `No GeoLocation found for lat: ${lat}, lon: ${lon}`, + }, + HttpStatus.INTERNAL_SERVER_ERROR, + ), + ); + try { await controller.getGeoRev(lat, lon); } catch (error) { @@ -99,7 +127,7 @@ describe('GeorevController', () => { expect(error.getStatus()).toBe(HttpStatus.INTERNAL_SERVER_ERROR); expect(error.getResponse()).toEqual({ status: 'fail', - error: 'No GeoLocation found for lat: 1.2345, lon: 2.3456', + error: `No GeoLocation found for lat: ${lat}, lon: ${lon}`, }); } }); diff --git a/src/modules/georev/georev.controller.ts b/src/modules/georev/georev.controller.ts index 6910318..076ff08 100644 --- a/src/modules/georev/georev.controller.ts +++ b/src/modules/georev/georev.controller.ts @@ -8,24 +8,38 @@ import { } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { GeorevService } from './georev.service'; -import { formatGeorevSuccessResponse } from '../..//utils/serializer/success'; +import { formatGeorevSuccessResponse } from '../../utils/serializer/success'; @ApiTags('/georev') @Controller('georev') export class GeorevController { private readonly logger = new Logger(GeorevController.name); - constructor(private readonly geoRevService: GeorevService) {} + constructor(private readonly geoRevService: GeorevService) { + } @Get() async getGeoRev(@Query('lat') lat: string, @Query('lon') lon: string) { try { if (!lat || !lon) { this.logger.error(`lat lon query missing`); - return { status: 'fail', error: `lat lon query missing` }; + throw new HttpException( + { status: 'fail', error: `lat lon query missing` }, + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + + if (!this.geoRevService.isValidLatitudeLongitude(lat, lon)) { + this.logger.error('Invalid latitude or longitude'); + throw new HttpException( + { 'status': 'fail', 'error': 'Invalid latitude or longitude' }, + HttpStatus.INTERNAL_SERVER_ERROR, + ); } - const resp = this.geoRevService.getGeoRev(lat, lon); + let resp = await this.geoRevService.getGeoRev(lat, lon); + resp = resp[0]; + if (!resp) { this.logger.error(`No GeoLocation found for lat: ${lat}, lon: ${lon}`); diff --git a/src/modules/georev/georev.module.ts b/src/modules/georev/georev.module.ts index 0cfac64..adf6695 100644 --- a/src/modules/georev/georev.module.ts +++ b/src/modules/georev/georev.module.ts @@ -1,11 +1,10 @@ import { Module } from '@nestjs/common'; import { GeorevController } from './georev.controller'; import { GeorevService } from './georev.service'; -import { GeojsonService } from '../../services/geojson/geojson.service'; import { GeoqueryService } from '../../services/geoquery/geoquery.service'; @Module({ controllers: [GeorevController], - providers: [GeorevService, GeojsonService, GeoqueryService], + providers: [GeorevService, GeoqueryService], }) export class GeorevModule {} diff --git a/src/modules/georev/georev.service.spec.ts b/src/modules/georev/georev.service.spec.ts index 26d4dc2..ab97a28 100644 --- a/src/modules/georev/georev.service.spec.ts +++ b/src/modules/georev/georev.service.spec.ts @@ -2,24 +2,14 @@ import { Test, TestingModule } from '@nestjs/testing'; import { GeorevService } from './georev.service'; import { ConfigService } from '@nestjs/config'; import { GeoqueryService } from '../../services/geoquery/geoquery.service'; -import { GeojsonService } from '../../services/geojson/geojson.service'; +import { PrismaService } from '../prisma/prisma.service'; const constants = { getGeoRev: { success: { - levelLocationName: 'Saraswati Vihar', - OBJECTID: 430, - stcode11: '07', - dtcode11: '090', - sdtcode11: '00431', - Shape_Length: 107706.63225163253, - Shape_Area: 199520680.70346165, - stname: 'DELHI', - dtname: 'North West', - sdtname: 'Saraswati Vihar', - Subdt_LGD: 431, - Dist_LGD: 82, - State_LGD: 7, + state_name: 'DELHI', + district_name: 'North West', + subdistrict_name: 'Saraswati Vihar', }, }, }; @@ -31,7 +21,19 @@ describe('GeorevService', () => { const module: TestingModule = await Test.createTestingModule({ providers: [ GeorevService, - GeojsonService, + { + provide: PrismaService, // Mock PrismaService here + useValue: { + // Mocked methods of PrismaService, if needed + $queryRawUnsafe: jest.fn((query: string) => { + return [{ + state_name: 'DELHI', + district_name: 'North West', + subdistrict_name: 'Saraswati Vihar', + }] + }), + }, + }, GeoqueryService, { provide: ConfigService, @@ -62,24 +64,16 @@ describe('GeorevService', () => { expect(service).toBeDefined(); }); - it('should call individualQuery method with correct parameters', () => { + it('should call individualQuery method with correct parameters', async () => { const lat = '10.12345'; const lon = '20.67890'; - jest - .spyOn(service, 'getGeoRev') - .mockReturnValue(constants.getGeoRev.success); - - const result = service.getGeoRev(lat, lon); + // jest + // .spyOn(service, 'getGeoRev') + // .mockReturnValue(constants.getGeoRev.success); - expect(result).toEqual(constants.getGeoRev.success); - }); + const result = await service.getGeoRev(lat, lon); - it('should handle a missing coordinate', () => { - jest - .spyOn(service, 'getGeoRev') - .mockReturnValue(Error('coordinates must contain numbers')); - const result = service.getGeoRev(null, '20.67890'); - expect(result).toEqual(Error('coordinates must contain numbers')); + expect(result).toEqual([constants.getGeoRev.success]); }); }); diff --git a/src/modules/georev/georev.service.ts b/src/modules/georev/georev.service.ts index f7aed16..08b4c07 100644 --- a/src/modules/georev/georev.service.ts +++ b/src/modules/georev/georev.service.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import { GeojsonService } from '../../services/geojson/geojson.service'; import { GeoqueryService } from '../../services/geoquery/geoquery.service'; @Injectable() @@ -9,24 +8,26 @@ export class GeorevService { constructor( private readonly geoQueryService: GeoqueryService, - private readonly geoJsonService: GeojsonService, private readonly configService: ConfigService, ) { - this.geoJsonFiles = geoJsonService.getGeoJsonFiles(); } - getGeoRev(lat: string, lon: string) { + async getGeoRev(lat: string, lon: string) { try { - // Searching for SUBDISTRICT GeoLocation Level - const response = this.geoQueryService.individualQuery( - this.configService.get('country'), - this.configService.get('geoLocationLevels.SUBDISTRICT'), - [parseFloat(lon), parseFloat(lat)], - this.geoJsonFiles, - ); - return response; + return await this.geoQueryService.querySubDistrictContains(parseFloat(lat), parseFloat(lon)); } catch (error) { throw error; } } + + + isValidLatitudeLongitude(lat: string, lon: string) { + const parsedLat = parseFloat(lat); + const parsedLon = parseFloat(lon); + + const isValidLat = !isNaN(parsedLat) && parsedLat >= -90 && parsedLat <= 90; + const isValidLon = !isNaN(parsedLon) && parsedLon >= -180 && parsedLon <= 180; + + return isValidLat && isValidLon; + } } diff --git a/src/modules/location/location.controller.spec.ts b/src/modules/location/location.controller.spec.ts index 1f7a76a..8df1b75 100644 --- a/src/modules/location/location.controller.spec.ts +++ b/src/modules/location/location.controller.spec.ts @@ -3,133 +3,115 @@ import { LocationController } from './location.controller'; import { ConfigService } from '@nestjs/config'; import { LocationSearchService } from './location.search-service'; import { LocationService } from './location.service'; -import { HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { HttpException, HttpStatus } from '@nestjs/common'; describe('LocationController', () => { - let controller: LocationController; - let configService: ConfigService; - let locationSearchService: LocationSearchService; + let locationController: LocationController; let locationService: LocationService; + let locationSearchService: LocationSearchService; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [LocationController], providers: [ { - provide: ConfigService, + provide: LocationService, useValue: { - get: jest.fn((key: string) => { - if (key === 'geoLocationLevels') { - return { - VILLAGE: 'VILLAGE', - SUBDISTRICT: 'SUBDISTRICT', - DISTRICT: 'DISTRICT', - STATE: 'STATE', - }; - } else if (key === 'levelsMapping') { - return { - STATE: { - name: 'state', - path: 'state', - depth: 0, - }, - DISTRICT: { - name: 'district', - path: 'state->district', - depth: 1, - }, - SUBDISTRICT: { - name: 'subDistrict', - path: 'state->district->subDistrict', - depth: 2, - }, - VILLAGE: { - name: 'village', - path: 'state->district->subDistrict->village', - depth: 3, - }, - }; - } - }), + getCentroid: jest.fn(), // Mock the method for LocationService }, }, { provide: LocationSearchService, useValue: { - fuzzySearch: jest.fn(() => ({ result: 'mocked result' })), + fuzzySearch: jest.fn(( + locationLevel, + query, + filter + ) => { + return [{"name": "Location 1"}] + }), // Mock the method for LocationSearchService }, }, { - provide: LocationService, + provide: ConfigService, useValue: { - getCentroid: jest.fn(() => ({ - properties: { - levelLocationName: 'Lucknow', - dtname: 'Lucknow', - stname: 'UTTAR PRADESH', - stcode11: '09', - dtcode11: '157', - year_stat: '2011_c', - SHAPE_Length: 424086.646831452, - SHAPE_Area: 3190740670.6066375, - OBJECTID: 229, - test: null, - Dist_LGD: 162, - State_LGD: 9, - }, - latitude: 26.830190863213858, - longitude: 80.89119983155268, - })), + get: jest.fn((key: string) => { + if (key === 'tableMeta') { + return { LEVEL1: 'Level 1', LEVEL2: 'Level 2' }; + } else if (key === 'levelsMapping') { + return { LEVEL1: 'Mapping 1', LEVEL2: 'Mapping 2' }; + } + return null; + }), }, }, ], }).compile(); - controller = module.get(LocationController); - configService = module.get(ConfigService); + locationController = module.get(LocationController); + locationService = module.get(LocationService); locationSearchService = module.get( LocationSearchService, ); - locationService = module.get(LocationService); }); it('should be defined', () => { - expect(controller).toBeDefined(); + expect(locationController).toBeDefined(); }); - it('should handle missing query parameter in getCentroid', () => { - expect(() => controller.getCentroid('state', null)).toThrow(HttpException); + describe('health', () => { + it('should return "up" message', () => { + expect(locationController.health()).toEqual({ message: 'up' }); + }); }); - it('should call getCentroid method with correct parameters', () => { - const result = controller.getCentroid('DISTRICT', 'lucknow'); - expect(result).toEqual({ - status: 'success', - state: 'UTTAR PRADESH', - district: 'Lucknow', - subDistrict: '', - city: '', - block: '', - village: '', - lat: 26.830190863213858, - lon: 80.89119983155268, + describe('getCentroid', () => { + it('should throw BAD_REQUEST if query is missing', async () => { + await expect( + locationController.getCentroid('LEVEL1', ''), + ).rejects.toThrow(HttpException); + await expect( + locationController.getCentroid('LEVEL1', ''), + ).rejects.toThrow(`No LEVEL1 query found`); }); - }); - it('should handle missing query parameter in fuzzySearch', () => { - expect(() => controller.fuzzySearch('state', { query: null })).toThrow( - HttpException, - ); - }); + it('should return centroid data when query is valid', async () => { + const mockCentroidResponse = { + properties: { name: 'Location' }, + latitude: 12.9716, + longitude: 77.5946, + }; - it('should handle unsupported GeoLocation Level in fuzzySearch', () => { - expect(() => - controller.fuzzySearch('country', { query: 'testQuery' }), - ).toThrow(HttpException); - }); + jest + .spyOn(locationService, 'getCentroid') + .mockResolvedValueOnce(mockCentroidResponse); + + const result = await locationController.getCentroid('LEVEL1', 'query'); + expect(result).toEqual({ + "block": "", + "city": "", + "district": "", + "lat": 12.9716, + "lon": 77.5946, + "state": "", + "status": "success", + "subDistrict": "", + "village": "", + }); + }); - it('should call fuzzySearch method with correct parameters', () => { - const result = controller.fuzzySearch('state', { query: 'testQuery' }); - expect(result).toEqual({ result: 'mocked result' }); + it('should throw NOT_FOUND if location service throws an error', async () => { + jest + .spyOn(locationService, 'getCentroid') + .mockRejectedValueOnce(new Error('NotFoundError')); + + await expect( + locationController.getCentroid('LEVEL1', 'query'), + ).rejects.toThrow(HttpException); + await expect( + locationController.getCentroid('LEVEL1', 'query'), + ).rejects.toThrow('TypeError'); + }); }); + }); diff --git a/src/modules/location/location.controller.ts b/src/modules/location/location.controller.ts index ec133c2..c178274 100644 --- a/src/modules/location/location.controller.ts +++ b/src/modules/location/location.controller.ts @@ -1,19 +1,10 @@ -import { - Controller, - Get, - Post, - Body, - Param, - Query, - HttpException, - HttpStatus, - Logger, -} from '@nestjs/common'; +import { Body, Controller, Get, HttpException, HttpStatus, Logger, Param, Post, Query } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { ApiTags } from '@nestjs/swagger'; import { formatCentroidResponse } from '../../utils/serializer/success'; import { LocationSearchService } from './location.search-service'; import { LocationService } from './location.service'; + @ApiTags('/location') @Controller('location') export class LocationController { @@ -27,15 +18,20 @@ export class LocationController { private readonly locationService: LocationService, ) { this.geoLocationLevels = this.configService.get<{ [key: string]: any }>( - 'geoLocationLevels', + 'tableMeta', ); this.levelsMapping = this.configService.get<{ [key: string]: any }>( 'levelsMapping', ); } + @Get() + health() { + return {"message": "up"} + } + @Get(':locationlevel/centroid') - getCentroid( + async getCentroid( @Param('locationlevel') locationLevel: string, @Query('query') query: string, ) { @@ -47,7 +43,8 @@ export class LocationController { ); } try { - const response = this.locationService.getCentroid(locationLevel, query); + const response = await this.locationService.getCentroid(locationLevel, query); + this.logger.log(response); return formatCentroidResponse( response.properties, response.latitude, @@ -62,9 +59,9 @@ export class LocationController { } @Post(':locationlevel/fuzzysearch') - fuzzySearch( + async fuzzySearch( @Param('locationlevel') locationLevel: string, - @Body() body: any, + @Body() body: any, // ) { try { if ( @@ -87,50 +84,14 @@ export class LocationController { } const filter = body.filter || {}; - const filterArray = []; - for (const filterKey of Object.keys(filter)) { - if ( - !Object.keys(this.geoLocationLevels).includes(filterKey.toUpperCase()) - ) { - throw new HttpException( - `Unsupported GeoLocation Level Filter: ${filterKey}`, - HttpStatus.BAD_REQUEST, - ); - } - filterArray.push({ - level: this.levelsMapping[filterKey.toUpperCase()], - query: filter[filterKey], - }); - } - - let searchLevel; - switch (locationLevel.toUpperCase()) { - case 'STATE': - searchLevel = this.levelsMapping.STATE; - break; - case 'DISTRICT': - searchLevel = this.levelsMapping.DISTRICT; - break; - case 'SUBDISTRICT': - searchLevel = this.levelsMapping.SUBDISTRICT; - break; - case 'VILLAGE': - searchLevel = this.levelsMapping.VILLAGE; - break; - default: - throw new HttpException( - 'Invalid location level', - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } - const queryResponse = this.locationSearchService.fuzzySearch( - searchLevel, + return this.locationSearchService.fuzzySearch( + locationLevel, query, - filterArray, + filter, ); - return queryResponse; } catch (error) { + this.logger.error(error) throw new HttpException(error.name, HttpStatus.INTERNAL_SERVER_ERROR); } } diff --git a/src/modules/location/location.module.ts b/src/modules/location/location.module.ts index a1c2e1c..49b605d 100644 --- a/src/modules/location/location.module.ts +++ b/src/modules/location/location.module.ts @@ -2,23 +2,14 @@ import { Module } from '@nestjs/common'; import { LocationController } from './location.controller'; import { LocationService } from './location.service'; import { LocationSearchService } from './location.search-service'; -import { GeojsonService } from '../../services/geojson/geojson.service'; import * as path from 'path'; +import { GeoqueryService } from '../../services/geoquery/geoquery.service'; @Module({ controllers: [LocationController], providers: [ - { - useFactory: () => { - const filePath = path.join( - process.cwd(), - './src/geojson-data/PARSED_MASTER_LOCATION_NAMES.json', - ); - return new LocationSearchService(filePath); - }, - provide: LocationSearchService, - }, - GeojsonService, + LocationSearchService, + GeoqueryService, LocationService, ], }) diff --git a/src/modules/location/location.search-service.spec.ts b/src/modules/location/location.search-service.spec.ts deleted file mode 100644 index 7820610..0000000 --- a/src/modules/location/location.search-service.spec.ts +++ /dev/null @@ -1,124 +0,0 @@ -import * as fs from 'fs'; -import { Level, LocationSearchService } from './location.search-service'; - -jest.mock('fs'); - -describe('LocationSearchService', () => { - let locationSearchService: LocationSearchService; - - const mockData = JSON.stringify([ - { - state: 'State1', - districts: [ - { - district: 'District1', - subDistricts: [ - { - subDistrict: 'SubDistrict1', - villages: ['Village1', 'Village2'], - }, - ], - }, - ], - }, - { - state: 'State2', - districts: [ - { - district: 'District2', - subDistricts: [ - { - subDistrict: 'SubDistrict2', - villages: ['Village3', 'Village4'], - }, - ], - }, - ], - }, - ]); - - beforeEach(() => { - (fs.readFileSync as jest.Mock).mockReturnValue(mockData); - locationSearchService = new LocationSearchService('mockFilePath'); - }); - - it('should be defined', () => { - expect(locationSearchService).toBeDefined(); - }); - - it('should preprocess data correctly', () => { - expect(locationSearchService['villagePreprocessedData']).toHaveLength(4); - expect(locationSearchService['subDistrictPreprocessedData']).toHaveLength( - 2, - ); - expect(locationSearchService['districtPreprocessedData']).toHaveLength(2); - expect(locationSearchService['statePreProcessedData']).toHaveLength(2); - }); - - it('should return correct results for state level search', () => { - const result = locationSearchService.search(Level.STATE, 'State1', null); - expect(result).toEqual([{ state: 'State1' }]); - }); - - it('should return correct results for district level search', () => { - const result = locationSearchService.search( - Level.DISTRICT, - 'District1', - null, - ); - expect(result).toEqual([{ state: 'State1', district: 'District1' }]); - }); - - it('should return correct results for sub-district level search', () => { - const result = locationSearchService.search( - Level.SUBDISTRICT, - 'SubDistrict1', - null, - ); - expect(result).toEqual([ - { state: 'State1', district: 'District1', subDistrict: 'SubDistrict1' }, - ]); - }); - - it('should return correct results for village level search', () => { - const result = locationSearchService.search( - Level.VILLAGE, - 'Village1', - null, - ); - expect(result).toEqual([ - { - state: 'State1', - district: 'District1', - subDistrict: 'SubDistrict1', - village: 'Village1', - }, - ]); - }); - - it('should apply filters correctly', () => { - const filters = [{ level: Level.STATE, query: 'State1' }]; - const result = locationSearchService.search( - Level.VILLAGE, - 'Village1', - filters, - ); - expect(result).toEqual([ - { - state: 'State1', - district: 'District1', - subDistrict: 'SubDistrict1', - village: 'Village1', - }, - ]); - }); - - it('should return no results if no matches are found', () => { - const result = locationSearchService.search( - Level.VILLAGE, - 'NonExistentVillage', - null, - ); - expect(result).toEqual([]); - }); -}); diff --git a/src/modules/location/location.search-service.ts b/src/modules/location/location.search-service.ts index 1d61206..213125f 100644 --- a/src/modules/location/location.search-service.ts +++ b/src/modules/location/location.search-service.ts @@ -1,150 +1,56 @@ import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { GeoqueryService } from '../../services/geoquery/geoquery.service'; -const Fuse = require('fuse.js'); -import * as fs from 'fs'; - -const logger = new Logger('ls-st'); - -type LevelType = { - name: string; - path: string; - depth: number; -}; - -export const Level = Object.freeze({ - STATE: { - name: 'state', - path: 'state', - depth: 0, - } as LevelType, - DISTRICT: { - name: 'district', - path: 'state->district', - depth: 1, - } as LevelType, - SUBDISTRICT: { - name: 'subDistrict', - path: 'state->district->subDistrict', - depth: 2, - } as LevelType, - VILLAGE: { - name: 'village', - path: 'state->district->subDistrict->village', - depth: 3, - } as LevelType, -}); - -type LevelKeys = keyof typeof Level; -type Level = (typeof Level)[LevelKeys]; @Injectable() export class LocationSearchService { - private villagePreprocessedData: any[]; - private subDistrictPreprocessedData: any[]; - private districtPreprocessedData: any[]; - private statePreProcessedData: any[]; + logger = new Logger(LocationSearchService.name); - constructor(filePath: string) { - this.villagePreprocessedData = []; - this.subDistrictPreprocessedData = []; - this.districtPreprocessedData = []; - this.statePreProcessedData = []; + constructor(private readonly config: ConfigService, private readonly geoquery: GeoqueryService) { - const jsonData = JSON.parse(fs.readFileSync(filePath, 'utf-8')); - - jsonData.forEach((stateData: any) => { - stateData.districts.forEach((districtData: any) => { - districtData.subDistricts.forEach((subDistrictData: any) => { - subDistrictData.villages.forEach((village: any) => { - if (village !== null) { - this.villagePreprocessedData.push({ - state: stateData.state, - district: districtData.district, - subDistrict: subDistrictData.subDistrict, - village, - }); - } - }); - this.subDistrictPreprocessedData.push({ - state: stateData.state, - district: districtData.district, - subDistrict: subDistrictData.subDistrict, - }); - }); - this.districtPreprocessedData.push({ - state: stateData.state, - district: districtData.district, - }); - }); - this.statePreProcessedData.push({ - state: stateData.state, - }); - }); } - fuzzySearch(level: Level, query: string, filters: any[] | null): any[] { + fuzzySearch(level: string, query: string, filters) { return this.querySearch(level, query, 0.1, 0, filters); } - search(level: Level, query: string, filters: any[] | null): any[] { + search(level: string, query: string, filters) { return this.querySearch(level, query, 0.0, 0, filters); } - private querySearch( - searchLevel: Level | any, + private async querySearch( + searchLevel: string, query: string, threshold: number, distance: number = 0, - filters: any[] | null, - ): any[] { - const options = { - keys: [searchLevel.name], - threshold, - distance, - isCaseSensitive: false, - }; - let processedData: any[]; + filters, + ) { + const { state_name, district_name, subdistrict_name } = filters; - switch (searchLevel.name) { - case Level.STATE.name: - processedData = this.statePreProcessedData; - break; - case Level.DISTRICT.name: - processedData = this.districtPreprocessedData; + let result; + + switch (searchLevel.toLowerCase()) { + case 'state': break; - case Level.SUBDISTRICT.name: - processedData = this.subDistrictPreprocessedData; + case 'district': + result = await this.geoquery.fuzzyDistrictSearch(query, { state_name }); break; - case Level.VILLAGE.name: - processedData = this.villagePreprocessedData; + case 'subdistrict': + result = await this.geoquery.fuzzySubDistrictSearch(query, { state_name, district_name }); break; - default: - processedData = []; + case 'village': + result = await this.geoquery.fuzzyVillageSearch(query, { state_name, district_name, subdistrict_name }); break; } - - if (filters !== null) { - for (let nodeDepth = 0; nodeDepth < searchLevel.depth; nodeDepth++) { - for (const filter of filters) { - if (filter.level.depth !== nodeDepth) continue; - const filteredData = []; - for (let index = 0; index < processedData.length; index++) { - if ( - processedData[index][`${filter.level.name}`] - .toLowerCase() - .includes(filter.query.toLowerCase()) - ) { - filteredData.push(processedData[index]); - } - } - processedData = filteredData; - } - } - } - - const fuse = new Fuse(processedData, options); - const result = fuse.search(query); - - return result.map((entry) => ({ ...entry.item })); + this.logger.log(result); + return { + matches: result.map((item: any) => ({ + state: item.state_name || state_name || null, + district: item.district_name || district_name || null, + subDistrict: item.subdistrict_name || subdistrict_name || null, + village: item.village_name || null, + })), + }; } } diff --git a/src/modules/location/location.service.spec.ts b/src/modules/location/location.service.spec.ts index c9592c4..d6754ce 100644 --- a/src/modules/location/location.service.spec.ts +++ b/src/modules/location/location.service.spec.ts @@ -1,17 +1,13 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; -import { Logger } from '@nestjs/common'; import { LocationService } from './location.service'; -import { GeojsonService } from '../../services/geojson/geojson.service'; -import * as turf from '@turf/turf'; - -jest.mock('@nestjs/common/services/logger.service'); -jest.mock('@turf/turf'); +import { GeoqueryService } from '../../services/geoquery/geoquery.service'; +import { Logger } from '@nestjs/common'; describe('LocationService', () => { - let service: LocationService; + let locationService: LocationService; let configService: ConfigService; - let geojsonService: GeojsonService; + let geoQueryService: GeoqueryService; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -20,75 +16,78 @@ describe('LocationService', () => { { provide: ConfigService, useValue: { - get: jest.fn().mockReturnValue('testCountry'), + get: jest.fn(), // Mock method }, }, { - provide: GeojsonService, + provide: GeoqueryService, useValue: { - getGeoJsonFiles: jest.fn().mockReturnValue({ - testCountry_state: { - features: [ - { - properties: { levelLocationName: 'testState' }, - geometry: { - type: 'Polygon', - coordinates: [ - [ - [0, 0], - [1, 0], - [1, 1], - [0, 1], - [0, 0], - ], - ], - }, - }, - ], - }, - }), + geometryCentroid: jest.fn(), // Mock method }, }, + Logger, // NestJS logger service ], }).compile(); - service = module.get(LocationService); + locationService = module.get(LocationService); configService = module.get(ConfigService); - geojsonService = module.get(GeojsonService); + geoQueryService = module.get(GeoqueryService); }); it('should be defined', () => { - expect(service).toBeDefined(); + expect(locationService).toBeDefined(); }); - it('should return the correct centroid for a given location', () => { - const centroidMock = { geometry: { coordinates: [0.5, 0.5] } }; - (turf.centroid as jest.Mock).mockReturnValue(centroidMock); + describe('getCentroid', () => { + it('should return centroid data for valid locationLevel and query', async () => { + const locationLevel = 'LEVEL1'; + const query = 'some query'; + + // Mock ConfigService.get to return a value for the tableMeta + jest.spyOn(configService, 'get').mockReturnValue('mockTableMeta'); + + // Mock GeoqueryService.geometryCentroid to return valid centroid data + const mockResponse = [ + { + coordinate: JSON.stringify({ + coordinates: [77.5946, 12.9716], // Longitude, Latitude + }), + name: 'Sample Location', + }, + ]; + jest + .spyOn(geoQueryService, 'geometryCentroid') + .mockResolvedValueOnce(mockResponse); - const result = service.getCentroid('state', 'testState'); + const result = await locationService.getCentroid(locationLevel, query); - expect(result).toEqual({ - properties: { levelLocationName: 'testState' }, - latitude: 0.5, - longitude: 0.5, + expect(result).toEqual({ + properties: mockResponse[0], + latitude: 12.9716, + longitude: 77.5946, + }); + expect(configService.get).toHaveBeenCalledWith(`tableMeta.${locationLevel}`); + expect(geoQueryService.geometryCentroid).toHaveBeenCalledWith('mockTableMeta', query); }); - expect(turf.centroid).toHaveBeenCalledWith( - turf.polygon([ - [ - [0, 0], - [1, 0], - [1, 1], - [0, 1], - [0, 0], - ], - ]), - ); - }); + it('should throw an error when geoQueryService fails', async () => { + const locationLevel = 'LEVEL1'; + const query = 'some query'; + + // Mock ConfigService.get to return a value for the tableMeta + jest.spyOn(configService, 'get').mockReturnValue('mockTableMeta'); - it('should throw an error if the location is not found', () => { - expect(() => service.getCentroid('state', 'invalidState')).toThrowError( - 'No state found with name: invalidState', - ); + // Mock GeoqueryService.geometryCentroid to throw an error + jest + .spyOn(geoQueryService, 'geometryCentroid') + .mockRejectedValueOnce(new Error('Service Error')); + + await expect( + locationService.getCentroid(locationLevel, query), + ).rejects.toThrow('Service Error'); + + expect(configService.get).toHaveBeenCalledWith(`tableMeta.${locationLevel}`); + expect(geoQueryService.geometryCentroid).toHaveBeenCalledWith('mockTableMeta', query); + }); }); }); diff --git a/src/modules/location/location.service.ts b/src/modules/location/location.service.ts index 38d578b..e034ee0 100644 --- a/src/modules/location/location.service.ts +++ b/src/modules/location/location.service.ts @@ -1,56 +1,28 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import * as turf from '@turf/turf'; -import { GeojsonService } from '../../services/geojson/geojson.service'; +import { GeoqueryService } from '../../services/geoquery/geoquery.service'; @Injectable() export class LocationService { private readonly logger = new Logger(LocationService.name); - private readonly geoJsonFiles: { [key: string]: any }; private readonly country: string; constructor( private readonly configService: ConfigService, - private readonly geoJsonService: GeojsonService, + private readonly geoQueryService: GeoqueryService, ) { - this.geoJsonFiles = this.geoJsonService.getGeoJsonFiles(); - this.country = this.configService.get('country'); } - getCentroid(locationLevel: string, query: string) { + async getCentroid(locationLevel: string, query: string) { try { - let queryFeature; - for (const feature of this.geoJsonFiles[ - `${this.country}_${locationLevel}` - ].features) { - if ( - feature.properties.levelLocationName.toLowerCase() === - query.toLowerCase() - ) { - queryFeature = feature; - break; - } - } - - if (!queryFeature) { - throw new Error(`No ${locationLevel} found with name: ${query}`); - } - - let polygonFeature; - if (queryFeature.geometry.type === 'Polygon') { - polygonFeature = turf.polygon(queryFeature.geometry.coordinates); - } else { - polygonFeature = turf.multiPolygon(queryFeature.geometry.coordinates); - } - - const centroid = turf.centroid(polygonFeature); - const longitude = centroid.geometry.coordinates[0]; - const latitude = centroid.geometry.coordinates[1]; - + const tableMeta = this.configService.get(`tableMeta.${locationLevel}`); + let resp: any = await this.geoQueryService.geometryCentroid(tableMeta, query); + resp = resp[0]; + const { coordinates } = JSON.parse(resp.coordinate); this.logger.log( - `Centroid Success Response: ${JSON.stringify(queryFeature.properties)}`, + `Centroid Success Response: ${resp}`, ); - return { properties: queryFeature.properties, latitude, longitude }; + return { properties: resp, latitude: coordinates[1], longitude: coordinates[0] }; } catch (error) { throw error; } diff --git a/src/modules/prisma/prisma.module.ts b/src/modules/prisma/prisma.module.ts new file mode 100644 index 0000000..d80c9f3 --- /dev/null +++ b/src/modules/prisma/prisma.module.ts @@ -0,0 +1,9 @@ +import { Global, Module } from '@nestjs/common'; +import { PrismaService } from './prisma.service'; + +@Global() +@Module({ + providers: [PrismaService], + exports: [PrismaService] +}) +export class PrismaModule {} diff --git a/src/modules/prisma/prisma.service.ts b/src/modules/prisma/prisma.service.ts new file mode 100644 index 0000000..359f950 --- /dev/null +++ b/src/modules/prisma/prisma.service.ts @@ -0,0 +1,9 @@ +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; + +@Injectable() +export class PrismaService extends PrismaClient implements OnModuleInit { + async onModuleInit() { + await this.$connect(); + } +} diff --git a/src/services/geojson/geojson.service.spec.ts b/src/services/geojson/geojson.service.spec.ts deleted file mode 100644 index 29b78b5..0000000 --- a/src/services/geojson/geojson.service.spec.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { GeojsonService } from './geojson.service'; -import { ConfigService } from '@nestjs/config'; - -describe('GeojsonService', () => { - let service: GeojsonService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - GeojsonService, - { - provide: ConfigService, - useValue: { - get: jest.fn().mockImplementation((key: string) => { - switch (key) { - case 'requiredGeoLocationLevels': - return ['SUBDISTRICT', 'DISTRICT', 'STATE']; - case 'country': - return 'INDIA'; - default: - return null; - } - }), - }, - }, - ], - }).compile(); - - service = module.get(GeojsonService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/services/geojson/geojson.service.ts b/src/services/geojson/geojson.service.ts deleted file mode 100644 index 8ac1ea4..0000000 --- a/src/services/geojson/geojson.service.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import * as fs from 'fs'; -import * as path from 'path'; - -@Injectable() -export class GeojsonService { - private readonly logger = new Logger(GeojsonService.name); - private readonly geoJsonFilesPath: string; - private readonly requiredGeoLocationLevels: Array; - private readonly country: string; - private geoJsonFiles: { [key: string]: any } = {}; - - constructor(private configService: ConfigService) { - this.geoJsonFilesPath = path.join(process.cwd(), './src/geojson-data'); // Adjust the path as needed - this.requiredGeoLocationLevels = this.configService.get>( - 'requiredGeoLocationLevels', - ); - this.country = this.configService.get('country'); - this.loadGeoJsonFiles(); - } - - private loadGeoJsonFiles(): void { - try { - const files = fs.readdirSync(this.geoJsonFilesPath); - for (const locationLevel of this.requiredGeoLocationLevels) { - const geoJsonFileName = `${this.country}_${locationLevel}.geojson`; - const geoJsonKeyName = `${this.country}_${locationLevel}`; - if (!files.includes(geoJsonFileName)) { - this.logger.error( - `Required GeoJson file: ${geoJsonFileName} not present`, - ); - process.exit(); - } else { - this.geoJsonFiles[geoJsonKeyName] = JSON.parse( - fs.readFileSync( - `${this.geoJsonFilesPath}/${geoJsonFileName}`, - 'utf8', - ), - ); - this.logger.log(`Loaded GeoJson file: ${geoJsonFileName}`); - } - } - } catch (err) { - this.logger.error(`Error loading GeoJson files: ${err}`); - } - } - - getGeoJsonFiles(): { [key: string]: any } { - return this.geoJsonFiles; - } -} diff --git a/src/services/geoquery/geoquery.service.spec.ts b/src/services/geoquery/geoquery.service.spec.ts deleted file mode 100644 index 8efef09..0000000 --- a/src/services/geoquery/geoquery.service.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { GeoqueryService } from './geoquery.service'; - -describe('GeoqueryService', () => { - let service: GeoqueryService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [GeoqueryService], - }).compile(); - - service = module.get(GeoqueryService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/src/services/geoquery/geoquery.service.ts b/src/services/geoquery/geoquery.service.ts index 083b5b8..88017c9 100644 --- a/src/services/geoquery/geoquery.service.ts +++ b/src/services/geoquery/geoquery.service.ts @@ -1,44 +1,176 @@ + + import { Injectable, Logger } from '@nestjs/common'; -import * as turf from '@turf/turf'; +import { PrismaService } from '../../modules/prisma/prisma.service'; @Injectable() export class GeoqueryService { private readonly logger = new Logger(GeoqueryService.name); - isPointInMultiPolygon(multiPolygon, point) { - this.logger.log(`Checking if point is in MultiPolygon`); - return multiPolygon.geometry.coordinates.some((polygonCoordinates) => { - const poly = turf.polygon(polygonCoordinates); - return turf.booleanContains(poly, point); - }); - } - - individualQuery( - country: string, - geoLocationLevel: string, - coordinates, - geoJsonFiles, - ) { - const pointToSearch = turf.point(coordinates); - for (const feature of geoJsonFiles[`${country}_${geoLocationLevel}`] - .features) { - if (feature.geometry.type === 'Polygon') { - this.logger.log(`Checking if point is in Polygon`); - const poly = turf.polygon( - feature.geometry.coordinates, - feature.properties, - ); - if (turf.booleanContains(poly, pointToSearch)) { - this.logger.log(`Point is in Polygon`); - return poly.properties; - } - } else if (feature.geometry.type === 'MultiPolygon') { - this.logger.log(`Checking if point is in MultiPolygon`); - if (this.isPointInMultiPolygon(feature, pointToSearch)) { - this.logger.log(`Point is in MultiPolygon`); - return feature.properties; - } - } + constructor(private readonly prisma: PrismaService) {} + + async queryStateContains(lat: number, lon: number) { + this.logger.log(`Querying state with lat: ${lat}, lon: ${lon}`); + + return this.prisma.$queryRawUnsafe(` + SELECT s.state_code AS state_code, + s.state_name AS state_name + FROM "State" s + WHERE ST_Contains(s.geometry, ST_SetSRID(ST_MakePoint(${lon}, ${lat}), 4326)); + `); + } + + async queryDistrictContains(lat: number, lon: number) { + this.logger.log(`Querying district with lat: ${lat}, lon: ${lon}`); + + return this.prisma.$queryRawUnsafe(` + SELECT s.state_code AS state_code, + s.state_name AS state_name, + d.district_code AS district_code, + d.district_name AS district_name + FROM "District" d + JOIN "State" s ON d.state_id = s.state_code + WHERE ST_Contains(d.geometry, ST_SetSRID(ST_MakePoint(${lon}, ${lat}), 4326)); + `); + } + + async querySubDistrictContains(lat: number, lon: number) { + this.logger.log(`Querying sub-district with lat: ${lat}, lon: ${lon}`); + + return this.prisma.$queryRawUnsafe(` + SELECT s.state_code AS state_code, + s.state_name AS state_name, + d.district_code AS district_code, + d.district_name AS district_name, + sd.subdistrict_code AS subdistrict_code, + sd.subdistrict_name AS subdistrict_name + FROM "SubDistrict" sd + JOIN "District" d ON sd.district_id = d.district_code + JOIN "State" s ON d.state_id = s.state_code + WHERE ST_Contains(sd.geometry, ST_SetSRID(ST_MakePoint(${lon}, ${lat}), 4326)); + `); + } + + async queryVillageContains(lat: number, lon: number) { + this.logger.log(`Querying village with lat: ${lat}, lon: ${lon}`); + + return this.prisma.$queryRawUnsafe(` + SELECT s.state_code AS state_code, + s.state_name AS state_name, + d.district_code AS district_code, + d.district_name AS district_name, + sd.subdistrict_code AS subdistrict_code, + sd.subdistrict_name AS subdistrict_name, + v.village_code AS village_code, + v.village_name AS village_name + FROM "Village" v + JOIN "SubDistrict" sd ON v.subdistrict_id = sd.subdistrict_code + JOIN "District" d ON sd.district_id = d.district_code + JOIN "State" s ON d.state_id = s.state_code + WHERE ST_Contains(v.geometry, ST_SetSRID(ST_MakePoint(${lon}, ${lat}), 4326)); + `); + } + + async geometryCentroid(tableMeta: { tname: string; fname: string }, fieldName: string) { + this.logger.log(`SELECT *, ST_AsGeoJson(ST_Centroid(geometry)) as coordinate + FROM "${tableMeta.tname}" + WHERE ${tableMeta.fname} ILIKE '${fieldName}';`); + return this.prisma.$queryRawUnsafe(` + SELECT ${tableMeta.fname}, ST_AsGeoJson(ST_Centroid(geometry)) as coordinate + FROM "${tableMeta.tname}" + WHERE ${tableMeta.fname} ILIKE '${fieldName}' LIMIT 1; + `); + } + + async fuzzyStateSearch(state_name: string) { + return this.prisma.$queryRawUnsafe(` + WITH input AS (SELECT '${state_name }'::VARCHAR AS input_name) + SELECT st.name AS state_name, st.code AS state_code, levenshtein(i.input_name, st.name) as levenshtein + FROM "State" st, + input i + WHERE levenshtein(i.input_name, st.name) <= 5 + ORDER BY levenshtein LIMIT 1; + `); + } + + async fuzzyDistrictSearch(district_name: string, filter: { state_name?: string }) { + const { state_name } = filter; + + let whereClause = ` + WHERE (levenshtein(i.input_name, dt.name) <= 5 + OR i.input_name ILIKE '%' || dt.name || '%' + OR dt.name ILIKE '%' || i.input_name || '%')`; + + if (state_name) { + whereClause += ` AND st.name ILIKE '${state_name}'`; + } + + return this.prisma.$queryRawUnsafe(` + WITH input AS (SELECT '${district_name}'::VARCHAR AS input_name) + SELECT dt.name AS district_name, dt.code AS district_code, levenshtein(i.input_name, dt.name) as levenshtein, st.name AS state_name + FROM "District" dt + JOIN "State" st ON dt.state_id = st.state_code, + input i ${whereClause} + ORDER BY levenshtein LIMIT 1; + `); + } + + async fuzzySubDistrictSearch(subdistrict_name: string, filter: { state_name?: string, district_name?: string }) { + const { state_name, district_name } = filter; + + let whereClause = ` + WHERE (levenshtein(i.input_name, sdt.name) <= 5 + OR i.input_name ILIKE '%' || sdt.name || '%' + OR sdt.name ILIKE '%' || i.input_name || '%')`; + + if (state_name) { + whereClause += ` AND st.name ILIKE '${state_name}'`; + } + + if (district_name) { + whereClause += ` AND dt.name ILIKE '${district_name}'`; + } + + return this.prisma.$queryRawUnsafe(` + WITH input AS (SELECT '${subdistrict_name}'::VARCHAR AS input_name) + SELECT sdt.name AS subdistrict_name, sdt.code AS subdistrict_code, levenshtein(i.input_name, sdt.name) as levenshtein, st.name AS state_name, dt.name AS district_name + FROM "SubDistrict" sdt + JOIN "District" dt ON sdt.district_id = dt.district_code + JOIN "State" st ON dt.state_id = st.state_code, + input i ${whereClause} + ORDER BY levenshtein LIMIT 1; + `); + } + + async fuzzyVillageSearch(village_name: string, filter: { state_name?: string, district_name?: string, subdistrict_name?: string }) { + const { state_name, district_name, subdistrict_name } = filter; + + let whereClause = ` + WHERE (levenshtein(i.input_name, v.village_name) <= 5 + OR i.input_name ILIKE '%' || v.village_name || '%' + OR v.village_name ILIKE '%' || i.input_name || '%')`; + + if (state_name) { + whereClause += ` AND st.name ILIKE '${state_name}'`; } + + if (district_name) { + whereClause += ` AND dt.name ILIKE '${district_name}'`; + } + + if (subdistrict_name) { + whereClause += ` AND sdt.name ILIKE '${subdistrict_name}'`; + } + + return this.prisma.$queryRawUnsafe(` + WITH input AS (SELECT '${village_name}'::VARCHAR AS input_name) + SELECT v.village_name AS village_name, v.village_code AS village_code, levenshtein(i.input_name, v.village_name) as levenshtein, st.name AS state_name, dt.name AS district_name, sdt.name AS subdistrict_name + FROM "Village" v + JOIN "SubDistrict" sdt ON v.subdistrict_id = sdt.subdistrict_code + JOIN "District" dt ON sdt.district_id = dt.district_code + JOIN "State" st ON dt.state_id = st.state_code, + input i ${whereClause} + ORDER BY levenshtein LIMIT 1; + `); } } diff --git a/src/utils/serializer/success.ts b/src/utils/serializer/success.ts index 3f52f86..6be7bf6 100644 --- a/src/utils/serializer/success.ts +++ b/src/utils/serializer/success.ts @@ -28,9 +28,9 @@ export const formatGeorevSuccessResponse = (data: any) => { logger.log(`GeoRev Success Response: ${JSON.stringify(data)}`); return { status: 'success', - state: data.stname ?? '', - district: data.dtname ?? '', - subDistrict: data.sdtname ?? '', + state: data.state_name ?? '', + district: data.district_name ?? '', + subDistrict: data.subdistrict_name ?? '', }; }; @@ -42,12 +42,12 @@ export const formatCentroidResponse = ( logger.log(`Centroid Success Response: ${JSON.stringify(data)}`); return { status: 'success', - state: data.stname ?? '', - district: data.dtname ?? '', - subDistrict: data.sdtname ?? '', + state: data.state_name ?? '', + district: data.district_name ?? '', + subDistrict: data.subdistrict_name ?? '', city: '', block: '', - village: '', + village: data.village_name ?? '', lat: latitude, lon: longitude, }; diff --git a/test/app.e2e-spec.ts b/test/app.e2e-spec.ts index e8a95b7..2b6520a 100644 --- a/test/app.e2e-spec.ts +++ b/test/app.e2e-spec.ts @@ -158,7 +158,7 @@ describe('AppController (e2e)', () => { expect(response.status).toBe(500); expect(response.body).toEqual({ status: 'fail', - error: 'coordinates must contain numbers', + error: 'Invalid latitude or longitude', }); }); @@ -169,14 +169,14 @@ describe('AppController (e2e)', () => { expect(response.status).toBe(200); expect(response.body).toEqual({ status: 'success', - state: 'UTTAR PRADESH', + state: '', district: 'Lucknow', subDistrict: '', city: '', block: '', village: '', - lat: 26.830190863213858, - lon: 80.89119983155268, + lat: 26.841984034, + lon: 80.905485485, }); }); @@ -186,4 +186,6 @@ describe('AppController (e2e)', () => { ); expect(response.status).toBe(404); }); + + }); diff --git a/test/jest-e2e.json b/test/jest-e2e.json index e9d912f..68080d3 100644 --- a/test/jest-e2e.json +++ b/test/jest-e2e.json @@ -5,5 +5,6 @@ "testRegex": ".e2e-spec.ts$", "transform": { "^.+\\.(t|j)s$": "ts-jest" - } + }, + "testTimeout": 10000 } From 60597c23d438d76bb0182e25b0b6e5a2694f5165 Mon Sep 17 00:00:00 2001 From: Dhruv Baliyan Date: Tue, 3 Dec 2024 09:25:32 +0530 Subject: [PATCH 08/11] feat: add create/search place APIs --- package-lock.json | 2 +- package.json | 2 +- .../20241203015556_add_place/migration.sql | 10 ++++ prisma/schema.prisma | 8 +++ src/app.module.ts | 2 + src/main.ts | 3 +- src/modules/place/dto/place.dto.ts | 36 +++++++++++++ src/modules/place/place.controller.ts | 20 +++++++ src/modules/place/place.module.ts | 9 ++++ src/modules/place/place.service.ts | 52 +++++++++++++++++++ 10 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 prisma/migrations/20241203015556_add_place/migration.sql create mode 100644 src/modules/place/dto/place.dto.ts create mode 100644 src/modules/place/place.controller.ts create mode 100644 src/modules/place/place.module.ts create mode 100644 src/modules/place/place.service.ts diff --git a/package-lock.json b/package-lock.json index 838d391..fcf7d0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -54,7 +54,7 @@ "supertest": "^6.3.3", "ts-jest": "^29.1.0", "ts-loader": "^9.4.3", - "ts-node": "^10.9.1", + "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", "typescript": "^5.1.3" } diff --git a/package.json b/package.json index 4656df1..568eeae 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "supertest": "^6.3.3", "ts-jest": "^29.1.0", "ts-loader": "^9.4.3", - "ts-node": "^10.9.1", + "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", "typescript": "^5.1.3" }, diff --git a/prisma/migrations/20241203015556_add_place/migration.sql b/prisma/migrations/20241203015556_add_place/migration.sql new file mode 100644 index 0000000..165b005 --- /dev/null +++ b/prisma/migrations/20241203015556_add_place/migration.sql @@ -0,0 +1,10 @@ +-- CreateTable +CREATE TABLE "Place" ( + "id" SERIAL NOT NULL, + "name" TEXT NOT NULL, + "type" TEXT NOT NULL, + "tag" TEXT NOT NULL, + "location" geometry(Point, 4326) NOT NULL, + + CONSTRAINT "Place_pkey" PRIMARY KEY ("id") +); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7e2e3ed..df535c8 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -74,3 +74,11 @@ model Village { state_id Int? state State? @relation(fields: [state_id], references: [state_code]) } + +model Place { + id Int @id @default(autoincrement()) + name String + type String + tag String + location Unsupported("geometry(Point, 4326)") +} diff --git a/src/app.module.ts b/src/app.module.ts index 9534cd0..7bb73b7 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -7,12 +7,14 @@ import { ConfigModule } from '@nestjs/config'; import { config } from './config/config'; import { AppController } from './app.controller'; import { AppService } from './app.service'; +import { PlaceModule } from './modules/place/place.module'; @Module({ imports: [ CityModule, GeorevModule, LocationModule, + PlaceModule, PrismaModule, ConfigModule.forRoot({ envFilePath: `.env`, diff --git a/src/main.ts b/src/main.ts index d36f4ea..b807fa0 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,7 +24,6 @@ async function bootstrap() { const configService = app.get(ConfigService); const port = configService.get('port'); const host = configService.get('host'); - const method = configService.get('method'); // Register plugins and middleware await app.register(multipart); @@ -46,7 +45,7 @@ async function bootstrap() { // Start the server await app.listen(port, host, (err, address) => { - logger.log(`Server running on ${method}://${host}:${port}`); + logger.log(`Server running on ${host}:${port}`); }); // Log additional information as needed diff --git a/src/modules/place/dto/place.dto.ts b/src/modules/place/dto/place.dto.ts new file mode 100644 index 0000000..83c445f --- /dev/null +++ b/src/modules/place/dto/place.dto.ts @@ -0,0 +1,36 @@ +import { IsString, IsNumber, IsLatitude, IsLongitude, ValidateIf, IsNotEmpty, IsArray, IsOptional } from 'class-validator'; + +export class CreatePlaceDto { + @IsString() + name: string; + + @IsString() + type: string; + + @IsString() + tag: string; + + @IsLatitude() + lat: number; + + @IsLongitude() + lon: number; +} + +export class SearchPlaceDto { + @IsArray() + @IsNotEmpty() + geofenceBoundary: number[][]; // Array of [lon, lat] pairs defining the geofence (polygon) + + @IsOptional() + @IsString() + name?: string; + + @IsOptional() + @IsString() + tag?: string; + + @IsOptional() + @IsString() + type?: string; + } \ No newline at end of file diff --git a/src/modules/place/place.controller.ts b/src/modules/place/place.controller.ts new file mode 100644 index 0000000..fb3edcf --- /dev/null +++ b/src/modules/place/place.controller.ts @@ -0,0 +1,20 @@ +import { Body, Controller, Get, Post } from "@nestjs/common"; +import { ApiTags } from "@nestjs/swagger"; +import { PlaceService } from "./place.service"; +import { CreatePlaceDto, SearchPlaceDto } from "./dto/place.dto"; + +@ApiTags('/place') +@Controller('place') +export class PlaceController { + constructor(private readonly placeService: PlaceService) {} + + @Post('search') + async searchPlace(@Body() searchPlaceDto: SearchPlaceDto) { + return this.placeService.searchPlaces(searchPlaceDto); + } + + @Post() + async addPlace(@Body() createPlaceDto: CreatePlaceDto) { + return this.placeService.createPlace(createPlaceDto); + } +} \ No newline at end of file diff --git a/src/modules/place/place.module.ts b/src/modules/place/place.module.ts new file mode 100644 index 0000000..b02e395 --- /dev/null +++ b/src/modules/place/place.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { PlaceController } from './place.controller'; +import { PlaceService } from './place.service'; + +@Module({ + controllers: [PlaceController], + providers: [PlaceService], +}) +export class PlaceModule {} diff --git a/src/modules/place/place.service.ts b/src/modules/place/place.service.ts new file mode 100644 index 0000000..229c728 --- /dev/null +++ b/src/modules/place/place.service.ts @@ -0,0 +1,52 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { CreatePlaceDto, SearchPlaceDto } from "./dto/place.dto"; +import { PrismaService } from "../prisma/prisma.service"; + +@Injectable() +export class PlaceService { + private readonly logger = new Logger(PlaceService.name); + + constructor(private readonly prisma: PrismaService) { } + + async createPlace(createPlaceDto: CreatePlaceDto): Promise { + const { name, type, tag, lat, lon } = createPlaceDto; + const point = `ST_SetSRID(ST_MakePoint(${lon}, ${lat}), 4326)`; + this.logger.debug(`Adding place ${createPlaceDto}`) + return this.prisma.$executeRawUnsafe( + `INSERT INTO "Place" (name, type, tag, location) VALUES ($1, $2, $3, ${point})`, + name, + type, + tag, + ); + } + + async searchPlaces(searchPlaceDto: SearchPlaceDto): Promise { + const { geofenceBoundary, name, tag, type } = searchPlaceDto; + + // Create a polygon for the geofence + const polygon = `ST_MakePolygon(ST_GeomFromText('LINESTRING(${geofenceBoundary + .map((point) => point.join(' ')) + .join(', ')}, ${geofenceBoundary[0].join(' ')})', 4326))`; + + // Base query for filtering places + let query = ` + SELECT id, name, type, tag, ST_AsText(location::geometry) as location + FROM "Place" + WHERE ST_Within(location, ${polygon}) + `; + + // Add optional filters + if (name) { + query += ` AND name ILIKE '%${name}%'`; + } + else if (tag) { + query += ` AND tag ILIKE '%${tag}%'`; + } + else if (type) { + query += ` AND type ILIKE '%${type}%'`; + } + + // Execute the query + return this.prisma.$queryRawUnsafe(query); + } +} From 1a411f78db771284a16d7b2338ea6797b2d9cfbd Mon Sep 17 00:00:00 2001 From: Dhruv Baliyan Date: Thu, 5 Dec 2024 09:47:57 +0530 Subject: [PATCH 09/11] feat: fix setup script --- .env.sample | 1 + setup.sh | 41 ++++++++++++++++++-- src/main.ts | 6 +-- src/modules/georev/georev.controller.spec.ts | 4 +- src/modules/georev/georev.controller.ts | 4 +- 5 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 .env.sample diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..f7c85ff --- /dev/null +++ b/.env.sample @@ -0,0 +1 @@ +DATABASE_URL=postgresql://admin:password@192.168.1.43:5555/gis \ No newline at end of file diff --git a/setup.sh b/setup.sh index 49657a3..b2a711b 100755 --- a/setup.sh +++ b/setup.sh @@ -1,7 +1,40 @@ mkdir ./src/geojson-data +is_wget2_installed() { + if command -v wget2 &> /dev/null; then + return 0 # wget2 is installed + else + return 1 # wget2 is not installed + fi +} + +if is_wget2_installed; then + echo "wget2 is already installed." +else + # Check if the OS is macOS or Linux + if [[ "$(uname)" == "Darwin" ]]; then + echo "macOS detected. Installing wget2 using Homebrew..." + # Check if Homebrew is installed, if not install it + if ! command -v brew &> /dev/null; then + echo "Homebrew not found. Installing Homebrew..." + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + fi + # Install wget2 using Homebrew + brew install wget + elif [[ "$(uname)" == "Linux" ]]; then + echo "Linux detected. Installing wget2 using apt..." + # Update package list and install wget2 using apt + sudo apt update + sudo apt install wget2 -y + else + echo "Unsupported OS detected." + exit 1 + fi +fi + +# curl -o ./db.mmdb -L --fail --compressed https://mmdbcdn.posthog.net # getting the latest db.mmdb -curl -o ./db.mmdb -L --fail --compressed https://mmdbcdn.posthog.net +wget2 -O db.mmdb https://mmdbcdn.posthog.net cd ./src @@ -45,6 +78,6 @@ cd ../.. # Updating geoJSON files through script to make them usable in src cd ./scripts npx ts-node parse.geojson.ts - -# Changing PWD back to /server/ -cd - &> /dev/null +npx ts-node ingestors/state.geojson.ts +npx ts-node ingestors/district.geojson.ts +npx ts-node ingestors/subdistrict.geojson.ts \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index b807fa0..10cc06e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -22,8 +22,6 @@ async function bootstrap() { const logger = new Logger('Main'); // 'Main' is the context name const configService = app.get(ConfigService); - const port = configService.get('port'); - const host = configService.get('host'); // Register plugins and middleware await app.register(multipart); @@ -44,8 +42,8 @@ async function bootstrap() { SwaggerModule.setup('api-docs', app, document); // Start the server - await app.listen(port, host, (err, address) => { - logger.log(`Server running on ${host}:${port}`); + await app.listen(3000, '0.0.0.0', (err, address) => { + logger.log(`Server running on 0.0.0.0:3000`); }); // Log additional information as needed diff --git a/src/modules/georev/georev.controller.spec.ts b/src/modules/georev/georev.controller.spec.ts index ef8bf2e..1e52a9a 100644 --- a/src/modules/georev/georev.controller.spec.ts +++ b/src/modules/georev/georev.controller.spec.ts @@ -80,7 +80,7 @@ describe('GeorevController', () => { const result = await controller.getGeoRev(lat, lon); } catch (error) { expect(error).toBeInstanceOf(HttpException); - expect(error.getStatus()).toBe(HttpStatus.INTERNAL_SERVER_ERROR); + expect(error.getStatus()).toBe(HttpStatus.BAD_REQUEST); expect(error.getResponse()).toEqual({ status: 'fail', error: 'lat lon query missing', @@ -98,7 +98,7 @@ describe('GeorevController', () => { } catch (error) { expect(error).toBeInstanceOf(HttpException); - expect(error.getStatus()).toBe(HttpStatus.INTERNAL_SERVER_ERROR); + expect(error.getStatus()).toBe(HttpStatus.BAD_REQUEST); expect(error.getResponse()).toEqual({ status: 'fail', error: 'Invalid latitude or longitude', diff --git a/src/modules/georev/georev.controller.ts b/src/modules/georev/georev.controller.ts index 076ff08..e61c2ca 100644 --- a/src/modules/georev/georev.controller.ts +++ b/src/modules/georev/georev.controller.ts @@ -25,7 +25,7 @@ export class GeorevController { this.logger.error(`lat lon query missing`); throw new HttpException( { status: 'fail', error: `lat lon query missing` }, - HttpStatus.INTERNAL_SERVER_ERROR, + HttpStatus.BAD_REQUEST, ); } @@ -33,7 +33,7 @@ export class GeorevController { this.logger.error('Invalid latitude or longitude'); throw new HttpException( { 'status': 'fail', 'error': 'Invalid latitude or longitude' }, - HttpStatus.INTERNAL_SERVER_ERROR, + HttpStatus.BAD_REQUEST, ); } From e87d8834f8cafd1db279d69423c68ed215610110 Mon Sep 17 00:00:00 2001 From: Dhruv Baliyan Date: Thu, 5 Dec 2024 14:37:54 +0530 Subject: [PATCH 10/11] fix: docker image fix --- Dockerfile | 98 +++++++++--------------------- docker-compose.yaml | 77 ++--------------------- package.json | 9 ++- setup.sh | 5 +- src/modules/place/place.service.ts | 26 +++++--- 5 files changed, 60 insertions(+), 155 deletions(-) diff --git a/Dockerfile b/Dockerfile index 802aaab..9e86d54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,79 +1,37 @@ -#FROM node:18.16.1-alpine -# -#COPY setup.sh -# -#RUN apk add --no-cache bash -#RUN npm i -g @nestjs/cli typescript ts-node -# -#COPY package*.json /tmp/app/ -#RUN cd /tmp/app && npm install -# -#COPY . /usr/src/app -#RUN cp -a /tmp/app/node_modules /usr/src/app -#COPY ./wait-for-it.sh /opt/wait-for-it.sh -#COPY ./startup.dev.sh /opt/startup.dev.sh -#RUN sed -i 's/ -#//g' /opt/wait-for-it.sh -#RUN sed -i 's/ -#//g' /opt/startup.dev.sh -# -#WORKDIR /usr/src/app -#RUN cp env-example .env -#RUN npx prisma generate -#RUN npm run build -# -#CMD ["/opt/startup.dev.sh"] -# -#EXPOSE 3000 +FROM node:18-slim as base +RUN apt-get update -y && apt-get install -y openssl - -#FROM node:18.16.1-alpine -# -#WORKDIR /usr/src/app -# -#COPY . . -# -#RUN apt-get update && apt-get install -y curl && apt-get install -y git -#CMD /bin/bash -#COPY ./package*.json ./ -#RUN ./setup.sh -# -#ENV NODE_ENV production -#CMD ["npm", "i"] -#CMD [ "npm", "run", "start:dev" ] -# -#EXPOSE 3000 - - -FROM node:20.11.0-alpine - -# Set the working directory -WORKDIR /usr/src/app - -# Install curl and git using apk -RUN apk update && apk add --no-cache curl git - -RUN #npm config set registry http://registry.npmjs.org/ - -# Copy package files first for better caching of npm install -COPY ./package*.json ./ - -# Install dependencies +FROM base AS install +WORKDIR /app +COPY package*.json ./ RUN npm install -# Copy the rest of the application code +FROM base as build +WORKDIR /app +COPY prisma ./prisma/ +COPY --from=install /app/node_modules ./node_modules +RUN npx prisma generate COPY . . +RUN npm run build -# Convert setup.sh to Unix-style line endings (LF) +FROM base as data +WORKDIR /app +COPY --from=install /app/node_modules ./node_modules +COPY . . RUN sed -i 's/\r$//' ./setup.sh - - -# Run any additional setup script RUN chmod +x ./setup.sh RUN ./setup.sh -# Set environment variable -ENV NODE_ENV production - -# Start the application -CMD ["npm", "run", "start:dev"] +FROM base +WORKDIR /app +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/dist ./dist +COPY --from=build /app/package*.json ./ +COPY --from=build /app/prisma ./prisma +COPY --from=data /app/db.mmdb ./db.mmdb +COPY --from=data /app/src/geojson-data ./src/geojson-data +COPY ./src ./src +COPY tsconfig.json ./tsconfig.json +EXPOSE 3000 + +CMD ["npm", "run", "migrate:ingest:start:prod"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index fe27d15..ba075fe 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,74 +1,9 @@ services: - fusionauth: - image: fusionauth/fusionauth-app:latest - depends_on: - postgres: - condition: service_healthy - environment: - DATABASE_URL: jdbc:postgresql://postgres:5432/fusionauth - DATABASE_ROOT_USERNAME: ${POSTGRES_USER} - DATABASE_ROOT_PASSWORD: ${POSTGRES_PASSWORD} - DATABASE_USERNAME: ${DATABASE_USERNAME} - DATABASE_PASSWORD: ${DATABASE_PASSWORD} - FUSIONAUTH_APP_MEMORY: ${FUSIONAUTH_APP_MEMORY} - FUSIONAUTH_APP_RUNTIME_MODE: ${FUSIONAUTH_APP_RUNTIME_MODE} - FUSIONAUTH_APP_URL: http://fusionauth:9011 - FUSIONAUTH_APP_KICKSTART_FILE: /usr/local/fusionauth/kickstarts/kickstart.json - env_file: - - ./env-example - volumes: - - fa-config:/usr/local/fusionauth/config - - ./kickstart:/usr/local/fusionauth/kickstarts - networks: - - default - restart: unless-stopped - ports: - - 9011:9011 - - postgres: - image: postgres:15.3-alpine + geoquery: + build: + context: . + dockerfile: Dockerfile ports: - - ${DATABASE_PORT}:5432 - volumes: - - ./.data/db:/var/lib/postgresql/data + - "3000:3000" environment: - POSTGRES_USER: ${DATABASE_USERNAME} - POSTGRES_PASSWORD: ${DATABASE_PASSWORD} - POSTGRES_DB: ${DATABASE_NAME} - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U postgres'] - interval: 5s - timeout: 5s - retries: 5 - - shadow-postgres: - image: postgres:15.3-alpine - ports: - - ${SHADOW_DATABASE_PORT}:5432 - volumes: - - ./.data/shadow-db:/var/lib/postgresql/data - environment: - POSTGRES_USER: ${SHADOW_DATABASE_USERNAME} - POSTGRES_PASSWORD: ${SHADOW_DATABASE_PASSWORD} - POSTGRES_DB: ${SHADOW_DATABASE_NAME} - - cache: - image: redis:6.2-alpine - restart: always - ports: - - '${CACHE_PORT}:6379' - command: redis-server --save 20 1 - volumes: - - cache:/data - - # api: - # build: - # context: . - # dockerfile: Dockerfile - # ports: - # - 3000:3000 -volumes: - fa-config: - cache: -networks: - default: + DATABASE_URL: postgresql://admin:password@192.168.1.43:5555/gis \ No newline at end of file diff --git a/package.json b/package.json index 568eeae..b54407a 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "start:dev": "stencil start --watch", "start:debug": "stencil start --debug --watch", "start:prod": "NODE_ENV=production node dist/main", + "migrate:ingest:start:prod": "npx ts-node src/scripts/ingestors/state.geojson.ts && npx ts-node src/scripts/ingestors/district.geojson.ts && npx ts-node src/scripts/ingestors/subdistrict.geojson.ts && npm run migrate:start:prod", + "migrate:start:prod": "npx prisma migrate deploy && npm run start:prod", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "test": "jest", "test:watch": "jest --watch", @@ -41,7 +43,12 @@ "multer": "^1.4.5-lts.1", "reflect-metadata": "^0.1.13", "request-ip": "^3.3.0", - "rxjs": "^7.8.1" + "rxjs": "^7.8.1", + "ts-jest": "^29.1.0", + "ts-loader": "^9.4.3", + "ts-node": "^10.9.2", + "tsconfig-paths": "^4.2.0", + "typescript": "^5.1.3" }, "devDependencies": { "@nestjs/testing": "^10.0.0", diff --git a/setup.sh b/setup.sh index b2a711b..5678fd1 100755 --- a/setup.sh +++ b/setup.sh @@ -77,7 +77,4 @@ cd ../.. # Updating geoJSON files through script to make them usable in src cd ./scripts -npx ts-node parse.geojson.ts -npx ts-node ingestors/state.geojson.ts -npx ts-node ingestors/district.geojson.ts -npx ts-node ingestors/subdistrict.geojson.ts \ No newline at end of file +npx ts-node parse.geojson.ts \ No newline at end of file diff --git a/src/modules/place/place.service.ts b/src/modules/place/place.service.ts index 229c728..a3d03d9 100644 --- a/src/modules/place/place.service.ts +++ b/src/modules/place/place.service.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger } from "@nestjs/common"; +import { HttpException, HttpStatus, Injectable, Logger } from "@nestjs/common"; import { CreatePlaceDto, SearchPlaceDto } from "./dto/place.dto"; import { PrismaService } from "../prisma/prisma.service"; @@ -12,12 +12,16 @@ export class PlaceService { const { name, type, tag, lat, lon } = createPlaceDto; const point = `ST_SetSRID(ST_MakePoint(${lon}, ${lat}), 4326)`; this.logger.debug(`Adding place ${createPlaceDto}`) - return this.prisma.$executeRawUnsafe( - `INSERT INTO "Place" (name, type, tag, location) VALUES ($1, $2, $3, ${point})`, - name, - type, - tag, - ); + try { + return this.prisma.$executeRawUnsafe( + `INSERT INTO "Place" (name, type, tag, location) VALUES ($1, $2, $3, ${point})`, + name, + type, + tag, + ); + } catch (error) { + throw new HttpException('Error adding places', HttpStatus.INTERNAL_SERVER_ERROR); + } } async searchPlaces(searchPlaceDto: SearchPlaceDto): Promise { @@ -46,7 +50,11 @@ export class PlaceService { query += ` AND type ILIKE '%${type}%'`; } - // Execute the query - return this.prisma.$queryRawUnsafe(query); + try { + // Execute the query + return this.prisma.$queryRawUnsafe(query); + } catch (error) { + throw new HttpException('Error querying database', HttpStatus.INTERNAL_SERVER_ERROR); + } } } From a193ac4ed16813dc0c6baebb48ffcf06203a6856 Mon Sep 17 00:00:00 2001 From: KDwevedi Date: Tue, 24 Dec 2024 09:40:21 +0530 Subject: [PATCH 11/11] using postgis geography instead of postgis geometry --- prisma/migrations/alter_place/migration.sql | 6 ++ prisma/schema.prisma | 2 +- src/modules/place/dto/place.dto.ts | 21 +++++-- src/modules/place/place.service.ts | 63 ++++++++++++--------- 4 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 prisma/migrations/alter_place/migration.sql diff --git a/prisma/migrations/alter_place/migration.sql b/prisma/migrations/alter_place/migration.sql new file mode 100644 index 0000000..2de094b --- /dev/null +++ b/prisma/migrations/alter_place/migration.sql @@ -0,0 +1,6 @@ +-- Enable PostGIS extension (if not already enabled) +CREATE EXTENSION IF NOT EXISTS postgis; + +-- Alter the "location" column to use the geography type +ALTER TABLE "Place" + ALTER COLUMN "location" TYPE geography(Point, 4326) USING "location"::geography; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index df535c8..20e0b98 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -80,5 +80,5 @@ model Place { name String type String tag String - location Unsupported("geometry(Point, 4326)") + location Unsupported("geography(Point, 4326)") } diff --git a/src/modules/place/dto/place.dto.ts b/src/modules/place/dto/place.dto.ts index 83c445f..ca7d066 100644 --- a/src/modules/place/dto/place.dto.ts +++ b/src/modules/place/dto/place.dto.ts @@ -18,19 +18,28 @@ export class CreatePlaceDto { } export class SearchPlaceDto { + @ValidateIf((dto) => !dto.geofenceBoundary) @IsArray() - @IsNotEmpty() - geofenceBoundary: number[][]; // Array of [lon, lat] pairs defining the geofence (polygon) - + origin?: [number, number]; // [longitude, latitude] + + @ValidateIf((dto) => !dto.geofenceBoundary) + @IsNumber() + distance?: number; // Distance in meters + + @ValidateIf((dto) => !dto.origin || !dto.distance) + @IsArray() + geofenceBoundary?: number[][]; // Polygon boundary + @IsOptional() @IsString() name?: string; - + @IsOptional() @IsString() tag?: string; - + @IsOptional() @IsString() type?: string; - } \ No newline at end of file +} + diff --git a/src/modules/place/place.service.ts b/src/modules/place/place.service.ts index 229c728..6c07321 100644 --- a/src/modules/place/place.service.ts +++ b/src/modules/place/place.service.ts @@ -1,6 +1,7 @@ import { Injectable, Logger } from "@nestjs/common"; import { CreatePlaceDto, SearchPlaceDto } from "./dto/place.dto"; import { PrismaService } from "../prisma/prisma.service"; +import { Prisma } from "@prisma/client"; @Injectable() export class PlaceService { @@ -10,43 +11,49 @@ export class PlaceService { async createPlace(createPlaceDto: CreatePlaceDto): Promise { const { name, type, tag, lat, lon } = createPlaceDto; - const point = `ST_SetSRID(ST_MakePoint(${lon}, ${lat}), 4326)`; - this.logger.debug(`Adding place ${createPlaceDto}`) - return this.prisma.$executeRawUnsafe( - `INSERT INTO "Place" (name, type, tag, location) VALUES ($1, $2, $3, ${point})`, - name, - type, - tag, - ); + + const query = ` + INSERT INTO "Place" (name, type, tag, location) + VALUES ($1, $2, $3, ST_SetSRID(ST_MakePoint($4, $5), 4326)::geography) + `; + + this.logger.debug(`Adding place: ${JSON.stringify(createPlaceDto)}`); + return this.prisma.$executeRawUnsafe(query, name, type, tag, lon, lat); } + + - async searchPlaces(searchPlaceDto: SearchPlaceDto): Promise { - const { geofenceBoundary, name, tag, type } = searchPlaceDto; - - // Create a polygon for the geofence - const polygon = `ST_MakePolygon(ST_GeomFromText('LINESTRING(${geofenceBoundary - .map((point) => point.join(' ')) - .join(', ')}, ${geofenceBoundary[0].join(' ')})', 4326))`; - // Base query for filtering places - let query = ` + async searchPlaces(searchPlaceDto: SearchPlaceDto): Promise { + const { origin, distance, name, tag, type } = searchPlaceDto; + + if (!origin || !distance) { + throw new Error('Origin and distance are required for radial searches.'); + } + + const [longitude, latitude] = origin; + + // Construct the base query + let query = Prisma.sql` SELECT id, name, type, tag, ST_AsText(location::geometry) as location FROM "Place" - WHERE ST_Within(location, ${polygon}) + WHERE ST_DWithin(location, ST_SetSRID(ST_MakePoint(${longitude}, ${latitude}), 4326)::geography, ${distance}) `; - - // Add optional filters + + // Add filters dynamically if (name) { - query += ` AND name ILIKE '%${name}%'`; + query = Prisma.sql`${query} AND name ILIKE ${`%${name}%`}`; } - else if (tag) { - query += ` AND tag ILIKE '%${tag}%'`; + if (tag) { + query = Prisma.sql`${query} AND tag ILIKE ${`%${tag}%`}`; } - else if (type) { - query += ` AND type ILIKE '%${type}%'`; + if (type) { + query = Prisma.sql`${query} AND type ILIKE ${`%${type}%`}`; } - - // Execute the query - return this.prisma.$queryRawUnsafe(query); + + this.logger.debug(`Executing search query: ${query.statement}`); + return this.prisma.$queryRaw(query); } + + }