chore: log progress when long-running loops

src/features/domain.ts

@@ -21,6 +21,7 @@ import { rawDomainTypeCsvToModel } from '@etabli/src/models/mappers/raw-domain';
 import { prisma } from '@etabli/src/prisma';
 import { watchGracefulExitInLoop } from '@etabli/src/server/system';
 import { getListDiff } from '@etabli/src/utils/comparaison';
+import { formatArrayProgress } from '@etabli/src/utils/format';
 import { containsHtml } from '@etabli/src/utils/html';
 import { sleep } from '@etabli/src/utils/sleep';
 
@@ -312,10 +313,10 @@ export async function updateRobotsTxtOnDomains() {
     },
   });
 
-  for (const rawDomain of rawDomains) {
+  for (const [rawDomainIndex, rawDomain] of Object.entries(rawDomains)) {
     watchGracefulExitInLoop();
 
-    console.log(`try to process robots.txt for domain ${rawDomain.name} (${rawDomain.id})`);
+    console.log(`try to process robots.txt for domain ${rawDomain.name} (${rawDomain.id}) ${formatArrayProgress(rawDomainIndex, rawDomains.length)}`);
 
     try {
       const rootUrl = new URL(`https://${rawDomain.name}`);
@@ -429,10 +430,12 @@ export async function updateWildcardCertificateOnDomains() {
     },
   });
 
-  for (const rawDomain of rawDomains) {
+  for (const [rawDomainIndex, rawDomain] of Object.entries(rawDomains)) {
     watchGracefulExitInLoop();
 
-    console.log(`try to process SSL certificate for domain ${rawDomain.name} (${rawDomain.id})`);
+    console.log(
+      `try to process SSL certificate for domain ${rawDomain.name} (${rawDomain.id}) ${formatArrayProgress(rawDomainIndex, rawDomains.length)}`
+    );
 
     const certificate = await new Promise<PeerCertificate | null>((resolve, reject) => {
       const request = https.request(
@@ -493,10 +496,12 @@ export async function updateWebsiteDataOnDomains() {
     },
   });
 
-  for (const rawDomain of rawDomains) {
+  for (const [rawDomainIndex, rawDomain] of Object.entries(rawDomains)) {
     watchGracefulExitInLoop();
 
-    console.log(`try to process website content for domain ${rawDomain.name} (${rawDomain.id})`);
+    console.log(
+      `try to process website content for domain ${rawDomain.name} (${rawDomain.id}) ${formatArrayProgress(rawDomainIndex, rawDomains.length)}`
+    );
 
     try {
       const url = new URL(`https://${rawDomain.name}`);
@@ -736,10 +741,15 @@ export async function matchDomains() {
     return (b.name.match(/\./g) || []).length - (a.name.match(/\./g) || []).length;
   });
 
-  for (const rawDomainToUpdate of rawDomainsToUpdate) {
+  for (const [rawDomainToUpdateIndex, rawDomainToUpdate] of Object.entries(rawDomainsToUpdate)) {
     watchGracefulExitInLoop();
 
-    console.log(`try to bind similar domains to domain ${rawDomainToUpdate.name} (${rawDomainToUpdate.id})`);
+    console.log(
+      `try to bind similar domains to domain ${rawDomainToUpdate.name} (${rawDomainToUpdate.id}) ${formatArrayProgress(
+        rawDomainToUpdateIndex,
+        rawDomainsToUpdate.length
+      )}`
+    );
 
     const rootRawDomain = sortedRootRawDomains.find((d) => {
       // Look for `.abc.com` in case of `subdomain.abc.com`

src/features/initiative.ts

@@ -37,6 +37,7 @@ import { analyzeWithSemgrep } from '@etabli/src/semgrep/index';
 import { watchGracefulExitInLoop } from '@etabli/src/server/system';
 import { getListDiff } from '@etabli/src/utils/comparaison';
 import { capitalizeFirstLetter } from '@etabli/src/utils/format';
+import { formatArrayProgress } from '@etabli/src/utils/format';
 import { languagesExtensions } from '@etabli/src/utils/languages';
 import { sleep } from '@etabli/src/utils/sleep';
 import { WappalyzerResultSchema } from '@etabli/src/wappalyzer';
@@ -465,10 +466,10 @@ export async function feedInitiativesFromDatabase() {
     const initiativeGptTemplate = handlebars.compile(initiativeGptTemplateContent);
 
     // To debug easily we write each result on disk (also, since using lot of CLIs we have no other choice :D)
-    for (const initiativeMap of initiativeMaps) {
+    for (const [initiativeMapIndex, initiativeMap] of Object.entries(initiativeMaps)) {
       watchGracefulExitInLoop();
 
-      console.log(`feed initiative ${initiativeMap.id}`);
+      console.log(`feed initiative ${initiativeMap.id} ${formatArrayProgress(initiativeMapIndex, initiativeMaps.length)}`);
 
       if (initiativeMap.RawDomainsOnInitiativeMaps.length > 0 && initiativeMap.RawRepositoriesOnInitiativeMaps.length > 0) {
         // TODO: for test for now we just skip those not having both types
@@ -502,10 +503,15 @@ export async function feedInitiativesFromDatabase() {
       }
 
       let mixedInitiativeTools: string[] = [];
-      for (const rawDomainOnIMap of initiativeMap.RawDomainsOnInitiativeMaps) {
+      for (const [rawDomainOnIMapIndex, rawDomainOnIMap] of Object.entries(initiativeMap.RawDomainsOnInitiativeMaps)) {
         watchGracefulExitInLoop();
 
-        console.log(`domain ${rawDomainOnIMap.rawDomain.name} ${rawDomainOnIMap.main ? '(main)' : ''} (${rawDomainOnIMap.rawDomain.id})`);
+        console.log(
+          `domain ${rawDomainOnIMap.rawDomain.name} ${rawDomainOnIMap.main ? '(main)' : ''} (${rawDomainOnIMap.rawDomain.id}) ${formatArrayProgress(
+            rawDomainOnIMapIndex,
+            initiativeMap.RawDomainsOnInitiativeMaps.length
+          )}`
+        );
 
         const rawDomain = rawDomainOnIMap.rawDomain;
 
@@ -567,13 +573,13 @@ export async function feedInitiativesFromDatabase() {
         }
       }
 
-      for (const rawRepositoryOnIMap of initiativeMap.RawRepositoriesOnInitiativeMaps) {
+      for (const [rawRepositoryOnIMapIndex, rawRepositoryOnIMap] of Object.entries(initiativeMap.RawRepositoriesOnInitiativeMaps)) {
         watchGracefulExitInLoop();
 
         console.log(
           `repository ${rawRepositoryOnIMap.rawRepository.repositoryUrl} ${rawRepositoryOnIMap.main ? '(main)' : ''} (${
             rawRepositoryOnIMap.rawRepository.id
-          })`
+          }) ${formatArrayProgress(rawRepositoryOnIMapIndex, initiativeMap.RawRepositoriesOnInitiativeMaps.length)}`
         );
 
         const rawRepository = rawRepositoryOnIMap.rawRepository;

src/features/repository.ts

@@ -12,6 +12,7 @@ import { rawRepositoryPlatformJsonToModel } from '@etabli/src/models/mappers/raw
 import { prisma } from '@etabli/src/prisma';
 import { watchGracefulExitInLoop } from '@etabli/src/server/system';
 import { getListDiff } from '@etabli/src/utils/comparaison';
+import { formatArrayProgress } from '@etabli/src/utils/format';
 import { emptyStringtoNullPreprocessor } from '@etabli/src/utils/validation';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -325,11 +326,13 @@ export async function updateInferredMetadataOnRepositories() {
     },
   });
 
-  for (const rawRepository of rawRepositories) {
+  for (const [rawRepositoryIndex, rawRepository] of Object.entries(rawRepositories)) {
     watchGracefulExitInLoop();
 
     console.log(
-      `try to locally infer metadata for repository {${rawRepository.platform}} ${rawRepository.organizationName}/${rawRepository.name} (${rawRepository.id})`
+      `try to locally infer metadata for repository {${rawRepository.platform}} ${rawRepository.organizationName}/${rawRepository.name} (${
+        rawRepository.id
+      }) ${formatArrayProgress(rawRepositoryIndex, rawRepositories.length)}`
     );
 
     // If there is a declared URL, take it as it comes
@@ -436,11 +439,13 @@ export async function matchRepositories() {
     },
   });
 
-  for (const rawRepositoryToUpdate of rawRepositoriesToUpdate) {
+  for (const [rawRepositoryToUpdateIndex, rawRepositoryToUpdate] of Object.entries(rawRepositoriesToUpdate)) {
     watchGracefulExitInLoop();
 
     console.log(
-      `try to bind similar repositories to repository {${rawRepositoryToUpdate.platform}} ${rawRepositoryToUpdate.organizationName}/${rawRepositoryToUpdate.name} (${rawRepositoryToUpdate.id})`
+      `try to bind similar repositories to repository {${rawRepositoryToUpdate.platform}} ${rawRepositoryToUpdate.organizationName}/${
+        rawRepositoryToUpdate.name
+      } (${rawRepositoryToUpdate.id}) ${formatArrayProgress(rawRepositoryToUpdateIndex, rawRepositoriesToUpdate.length)}`
     );
 
     // The repositories to look for should be under the same platform and organization (unlikely people would set code for the same application at multiple locations)

src/utils/format.ts

@@ -1,3 +1,9 @@
 export function capitalizeFirstLetter(value: string) {
   return value.charAt(0).toUpperCase() + value.slice(1);
 }
+
+export function formatArrayProgress(arrayIndex: string | number, arrayLength: number) {
+  const currentIndex = typeof arrayIndex === 'string' ? parseInt(arrayIndex, 10) : arrayIndex;
+
+  return `(${currentIndex + 1}/${arrayLength})`;
+}