Update domain table to include line number as part of unique constrai…

…nt, like access.
johnspurlock · Sep 22, 2021 · 7b31df4 · 7b31df4
1 parent 2412a74
commit 7b31df4
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 11 deletions.
diff --git a/src/database.ts b/src/database.ts
@@ -1,7 +1,7 @@
 import { DB } from 'https://deno.land/x/sqlite@v2.4.2/mod.ts';
 import { AccessRecord, checkAccessRecord, checkDomainRecord, DomainRecord, isAccessRecordBeta1, isDomainRecordBeta1 } from './model.ts';
 
-const VERSION = 3;
+const VERSION = 4;
 
 export class Database {
     private readonly _db: DB;
@@ -26,6 +26,7 @@ export class Database {
 
         this._db.query(`create table if not exists domain${VERSION}(
             filename text not null,
+            line integer not null,
             bundleId text not null,
             domain text not null,
             context text not null,
@@ -37,7 +38,7 @@ export class Database {
             hits integer not null,
             domainOwner string not null,
             firstTimeStamp string not null,
-            primary key (filename, bundleId, domain, context, initiatedType)) without rowid`);
+            primary key (filename, line)) without rowid`);
     }
 
     getFilenames(): string[] {
@@ -80,12 +81,12 @@ export class Database {
         this._db.query(`delete from domain${VERSION} where filename = ?`, [ filename ]);
     }
 
-    insertDomain(filename: string, bundleId: string, record: DomainRecord) {
+    insertDomain(filename: string, line: number, bundleId: string, record: DomainRecord) {
         checkDomainRecord(record);
         const effectiveUserId = isDomainRecordBeta1(record) ? record.effectiveUserId : undefined;
         const hasAppBundleName = isDomainRecordBeta1(record) ? record['hasApp.bundleName'] : undefined;
-        this._db.query(`insert into domain${VERSION}(filename, bundleId, domain, effectiveUserId, domainType, timeStamp, hasAppBundleName, context, hits, domainOwner, initiatedType, firstTimeStamp) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, 
-                    [filename, bundleId, record.domain, effectiveUserId, record.domainType, record.timeStamp, hasAppBundleName, record.context, record.hits, record.domainOwner, record.initiatedType, record.firstTimeStamp]);
+        this._db.query(`insert into domain${VERSION}(filename, line, bundleId, domain, effectiveUserId, domainType, timeStamp, hasAppBundleName, context, hits, domainOwner, initiatedType, firstTimeStamp) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, 
+                    [filename, line, bundleId, record.domain, effectiveUserId, record.domainType, record.timeStamp, hasAppBundleName, record.context, record.hits, record.domainOwner, record.initiatedType, record.firstTimeStamp]);
         if (this._db.changes !== 1) throw new Error(`Failed to insert domain record`);
     }
 

diff --git a/src/importer.ts b/src/importer.ts
@@ -10,7 +10,7 @@ export function importReportFile(text: string, filename: string, db: Database) {
     let recordType = '';
     let recordTypeVersion = 0;
     let foundEndOfSection = false;
-    let nextLine = 1;
+    let lineNumber = 1;
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
         if (line === '') continue;
@@ -51,13 +51,13 @@ export function importReportFile(text: string, filename: string, db: Database) {
                 const timestamp = convertZonedTimestampToUtc(parsed.timeStamp);
                 const record = { ...parsed, timestamp };
                 const outOfProcess = record.outOfProcess !== undefined ? record.outOfProcess.toString() : undefined;
-                db.insertAccess(filename, nextLine++, record, record.category, outOfProcess);
+                db.insertAccess(filename, lineNumber, record, record.category, outOfProcess);
             } else if (obj.type === 'networkActivity') {
                 const parsed = parseDomainRecordBeta3(obj, line);
                 const timeStamp = convertZonedTimestampToUtc(parsed.timeStamp);
                 const firstTimeStamp = convertZonedTimestampToUtc(parsed.firstTimeStamp);
                 const record = { ...parsed, timeStamp, firstTimeStamp };
-                db.insertDomain(filename, record.bundleID, record);
+                db.insertDomain(filename, lineNumber, record.bundleID, record);
             } else {
                 throw new Error(`Unexpected type: ${obj.type}`);
             }
@@ -68,17 +68,18 @@ export function importReportFile(text: string, filename: string, db: Database) {
                 const timestamp = convertZonedTimestampToUtc(parsed.timestamp);
                 const record = { ...parsed, timestamp };
                 const category = isAccessRecordBeta2(parsed) ? parsed.category : undefined;
-                db.insertAccess(filename, nextLine++, record, category, undefined);
+                db.insertAccess(filename, lineNumber, record, category, undefined);
             } else if (recordType === 'networkActivity') {
                 const parsed = parseDomainRecordBeta2(obj, line);
                 const timeStamp = convertZonedTimestampToUtc(parsed.timeStamp);
                 const firstTimeStamp = convertZonedTimestampToUtc(parsed.firstTimeStamp);
                 const record = { ...parsed, timeStamp, firstTimeStamp };
-                db.insertDomain(filename, record.bundleID, record);
+                db.insertDomain(filename, lineNumber, record.bundleID, record);
             }
         } else {
             throw new Error(`Bad line, expected access record: ${line}`);
         }
+        lineNumber++;
     }
 }
 
@@ -121,12 +122,13 @@ function importDomainRecordsBeta1(json: string, db: Database, filename: string)
     const obj = JSON.parse(json);
     if (typeof obj !== 'object') throw new Error(`importDomainRecordsBeta1: expected object, found ${typeof obj}`);
     db.clearDomain(filename);
+    let lineNumber = 1;
     for (const bundleId of Object.keys(obj)) {
         const records = obj[bundleId];
         if (!Array.isArray(records)) throw new Error(`importDomainRecordsBeta1: expected records array, found ${typeof records}`);
         for (const record of records) {
             if (!isDomainRecordBeta1(record)) throw new Error(`importDomainRecordsBeta1: Bad record: ${JSON.stringify(record)}`);
-            db.insertDomain(filename, bundleId, record);
+            db.insertDomain(filename, lineNumber++, bundleId, record);
         }
     }
 }