Add passive database checkpoints (#2944)

* Change performCheckpoint() to accept a string as to what sort of database checkpoint to perform to reduce system I/O when performing a checkpoint
* Add passive checkpoints at stages when performing certain operations
* Perform a checkpoint before database shutdown
* Change database synchronous mode to 'normal' given all the database synchronized(lock) for each operation to reduce system I/O where possible

src/itemdb.d

@@ -328,7 +328,7 @@ final class ItemDatabase {
 			// The synchronous setting determines how carefully SQLite writes data to disk, balancing between performance and data safety.
 			// https://sqlite.org/pragma.html#pragma_synchronous
 			// PRAGMA synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
-			db.exec("PRAGMA synchronous=FULL;");
+			db.exec("PRAGMA synchronous=NORMAL;");
 		} catch (SqliteException exception) {
 			detailSQLErrorMessage(exception);
 		} 
@@ -1052,8 +1052,8 @@ final class ItemDatabase {
 		}
 	}
 
-	// Perform a checkpoint by writing the data into to the database from the WAL file
-	void performCheckpoint() {
+	// Perform a checkpoint (either TRUNCATE or PASSIVE) by writing the data into to the database from the WAL file
+	void performCheckpoint(string checkpointType) {
 		synchronized(databaseLock) {
 			// Log what we are attempting to do
 			if (debugLogging) {addLogEntry("Attempting to perform a database checkpoint to merge temporary data", ["debug"]);}
@@ -1078,7 +1078,8 @@ final class ItemDatabase {
 				}
 
 				// Ensure there are no pending operations by performing a checkpoint
-				db.exec("PRAGMA wal_checkpoint(TRUNCATE);");
+				string databaseCommand = format("PRAGMA wal_checkpoint(%s);" , checkpointType);
+				db.exec(databaseCommand);
 				if (debugLogging) {addLogEntry("Database checkpoint is complete", ["debug"]);}
 
 			} catch (SqliteException exception) {

src/main.d

@@ -1070,7 +1070,7 @@ int main(string[] cliArgs) {
 
 						// Write WAL and SHM data to file for this loop and release memory used by in-memory processing
 						if (debugLogging) {addLogEntry("Merge contents of WAL and SHM files into main database file", ["debug"]);}
-						itemDB.performCheckpoint();
+						itemDB.performCheckpoint("TRUNCATE");
 					} else {
 						// Not online
 						addLogEntry("Microsoft OneDrive service is not reachable at this time. Will re-try on next sync attempt.");
@@ -1583,12 +1583,20 @@ void shutdownSyncEngine() {
 void shutdownDatabase() {
     if (itemDB !is null && itemDB.isDatabaseInitialised()) {
 		if (debugLogging) {addLogEntry("Shutting down Database instance", ["debug"]);}
+
+		// Write WAL and SHM data to file
+		if (debugLogging) {addLogEntry("Merge contents of WAL and SHM files into main database file before shutting down database", ["debug"]);}
+		itemDB.performCheckpoint("TRUNCATE");
+
+		// Do we perform a database vacuum?
 		if (performDatabaseVacuum) {
 			// Logging to attempt this is denoted from performVacuum() - so no need to confirm here
 			itemDB.performVacuum();
 			// If this completes, it is denoted from performVacuum() - so no need to confirm here
 		}
-		itemDB.closeDatabaseFile(); // Close the DB File Handle
+
+		 // Close the DB File Handle
+		itemDB.closeDatabaseFile();
 		object.destroy(itemDB);
 		cleanupDatabaseFiles(runtimeDatabaseFile);
 		itemDB = null;

src/sync.d

@@ -866,9 +866,7 @@ class SyncEngine {
 			// When using 'sync_list' we need to do this
 			sharedFolderDeltaGeneration = true;
 			currentSharedFolderName = sharedFolderName;
-
 			generateSimulatedDeltaResponse = true;
-
 		}
 
 		// Reset latestDeltaLink & deltaLinkCache
@@ -1195,6 +1193,9 @@ class SyncEngine {
 
 				// To finish off the JSON processing items, this is needed to reflect this in the log
 				if (debugLogging) {addLogEntry("------------------------------------------------------------------", ["debug"]);}
+
+				// For this set of items, perform a DB PASSIVE checkpoint
+				itemDB.performCheckpoint("PASSIVE");
 			}
 
 			if (appConfig.verbosityCount == 0) {
@@ -2369,6 +2370,9 @@ class SyncEngine {
 			// send an array containing 'appConfig.getValueLong("threads")' JSON items to download
 			downloadOneDriveItemsInParallel(chunk);
 		}
+
+		// For this set of items, perform a DB PASSIVE checkpoint
+		itemDB.performCheckpoint("PASSIVE");
 	}
 
 	// Download items in parallel
@@ -4233,8 +4237,10 @@ class SyncEngine {
 		// For each batch of files to upload, upload the changed data to OneDrive
 		foreach (chunk; databaseItemsWhereContentHasChanged.chunks(batchSize)) {
 			processChangedLocalItemsToUploadInParallel(chunk);
-			chunk = null;
 		}
+
+		// For this set of items, perform a DB PASSIVE checkpoint
+		itemDB.performCheckpoint("PASSIVE");
 	}
 
 	// Process all the changed local items in parallel
@@ -5642,6 +5648,9 @@ class SyncEngine {
 		foreach (chunk; newLocalFilesToUploadToOneDrive.chunks(batchSize)) {
 			uploadNewLocalFileItemsInParallel(chunk);
 		}
+
+		// For this set of items, perform a DB PASSIVE checkpoint
+		itemDB.performCheckpoint("PASSIVE");
 	}
 
 	// Upload the file batches in parallel
@@ -8578,6 +8587,9 @@ class SyncEngine {
 				// send an array containing 'appConfig.getValueLong("threads")' JSON items to resume upload
 				resumeSessionUploadsInParallel(chunk);
 			}
+
+			// For this set of items, perform a DB PASSIVE checkpoint
+			itemDB.performCheckpoint("PASSIVE");
 		}
 	}