Merge pull request #396 from the-qa-company/GH-395-dump-getter

GH-395 dump getter, config with a little fix
the-qa-company · Aug 17, 2023 · c808627 · c808627
2 parents 42d3aa6 + f869c3f
commit c808627
Show file tree

Hide file tree

Showing 8 changed files with 101 additions and 25 deletions.
diff --git a/...int-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java b/...int-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java
@@ -183,6 +183,11 @@ public ResponseEntity<Sparql.MergeRequestResult> dumpStore() {
 		return ResponseEntity.status(HttpStatus.OK).body(sparql.askForADump());
 	}
 
+	@GetMapping("/is_dumping")
+	public ResponseEntity<Sparql.IsDumpingResult> isDumping() {
+		return ResponseEntity.status(HttpStatus.OK).body(sparql.isDumping());
+	}
+
 	@GetMapping("/reindex")
 	public ResponseEntity<Sparql.LuceneIndexRequestResult> reindex() throws Exception {
 		return ResponseEntity.status(HttpStatus.OK).body(sparql.reindexLucene());

diff --git a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java
@@ -167,6 +167,18 @@ public boolean isMerging() {
 		}
 	}
 
+	public static class IsDumpingResult {
+		private final boolean dumping;
+
+		public IsDumpingResult(boolean dumping) {
+			this.dumping = dumping;
+		}
+
+		public boolean isDumping() {
+			return dumping;
+		}
+	}
+
 	public static class LoadFileResult {
 		private boolean loaded;
 
@@ -482,17 +494,28 @@ public MergeRequestResult askForADump() {
 		if (endpoint == null) {
 			throw new ServerWebInputException("No endpoint store, bad config?");
 		}
-		Path outLocation = Path.of("dump")
+
+		Path outLocation = compiledSail.getOptions().getDumpLocation()
 				.resolve(DateTimeFormatter.ofPattern("yyy-MM-dd HHmmss").format(LocalDateTime.now()));
 		return new MergeRequestResult(this.sparqlRepository.askDump(outLocation));
 	}
 
+	/**
+	 * @return if the store is dumping
+	 */
+	public IsDumpingResult isDumping() {
+		if (endpoint == null) {
+			throw new ServerWebInputException("No endpoint store, bad config?");
+		}
+		return new IsDumpingResult(endpoint.isDumping());
+	}
+
 	/**
 	 * @return if the store is merging
 	 */
 	public IsMergingResult isMerging() {
 		if (endpoint == null) {
-			throw new ServerWebInputException("No enpoint store, bad config?");
+			throw new ServerWebInputException("No endpoint store, bad config?");
 		}
 		return new IsMergingResult(endpoint.isMergeTriggered);
 	}

diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/CompiledSailOptions.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/CompiledSailOptions.java
@@ -29,6 +29,7 @@ public class CompiledSailOptions {
 	private boolean debugShowPlans;
 	private boolean debugShowCount;
 	private boolean optimization;
+	private Path dumpLocation;
 	private IRI storageMode;
 	private IRI hdtReadMode;
 	private IRI passMode;
@@ -58,6 +59,7 @@ public CompiledSailOptions() {
 			timeoutUpdate = debugOptions.timeoutUpdate;
 			timeoutQuery = debugOptions.timeoutQuery;
 			hdtOptions = debugOptions.hdtOptions;
+			dumpLocation = debugOptions.dumpLocation;
 			return;
 		}
 		// set default values
@@ -76,6 +78,7 @@ public CompiledSailOptions() {
 		timeoutUpdate = SailCompilerSchema.TIMEOUT_UPDATE.getHandler().defaultValue();
 		timeoutQuery = SailCompilerSchema.TIMEOUT_QUERY.getHandler().defaultValue();
 		hdtOptions = Map.of();
+		dumpLocation = Path.of("dump");
 	}
 
 	/**
@@ -84,7 +87,8 @@ public CompiledSailOptions() {
 	 * @param reader the reader
 	 */
 	void readOptions(SailCompiler.SailCompilerReader reader) {
-		reader.search(SailCompilerSchema.MAIN, SailCompilerSchema.OPTION).forEach(this::add);
+		reader.search(SailCompilerSchema.MAIN, SailCompilerSchema.OPTION)
+				.forEach(v -> add(v, reader.getSailCompiler()));
 		storageMode = reader.searchPropertyValue(SailCompilerSchema.MAIN, SailCompilerSchema.STORAGE_MODE_PROPERTY);
 		passMode = reader.searchPropertyValue(SailCompilerSchema.MAIN, SailCompilerSchema.HDT_PASS_MODE_PROPERTY);
 		rdf4jSplitUpdate = reader.searchPropertyValue(SailCompilerSchema.MAIN,
@@ -106,11 +110,12 @@ void readOptions(SailCompiler.SailCompilerReader reader) {
 										.map(reader.getSailCompiler()::asLitString)
 										.orElseThrow(() -> new SailCompiler.SailCompilerException(
 												"Found HDT param without value!"))));
-
+		dumpLocation = Path.of(reader.searchPropertyValue(SailCompilerSchema.MAIN, SailCompilerSchema.DUMP_LOCATION));
 	}
 
-	private void add(Value value) {
-		IRI iri = SailCompilerSchema.OPTION_PROPERTY.throwIfNotValidValue(value);
+	private void add(Value value, SailCompiler compiler) {
+		// an iri parsing
+		IRI iri = SailCompilerSchema.OPTION_PROPERTY.throwIfNotValidValue(value, compiler);
 
 		if (SailCompilerSchema.DEBUG_SHOW_TIME.equals(iri)) {
 			debugShowTime = true;
@@ -179,6 +184,10 @@ public IRI getHdtReadMode() {
 		return hdtReadMode;
 	}
 
+	public Path getDumpLocation() {
+		return dumpLocation;
+	}
+
 	public void setHdtReadMode(IRI hdtReadMode) {
 		this.hdtReadMode = hdtReadMode;
 	}
@@ -219,6 +228,10 @@ public String getHdtSpec() {
 		return hdtSpec;
 	}
 
+	public void setDumpLocation(Path dumpLocation) {
+		this.dumpLocation = dumpLocation;
+	}
+
 	/**
 	 * create {@link HDTOptions} from the config
 	 *

diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SailCompiler.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SailCompiler.java
@@ -445,14 +445,15 @@ public Optional<Value> searchOneOpt(Resource subject, IRI predicate) throws Sail
 		 *
 		 * @param subject  the subject to read
 		 * @param property the property to read and check values
-		 * @return the value
+		 * @return the value or the default property value if no value was found
 		 * @throws SailCompilerException if the value isn't a valid value for
 		 *                               this property
 		 */
 		public <T> T searchPropertyValue(Resource subject,
 				SailCompilerSchema.Property<T, ? extends SailCompilerSchema.ValueHandler<T>> property)
 				throws SailCompilerException {
-			return searchOneOpt(subject, property.getIri()).map(property::throwIfNotValidValue)
+			return searchOneOpt(subject, property.getIri())
+					.map(v -> property.throwIfNotValidValue(v, getSailCompiler()))
 					.orElseGet(property.getHandler()::defaultValue);
 		}
 

diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SailCompilerSchema.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/compiler/SailCompilerSchema.java
@@ -307,6 +307,12 @@ public class SailCompilerSchema {
 	public static final IRI NO_OPTIMIZATION = OPTION_PROPERTY.getHandler().createValue("noOptimization",
 			"Disable optimization for native stores");
 
+	/**
+	 * mdlc:dumpLocation
+	 */
+	public static final Property<String, StringTypeValueHandler> DUMP_LOCATION = propertyStr("dumpLocation",
+			"Dump location", "dump");
+
 	private static IRI iri(String name, String desc) {
 		return propertyVoid(name, desc).getIri();
 	}
@@ -410,12 +416,11 @@ public static void writeToMarkdown(OutputStream stream) {
 					}
 					w.println();
 				}
-			} else if (property.getHandler() instanceof NumberTypeValueHandler) {
+			} else if (property.getHandler() instanceof NumberTypeValueHandler h) {
 				w.println("### Value");
 				w.println();
 				w.println("Number value");
 				w.println();
-				NumberTypeValueHandler h = (NumberTypeValueHandler) property.getHandler();
 
 				w.println("- default value: " + h.defaultValue());
 
@@ -488,14 +493,15 @@ public String getDescription() {
 		}
 
 		/**
-		 * throw if an value isn't in the valid values
+		 * throw if a value isn't in the valid values
 		 *
-		 * @param value the value to test
+		 * @param value    the value to test
+		 * @param compiler the current compiler
 		 * @return the value
 		 * @throws SailCompiler.SailCompilerException if the value isn't valid
 		 */
-		public V throwIfNotValidValue(Value value) throws SailCompiler.SailCompilerException {
-			return handler.validate(value);
+		public V throwIfNotValidValue(Value value, SailCompiler compiler) throws SailCompiler.SailCompilerException {
+			return handler.validate(value, compiler);
 		}
 	}
 
@@ -506,17 +512,18 @@ public V throwIfNotValidValue(Value value) throws SailCompiler.SailCompilerExcep
 	 */
 	public interface ValueHandler<T> {
 		static ValueHandler<Value> id() {
-			return v -> v;
+			return (v, c) -> v;
 		}
 
 		/**
 		 * validate and return the converted value
 		 *
-		 * @param v the value to check
+		 * @param v        the value to check
+		 * @param compiler the current compiler
 		 * @return the converted value
 		 * @throws SailCompiler.SailCompilerException if the value isn't valid
 		 */
-		T validate(Value v) throws SailCompiler.SailCompilerException;
+		T validate(Value v, SailCompiler compiler) throws SailCompiler.SailCompilerException;
 
 		default T defaultValue() {
 			throw new SailCompiler.SailCompilerException("No default value for this property");
@@ -556,12 +563,13 @@ public String defaultValue() {
 		}
 
 		@Override
-		public String validate(Value v) throws SailCompiler.SailCompilerException {
-			if (!(v instanceof Literal)) {
+		public String validate(Value v, SailCompiler compiler) throws SailCompiler.SailCompilerException {
+			if (!v.isLiteral()) {
 				throw new SailCompiler.SailCompilerException(
 						v + " is not a valid literal value for the property " + parent);
 			}
-			String val = v.stringValue();
+			String val = compiler.asLitString(v);
+
 			if (regex != null) {
 				if (!regex.matcher(val).matches()) {
 					throw new SailCompiler.SailCompilerException(
@@ -623,7 +631,7 @@ public Set<IRI> getValues() {
 		}
 
 		@Override
-		public IRI validate(Value v) throws SailCompiler.SailCompilerException {
+		public IRI validate(Value v, SailCompiler compiler) throws SailCompiler.SailCompilerException {
 			if (!(v instanceof IRI)) {
 				throw new SailCompiler.SailCompilerException(
 						v + " is not a valid iri value for the property " + parent);
@@ -677,18 +685,27 @@ public NumberTypeValueHandler withRange(int min, int max) {
 		}
 
 		@Override
-		public Integer validate(Value v) throws SailCompiler.SailCompilerException {
-			if (!(v instanceof Literal)) {
+		public Integer validate(Value v, SailCompiler compiler) throws SailCompiler.SailCompilerException {
+			if (!v.isLiteral()) {
 				throw new SailCompiler.SailCompilerException(
 						v + " is not a valid literal value for the property " + parent);
 			}
 			Literal l = (Literal) v;
-			if (!l.getCoreDatatype().asXSDDatatype().orElseThrow(() -> new SailCompiler.SailCompilerException(
+			int value;
+			if (PARSED_STRING_DATATYPE.equals(l.getDatatype())) {
+				try {
+					value = Integer.parseInt(compiler.asLitString(l));
+				} catch (NumberFormatException e) {
+					throw new SailCompiler.SailCompilerException(
+							"Can't parse int for property " + parent + ": " + e.getMessage());
+				}
+			} else if (l.getCoreDatatype().asXSDDatatype().orElseThrow(() -> new SailCompiler.SailCompilerException(
 					l + " is not a valid number xsd literal for the property " + parent)).isIntegerDatatype()) {
+				value = l.intValue();
+			} else {
 				throw new SailCompiler.SailCompilerException(
 						l + " is not a valid number literal for the property " + parent);
 			}
-			int value = ((Literal) v).intValue();
 
 			if (value > max || value < min) {
 				throw new SailCompiler.SailCompilerException(

diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java
@@ -60,6 +60,7 @@
 import java.nio.file.Path;
 import java.util.Iterator;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -140,6 +141,7 @@ public class EndpointStore extends AbstractNotifyingSail {
 	private final EndpointFiles endpointFiles;
 	private MergeRunnable.MergeThread<?> mergerThread;
 	private final AtomicReference<EndpointStoreDump> dump = new AtomicReference<>();
+	private final AtomicBoolean dumping = new AtomicBoolean();
 
 	public void deleteNativeLocks() throws IOException {
 		// remove lock files of a hard shutdown (SAIL is already locked by
@@ -880,6 +882,7 @@ public boolean dump(EndpointStoreDump dump) {
 		if (this.dump.getAndUpdate(old -> old == null ? dump : old) != null) {
 			return false;
 		}
+		dumping.set(true);
 		mergeStore(true, false);
 		return true;
 	}
@@ -1070,6 +1073,14 @@ public AtomicReference<EndpointStoreDump> getDumpRef() {
 		return dump;
 	}
 
+	public boolean isDumping() {
+		return dumping.get();
+	}
+
+	public void setDumping(boolean val) {
+		dumping.set(val);
+	}
+
 	long getDebugId() {
 		return debugId;
 	}

diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java
@@ -566,6 +566,8 @@ private synchronized void step1(boolean restarting, Lock switchLock) throws Inte
 
 		// #391: stop the lucene updates
 		if (dumpInfo != null) {
+			// "fix" to the race condition during step 2 mfl
+			endpoint.setDumping(true);
 			dumpInfo.beforeMerge(endpoint);
 		}
 
@@ -626,6 +628,7 @@ private synchronized void step2(boolean restarting, Lock lock, EndpointStoreDump
 		// #391: save DUMP HDT
 		if (dumpInfo != null) {
 			dumpInfo.afterMerge(endpoint, Path.of(endpointFiles.getHDTNewIndex()));
+			endpoint.setDumping(endpoint.getDumpRef().get() != null);
 		}
 
 		debugStepPoint(MergeRunnableStopPoint.STEP2_END);

diff --git a/qendpoint-store/src/main/resources/repo_model.ttl b/qendpoint-store/src/main/resources/repo_model.ttl
@@ -28,6 +28,9 @@
 # Enable to show the plan of the executed queries
 # mdlc:main mdlc:option mdlc:debugShowPlan .
 
+# Set the dump location for the askDump functions, by default "dump" inside the root directory is used
+mdlc:main mdlc:dumpLocation "${locationHDT}dump"^^mdlc:parsedString .
+
 # Describe custom strings for the mdlc:parsedString format, will be available with ${key}
 mdlc:main mdlc:parsedStringParam [ mdlc:paramKey "luceneEvalMode" ; mdlc:paramValue "NATIVE" ] .
 mdlc:main mdlc:parsedStringParam [ mdlc:paramKey "luceneWktFields" ; mdlc:paramValue "http://nuts.de/geometry https://linkedopendata.eu/prop/direct/P127" ] .