jersey-servlet
@@ -29,21 +25,22 @@
/ui5/*
- connectorview
+ rulesview
+ BASIC
+
+
- connectorconfig
- connectorview
- connectorschema
- connectoroperator
+ rulesview
+ rulesadmin
diff --git a/WebContent/index.html b/WebContent/index.html
new file mode 100644
index 0000000..36d9d0e
--- /dev/null
+++ b/WebContent/index.html
@@ -0,0 +1,37 @@
+
+
+
+ Application Directory
+
+
+
+
+
+
+
+ Loading OpenUI5 from here
+
+
diff --git a/WebContent/ui5/Config.controller.js b/WebContent/ui5/Config.controller.js
new file mode 100644
index 0000000..58253ce
--- /dev/null
+++ b/WebContent/ui5/Config.controller.js
@@ -0,0 +1,31 @@
+sap.ui.define(["sap/ui/core/mvc/Controller"],
+function(Controller) {"use strict";
+return Controller.extend("io.rtdi.bigdata.rulesservice.ui5.Config", {
+ onInit : function() {
+ var model = new sap.ui.model.json.JSONModel();
+ model.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading config failed: " + text);
+ });
+ model.loadData("../rest/config");
+ this.getView().setModel(model);
+ },
+ onSave : function(event) {
+ var model = this.getView().getModel();
+ var post = new sap.ui.model.json.JSONModel();
+ post.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Save failed: " + text);
+ });
+ post.attachRequestCompleted(function() {
+ console.log(post.getProperty("/"));
+ });
+ var json = JSON.stringify(model.getProperty("/"));
+ var headers = {
+ "Content-Type": "application/json;charset=utf-8"
+ }
+ post.loadData("../rest/config", json, true, "POST", false, true, headers);
+ },
+});
+});
+
diff --git a/WebContent/ui5/Config.html b/WebContent/ui5/Config.html
new file mode 100644
index 0000000..a0002eb
--- /dev/null
+++ b/WebContent/ui5/Config.html
@@ -0,0 +1,35 @@
+
+
+
+ Connector
+
+
+
+
+
+
+ Loading OpenUI5 from here
+
+
+
diff --git a/WebContent/ui5/Config.view.xml b/WebContent/ui5/Config.view.xml
new file mode 100644
index 0000000..cccb9ae
--- /dev/null
+++ b/WebContent/ui5/Config.view.xml
@@ -0,0 +1,38 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/WebContent/ui5/Rule.controller.js b/WebContent/ui5/Rule.controller.js
new file mode 100644
index 0000000..2043ee2
--- /dev/null
+++ b/WebContent/ui5/Rule.controller.js
@@ -0,0 +1,257 @@
+sap.ui.define(["sap/ui/core/mvc/Controller"],
+function(Controller) {"use strict";
+return Controller.extend("io.rtdi.bigdata.rulesservice.ui5.Rule", {
+ onInit : function() {
+ var model = new sap.ui.model.json.JSONModel();
+ var subjectname = jQuery.sap.getUriParameters().get("subject");
+ var rulename = jQuery.sap.getUriParameters().get("rule");
+ var isnew = jQuery.sap.getUriParameters().get("new");
+ this.getView().setModel(model);
+ model.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading rule file failed: " + text);
+ });
+ if (isnew) {
+ model.loadData("../rest/subjects/" + encodeURI(subjectname) + "/defaultrule");
+ } else {
+ model.loadData("../rest/subjects/" + encodeURI(subjectname) + "/rules/" + rulename);
+ }
+ var topicmodel = new sap.ui.model.json.JSONModel();
+ this.getView().setModel(topicmodel, "topics");
+ topicmodel.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading topics failed: " + text);
+ });
+ topicmodel.loadData("../rest/topics");
+ this.onSampleRefresh();
+ },
+ onSampleRefresh : function(event) {
+ var samplemodel = new sap.ui.model.json.JSONModel();
+ var samplecontrol = this.getView().byId("samplefile");
+ samplecontrol.setModel(samplemodel, "samplefiles");
+ var subjectname = jQuery.sap.getUriParameters().get("subject");
+ samplemodel.loadData("../rest/sample/" + encodeURI(subjectname));
+ },
+ onAddTab : function(event) {
+ var model = this.getView().getModel();
+ var steps = model.getProperty("/rulesteps");
+ if (steps && steps.length > 0) {
+ var rules = {
+ "type" : "RuleStep",
+ "fieldname" : "next step",
+ "rules" : [ ]
+ };
+ this.addDefaultRules(steps[0]["rules"], rules["rules"]);
+ model.setProperty("/rulesteps/" + steps.length, rules);
+ } else {
+ var subjectname = jQuery.sap.getUriParameters().get("subject");
+ model.loadData("../rest/subjects/" + encodeURI(subjectname) + "/defaultrule");
+ }
+ },
+ onCloseTab : function(event) {
+ var item = event.getParameter("item");
+ var path = item.getBindingContext().getPath();
+ var index = path.substring(path.lastIndexOf('/')+1);
+ var model = this.getView().getModel();
+ var steps = model.getProperty("/rulesteps");
+ steps.pop(index);
+ model.setProperty("/rulesteps", steps);
+ },
+ addDefaultRules : function(from, to) {
+ for ( var r of from) {
+ if (r["type"] === "RecordRule" || r["type"] === "GenericRules" || r["type"] === "ArrayRule") {
+ var n = [];
+ to.push({
+ "type" : r["type"],
+ "fieldname" : r["fieldname"],
+ "rules" : n
+ });
+ this.addDefaultRules(r["rules"], n);
+ } else {
+ to.push({
+ "type" : "EmptyRule",
+ "fieldname" : r["fieldname"],
+ "fielddatatype" : r["fielddatatype"]
+ });
+ }
+ }
+ },
+ resulttypeFormatter : function(resulttype) {
+ if (resulttype === "PASS") {
+ return "sap-icon://message-success";
+ } else if (resulttype === "FAIL") {
+ return "sap-icon://message-warning";
+ } else if (resulttype === "WARN") {
+ return "sap-icon://question-mark";
+ } else {
+ return "sap-icon://message-error";
+ }
+ },
+ onChangeRuleType: function(event) {
+ var targetmodel = this.getView().getModel();
+ var rulepath = event.getParameter("selectedItem").getBindingContext().getPath();
+ var targetrow = targetmodel.getProperty(rulepath);
+ var ruletype = targetrow.type;
+ var parentpath = rulepath.substring(0, rulepath.lastIndexOf("/rules/"));
+ if (parentpath && parentpath.length > 0) {
+ /*
+ * Within a TestSet, there should be always at least on additional item to enter a new rule
+ */
+ var record = targetmodel.getProperty(parentpath);
+ if (record.type.startsWith("TestSet")) {
+ if (record.rules && record.rules.length > 0) {
+ var oLastEntity = record.rules[record.rules.length-1];
+ if (oLastEntity.type !== "EmptyRule") {
+ targetmodel.setProperty(parentpath + "/rules/" + record.rules.length,
+ {fieldname: targetrow.fieldname, type: "EmptyRule", iffalse: "FAIL", sampleresult: null});
+ }
+ }
+ }
+ }
+ if (ruletype === 'EmptyRule') {
+ targetmodel.setProperty(rulepath + "/rules", null);
+ } else if (ruletype === 'PrimitiveRule') {
+ targetmodel.setProperty(rulepath + "/rules", null);
+ if (!!!targetrow.iffalse) {
+ targetmodel.setProperty(rulepath + "/iffalse", "FAIL");
+ }
+ targetmodel.setProperty(rulepath + "/sampleresult", null);
+ } else {
+ var childrules = targetmodel.getProperty(rulepath + "/rules");
+ if (!childrules || childrules.length === 0) {
+ targetmodel.setProperty(rulepath + "/rules",
+ [ { fieldname: targetrow.fieldname, type: 'PrimitiveRule', iffalse: "FAIL", sampleresult: null },
+ { fieldname: targetrow.fieldname, type: 'EmptyRule', iffalse: "FAIL", sampleresult: null } ] );
+ }
+ }
+ },
+ onSave : function(event) {
+ var rulename = jQuery.sap.getUriParameters().get("rule");
+ var model = this.getView().getModel();
+ var newrulename = model.getProperty("/name");
+ var post = new sap.ui.model.json.JSONModel();
+ post.attachRequestCompleted(function() {
+ console.log(post.getProperty("/"));
+ if (rulename !== newrulename) {
+ const url = new URL(window.location)
+ url.searchParams.set("rule", newrulename);
+ url.searchParams.delete("new");
+ history.pushState(null, '', url);
+ }
+ });
+ post.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Saving failed: " + text);
+ });
+ var subjectname = jQuery.sap.getUriParameters().get("subject");
+ if (!rulename) {
+ rulename = "new";
+ }
+ var json = JSON.stringify(model.getProperty("/"));
+ var headers = {
+ "Content-Type": "application/json;charset=utf-8"
+ }
+ post.loadData("../rest/subjects/" + encodeURI(subjectname) + "/rules/" + rulename, json, true, "POST", false, true, headers);
+ },
+ onActivate : function(event) {
+ var rulename = jQuery.sap.getUriParameters().get("rule");
+ var post = new sap.ui.model.json.JSONModel();
+ post.attachRequestCompleted(function() {
+ console.log(post.getProperty("/"));
+ });
+ post.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Activate failed: " + text);
+ });
+ var subjectname = jQuery.sap.getUriParameters().get("subject");
+ var headers = {
+ "Content-Type": "application/json;charset=utf-8"
+ }
+ post.loadData("../rest/subjects/" + encodeURI(subjectname) + "/rules/" + rulename, undefined, true, "PATCH", false, true, headers);
+ },
+ onConditionChange: function(event) {
+ var source = event.getSource();
+ var rulepath = source.getBindingContext().getPath();
+ var editor = this.getView().byId("formulaeditor");
+ editor.setValue(event.getParameter("value"));
+ editor.setEditable(true);
+ editor.data("path", rulepath + "/condition");
+ },
+ onSubstituteChange: function(event) {
+ var source = event.getSource();
+ var rulepath = source.getBindingContext().getPath();
+ var editor = this.getView().byId("formulaeditor");
+ editor.setValue(event.getParameter("value"));
+ editor.setEditable(true);
+ editor.data("path", rulepath + "/substitute");
+ },
+ onEditorChange: function(event) {
+ var source = event.getSource();
+ var rulepath = source.data("path")
+ if (rulepath) {
+ var model = this.getView().getModel();
+ model.setProperty(rulepath, event.getParameter("value"));
+ }
+ },
+ onReadSample : function(event) {
+ var model = this.getView().getModel();
+ if (model.getProperty("/samplefile")) {
+ var json = JSON.stringify(model.getProperty("/"));
+ var headers = {
+ "Content-Type": "application/json;charset=utf-8"
+ }
+ model.loadData("../rest/calculate", json, true, "POST", false, true, headers);
+ } else {
+ sap.m.MessageToast.show("No sample file to read from was selected");
+ }
+ },
+ onSampleReapply : function(event) {
+ this.onReadSample(event);
+ },
+ onStepCalc : function(event) {
+ var model = this.getView().getModel();
+ var source = event.getSource().getParent().getParent();
+ var steppath = source.getBindingContext().getPath(); // i.e. steppath: /rulesteps/0
+ var stepindex = steppath.substring(steppath.lastIndexOf('/') + 1);
+ var json = JSON.stringify(model.getProperty("/"));
+ var headers = {
+ "Content-Type": "application/json;charset=utf-8"
+ }
+ var stepmodel = new sap.ui.model.json.JSONModel();
+ stepmodel.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Calculate failed: " + text);
+ });
+ stepmodel.attachRequestCompleted(function(event) {
+ var successful = event.getParameter("success");
+ if (successful) {
+ model.setProperty("/rulesteps/" + stepindex + "/", stepmodel.getProperty("/"));
+ }
+ });
+ stepmodel.loadData("../rest/calculate/" + stepindex, json, true, "POST", false, true, headers);
+ },
+ onMoveLeft : function(event) {
+ var source = event.getSource().getParent().getParent();
+ var steppath = source.getBindingContext().getPath(); // i.e. steppath: /rulesteps/0
+ var stepindex = parseInt(steppath.substring(steppath.lastIndexOf('/') + 1));
+ if (stepindex > 0) {
+ var model = this.getView().getModel();
+ var steps = model.getProperty("/rulesteps");
+ steps.splice(stepindex-1, 0, steps.splice(stepindex, 1)[0]);
+ model.setProperty("/rulesteps", steps);
+ }
+ },
+ onMoveRight : function(event) {
+ var source = event.getSource().getParent().getParent();
+ var steppath = source.getBindingContext().getPath(); // i.e. steppath: /rulesteps/0
+ var stepindex = parseInt(steppath.substring(steppath.lastIndexOf('/') + 1));
+ var model = this.getView().getModel();
+ var steps = model.getProperty("/rulesteps");
+ if (stepindex < steps.length-1) {
+ steps.splice(stepindex+1, 0, steps.splice(stepindex, 1)[0]);
+ model.setProperty("/rulesteps", steps);
+ }
+ }
+});
+});
+
diff --git a/WebContent/ui5/Rule.html b/WebContent/ui5/Rule.html
new file mode 100644
index 0000000..37d78f4
--- /dev/null
+++ b/WebContent/ui5/Rule.html
@@ -0,0 +1,35 @@
+
+
+
+ Rule definition
+
+
+
+
+
+
+ Loading OpenUI5 from here
+
+
+
diff --git a/WebContent/ui5/Rule.view.xml b/WebContent/ui5/Rule.view.xml
new file mode 100644
index 0000000..378c588
--- /dev/null
+++ b/WebContent/ui5/Rule.view.xml
@@ -0,0 +1,177 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/WebContent/ui5/Rules.controller.js b/WebContent/ui5/Rules.controller.js
new file mode 100644
index 0000000..d1a8b1d
--- /dev/null
+++ b/WebContent/ui5/Rules.controller.js
@@ -0,0 +1,52 @@
+sap.ui.define(["sap/ui/core/mvc/Controller"],
+function(Controller) {"use strict";
+return Controller.extend("io.rtdi.bigdata.rulesservice.ui5.Rules", {
+ onInit : function() {
+ var subjectsmodel = new sap.ui.model.json.JSONModel();
+ subjectsmodel.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading subjects failed: " + text);
+ });
+ this.getView().setModel(subjectsmodel);
+ subjectsmodel.loadData("../rest/subjects");
+ var rulesmodel = new sap.ui.model.json.JSONModel();
+ rulesmodel.attachRequestCompleted(function() {
+ this.setProperty("/" + this.getProperty("/").length, {"name": "create new", "new": true})
+ });
+ rulesmodel.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading rules failed: " + text);
+ });
+ this.getView().setModel(rulesmodel, "rules");
+ },
+ onSubjectSelected : function(event) {
+ var listitem = event.getSource();
+ var context = listitem.getBindingContext();
+ var record = context.getObject();
+ var subjectname = record.name;
+ var rulesmodel = this.getView().getModel("rules");
+ var ruleslist = this.getView().byId("rules");
+ ruleslist.data("subjectname", subjectname);
+ rulesmodel.loadData("../rest/subjects/" + encodeURI(subjectname) + "/rules");
+ },
+ onRuleSelected : function(event) {
+ var listitem = event.getSource();
+ var context = listitem.getBindingContext("rules");
+ var record = context.getObject();
+ var rulename = record.name;
+ var ruleslist = this.getView().byId("rules");
+ var subjectname = ruleslist.data("subjectname");
+ var url = window.location.href;
+ var rulelink;
+ if (record.new) {
+ rulelink = "&new=true";
+ } else {
+ rulelink = "&rule=" + encodeURI(rulename);
+ }
+
+ var ruleurl = url.replace("Rules.html", "Rule.html?subject=" + encodeURI(subjectname) + rulelink);
+ window.open(ruleurl, "_blank");
+ }
+});
+});
+
diff --git a/WebContent/ui5/Rules.html b/WebContent/ui5/Rules.html
new file mode 100644
index 0000000..f6c862a
--- /dev/null
+++ b/WebContent/ui5/Rules.html
@@ -0,0 +1,35 @@
+
+
+
+ Schema and rules
+
+
+
+
+
+
+ Loading OpenUI5 from here
+
+
+
diff --git a/WebContent/ui5/Rules.view.xml b/WebContent/ui5/Rules.view.xml
new file mode 100644
index 0000000..8b58cb1
--- /dev/null
+++ b/WebContent/ui5/Rules.view.xml
@@ -0,0 +1,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/WebContent/ui5/Sample.controller.js b/WebContent/ui5/Sample.controller.js
new file mode 100644
index 0000000..a476804
--- /dev/null
+++ b/WebContent/ui5/Sample.controller.js
@@ -0,0 +1,128 @@
+sap.ui.define(["sap/ui/core/mvc/Controller"],
+ function(Controller) {
+ "use strict";
+ return Controller.extend("io.rtdi.bigdata.rulesservice.ui5.Sample", {
+ onInit: function() {
+ var model = new sap.ui.model.json.JSONModel();
+ this.getView().setModel(model);
+ model.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading topics failed: " + text);
+ });
+ model.loadData("../rest/topics");
+ var table = this.getView().byId("tabledata");
+ var tablemodel = new sap.ui.model.json.JSONModel();
+ table.setModel(tablemodel);
+ },
+ onExecute: function(event) {
+ var table = this.getView().byId("tabledata");
+ var model = table.getModel();
+ var topicselector = this.getView().byId("topicselector");
+ var selectedtopics = topicselector.getSelectedKeys();
+ if (!selectedtopics || selectedtopics.length === 0) {
+ sap.m.MessageToast.show("No input topic(s) selected");
+ } else {
+ fetch("../rest/sample", {
+ method: "POST",
+ body: JSON.stringify(selectedtopics),
+ headers: {
+ "Content-Type": "application/json",
+ }
+ })
+ .then((response) => {
+ if (response.ok) {
+ return response;
+ } else {
+ return Promise.reject(response);
+ }
+ })
+ .then(response => {
+ const reader = response.body.getReader();
+ var utf8decoder = new TextDecoder('utf-8');
+ var messages = [];
+ model.setProperty("/", []);
+ var line = "";
+ return new ReadableStream({
+ start(controller) {
+ function update() {
+ reader.read().then(({ done, value }) => {
+ if (done) {
+ controller.close();
+ return;
+ }
+ let chunk = utf8decoder.decode(value);
+ var pos;
+ do {
+ pos = chunk.indexOf("\n");
+ if (pos >= 0) {
+ line = line + chunk.substring(0, pos);
+ var data = JSON.parse(line);
+ if (data.hasOwnProperty("offset")) {
+ model.setProperty("/" + messages.length, data);
+ messages.push(data);
+ }
+ line = "";
+ chunk = chunk.substring(pos+1);
+ } else {
+ line = line + chunk;
+ }
+
+ } while (pos >= 0);
+ update();
+ });
+ }
+ update();
+ }
+ });
+ })
+ .catch((err) => err.text()
+ .then((s) => {
+ try {
+ var json = JSON.parse(s);
+ sap.m.MessageToast.show(json["errormessage"]);
+ } catch (parseerror) {
+ sap.m.MessageToast.show(s);
+ }
+ })
+ );
+ }
+ },
+ onSavePayload: function(event) {
+ var control = event.getSource().getParent();
+ var path = control.getBindingContext().getPath();
+ var model = control.getModel();
+ var i = path.substring(path.lastIndexOf('/')+1);
+ this.savepayload(model, i);
+ },
+ onSaveSelected: function(event) {
+ var table = this.getView().byId("tabledata");
+ var model = table.getModel();
+ var data = model.getProperty("/");
+ for ( var i=0; i
+
+
+ Collect Sample Data
+
+
+
+
+
+
+ Loading OpenUI5 from here
+
+
+
diff --git a/WebContent/ui5/Sample.view.xml b/WebContent/ui5/Sample.view.xml
new file mode 100644
index 0000000..b27f499
--- /dev/null
+++ b/WebContent/ui5/Sample.view.xml
@@ -0,0 +1,74 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/WebContent/ui5/Topics.controller.js b/WebContent/ui5/Topics.controller.js
new file mode 100644
index 0000000..4179b8c
--- /dev/null
+++ b/WebContent/ui5/Topics.controller.js
@@ -0,0 +1,37 @@
+sap.ui.define(["sap/ui/core/mvc/Controller"],
+function(Controller) {"use strict";
+return Controller.extend("io.rtdi.bigdata.rulesservice.ui5.Topics", {
+ onInit : function() {
+ var model = new sap.ui.model.json.JSONModel();
+ this.getView().setModel(model);
+ model.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading topic rules failed: " + text);
+ });
+ model.loadData("../rest/topicrules");
+ var model2 = new sap.ui.model.json.JSONModel();
+ this.getView().setModel(model2, "rulegroups");
+ model2.attachRequestFailed(function(event) {
+ var text = event.getParameter("responseText");
+ sap.m.MessageToast.show("Reading rules failed: " + text);
+ });
+ model2.loadData("../rest/rules");
+ },
+ onSave : function(event) {
+ var model = this.getView().getModel();
+ var json = JSON.stringify(model.getProperty("/"));
+ var headers = {
+ "Content-Type": "application/json;charset=utf-8"
+ }
+ model.loadData("../rest/topicrules", json, true, "POST", false, true, headers);
+ },
+ onChange: function(event) {
+ var model = this.getView().getModel();
+ var context = event.getSource().getBindingContext();
+ var path = context.getPath();
+ model.setProperty(path + "/modified", true);
+ }
+
+});
+});
+
diff --git a/WebContent/ui5/Topics.html b/WebContent/ui5/Topics.html
new file mode 100644
index 0000000..5413dd7
--- /dev/null
+++ b/WebContent/ui5/Topics.html
@@ -0,0 +1,35 @@
+
+
+
+ Topics and rules
+
+
+
+
+
+
+ Loading OpenUI5 from here
+
+
+
diff --git a/WebContent/ui5/Topics.view.xml b/WebContent/ui5/Topics.view.xml
new file mode 100644
index 0000000..141ee5e
--- /dev/null
+++ b/WebContent/ui5/Topics.view.xml
@@ -0,0 +1,105 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/WebContent/ui5/lib/.library b/WebContent/ui5/lib/.library
new file mode 100644
index 0000000..1a5c7e9
--- /dev/null
+++ b/WebContent/ui5/lib/.library
@@ -0,0 +1,18 @@
+
+
+
+ io.rtdi.bigdata.rulesservice.ui5
+ rtdi.io
+
+ Helper controls for the rulesservice.ui5 app
+
+
+
+ sap.ui.core
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/WebContent/ui5/lib/FocusInput.js b/WebContent/ui5/lib/FocusInput.js
new file mode 100644
index 0000000..7f173d8
--- /dev/null
+++ b/WebContent/ui5/lib/FocusInput.js
@@ -0,0 +1,20 @@
+sap.ui.define([
+ 'sap/m/Input',
+ './library'
+], function(Input) {
+ return Input.extend("io.rtdi.bigdata.rulesservice.ui5.lib.FocusInput", {
+ metadata : {
+ properties : {
+ },
+ aggregations : {
+ },
+ events : {
+ "focusIn" : {"value": {type: "string"}}
+ }
+ },
+ renderer : {},
+ onfocusin : function() {
+ this.fireFocusIn({ "value" : this.getValue()});
+ },
+ });
+});
diff --git a/WebContent/ui5/lib/JSONViewer.js b/WebContent/ui5/lib/JSONViewer.js
new file mode 100644
index 0000000..e2371fc
--- /dev/null
+++ b/WebContent/ui5/lib/JSONViewer.js
@@ -0,0 +1,158 @@
+sap.ui.define(
+[ 'sap/ui/core/Control',
+ './library'
+],
+function(Control) {
+
+ var counter = 0;
+
+ /**
+ * Transform a json object into html representation
+ *
+ * @return string
+ */
+ function json2html(json, controlid, key, collapsed) {
+ var html = '';
+ if (typeof json === 'string') {
+ // Escape tags and quotes
+ json = json.replace(/&/g, '&').replace(//g, '>').replace(/'/g,
+ ''').replace(/"/g, '"');
+
+ // Escape double quotes in the rendered non-URL
+ // string.
+ json = json.replace(/"/g, '\\"');
+ if (key) {
+ html += '"' + key + '" :';
+ }
+ html += '"' + json + '" ';
+ } else if (typeof json === 'number') {
+ if (key) {
+ html += '"' + key + '" :';
+ }
+ html += '' + json + ' ';
+ } else if (typeof json === 'boolean') {
+ if (key) {
+ html += '"' + key + '" :';
+ }
+ html += '' + json + ' ';
+ } else if (json === null) {
+ if (key) {
+ html += '"' + key + '" :';
+ }
+ html += 'null ';
+ } else if (json instanceof Array) {
+ if (key) {
+ html += '↕"' + key + '" :';
+ }
+ if (json.length > 0) {
+ var placeholder = json.length + (json.length > 1 ? ' items' : ' item');
+ html += '[';
+ counter++;
+ for (var i = 0; i < json.length; ++i) {
+ html += '';
+ html += json2html(json[i], controlid, undefined, collapsed);
+ // Add comma if item is not last
+ if (i < json.length - 1) {
+ html += ',';
+ }
+ html += ' ';
+ }
+ html += ' ';
+ html += '' + placeholder + ' ';
+ html += ']';
+ } else {
+ html += '[]';
+ }
+ } else if (typeof json === 'object') {
+ if (key) {
+ html += '↕"' + key + '" :';
+ }
+ var keyCount = Object.keys(json).length;
+ var placeholder = keyCount + (keyCount > 1 ? ' items' : ' item');
+ if (keyCount > 0) {
+ html += '{';
+ for ( var childkey in json) {
+ if (Object.prototype.hasOwnProperty.call(json, childkey)) {
+ html += '';
+ html += json2html(json[childkey], controlid, childkey, collapsed);
+ // Add comma if item is not last
+ if (--keyCount > 0) {
+ html += ',';
+ }
+ html += ' ';
+ }
+ }
+ html += ' ';
+ html += '' + placeholder + ' ';
+ html += '}';
+ } else {
+ html += '{}';
+ }
+ }
+ return html;
+ };
+
+
+ var control = Control.extend(
+ "io.rtdi.bigdata.rulesservice.ui5.lib.JSONViewer",
+ {
+ metadata : {
+ properties : {
+ text : {
+ type : "string",
+ defaultValue : ""
+ },
+ collapsed : {
+ type : "boolean",
+ defaultValue : false
+ }
+ },
+ aggregations : {},
+ },
+
+ renderer : function(oRm, oControl) {
+ oRm.write(""); // will carry the scrollbar
+
+ var outputtext = "";
+ var inputtext = oControl.getText();
+ if (inputtext) {
+ var json = JSON.parse(inputtext);
+ outputtext = json2html(json, oControl.getId(), undefined, oControl.getCollapsed());
+ }
+
+ oRm.write(outputtext);
+
+ oRm.write("
")
+ },
+
+ init : function() {
+ },
+
+ onAfterRendering : function(arguments) {
+ if (sap.ui.core.Control.prototype.onAfterRendering) {
+ sap.ui.core.Control.prototype.onAfterRendering.apply(this, arguments);
+ }
+ },
+
+ });
+
+ control.prototype.onclick = function (oEvent) {
+ var target1 = $(oEvent.target).siblings('ul.json-dict, ol.json-array');
+ var target2 = $(oEvent.target).siblings('.json-placeholder');
+ if (target1.length == 0 || target2.length == 0) {
+ // ignore
+ } else if (target1[0].style.display === 'none') {
+ target1[0].style.display = 'block';
+ target2[0].style.display = 'none';
+ } else {
+ target1[0].style.display = 'none';
+ target2[0].style.display = 'inline';
+ }
+ return false;
+ };
+
+ return control;
+});
diff --git a/WebContent/ui5/lib/RulesToolPage.js b/WebContent/ui5/lib/RulesToolPage.js
new file mode 100644
index 0000000..c0bb19c
--- /dev/null
+++ b/WebContent/ui5/lib/RulesToolPage.js
@@ -0,0 +1,59 @@
+sap.ui.define([
+ 'sap/tnt/ToolPage',
+ './library'
+], function(ToolPage) {
+ return ToolPage.extend("io.rtdi.bigdata.rulesservice.ui5.lib.RulesToolPage", {
+ metadata : {
+ properties : {
+ },
+ aggregations : {
+ },
+ events : {
+ }
+ },
+ renderer : {},
+ init : function() {
+ ToolPage.prototype.init.apply(this, arguments);
+ this.setAggregation("sideContent",
+ new sap.tnt.SideNavigation( {
+ id : "sideNavigation",
+ expanded: false,
+ item : new sap.tnt.NavigationList ( {
+ items: [
+ new sap.tnt.NavigationListItem({
+ text: "Home",
+ icon: "sap-icon://home",
+ href: "../index.html",
+ tooltip: "Home screen"
+ }),
+ new sap.tnt.NavigationListItem({
+ text: "Topics",
+ icon: "sap-icon://inventory",
+ href: "./Topics.html",
+ tooltip: "Rules applied to topic data"
+ }),
+ new sap.tnt.NavigationListItem({
+ text: "Rules",
+ icon: "sap-icon://document-text",
+ href: "./Rules.html",
+ tooltip: "List or create rules for a subject"
+ }),
+ new sap.tnt.NavigationListItem({
+ text: "Sample",
+ icon: "sap-icon://download",
+ href: "./Sample.html",
+ tooltip: "Generate sample data"
+ }),
+ new sap.tnt.NavigationListItem({
+ text: "Config",
+ icon: "sap-icon://wrench",
+ href: "./Config.html",
+ tooltip: "Configure the connections"
+ })
+ ]}
+ )
+ })
+ );
+ }
+ });
+});
diff --git a/WebContent/ui5/lib/library.js b/WebContent/ui5/lib/library.js
new file mode 100644
index 0000000..b388d98
--- /dev/null
+++ b/WebContent/ui5/lib/library.js
@@ -0,0 +1,32 @@
+sap.ui.define([
+ 'sap/ui/core/Lib',
+ 'sap/ui/core/library'
+ ], function(Library) {
+
+ "use strict";
+
+ /**
+ *
+ * @namespace
+ * @alias io.rtdi.bigdata.rulesservice.ui5.lib
+ * @author rtdi.io GmbH
+ * @public
+ */
+
+ const thisLib = Library.init({
+ version: "${version}",
+ name : "io.rtdi.bigdata.rulesservice.ui5.lib",
+ apiVersion: 2,
+ dependencies : ["sap.ui.core"],
+ types: [
+ ],
+ interfaces: [],
+ controls: [
+ "io.rtdi.bigdata.rulesservice.ui5.lib.FocusInput",
+ "io.rtdi.bigdata.rulesservice.ui5.lib.JSONViewer"
+ ],
+ elements: []
+ });
+
+ return thisLib;
+});
\ No newline at end of file
diff --git a/WebContent/ui5/lib/themes/sap_fiori_3/library.css b/WebContent/ui5/lib/themes/sap_fiori_3/library.css
new file mode 100644
index 0000000..680c4e5
--- /dev/null
+++ b/WebContent/ui5/lib/themes/sap_fiori_3/library.css
@@ -0,0 +1,45 @@
+/* Root element */
+.json-document {
+ padding: 1em 2em;
+}
+
+/* Syntax highlighting for JSON objects */
+ul.json-dict, ol.json-array {
+ list-style-type: none;
+ margin: 0 0 0 1px;
+ border-left: 1px dotted #ccc;
+ padding-left: 2em;
+}
+
+.json-string {
+ color: black;
+}
+
+.json-key {
+ color: blue;
+}
+
+.json-key-array {
+ color: blue;
+ cursor: row-resize;
+}
+
+.json-key-record {
+ color: blue;
+ cursor: row-resize;
+}
+
+.json-literal {
+ color: red;
+}
+
+/* Collapsable placeholder links */
+a.json-placeholder {
+ color: #aaa;
+ padding: 0 1em;
+ text-decoration: none;
+}
+
+a.json-placeholder:hover {
+ text-decoration: underline;
+}
\ No newline at end of file
diff --git a/docs/audit-schema.md b/docs/audit-schema.md
new file mode 100644
index 0000000..d81ad15
--- /dev/null
+++ b/docs/audit-schema.md
@@ -0,0 +1,98 @@
+This is the `__audit` field definition within the Avro schema.
+
+```
+ {
+ "__internal": true,
+ "__originalname": "__audit",
+ "default": null,
+ "doc": "If data is transformed this information is recorded here",
+ "name": "__audit",
+ "type": [
+ "null",
+ {
+ "__originalname": "__audit",
+ "doc": "If data is transformed this information is recorded here",
+ "fields": [
+ {
+ "__originalname": "__transformresult",
+ "doc": "Is the record PASS, FAILED or WARN?",
+ "name": "__transformresult",
+ "type": {
+ "length": 4,
+ "logicalType": "VARCHAR",
+ "type": "string"
+ }
+ },
+ {
+ "__originalname": "__details",
+ "default": null,
+ "doc": "Details of all transformations",
+ "name": "__details",
+ "type": [
+ "null",
+ {
+ "items": {
+ "__originalname": "__audit_details",
+ "doc": "Details of all transformations",
+ "fields": [
+ {
+ "__originalname": "__transformationname",
+ "doc": "A name identifying the applied transformation",
+ "name": "__transformationname",
+ "type": {
+ "length": 1024,
+ "logicalType": "NVARCHAR",
+ "type": "string"
+ }
+ },
+ {
+ "__originalname": "__transformresult",
+ "doc": "Is the record PASS, FAIL or WARN?",
+ "name": "__transformresult",
+ "type": {
+ "length": 4,
+ "logicalType": "VARCHAR",
+ "type": "string"
+ }
+ },
+ {
+ "__originalname": "__transformresult_text",
+ "doc": "Transforms can optionally describe what they did",
+ "name": "__transformresult_text",
+ "type": [
+ "null",
+ {
+ "length": 1024,
+ "logicalType": "NVARCHAR",
+ "type": "string"
+ }
+ ]
+ },
+ {
+ "__originalname": "__transformresult_quality",
+ "doc": "Transforms can optionally return a percent value from 0 (FAIL) to 100 (PASS)",
+ "name": "__transformresult_quality",
+ "type": [
+ "null",
+ {
+ "logicalType": "BYTE",
+ "type": "int"
+ }
+ ]
+ }
+ ],
+ "name": "__audit_details",
+ "type": "record"
+ },
+ "type": "array"
+ }
+ ]
+ }
+ ],
+ "name": "__audit",
+ "namespace": "",
+ "type": "record"
+ }
+ ]
+ }
+```
\ No newline at end of file
diff --git a/docs/media/Config.png b/docs/media/Config.png
new file mode 100644
index 0000000..550521f
Binary files /dev/null and b/docs/media/Config.png differ
diff --git a/docs/media/Rule.png b/docs/media/Rule.png
new file mode 100644
index 0000000..a6a4370
Binary files /dev/null and b/docs/media/Rule.png differ
diff --git a/docs/media/RulesService-RuleResult.png b/docs/media/RuleResult.png
similarity index 100%
rename from docs/media/RulesService-RuleResult.png
rename to docs/media/RuleResult.png
diff --git a/docs/media/RulesService-PipelineConfig.png b/docs/media/RulesService-PipelineConfig.png
deleted file mode 100644
index c1fc8a6..0000000
Binary files a/docs/media/RulesService-PipelineConfig.png and /dev/null differ
diff --git a/docs/media/RulesService-RuleDefinition-Level1.png b/docs/media/RulesService-RuleDefinition-Level1.png
deleted file mode 100644
index ef7c91a..0000000
Binary files a/docs/media/RulesService-RuleDefinition-Level1.png and /dev/null differ
diff --git a/docs/media/RulesService-RuleDefinition-Level2.png b/docs/media/RulesService-RuleDefinition-Level2.png
deleted file mode 100644
index 87f2896..0000000
Binary files a/docs/media/RulesService-RuleDefinition-Level2.png and /dev/null differ
diff --git a/docs/media/RulesService-RuleDefinition-Level3.png b/docs/media/RulesService-RuleDefinition-Level3.png
deleted file mode 100644
index dd14b32..0000000
Binary files a/docs/media/RulesService-RuleDefinition-Level3.png and /dev/null differ
diff --git a/docs/media/RulesService-RuleInstances.png b/docs/media/RulesService-RuleInstances.png
deleted file mode 100644
index 7337210..0000000
Binary files a/docs/media/RulesService-RuleInstances.png and /dev/null differ
diff --git a/docs/media/SampleData.png b/docs/media/SampleData.png
new file mode 100644
index 0000000..093dd7b
Binary files /dev/null and b/docs/media/SampleData.png differ
diff --git a/docs/media/Topics.png b/docs/media/Topics.png
new file mode 100644
index 0000000..2af57ed
Binary files /dev/null and b/docs/media/Topics.png differ
diff --git a/docs/rule-syntax.md b/docs/rule-syntax.md
new file mode 100644
index 0000000..986a447
--- /dev/null
+++ b/docs/rule-syntax.md
@@ -0,0 +1 @@
+Note: The library used here is [Apache JEXL](https://commons.apache.org/proper/commons-jexl/reference/syntax.html).
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index a2e8273..361ce68 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,11 +4,15 @@
4.0.0
io.rtdi.bigdata.service
rulesservice
- 0.9.26
+ ${revision}
war
UTF-8
- 0.10.16
+ 0.9.27
+ 2.24.1
+ 3.8.0
+ 3.1.8
+ 2.2.22
${project.artifactId}
@@ -17,8 +21,8 @@
maven-compiler-plugin
3.8.0
- 12
- 12
+ 21
+ 21
@@ -30,28 +34,142 @@
+
+
+ confluent
+ confluent
+ https://packages.confluent.io/maven/
+
+
- io.rtdi.bigdata.connector
- webappfoundation
- ${io.rtdi.bigdata.connector.version}
+ org.junit.jupiter
+ junit-jupiter-api
+ 5.11.2
+ test
- io.rtdi.bigdata.connector
- pipelinekafkatcp
- ${io.rtdi.bigdata.connector.version}
+ org.apache.kafka
+ kafka-streams-test-utils
+ ${kafka.version}
+ test
- junit
- junit
- 4.13.1
- test
+ org.glassfish.jersey.containers
+ jersey-container-servlet
+ ${jersey}
+
+
+ org.glassfish.jersey.core
+ jersey-client
+ ${jersey}
+
+
+ org.glassfish.jersey.inject
+ jersey-hk2
+ ${jersey}
+
+
+ org.glassfish.jaxb
+ jaxb-runtime
+ 4.0.5
+
+
+ org.glassfish.jersey.media
+ jersey-media-json-jackson
+ ${jersey}
+
+
+ org.glassfish.jersey.media
+ jersey-media-jaxb
+ ${jersey}
+
+
+ com.fasterxml.jackson.datatype
+ jackson-datatype-jsr310
+ 2.18.0
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-avro
+ 2.18.0
+
+
+ jakarta.servlet
+ jakarta.servlet-api
+ 6.1.0
+ provided
+
+
+ org.apache.kafka
+ kafka-clients
+ ${kafka.version}
+
+
+ org.apache.kafka
+ kafka-streams
+ ${kafka.version}
+
+
+ io.confluent
+ kafka-schema-registry-client
+ 7.7.1
+
+
+ io.confluent
+ kafka-avro-serializer
+ 7.7.1
+
+
+ org.apache.commons
+ commons-jexl3
+ 3.4.0
+
+
+ com.github.ben-manes.caffeine
+ caffeine
+ 3.1.8
+
+
+ io.rtdi.bigdata.kafka
+ kafkaavro
+ 0.10.15
+
+
+ org.apache.logging.log4j
+ log4j-api
+ ${log4j.version}
+
+
+ org.apache.logging.log4j
+ log4j-core
+ ${log4j.version}
+
+
+ org.apache.logging.log4j
+ log4j-slf4j2-impl
+ runtime
+ ${log4j.version}
+
+
+ org.slf4j
+ slf4j-api
+ 2.1.0-alpha1
+
+
+ io.swagger.core.v3
+ swagger-jaxrs2-jakarta
+ ${swagger}
+
+
+ io.swagger.core.v3
+ swagger-jaxrs2-servlet-initializer-jakarta
+ ${swagger}
- jakarta.servlet
- jakarta.servlet-api
- 5.0.0
- provided
+ io.swagger.core.v3
+ swagger-annotations
+ ${swagger}
diff --git a/src/main/java/io/rtdi/appcontainer/utils/HttpUtil.java b/src/main/java/io/rtdi/appcontainer/utils/HttpUtil.java
new file mode 100644
index 0000000..c1465de
--- /dev/null
+++ b/src/main/java/io/rtdi/appcontainer/utils/HttpUtil.java
@@ -0,0 +1,62 @@
+package io.rtdi.appcontainer.utils;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+
+public class HttpUtil {
+
+ public static final String COOKIES_HEADER = "Set-Cookie";
+
+ public static SSLContext sslcontext;
+ public static HostnameVerifier verifier;
+ private HttpURLConnection conn = null;
+ static {
+ try {
+ verifier = new SSLHostnameVerifierNoop();
+ // HttpsURLConnection.setDefaultHostnameVerifier(verifier);
+ sslcontext = SSLContext.getInstance("TLS");
+ sslcontext.init(null, new TrustManager[]{new SSLTrustManagerNoop()}, new java.security.SecureRandom());
+ } catch (KeyManagementException e) {
+ e.printStackTrace();
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ }
+ }
+
+
+ public HttpUtil() {
+ }
+
+ public void getHttpConnection(URL url, String requestmethod) throws IOException {
+ conn = (HttpURLConnection) url.openConnection();
+ if (conn instanceof HttpsURLConnection) {
+ ((HttpsURLConnection) conn).setSSLSocketFactory(sslcontext.getSocketFactory());
+ ((HttpsURLConnection) conn).setHostnameVerifier(verifier);
+ }
+ conn.setRequestMethod(requestmethod);
+ conn.setDoOutput(true);
+ conn.setDoInput(true);
+ conn.setReadTimeout(60000);
+ }
+
+ public HttpURLConnection getConnection() {
+ return conn;
+ }
+
+ public SSLContext getSSLContext() {
+ return sslcontext;
+ }
+
+ public HostnameVerifier getHostnameVerifier() {
+ return verifier;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/appcontainer/utils/SSLHostnameVerifierNoop.java b/src/main/java/io/rtdi/appcontainer/utils/SSLHostnameVerifierNoop.java
new file mode 100644
index 0000000..5a3fe78
--- /dev/null
+++ b/src/main/java/io/rtdi/appcontainer/utils/SSLHostnameVerifierNoop.java
@@ -0,0 +1,11 @@
+package io.rtdi.appcontainer.utils;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLSession;
+
+public class SSLHostnameVerifierNoop implements HostnameVerifier {
+ @Override
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/appcontainer/utils/SSLTrustManagerNoop.java b/src/main/java/io/rtdi/appcontainer/utils/SSLTrustManagerNoop.java
new file mode 100644
index 0000000..6ad6d98
--- /dev/null
+++ b/src/main/java/io/rtdi/appcontainer/utils/SSLTrustManagerNoop.java
@@ -0,0 +1,21 @@
+package io.rtdi.appcontainer.utils;
+
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.X509TrustManager;
+
+public class SSLTrustManagerNoop implements X509TrustManager {
+ @Override
+ public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+ }
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return new X509Certificate[0];
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/appcontainer/utils/UsageStatisticSender.java b/src/main/java/io/rtdi/appcontainer/utils/UsageStatisticSender.java
new file mode 100644
index 0000000..81b3be7
--- /dev/null
+++ b/src/main/java/io/rtdi/appcontainer/utils/UsageStatisticSender.java
@@ -0,0 +1,62 @@
+package io.rtdi.appcontainer.utils;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.rtdi.bigdata.rulesservice.RulesService;
+import jakarta.ws.rs.ProcessingException;
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.client.ClientBuilder;
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.client.Invocation;
+import jakarta.ws.rs.client.WebTarget;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+public class UsageStatisticSender implements Runnable {
+
+ private UsageStatistics data;
+ private boolean success;
+ private Invocation.Builder invocationBuilder;
+ protected final Logger logger;
+ private HttpUtil httputil;
+
+ public UsageStatisticSender(RulesService serviceListener) {
+ logger = LogManager.getLogger(this.getClass().getName());
+ try {
+ httputil = new HttpUtil();
+ Client client = ClientBuilder.newBuilder().sslContext(httputil.getSSLContext()).hostnameVerifier(httputil.getHostnameVerifier()).build();
+ WebTarget serviceroot = client.target("https://pw2djf8u01.execute-api.eu-central-1.amazonaws.com/Prod");
+ WebTarget usageendpoint = serviceroot.path("usagedata");
+ invocationBuilder = usageendpoint.request(MediaType.APPLICATION_JSON);
+ data = new UsageStatistics(serviceListener);
+ } catch (ProcessingException | IllegalArgumentException e) {
+ logger.error("Failed to create the connection to the UsageServer URL", e);
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ success = false;
+ data.update();
+ Response response = invocationBuilder.post(Entity.entity(data, MediaType.APPLICATION_JSON));
+ if (response.getStatus() == 200) {
+ String result = response.readEntity(String.class);
+ success = "\"saved\"".equals(result);
+ if (success) {
+ logger.info("Sent the usage statistics data successfully");
+ } else {
+ logger.warn("Data to the UsageServer was sent but server returned \"{}\"", result);
+ }
+ }
+ } catch (ProcessingException | IllegalArgumentException e) {
+ logger.error("Failed to send the data to the UsageServer", e);
+ }
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/appcontainer/utils/UsageStatistics.java b/src/main/java/io/rtdi/appcontainer/utils/UsageStatistics.java
new file mode 100644
index 0000000..cb01d7b
--- /dev/null
+++ b/src/main/java/io/rtdi/appcontainer/utils/UsageStatistics.java
@@ -0,0 +1,126 @@
+package io.rtdi.appcontainer.utils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import io.rtdi.bigdata.rulesservice.RulesService;
+
+public class UsageStatistics {
+
+ private List connections;
+ private long starttime;
+ private long endtime;
+
+ public UsageStatistics(RulesService serviceListener) {
+ connections = Collections.singletonList(new Connection(serviceListener));
+ starttime = System.currentTimeMillis();
+ }
+
+ public String getConnectorname() {
+ return "RulesService";
+ }
+
+ public List getConnections() {
+ return connections;
+ }
+
+ public long getStarttime() {
+ return starttime;
+ }
+
+ public long getEndtime() {
+ return endtime;
+ }
+
+ public void update() {
+ connections.get(0).update();
+ starttime = endtime;
+ endtime = System.currentTimeMillis();
+ }
+
+ public static class Connection {
+ private String connectionname;
+ private List producers;
+
+ public Connection(RulesService serviceListener) {
+ this.connectionname = serviceListener.getBootstrapServers();
+ producers = new ArrayList<>();
+ producers.add(new Producer("Rules", serviceListener));
+ update();
+ }
+
+ public String getConnectionname() {
+ return connectionname;
+ }
+
+ public List getProducers() {
+ return producers;
+ }
+
+ public void update() {
+ producers.get(0).update();
+ }
+
+ }
+
+ public static class Producer {
+
+ private String producername;
+ private List instances;
+
+ public Producer(String name, RulesService serviceListener) {
+ this.producername = name;
+ instances = Collections.singletonList(new ProducerInstance(serviceListener));
+ }
+
+ public String getProducername() {
+ return producername;
+ }
+
+ public List getInstances() {
+ return instances;
+ }
+
+ public int getInstancecount() {
+ return 1;
+ }
+
+ public void update() {
+ instances.get(0).update();
+ }
+
+ }
+
+ public static class ProducerInstance {
+
+ private Long lastdatatimestamp;
+ private String state;
+ private long rowsproduced;
+ private RulesService service;
+
+ public ProducerInstance(RulesService serviceListener) {
+ this.service = serviceListener;
+ }
+
+ public void update() {
+ rowsproduced = service.getRowsProduced();
+ state = service.getState();
+ lastdatatimestamp = service.getLastDataTimestamp();
+ }
+
+ public Long getLastdatatimestamp() {
+ return lastdatatimestamp;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public long getRowsproduced() {
+ return rowsproduced;
+ }
+
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/JerseyApplicationSettings.java b/src/main/java/io/rtdi/bigdata/rulesservice/JerseyApplicationSettings.java
index 9eb6b42..7a31e95 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/JerseyApplicationSettings.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/JerseyApplicationSettings.java
@@ -1,16 +1,56 @@
package io.rtdi.bigdata.rulesservice;
-import io.rtdi.bigdata.connector.connectorframework.JerseyApplication;
+import java.nio.file.Path;
-public class JerseyApplicationSettings extends JerseyApplication {
+import org.glassfish.jersey.server.ResourceConfig;
+
+import com.fasterxml.jackson.core.util.JacksonFeature;
+import com.fasterxml.jackson.module.jakarta.xmlbind.JakartaXmlBindAnnotationModule;
+
+import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
+import io.swagger.v3.oas.annotations.OpenAPIDefinition;
+import io.swagger.v3.oas.annotations.info.Info;
+import io.swagger.v3.oas.annotations.servers.Server;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.servlet.ServletContext;
+import jakarta.ws.rs.ApplicationPath;
+
+
+/**
+ * Enable OpenAPI
+ *
+ */
+@OpenAPIDefinition(
+ info = @Info(
+ title = "Rules Service",
+ version = "1.0",
+ description = "A Kafka KStream Rules Service"
+ ),
+ tags = {
+ @Tag(name = "Information", description = "APIs to interact with rules")
+ },
+ servers = {
+ @Server(
+ description = "Rule Service",
+ url = "../../"
+ )
+ }
+ )
+@ApplicationPath("/protected/rest")
+public class JerseyApplicationSettings extends ResourceConfig {
public JerseyApplicationSettings() {
super();
+ packages("io.rtdi.bigdata.rulesservice");
+ register(JacksonFeature.class);
+ register(ObjectMapperContextResolver.class);
+ register(RolesAllowedDynamicFeature2.class);
+ register(new OpenApiResource());
+ register(JakartaXmlBindAnnotationModule.class);
}
- @Override
- protected String[] getPackages() {
- return new String[] {"io.rtdi.bigdata.rulesservice.rest"};
+ public static Path getWebAppRootPath(ServletContext servletcontext) {
+ return Path.of(servletcontext.getRealPath("/"));
}
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/LoggingUtil.java b/src/main/java/io/rtdi/bigdata/rulesservice/LoggingUtil.java
new file mode 100644
index 0000000..dff36a4
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/LoggingUtil.java
@@ -0,0 +1,61 @@
+package io.rtdi.bigdata.rulesservice;
+
+import org.apache.logging.log4j.Logger;
+
+import io.rtdi.bigdata.rulesservice.rest.ErrorResponse;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.Response.Status;
+
+public class LoggingUtil {
+
+ public static void logRequestBegin(Logger log, HttpServletRequest request) {
+ log.info("Execution started: <{}> called by <{}>",
+ request != null ? request.getRequestURI() : "?",
+ request != null && request.getUserPrincipal() != null ? request.getUserPrincipal().getName() : "unknown");
+ }
+
+ public static void logRequestEnd(Logger log, HttpServletRequest request) {
+ log.info("Execution completed: <{}> called by <{}>",
+ request != null ? request.getRequestURI() : "?",
+ request != null && request.getUserPrincipal() != null ? request.getUserPrincipal().getName() : "unknown");
+ }
+
+ public static Response requestEnd(Logger log, HttpServletRequest request, Object payload) {
+ Response response = Response.ok(payload).build();
+ logRequestEnd(log, request);
+ return response;
+ }
+
+ public static Response requestEndTechnicalError(Logger log, HttpServletRequest request, Exception e) {
+ Response response = ErrorResponse.createErrorResponse(e);
+ log.error("Exception", e);
+ logRequestEnd(log, request);
+ return response;
+ }
+
+ public static Response requestEndInputError(Logger log, HttpServletRequest request, Exception e) {
+ Response response = ErrorResponse.createErrorResponse(e, Status.BAD_REQUEST);
+ log.warn("Exception", e);
+ logRequestEnd(log, request);
+ return response;
+ }
+
+ public static Response requestEndInputError(Logger log, HttpServletRequest request, Exception e, Status status) {
+ Response response = ErrorResponse.createErrorResponse(e, status);
+ log.warn("Exception", e);
+ logRequestEnd(log, request);
+ return response;
+ }
+
+ public static void logRequestEndTechnicalError(Logger log, HttpServletRequest request, Exception e) {
+ log.error("Exception", e);
+ logRequestEnd(log, request);
+ }
+
+ public static void logRequestEndInputError(Logger log, HttpServletRequest request, Exception e) {
+ log.warn("Exception", e);
+ logRequestEnd(log, request);
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/ObjectMapperContextResolver.java b/src/main/java/io/rtdi/bigdata/rulesservice/ObjectMapperContextResolver.java
new file mode 100644
index 0000000..0399342
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/ObjectMapperContextResolver.java
@@ -0,0 +1,27 @@
+package io.rtdi.bigdata.rulesservice;
+
+import com.fasterxml.jackson.core.StreamReadFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import jakarta.ws.rs.ext.ContextResolver;
+import jakarta.ws.rs.ext.Provider;
+
+@Provider
+public class ObjectMapperContextResolver implements ContextResolver {
+ private final ObjectMapper MAPPER;
+
+ public ObjectMapperContextResolver() {
+ MAPPER = JsonMapper.builder()
+ .enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION)
+ .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
+ .addModule(new JavaTimeModule())
+ .build();
+ }
+
+ @Override
+ public ObjectMapper getContext(Class type) {
+ return MAPPER;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/PropertiesException.java b/src/main/java/io/rtdi/bigdata/rulesservice/PropertiesException.java
new file mode 100644
index 0000000..449b544
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/PropertiesException.java
@@ -0,0 +1,71 @@
+package io.rtdi.bigdata.rulesservice;
+
+import java.io.IOException;
+
+/**
+ * Something wrong with the properties.
+ *
+ */
+public class PropertiesException extends IOException {
+
+ private static final long serialVersionUID = 116063643000518740L;
+ private String hint = null;
+ private String causingobject = null;
+
+ public PropertiesException() {
+ super();
+ }
+
+ public PropertiesException(String message) {
+ super(message);
+ }
+
+ public PropertiesException(Throwable cause) {
+ super(cause);
+ }
+
+ public PropertiesException(String message, String hint, String causingobject) {
+ this(message);
+ this.hint = hint;
+ this.causingobject = causingobject;
+ }
+
+ public PropertiesException(String message, Throwable cause, String hint) {
+ this(message, cause, hint, null);
+ }
+
+ public PropertiesException(String message, Throwable cause) {
+ this(message, cause, null, null);
+ }
+
+ /**
+ * The preferred constructor, where an error message, the causing exception, a hint of what the user can do and the object causing the problem.
+ *
+ * @param message as free form text
+ * @param cause is the causing exception, optional
+ * @param hint provides optional guidance to the user
+ * @param causingobject identifies the input causing the exception
+ */
+ public PropertiesException(String message, Throwable cause, String hint, String causingobject) {
+ super(message, cause);
+ this.hint = hint;
+ this.causingobject = causingobject;
+ }
+
+ public String getHint() {
+ return hint;
+ }
+
+ public String getCausingObject() {
+ return causingobject;
+ }
+
+ @Override
+ public String toString() {
+ String ret = super.toString();
+ if (causingobject != null) {
+ ret += "\r\nCausing Object: " + causingobject;
+ }
+ return ret;
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/RolesAllowedDynamicFeature2.java b/src/main/java/io/rtdi/bigdata/rulesservice/RolesAllowedDynamicFeature2.java
new file mode 100644
index 0000000..9b8bf3c
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/RolesAllowedDynamicFeature2.java
@@ -0,0 +1,155 @@
+package io.rtdi.bigdata.rulesservice;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.glassfish.jersey.server.internal.LocalizationMessages;
+import org.glassfish.jersey.server.model.AnnotatedMethod;
+
+import jakarta.annotation.Priority;
+import jakarta.annotation.security.DenyAll;
+import jakarta.annotation.security.PermitAll;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.ForbiddenException;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Priorities;
+import jakarta.ws.rs.container.ContainerRequestContext;
+import jakarta.ws.rs.container.ContainerRequestFilter;
+import jakarta.ws.rs.container.DynamicFeature;
+import jakarta.ws.rs.container.ResourceInfo;
+import jakarta.ws.rs.core.FeatureContext;
+
+public class RolesAllowedDynamicFeature2 implements DynamicFeature {
+
+ private static Map>>> roles = new HashMap<>();
+
+ public RolesAllowedDynamicFeature2() {
+ super();
+ }
+
+ /**
+ * @return Map of roles
+ */
+ public static Map>>> getRoles() {
+ return roles;
+ }
+
+ @Override
+ public void configure(final ResourceInfo resourceInfo, final FeatureContext configuration) {
+ final AnnotatedMethod am = new AnnotatedMethod(resourceInfo.getResourceMethod());
+
+ // DenyAll on the method take precedence over RolesAllowed and PermitAll
+ if (am.isAnnotationPresent(DenyAll.class)) {
+ configuration.register(new RolesAllowedRequestFilter());
+ return;
+ }
+
+ // RolesAllowed on the method takes precedence over PermitAll
+ RolesAllowed ra = am.getAnnotation(RolesAllowed.class);
+ if (ra != null) {
+ add(resourceInfo, am);
+ configuration.register(new RolesAllowedRequestFilter(ra.value()));
+ return;
+ }
+
+ // PermitAll takes precedence over RolesAllowed on the class
+ if (am.isAnnotationPresent(PermitAll.class)) {
+ // Do nothing.
+ return;
+ }
+
+ // DenyAll can't be attached to classes
+
+ // RolesAllowed on the class takes precedence over PermitAll
+ ra = resourceInfo.getResourceClass().getAnnotation(RolesAllowed.class);
+ if (ra != null) {
+ add(resourceInfo, am);
+ configuration.register(new RolesAllowedRequestFilter(ra.value()));
+ }
+ }
+
+ private void add(ResourceInfo resourceInfo, AnnotatedMethod am) {
+ String classname = resourceInfo.getResourceClass().getSimpleName();
+ if (classname != null) {
+ RolesAllowed ra = am.getAnnotation(RolesAllowed.class);
+ if (ra != null) {
+ for (String role : ra.value()) {
+ Map>> r = roles.get(role);
+ if (r == null) {
+ r = new HashMap<>();
+ roles.put(role, r);
+ }
+ Map> c = r.get(classname);
+ if (c == null) {
+ c = new HashMap<>();
+ r.put(classname, c);
+ }
+ Path p = am.getAnnotation(Path.class);
+ if (p != null) {
+ Set m = c.get(p.value());
+ if (m == null) {
+ m = new HashSet<>();
+ c.put(p.value(), m);
+ }
+ if (am.getAnnotation(GET.class) != null) {
+ m.add("GET");
+ } else if (am.getAnnotation(POST.class) != null) {
+ m.add("POST");
+ } else if (am.getAnnotation(DELETE.class) != null) {
+ m.add("DELETE");
+ } else if (am.getAnnotation(PUT.class) != null) {
+ m.add("PUT");
+ }
+ }
+ }
+ }
+ }
+
+ }
+
+ @Priority(Priorities.AUTHORIZATION) // authorization filter - should go after any authentication filters
+ private static class RolesAllowedRequestFilter implements ContainerRequestFilter {
+
+ private final boolean denyAll;
+ private final String[] rolesAllowed;
+
+ RolesAllowedRequestFilter() {
+ this.denyAll = true;
+ this.rolesAllowed = null;
+ }
+
+ RolesAllowedRequestFilter(final String[] rolesAllowed) {
+ this.denyAll = false;
+ this.rolesAllowed = (rolesAllowed != null) ? rolesAllowed : new String[] {};
+ }
+
+ @Override
+ public void filter(final ContainerRequestContext requestContext) throws IOException {
+ if (!denyAll) {
+ if (rolesAllowed.length > 0 && !isAuthenticated(requestContext)) {
+ throw new ForbiddenException(LocalizationMessages.USER_NOT_AUTHORIZED());
+ }
+
+ for (final String role : rolesAllowed) {
+ if (requestContext.getSecurityContext().isUserInRole(role)) {
+ return;
+ }
+ }
+ }
+
+ throw new ForbiddenException(LocalizationMessages.USER_NOT_AUTHORIZED());
+ }
+
+ private static boolean isAuthenticated(final ContainerRequestContext requestContext) {
+ return requestContext.getSecurityContext().getUserPrincipal() != null;
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/RuleFileKStream.java b/src/main/java/io/rtdi/bigdata/rulesservice/RuleFileKStream.java
new file mode 100644
index 0000000..dc679f6
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/RuleFileKStream.java
@@ -0,0 +1,245 @@
+package io.rtdi.bigdata.rulesservice;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.avro.Schema;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.KStream;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.confluent.kafka.schemaregistry.ParsedSchema;
+import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider;
+import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaString;
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
+import io.rtdi.bigdata.kafka.avro.AvroDeserializer;
+import io.rtdi.bigdata.kafka.avro.AvroSerializer;
+import io.rtdi.bigdata.kafka.avro.recordbuilders.SchemaBuilder;
+import io.rtdi.bigdata.kafka.avro.recordbuilders.ValueSchema;
+import io.rtdi.bigdata.rulesservice.config.RuleFileDefinition;
+import io.rtdi.bigdata.rulesservice.jexl.JexlAvroDeserializer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+
+public class RuleFileKStream extends Thread {
+
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ private Map ruledefinitions = new HashMap<>();
+ private Map> inoutschema = new HashMap<>();
+ private RulesService service;
+ private String inputtopicname;
+ private String outputtopicname;
+ private AtomicLong rowsprocessed = new AtomicLong();
+ private volatile long lastprocessedtimestamp = 0L;
+ private AvroSchemaProvider provider = new AvroSchemaProvider();
+ private Exception error;
+ private Integer instances;
+ private List rulefiles;
+ private AtomicLong processingtime = new AtomicLong();
+
+
+ public RuleFileKStream(List rulefiles, RulesService service, String inputtopicname, String outputtopicname, Integer instances) throws IOException {
+ this.service = service;
+ this.inputtopicname = inputtopicname;
+ this.outputtopicname = outputtopicname;
+ if (instances == null) {
+ this.instances = 1;
+ } else {
+ this.instances = instances;
+ }
+ this.rulefiles = rulefiles;
+ for (String location : rulefiles) {
+ int pos = location.indexOf('/');
+ if (pos <= 0) {
+ throw new IOException("Rulefile path does not follow the convention /");
+ }
+ String subjectname = location.substring(0, pos);
+ String path = location.substring(pos+1);
+ RuleFileDefinition rule = RuleFileDefinition.load(service.getRuleFileRootDir(), subjectname, Path.of(path), true);
+ if (rule != null) {
+ this.ruledefinitions.put(rule.schema().getFullName(), rule);
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ if (ruledefinitions.size() == 0) {
+ this.error = new IOException("Stream has no active rule files");
+ } else {
+ StreamsBuilder builder = new StreamsBuilder();
+ KStream transformation = builder.stream(
+ inputtopicname,
+ Consumed.with(Serdes.Bytes(), Serdes.Bytes()));
+ KStream result = transformation.mapValues(value -> mapper(value));
+ result.to(outputtopicname, Produced.with(Serdes.Bytes(), Serdes.Bytes()));
+ HashMap map = new HashMap<>(service.getKafkaProperties());
+ map.put(StreamsConfig.APPLICATION_ID_CONFIG, "rulesservice_" + inputtopicname);
+ map.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, String.valueOf(instances));
+ StreamsConfig streamsConfig = new StreamsConfig(map);
+ try (KafkaStreams streams = new KafkaStreams(builder.build(), streamsConfig);) {
+ streams.start();
+ try {
+ while (streams.state().isRunningOrRebalancing()) {
+ Thread.sleep(Duration.ofMinutes(1));
+ }
+ } catch (InterruptedException e) {
+ // exit loop and stop thread
+ }
+ } catch (Exception e) {
+ this.error = e;
+ }
+ }
+ }
+
+ private Bytes mapper(Bytes value) {
+ long startts = System.currentTimeMillis();
+ byte[] data = value.get();
+
+ int schemaid;
+ try {
+ schemaid = AvroDeserializer.getSchemaId(data);
+ } catch (IOException e) {
+ log.error("This input is not a valid Avro record with a magic byte", e);
+ throw new KafkaException("This input is not a valid Avro record with a magic byte", e);
+ }
+
+ ParsedSchema writerparsedschema;
+ try {
+ writerparsedschema = service.getSchemaclient().getSchemaById(schemaid);
+ } catch (IOException | RestClientException e) {
+ log.error("Exception when reading the schema <{}> from the schema registry by id", schemaid, e);
+ throw new KafkaException("Exception when reading the schema <" + schemaid + "> from the schema registry by id", e);
+ }
+ Schema writerschema = (Schema) writerparsedschema.rawSchema();
+
+ RuleFileDefinition ruledefinition = ruledefinitions.get(writerschema.getFullName());
+ if (ruledefinition == null) {
+ /*
+ * No transformations for that schema, hence a pass-through
+ */
+ return value;
+ } else {
+ JexlRecord inputrecord;
+ if (writerschema.getField(ValueSchema.AUDIT) == null) {
+ synchronized (this) {
+ Entry schemadata = inoutschema.get(schemaid);
+ if (schemadata == null) {
+ // if no, derive the new schema
+ SchemaBuilder outputschemabuilder = new SchemaBuilder(writerschema.getName() + "_CLEANSED", writerschema, false);
+ ValueSchema.addAuditField(outputschemabuilder);
+ outputschemabuilder.build();
+ Schema outputschema = outputschemabuilder.getSchema();
+ io.confluent.kafka.schemaregistry.client.rest.entities.Schema schemaentity = new io.confluent.kafka.schemaregistry.client.rest.entities.Schema(
+ ruledefinition.getOutputsubjectname(), 0, 0, new SchemaString(outputschema.toString()));
+ ParsedSchema parsed = provider.parseSchemaOrElseThrow(schemaentity , false, false);
+ // register the schema - if already known the API call will not create it and just return the schema id
+ int outputschemaid;
+ try {
+ outputschemaid = service.getSchemaclient().register(ruledefinition.getOutputsubjectname(), parsed);
+ } catch (IOException | RestClientException e) {
+ log.error("Exception when registering the schema for subject name <{}>\nschema: {}", ruledefinition.getOutputsubjectname(), outputschema.toString(true), e);
+ throw new KafkaException("Exception when writing the schema for subject name <" + ruledefinition.getOutputsubjectname() + ">", e);
+ }
+ schemadata = new SimpleEntry<>(outputschemaid, outputschema);
+ inoutschema.put(schemaid, schemadata);
+ schemaid = outputschemaid;
+ } else {
+ schemaid = schemadata.getKey();
+ }
+ try {
+ inputrecord = JexlAvroDeserializer.deserialize(data, writerschema, schemadata.getValue());
+ } catch (IOException e) {
+ log.error("Exception when deserializing the byte[] into an Avro record\ndata: {}\ninputschema: {}\noutputschema: {}", data, writerschema.toString(true), writerschema.toString(true), e);
+ throw new KafkaException("Exception when deserializing the byte[] into an Avro record", e);
+ }
+ }
+ } else {
+ /*
+ * The input has an AUDIT structure already, deserialize it into a JexRecord, apply the rules and serialize it into the same schema
+ */
+ try {
+ inputrecord = JexlAvroDeserializer.deserialize(data, writerschema);
+ } catch (IOException e) {
+ log.error("Exception when deserializing the byte[] into an Avro record\ndata: {}\ninputschema: {}\noutputschema: {}", data, writerschema.toString(true), writerschema.toString(true), e);
+ throw new KafkaException("Exception when deserializing the byte[] into an Avro record", e);
+ }
+ }
+
+ try {
+ ruledefinition.apply(inputrecord, false);
+ } catch (IOException e) {
+ log.error("Exception when applying the rules to the record\\nrulefile: {}\nrecord: {}", ruledefinition.getName(), inputrecord, e);
+ throw new KafkaException("Exception when applying the rules to the record", e);
+ }
+
+ byte[] out;
+ try {
+ out = AvroSerializer.serialize(schemaid, inputrecord);
+ rowsprocessed.incrementAndGet();
+ lastprocessedtimestamp = System.currentTimeMillis();
+ processingtime.addAndGet(lastprocessedtimestamp - startts);
+ return Bytes.wrap(out);
+ } catch (IOException e) {
+ log.error("Exception when serializing the Avro record into a byte[]", e);
+ throw new KafkaException("Exception when serializing the Avro record into a byte[]", e);
+ }
+ }
+ }
+
+ public long getRowsprocessed() {
+ return rowsprocessed.get();
+ }
+
+ public long getLastprocessedtimestamp() {
+ return lastprocessedtimestamp;
+ }
+
+ public Map getRulefiledefinitions() {
+ return ruledefinitions;
+ }
+
+ public String getInputtopicname() {
+ return inputtopicname;
+ }
+
+ public String getOutputtopicname() {
+ return outputtopicname;
+ }
+
+ public Exception getError() {
+ return error;
+ }
+
+ public int getInstances() {
+ return instances;
+ }
+
+ public List getRulefiles() {
+ return rulefiles;
+ }
+
+ public Long getAvgProcessingtime() {
+ if (rowsprocessed.get() != 0) {
+ return processingtime.get() / rowsprocessed.get();
+ } else {
+ return null;
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/RuleStep.java b/src/main/java/io/rtdi/bigdata/rulesservice/RuleStep.java
deleted file mode 100644
index db89df5..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/RuleStep.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package io.rtdi.bigdata.rulesservice;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import io.rtdi.bigdata.connector.pipeline.foundation.MicroServiceTransformation;
-import io.rtdi.bigdata.connector.pipeline.foundation.SchemaRegistryName;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-import io.rtdi.bigdata.connector.pipeline.foundation.utils.FileNameEncoder;
-import io.rtdi.bigdata.connector.pipeline.foundation.utils.SchemaNameEncoder;
-
-public class RuleStep extends MicroServiceTransformation {
- /*
- * Index over the encoded schema name
- */
- private Map rs = new HashMap<>();
-
- public RuleStep(String name) {
- super(name);
- }
-
- public RuleStep(File dir) throws PropertiesException {
- this(dir.getName());
- for (File rulefile : dir.listFiles()) {
- if (rulefile.isFile() && rulefile.getName().endsWith(".json")) {
- SchemaRegistryName schemaname = SchemaRegistryName.create(rulefile.getName().substring(0, rulefile.getName().length()-5));
- SchemaRuleSet r = new SchemaRuleSet(schemaname, dir);
- rs.put(SchemaNameEncoder.encodeName(schemaname.getEncodedName()), r);
- }
- }
- }
-
- @Override
- public JexlRecord applyImpl(JexlRecord valuerecord) throws IOException {
- SchemaRuleSet r = rs.get(valuerecord.getSchema().getFullName());
- if (r != null && r.getRules() != null) {
- addOperationLogLine("Applied rule for schema " + r.getSchemaname());
- return r.apply(valuerecord);
- } else {
- addOperationLogLine("Record with schema " + valuerecord.getSchema().getFullName() + " has no rules defined - data passed through");
- return valuerecord;
- }
- }
-
- public SchemaRuleSet getSchemaRule(SchemaRegistryName schemaname) {
- return rs.get(schemaname.getEncodedName());
- }
-
- public void setSchemaRuleSet(Map data) {
- rs = data;
- }
-
- public Map getSchemaRules() {
- return rs;
- }
-
- public void addSchemaRuleSet(SchemaRuleSet r) {
- rs.put(r.getSchemaName().getEncodedName(), r);
- }
-
- public void removeSchemaRuleSet(SchemaRegistryName schemaname) {
- rs.remove(schemaname.getEncodedName());
- }
-
- public void deleteSchemaRuleSetDefinition(SchemaRegistryName schemaname, File dir) {
- rs.remove(schemaname.getEncodedName());
- File file = new File(dir, FileNameEncoder.encodeName(schemaname.getName()) + ".json");
- if (file.exists()) {
- file.delete();
- }
- }
-
-}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/RulesService.java b/src/main/java/io/rtdi/bigdata/rulesservice/RulesService.java
index 4d9d999..ad2ff0f 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/RulesService.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/RulesService.java
@@ -1,110 +1,559 @@
package io.rtdi.bigdata.rulesservice;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
-import java.util.Collections;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import javax.naming.Context;
+import javax.naming.InitialContext;
import org.apache.avro.Schema;
-import org.apache.avro.Schema.Field;
-import org.apache.avro.SchemaBuilderException;
-
-import io.rtdi.bigdata.connector.connectorframework.Service;
-import io.rtdi.bigdata.connector.connectorframework.controller.ServiceController;
-import io.rtdi.bigdata.connector.pipeline.foundation.IPipelineAPI;
-import io.rtdi.bigdata.connector.pipeline.foundation.MicroServiceTransformation;
-import io.rtdi.bigdata.connector.pipeline.foundation.SchemaHandler;
-import io.rtdi.bigdata.connector.pipeline.foundation.entity.ServiceEntity;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-import io.rtdi.bigdata.connector.pipeline.foundation.utils.IOUtils;
-import io.rtdi.bigdata.kafka.avro.SchemaConstants;
-import io.rtdi.bigdata.kafka.avro.recordbuilders.ValueSchema;
-
-public class RulesService extends Service {
-
- public RulesService(ServiceController controller) throws PropertiesException {
- super(controller);
+import org.apache.avro.Schema.Parser;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.DescribeClusterResult;
+import org.apache.kafka.clients.admin.ListTopicsResult;
+import org.apache.kafka.common.KafkaFuture;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient;
+import io.confluent.kafka.schemaregistry.client.SchemaMetadata;
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
+import io.rtdi.appcontainer.utils.UsageStatisticSender;
+import io.rtdi.bigdata.rulesservice.config.RuleFileDefinition;
+import io.rtdi.bigdata.rulesservice.config.RuleFileName;
+import io.rtdi.bigdata.rulesservice.config.RuleStep;
+import io.rtdi.bigdata.rulesservice.config.ServiceSettings;
+import io.rtdi.bigdata.rulesservice.config.ServiceStatus;
+import io.rtdi.bigdata.rulesservice.config.TopicRule;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+import io.rtdi.bigdata.rulesservice.rest.SampleData;
+import io.rtdi.bigdata.rulesservice.rules.RuleUtils;
+import jakarta.servlet.ServletContext;
+import jakarta.servlet.ServletContextEvent;
+import jakarta.servlet.ServletContextListener;
+import jakarta.servlet.annotation.WebListener;
+
+/**
+ * Main java class running the rules service
+ *
+ */
+@WebListener
+public class RulesService implements ServletContextListener {
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ private ScheduledExecutorService executor;
+ private CachedSchemaRegistryClient schemaclient;
+ private ServletContext context;
+ private Path settingsdir;
+ private Path rulegroupdir;
+ private Exception globalexception;
+ private String properties;
+ private Map propertiesmap;
+ private Admin admin;
+ /**
+ * Map>
+ */
+ private Map services = new HashMap<>();
+
+ /**
+ * Default constructor.
+ */
+ public RulesService() {
+ super();
+ }
+
+ public Map getKafkaProperties() {
+ return propertiesmap;
+ }
+
+ public String getBootstrapServers() {
+ if (propertiesmap != null) {
+ return propertiesmap.get("bootstrap.servers");
+ } else {
+ return null;
+ }
+ }
+
+ public long getRowsProduced() {
+ if (services != null) {
+ long count = 0L;
+ for (RuleFileKStream service : services.values()) {
+ count += service.getRowsprocessed();
+ }
+ return count;
+ } else {
+ return 0L;
+ }
+ }
+
+ public String getState() {
+ if (services != null && services.size() > 0) {
+ return "RUNNING";
+ } else {
+ return "IDLE";
+ }
+ }
+
+ public Long getLastDataTimestamp() {
+ if (services != null) {
+ long ts = 0L;
+ for (RuleFileKStream service : services.values()) {
+ if (ts < service.getLastprocessedtimestamp()) {
+ ts = service.getLastprocessedtimestamp();
+ }
+ }
+ return ts;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * @see ServletContextListener#contextDestroyed(ServletContextEvent)
+ */
+ @Override
+ public void contextDestroyed(ServletContextEvent sce) {
+ if (executor != null) {
+ executor.shutdownNow();
+ try {
+ executor.awaitTermination(20, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ }
+ }
+ close();
+ }
+
+ /**
+ * @see ServletContextListener#contextInitialized(ServletContextEvent)
+ */
+ @Override
+ public void contextInitialized(ServletContextEvent sce) {
+ if (System.getenv("STATISTICS") == null
+ || !System.getenv("STATISTICS").equalsIgnoreCase("FALSE")) {
+ this.executor = Executors.newSingleThreadScheduledExecutor();
+ executor.scheduleAtFixedRate(new UsageStatisticSender(this), 1, 10, TimeUnit.MINUTES);
+ this.context = sce.getServletContext();
+ this.context.setAttribute("service", this);
+ }
+ try {
+ Context initCtx = new InitialContext();
+ Context envCtx = (Context) initCtx.lookup("java:comp/env");
+ Object o = envCtx.lookup("rulesettings");
+ if (o != null) {
+ settingsdir = Path.of(o.toString());
+ log.info("Found JNDI resource name , hence the settings directory is <{}>", settingsdir);
+ } else {
+ settingsdir = Path.of("/apps/rulesservice/settings");
+ log.info("No JNDI resource found for name , hence the settings directory is the default <{}>", settingsdir);
+ }
+ o = envCtx.lookup("rulegroups");
+ if (o != null) {
+ rulegroupdir = Path.of(o.toString());
+ log.info("Found JNDI resource name , hence the rules root directory is <{}>", rulegroupdir);
+ } else {
+ rulegroupdir = Path.of("/apps/rulesservice/definitions");
+ log.info("No JNDI resource found for name , hence the rules root directory is the default <{}>", rulegroupdir);
+ }
+ configure();
+ startService();
+ } catch (Exception e) {
+ this.globalexception = e;
+ }
+ }
+
+ public void startService() throws IOException {
+ stopService();
+ Map topicrulefiles = getTopicRuleFiles();
+ for (Entry entry : topicrulefiles.entrySet()) {
+ TopicRule r = entry.getValue();
+ RuleFileKStream rule = new RuleFileKStream(r.getRulefiles(), this, r.getInputtopicname(), r.getOutputtopicname(), r.getInstances());
+ if (rule.getRulefiledefinitions().size() > 0) {
+ services.put(r.getInputtopicname(), rule);
+ rule.start();
+ log.info("Thread for topic {} started", r.getInputtopicname());
+ } else {
+ log.info("Topic {} has no active rules", r.getInputtopicname());
+ }
+ }
+ }
+
+ public void startService(String topicname) throws IOException, InterruptedException {
+ stopService(topicname);
+ Map topicrules = getTopicRuleFiles();
+ TopicRule r = topicrules.get(topicname);
+ if (r != null) {
+ RuleFileKStream rule = new RuleFileKStream(r.getRulefiles(), this, r.getInputtopicname(), r.getOutputtopicname(), r.getInstances());
+ if (rule.getRulefiledefinitions().size() > 0) {
+ services.put(r.getInputtopicname(), rule);
+ rule.start();
+ log.info("Thread for topic {} started", r.getInputtopicname());
+ } else {
+ log.info("Topic {} has no active rules", r.getInputtopicname());
+ }
+ }
+ }
+
+ public void startFailedServices() throws IOException {
+ for (Entry entry : services.entrySet()) {
+ RuleFileKStream stream = entry.getValue();
+ if (! stream.isAlive()) {
+ RuleFileKStream rule = new RuleFileKStream(stream.getRulefiles(), this, stream.getInputtopicname(), stream.getOutputtopicname(), stream.getInstances());
+ if (rule.getRulefiledefinitions().size() > 0) {
+ services.put(rule.getInputtopicname(), rule);
+ rule.start();
+ log.info("Thread for topic {} re-started", rule.getInputtopicname());
+ } else {
+ log.info("Topic {} has no active rules", rule.getInputtopicname());
+ }
+ }
+ }
+ }
+
+ public ServiceStatus getServiceStatus() throws IOException {
+ return new ServiceStatus(this);
+ }
+
+ public void stopService() throws IOException {
+ /*
+ * Signal all to stop
+ */
+ for (RuleFileKStream service : services.values()) {
+ service.interrupt();
+ }
/*
- * Validate that the last schema version has the AUDIT structure where rules write their results into.
- * If data was produced via this framework it will, but data produced via other means might not.
+ * Remove all stopped instances within 1 minute
*/
- RulesServiceProperties props = (RulesServiceProperties) controller.getServiceProperties();
- IPipelineAPI api = controller.getPipelineAPI();
- if (props.getMicroserviceTransformations() != null) {
- for (MicroServiceTransformation microservice : props.getMicroserviceTransformations()) {
- RuleStep rulestep = (RuleStep) microservice;
- for (SchemaRuleSet rs : rulestep.getSchemaRules().values()) {
- SchemaHandler handler = api.getSchema(rs.getSchemaName());
- Schema valueschema = handler.getValueSchema();
- if (valueschema.getField(ValueSchema.AUDIT) == null) {
- api.registerSchema(handler.getSchemaName(), null, handler.getKeySchema(), createNewSchemaVersion(valueschema));
+ long until = System.currentTimeMillis() + 60000L;
+ while (System.currentTimeMillis() < until && services.size() > 0) {
+ Iterator> iter = services.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry service = iter.next();
+ try {
+ if (service.getValue().join(Duration.ofSeconds(2))) {
+ iter.remove();
+ log.info("Thread for topic {} stopped", service.getKey());
}
+ } catch (InterruptedException e) {
+ // NOOP as we have to close all resources, no matter what
}
}
}
}
- private Schema createNewSchemaVersion(Schema schema) throws PropertiesException {
+ public void stopService(String topicname) throws IOException, InterruptedException {
+ RuleFileKStream service = services.get(topicname);
+ if (service != null) {
+ /*
+ * Signal all to stop
+ */
+ service.interrupt();
+ /*
+ * Remove within 1 minute
+ */
+ if (service.join(Duration.ofSeconds(60))) {
+ services.remove(topicname);
+ log.info("Thread for topic {} stopped", topicname);
+ }
+ }
+ }
+
+ public static RulesService getRulesService(ServletContext context) {
+ return (RulesService) context.getAttribute("service");
+ }
+
+ private void configure() throws IOException, RestClientException {
+ this.globalexception = null;
+ File propertiesfile = new File(settingsdir.toFile(), "kafka.properties");
+ if (!propertiesfile.isFile()) {
+ log.error("The mandatory kafka.properties file does not exist at <{}>", propertiesfile.toString());
+ throw new IOException("The mandatory kafka.properties file does not exist at <" + propertiesfile.toString() + ">");
+ } else {
+ log.info("Found the kafka.properties file as <{}>", propertiesfile.toString());
+ }
+ Properties kafkaproperties = new Properties();
+ try (InputStream is = new FileInputStream(propertiesfile)) {
+ kafkaproperties.load(is);
+ }
+ this.properties = Files.readString(settingsdir.resolve("kafka.properties"), StandardCharsets.UTF_8);
+ String schemaurls = kafkaproperties.getProperty("schema.registry.url");
+ if (schemaurls == null) {
+ throw new IOException("The kafka.properties file does not contain a ");
+ }
+ String[] urls = schemaurls.split(",");
+ List schemaRegistryUrls = new ArrayList<>();
+ for (String url : urls) {
+ schemaRegistryUrls.add(url.trim());
+ }
+ propertiesmap = kafkaproperties.entrySet().stream().collect(
+ Collectors.toMap(
+ e -> e.getKey().toString(),
+ e -> e.getValue().toString()
+ )
+ );
+ schemaclient = new CachedSchemaRegistryClient(schemaRegistryUrls, 100, propertiesmap);
+ schemaclient.getMode(); // invoke schema registry to validate connection
+ admin = Admin.create(kafkaproperties);
+ admin.describeCluster();
+ }
+
+ public List getRuleFiles(String schemaname) {
+ List groups = RuleFileDefinition.getAllRuleFiles(rulegroupdir, schemaname);
+ return groups;
+ }
+
+ public List getAllRuleFiles() {
+ List groups = RuleFileDefinition.getAllRuleFiles(rulegroupdir);
+ return groups;
+ }
+
+
+ public Collection getSubjects() throws IOException, RestClientException {
+ Collection subjects = schemaclient.getAllSubjects();
+ return subjects;
+ }
+
+ public Schema getLatestSchema(String subjectname) throws IOException, RestClientException {
+ SchemaMetadata metadata = schemaclient.getLatestSchemaMetadata(subjectname);
+ String schemastring = metadata.getSchema();
+ org.apache.avro.Schema.Parser parser = new org.apache.avro.Schema.Parser();
+ org.apache.avro.Schema schema = parser.parse(schemastring);
+ return schema;
+ }
+
+ public Exception getGlobalException() {
+ return globalexception;
+ }
+
+ public Path getRuleFileRootDir() {
+ return this.rulegroupdir;
+ }
+
+ public ServiceSettings getConfig(boolean isadmin) {
+ ServiceSettings settings = new ServiceSettings();
+ settings.setAdminuser(isadmin);
+ if (isadmin) {
+ // The call has been made by someone with admin permissions
+ settings.setProperties(properties);
+ } else {
+ settings.setProperties("# Data not exposed; only users with the role can view/edit");
+ }
+ settings.setSchemaregconnected(false);
+ if (schemaclient != null) {
+ try {
+ Collection schemas = getSubjects();
+ settings.setSchemaregsubjects(schemas.size());
+ settings.setSchemaregconnected(true);
+ } catch (IOException | RestClientException e) {
+ log.error("getConfig() ran into an error", e);
+ globalexception = e;
+ }
+ }
+ settings.setKafkaconnected(false);
+ if (admin != null) {
+ try {
+ DescribeClusterResult result = admin.describeCluster();
+ result.nodes().get(2, TimeUnit.SECONDS);
+ settings.setKafkaconnected(true);
+ } catch (Exception e) {
+ log.error("getConfig() ran into an error", e);
+ globalexception = e;
+ }
+ }
+ if (services.size() > 0) {
+ settings.setServicerunning(true);
+ } else {
+ settings.setServicerunning(false);
+ }
+ settings.setErrormessage(globalexception != null ? globalexception.getMessage() : null);
+ return settings;
+ }
+
+ public Collection getTopicsAndRules() throws IOException {
+ ListTopicsResult topicsresult = admin.listTopics();
+ KafkaFuture> future = topicsresult.names();
+ Map fileset = getTopicRuleFiles();
+ Collection l;
try {
- ValueSchema n = new ValueSchema(schema.getName(), schema.getNamespace(), schema.getDoc());
- for (Field f : schema.getFields()) {
- switch (f.name()) {
- case SchemaConstants.SCHEMA_COLUMN_CHANGE_TYPE:
- case SchemaConstants.SCHEMA_COLUMN_CHANGE_TIME:
- case SchemaConstants.SCHEMA_COLUMN_SOURCE_ROWID:
- case SchemaConstants.SCHEMA_COLUMN_SOURCE_TRANSACTION:
- case SchemaConstants.SCHEMA_COLUMN_SOURCE_SYSTEM:
- case SchemaConstants.SCHEMA_COLUMN_EXTENSION:
- case ValueSchema.AUDIT:
- // do nothing, use the ValueSchema schema definition for the fixed columns
- default:
- n.add(f);
+ l = future.get(20, TimeUnit.SECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ throw new IOException("Failed to read the list of topics from Kafka within 20 seconds", e);
+ }
+ TreeSet out = new TreeSet<>();
+ for (String topicname : l) {
+ TopicRule tr = fileset.get(topicname);
+ if (tr != null) {
+ out.add(tr);
+ } else {
+ out.add(new TopicRule(topicname));
+ }
+ }
+ return out;
+ }
+
+ /**
+ * @return a map data
+ * @throws IOException
+ */
+ public Map getTopicRuleFiles() throws IOException {
+ ObjectMapper om = new ObjectMapper();
+ Path topicrules = rulegroupdir.resolve("topics");
+ File topicruledir = topicrules.toFile();
+ Map fileset = new HashMap<>();
+ if (topicruledir.isDirectory()) {
+ File[] files = topicruledir.listFiles();
+ for(File f : files) {
+ if (f.isFile() && f.getName().endsWith(".json")) {
+ TopicRule tr = om.readValue(f, TopicRule.class);
+ fileset.put(f.getName().substring(0, f.getName().length()-5), tr);
}
}
- n.build();
- return n.getSchema();
- } catch (SchemaBuilderException e) {
- throw new PropertiesException("The most recent schema version does not have a " + ValueSchema.AUDIT + " field - tried to derive a new schema and failed", e);
}
+ return fileset;
}
- @Override
- public void updateLandscape() {
- RulesServiceProperties props = (RulesServiceProperties) getController().getServiceProperties();
- List consumertopics = Collections.singletonList(props.getSourceTopic());
-
- Map> producertopics = new HashMap<>();
- Set usedschemas = new HashSet<>();
-
- if (props.getMicroserviceTransformations() != null) {
- for ( MicroServiceTransformation microservice : props.getMicroserviceTransformations()) {
- RuleStep rulestep = (RuleStep) microservice;
- for (SchemaRuleSet rs : rulestep.getSchemaRules().values()) {
- usedschemas.add(rs.getSchemaName().getName());
+
+ public List saveTopicRules(List input) {
+ Path topicruledir = rulegroupdir.resolve("topics");
+ File f = topicruledir.toFile();
+ if (!f.exists()) {
+ f.mkdirs();
+ }
+ for ( TopicRule topicrule : input) {
+ if (topicrule.getModified() != null && topicrule.getModified() == Boolean.TRUE) {
+ if (topicrule.getInputtopicname() == null) {
+ // ignore
+ } else if (topicrule.getOutputtopicname() == null) {
+ topicrule.delete(topicruledir);
+ topicrule.setModified(null);
+ topicrule.setInfo("deleted");
+ } else if (topicrule.getInputtopicname().equals(topicrule.getOutputtopicname())) {
+ topicrule.setInfo("The rule for topic <" + topicrule.getInputtopicname() + "> cannot write into itself, output topic must be different");
+ } else {
+ try {
+ topicrule.save(topicruledir);
+ topicrule.setModified(null);
+ topicrule.setInfo("saved");
+ } catch (IOException e) {
+ log.error("Failed to save the file for topic <{}>", topicrule.getInputtopicname(), e);
+ topicrule.setInfo("Failed to save the topic file (" + e.getMessage() + ")");
+ }
+ try {
+ startService(topicrule.getInputtopicname());
+ } catch (IOException | InterruptedException e) {
+ log.error("Failed to start the service for the topic <{}>", topicrule.getInputtopicname(), e);
+ topicrule.setInfo("Failed to start the service for the topic <" + topicrule.getInputtopicname() + ">: " + e.getMessage());
+ }
}
+ } else {
+ topicrule.setInfo(null);
+ }
+ }
+ return input;
+ }
+
+ public void saveConfig(ServiceSettings input) throws IOException {
+ String properties = input.getProperties();
+ Files.writeString(settingsdir.resolve("kafka.properties"), properties, StandardCharsets.UTF_8);
+ close();
+ try {
+ configure();
+ } catch (Exception e) {
+ this.globalexception = e;
+ }
+ }
+
+ private void close() {
+ log.info("Closing all resources");
+ try {
+ stopService();
+ } catch (IOException e) {
+ log.error("Stopping the services failed - ignored", e);
+ }
+ if (admin != null) {
+ try {
+ admin.close(Duration.ofSeconds(10));
+ } catch (Exception e) {
+ log.error("Closing the Kafka admin client within 10 seconds failed - ignored", e);
}
-
- producertopics.put(props.getTargetTopic(), usedschemas);
-
+ }
+ if (schemaclient != null) {
try {
- getController().getPipelineAPI().addServiceMetadata(
- new ServiceEntity(
- getController().getName(),
- IOUtils.getHostname(),
- getController().getPipelineAPI().getConnectionLabel(),
- consumertopics,
- producertopics));
- } catch (IOException e) {
- logger.error("Failed to update the Landscape metadata in the server", e);
+ schemaclient.close();
+ } catch (Exception e) {
+ log.error("Closing the schema registry client failed - ignored", e);
}
}
}
- @Override
- public void updateSchemaCache() {
+ public Collection getTopics() throws IOException {
+ ListTopicsResult topicsresult = admin.listTopics();
+ KafkaFuture> future = topicsresult.names();
+ try {
+ return future.get(20, TimeUnit.SECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ throw new IOException("Failed to read the list of topics from Kafka within 20 seconds", e);
+ }
+ }
+
+ public RuleStep applySampleData(RuleFileDefinition input, Integer stepindex) throws IOException {
+ if (input.getSchema() == null) {
+ throw new PropertiesException("The Rule file does not contain the schema information");
+ }
+ Parser parser = new Schema.Parser();
+ Schema schema = parser.parse(input.getSchema());
+ RuleStep step = input.getRulesteps().get(stepindex);
+ JexlRecord rec = (JexlRecord) RuleUtils.getSampleValue(step, schema);
+ step.apply(rec, rec, true);
+ rec.mergeReplacementValues();
+ step.updateSampleOutput(rec);
+ return step;
+ }
+
+ public RuleFileDefinition applySampleFile(RuleFileDefinition input) throws IOException {
+ if (input.getSchema() == null) {
+ throw new PropertiesException("The Rule file does not contain the schema information");
+ }
+ if (input.getSamplefile() == null) {
+ throw new PropertiesException("The Rule file does not specify a sample file");
+ }
+ Parser parser = new Schema.Parser();
+ Schema schema = parser.parse(input.getSchema());
+ SampleData sample = SampleData.load(getRuleFileRootDir(), input.getInputsubjectname(), Path.of(input.getSamplefile()));
+ JexlRecord samplerecord = sample.createRecord(schema);
+ input.apply(samplerecord, true);
+ return input;
+ }
+
+ public CachedSchemaRegistryClient getSchemaclient() {
+ return schemaclient;
+ }
+
+ public Map getServices() {
+ return services;
}
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/RulesServiceFactory.java b/src/main/java/io/rtdi/bigdata/rulesservice/RulesServiceFactory.java
deleted file mode 100644
index 217040d..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/RulesServiceFactory.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package io.rtdi.bigdata.rulesservice;
-
-import java.io.IOException;
-
-import io.rtdi.bigdata.connector.connectorframework.BrowsingService;
-import io.rtdi.bigdata.connector.connectorframework.ConnectorFactory;
-import io.rtdi.bigdata.connector.connectorframework.IConnectorFactoryService;
-import io.rtdi.bigdata.connector.connectorframework.Service;
-import io.rtdi.bigdata.connector.connectorframework.controller.ConnectionController;
-import io.rtdi.bigdata.connector.connectorframework.controller.ServiceController;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-import io.rtdi.bigdata.connector.properties.ConnectionProperties;
-import io.rtdi.bigdata.connector.properties.ServiceProperties;
-
-public class RulesServiceFactory extends ConnectorFactory implements IConnectorFactoryService {
-
- public RulesServiceFactory() {
- super("RulesService");
- }
-
- @Override
- public ConnectionProperties createConnectionProperties(String name) throws PropertiesException {
- return null;
- }
-
- @Override
- public BrowsingService createBrowsingService(ConnectionController controller) throws IOException {
- return null;
- }
-
- @Override
- public Service createService(ServiceController instance) throws PropertiesException {
- return new RulesService(instance);
- }
-
- @Override
- public ServiceProperties createServiceProperties(String servicename) throws PropertiesException {
- return new RulesServiceProperties(servicename);
- }
-
- @Override
- public boolean supportsBrowsing() {
- return false;
- }
-
-}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/RulesServiceProperties.java b/src/main/java/io/rtdi/bigdata/rulesservice/RulesServiceProperties.java
deleted file mode 100644
index 256ed9c..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/RulesServiceProperties.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package io.rtdi.bigdata.rulesservice;
-
-import java.io.File;
-import java.io.IOException;
-
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-import io.rtdi.bigdata.connector.properties.ServiceProperties;
-
-public class RulesServiceProperties extends ServiceProperties {
-
- public RulesServiceProperties(String name) throws PropertiesException {
- super(name);
- }
-
- public RulesServiceProperties(File dir, String name) throws IOException {
- super(dir, name);
- }
-
- @Override
- protected RuleStep readMicroservice(File dir) throws PropertiesException {
- RuleStep m = new RuleStep(dir);
- return m;
- }
-
- @Override
- protected void writeMicroservice(File stepdir) throws IOException {
- }
-
-}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/SchemaRuleSet.java b/src/main/java/io/rtdi/bigdata/rulesservice/SchemaRuleSet.java
deleted file mode 100644
index d435d86..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/SchemaRuleSet.java
+++ /dev/null
@@ -1,202 +0,0 @@
-package io.rtdi.bigdata.rulesservice;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlTransient;
-
-import org.apache.avro.Schema;
-import org.apache.avro.Schema.Type;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.SchemaRegistryName;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.AvroUtils;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-import io.rtdi.bigdata.connector.pipeline.foundation.utils.FileNameEncoder;
-import io.rtdi.bigdata.kafka.avro.RowType;
-import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.rulesservice.rules.RecordRule;
-import io.rtdi.bigdata.rulesservice.rules.Rule;
-
-public class SchemaRuleSet extends RecordRule {
- private static ObjectMapper mapper;
- private SchemaRegistryName schemaname;
-
- static {
- mapper = new ObjectMapper();
- mapper.setSerializationInclusion(Include.NON_NULL);
- }
-
- public SchemaRuleSet(SchemaRegistryName schemaname, File schemadir) throws PropertiesException {
- this(schemaname);
- read(schemadir);
- }
-
- public SchemaRuleSet(SchemaRegistryName schemaname) {
- this();
- this.schemaname = schemaname;
- }
-
- public SchemaRuleSet() {
- super();
- }
-
- public JexlRecord apply(JexlRecord valuerecord) throws IOException {
- RowType changetype = AvroUtils.getChangeType(valuerecord);
- // Rules are applied to new or changed records but not deleted rows
- if (changetype == null || (changetype != RowType.EXTERMINATE && changetype != RowType.DELETE && changetype != RowType.TRUNCATE)) {
- List ruleresults = new ArrayList<>();
- apply(valuerecord, ruleresults);
- AvroUtils.mergeResults(valuerecord, ruleresults);
- }
- return valuerecord;
- }
-
- @Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException {
- if (getRules() != null) {
- for ( Rule rule : getRules()) {
- rule.apply(valuerecord, ruleresults);
- }
- }
- return null;
- }
-
- @Override
- public void assignSamplevalue(JexlRecord sampledata) {
- if (sampledata != null && this.getRules() != null) {
- for ( Rule r : this.getRules()) {
- r.assignSamplevalue(sampledata);
- }
- }
- }
-
- @Override
- public RuleResult validateRule(JexlRecord valuerecord) {
- if (valuerecord != null && this.getRules() != null) {
- for ( Rule r : this.getRules()) {
- r.validateRule(valuerecord);
- }
- }
- return null;
- }
-
-
- @Override
- public SchemaRuleSet createUIRuleTree(Schema schema) throws PropertiesException {
- return createUIRuleTree(schemaname, getFieldname(), schema, getRules());
- }
-
- public void updateSchema(Schema schema) throws PropertiesException {
- addFields(this, schema, getRules());
- }
-
- public static SchemaRuleSet createUIRuleTree(SchemaRegistryName schemaname, String fieldname, Schema schema, List originalrules) throws PropertiesException {
- if (schema.getType() == Type.RECORD) {
- SchemaRuleSet r = new SchemaRuleSet(schemaname);
- addFields(r, schema, originalrules);
- return r;
- } else {
- throw new PropertiesException("Provided Schema is not a record schema", (String) null, schema.getFullName());
- }
- }
-
-
- private void read(File directory) throws PropertiesException {
- if (!directory.exists()) {
- throw new PropertiesException("Directory for the rule schema files does not exist", "Use the UI or create the file manually", directory.getAbsolutePath());
- } else if (!directory.isDirectory()) {
- throw new PropertiesException("Specified location exists and is no directory", (String) null, directory.getAbsolutePath());
- } else {
- File file = new File(directory, FileNameEncoder.encodeName(schemaname + ".json"));
- if (!file.exists()) {
- // Do nothing if directory exists but file does not
- } else if (!file.canRead()) {
- throw new PropertiesException("Rules file is not read-able", "Check file permissions and users", file.getAbsolutePath());
- } else {
- try {
- RecordRule pg = mapper.readValue(file, SchemaRuleSet.class);
- pg.postSerialization();
- if (pg.getRules() != null) {
- // A file without a rule should not throw an exception
- addAll(pg.getRules());
- }
- } catch (PropertiesException e) {
- throw e; // to avoid nesting the exception
- } catch (IOException e) {
- throw new PropertiesException("Cannot parse the json file with the schema rules", e, "check filename and format", file.getName());
- }
- }
- }
- }
-
- public void write(File directory) throws PropertiesException {
- if (!directory.exists()) {
- throw new PropertiesException("Directory for the schema rule files does not exist", "Use the UI or create the file manually", directory.getAbsolutePath());
- } else if (!directory.isDirectory()) {
- throw new PropertiesException("Specified location exists and is no directory", (String) null, directory.getAbsolutePath());
- } else {
- File file = new File(directory, FileNameEncoder.encodeName(schemaname + ".json"));
- if (file.exists() && !file.canWrite()) { // Either the file does not exist or it exists and is write-able
- throw new PropertiesException("Schema rule file is not write-able", "Check file permissions and users", file.getAbsolutePath());
- } else {
- try {
- mapper.writeValue(file, this.createSimplified());
- } catch (IOException e) {
- throw new PropertiesException("Failed to write the json schema rule file", e, "check filename", file.getName());
- }
-
- }
- }
- }
-
- @Override
- protected Rule createSimplified() throws IOException {
- Rule n = createNewInstance();
- addAllMandatory(n);
- return n;
- }
-
-
- @XmlTransient
- @JsonIgnore
- public void setSchemaName(SchemaRegistryName name) {
- this.schemaname = name;
- }
-
- @XmlTransient
- @JsonIgnore
- public void setSchemaname(String name) {
- this.schemaname = SchemaRegistryName.create(name);
- }
-
- @Override
- protected Rule createNewInstance() throws ConnectorCallerException {
- return new SchemaRuleSet();
- }
-
- @Override
- public String toString() {
- return "Rules for schema";
- }
-
- @XmlTransient
- @JsonIgnore
- public SchemaRegistryName getSchemaName() {
- return schemaname;
- }
-
- @XmlTransient
- @JsonIgnore
- public String getSchemaname() {
- return schemaname.getName();
- }
-
-}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleFileDefinition.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleFileDefinition.java
new file mode 100644
index 0000000..a3be1fe
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleFileDefinition.java
@@ -0,0 +1,326 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.avro.Schema;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
+import io.rtdi.bigdata.kafka.avro.RowType;
+import io.rtdi.bigdata.rulesservice.RulesService;
+import io.rtdi.bigdata.rulesservice.jexl.AvroRuleUtils;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+import io.rtdi.bigdata.rulesservice.rules.RuleUtils;
+
+public class RuleFileDefinition {
+ private Path name;
+ private String inputsubjectname;
+ private String outputsubjectname;
+ private String samplefile;
+ private String schema;
+ private Schema schemadef;
+
+ private List rulesteps;
+ /**
+ * A Definition can be saved while still not complete - then the previous version is to be used
+ */
+ private boolean active = false;
+
+ /**
+ * @param name depicts a relative path to the json file
+ * @param inputsubjectname is the name of the subject this rule group assumes as input
+ */
+ public RuleFileDefinition(Path name, String inputsubjectname) {
+ this.name = name;
+ this.inputsubjectname = inputsubjectname;
+ }
+
+ public RuleFileDefinition() {
+ }
+
+ public RuleFileDefinition(Schema schema) {
+ schemadef = schema;
+ this.schema = schema.toString();
+ }
+
+ public List getRulesteps() {
+ return rulesteps;
+ }
+
+ public void setRulesteps(List rulesteps) {
+ this.rulesteps = rulesteps;
+ }
+
+ public JexlRecord apply(JexlRecord valuerecord, boolean test) throws IOException {
+ RowType changetype = AvroRuleUtils.getChangeType(valuerecord);
+ // Rules are applied to new or changed records but not deleted rows
+ if (rulesteps != null && rulesteps.size() > 0 && (changetype == null || (changetype != RowType.EXTERMINATE && changetype != RowType.DELETE && changetype != RowType.TRUNCATE))) {
+ for (RuleStep step : rulesteps) {
+ /*
+ * Go through one rule step, collect the rule results to be stored in the _audit record and also the changed values.
+ * The changed values are collected first and applied in a second step to avoid side effects when one rule modifies
+ * a value and another reads it.
+ */
+ step.apply(valuerecord, valuerecord, test);
+ valuerecord.mergeReplacementValues();
+ if (test) {
+ step.updateSampleOutput(valuerecord);
+ }
+ }
+ if (!test) {
+ /*
+ * Take all collected rule results and add them to the record.
+ */
+ AvroRuleUtils.mergeResults(valuerecord);
+ }
+ }
+ return valuerecord;
+ }
+
+ /**
+ * @param rootdir where all files are located
+ * @param schemanname the rule requires as input
+ * @param name depicts a relative path to the json file
+ * @param active if true, it goes into the active folder and returns that version, else the inactive folder with intermediate files
+ * @return the RuleFileDefinition or null, if it not exists as active/inactive as requested
+ * @throws IOException
+ */
+ public static RuleFileDefinition load(Path rootdir, String subjectname, Path name, boolean active) throws IOException {
+ /*
+ * The directory structure is ////..../
+ */
+ if (name.isAbsolute()) {
+ throw new IOException("name path <" + name + "> cannot be an absolute path");
+ }
+ String activedir = getActivePath(active);
+ Path filepath = rootdir.resolve(subjectname).resolve(activedir).resolve(name);
+ if (filepath.toFile().isFile()) {
+ ObjectMapper om = new ObjectMapper();
+ RuleFileDefinition rg = om.readValue(filepath.toFile(), RuleFileDefinition.class);
+ rg.name = name;
+ rg.inputsubjectname = subjectname;
+ rg.setActive(active);
+ return rg;
+ } else {
+ return null;
+ }
+ }
+
+ private static String getActivePath(boolean active) {
+ return active ? "active" : "inactive";
+ }
+
+ public void save(Path rootdir, Path filename) throws IOException {
+ if (filename == null) {
+ throw new IOException("The filename cannot be null");
+ } else if (filename.toString().contains("..")) {
+ throw new IOException("The filename can contain subdirectories but not <..>");
+ }
+ if (name == null) {
+ throw new IOException("The name field cannot be null");
+ } else if (name.toString().contains("..")) {
+ throw new IOException("The name field can contain subdirectories but not <..>");
+ }
+ ObjectMapper om = new ObjectMapper();
+ String activedir = getActivePath(active);
+ Path path = rootdir.resolve(inputsubjectname).resolve(activedir).resolve(name); // name can have directories too
+ File dir = path.getParent().toFile();
+ dir.mkdirs(); // create all directories in case this is a new rule group
+
+ om.writer()
+ .withDefaultPrettyPrinter()
+ .writeValue(path.toFile(), this); // save the file with the path in the object
+ if (filename != null && !filename.equals(this.name)) {
+ // file was renamed, delete the old
+ path = rootdir.resolve(inputsubjectname).resolve(activedir).resolve(filename);
+ File file = path.toFile();
+ if (file.exists()) {
+ file.delete();
+ }
+ }
+ }
+
+ @JsonIgnore
+ public boolean isActive() {
+ return active;
+ }
+
+ public void setActive(boolean active) {
+ this.active = active;
+ }
+
+ public String getName() {
+ if (name == null) {
+ return "new";
+ } else if (File.separatorChar == '\\') {
+ return name.toString().replace('\\', '/');
+ } else {
+ return name.toString();
+ }
+ }
+
+ public void setName(String name) {
+ if (name.startsWith("/")) {
+ name = name.substring(1);
+ }
+ this.name = Path.of(name);
+ }
+
+ @JsonIgnore
+ public void setPath(Path path) {
+ this.name = path;
+ }
+
+ public String getInputsubjectname() {
+ return inputsubjectname;
+ }
+
+ public void setInputsubjectname(String inputsubjectname) {
+ this.inputsubjectname = inputsubjectname;
+ }
+
+ public void addRuleStep(RuleStep step) {
+ if (rulesteps == null) {
+ rulesteps = new ArrayList<>();
+ }
+ rulesteps.add(step);
+ }
+
+ public static List getAllRuleFiles(Path rootdir, String subjectname) {
+ String activedir = getActivePath(false);
+ Path path = rootdir.resolve(subjectname).resolve(activedir);
+ List files = new ArrayList<>();
+ List ret = new ArrayList<>();
+ collectFiles(path, path.toFile(), files);
+ for (Path p : files) {
+ FileTime filedateinactive = null;
+ try {
+ Path inactivepath = path.resolve(p);
+ BasicFileAttributes attr = Files.readAttributes(inactivepath, BasicFileAttributes.class);
+ filedateinactive = attr.lastModifiedTime();
+ } catch (IOException e) {
+ }
+ String name = p.toString();
+ if (File.separatorChar == '\\') {
+ name = name.replace('\\', '/');
+ }
+ Path activepath = rootdir.resolve(subjectname).resolve(getActivePath(true)).resolve(p);
+ FileTime filedateactive = null;
+ try {
+ BasicFileAttributes activeattr = Files.readAttributes(activepath, BasicFileAttributes.class);
+ filedateactive = activeattr.lastModifiedTime();
+ } catch (IOException e) {
+ }
+ RuleFileName n = new RuleFileName(name, subjectname, filedateinactive, filedateactive);
+ ret.add(n);
+ }
+ return ret;
+ }
+
+ public static List getAllRuleFiles(Path rootdir) {
+ File[] subjectnamedirs = rootdir.toFile().listFiles();
+ List ret = new ArrayList<>();
+ for (File f : subjectnamedirs) {
+ if (f.isDirectory()) {
+ ret.addAll(getAllRuleFiles(rootdir, f.getName()));
+ }
+ }
+ return ret;
+ }
+
+
+ private static void collectFiles(Path startdir, File dir, List files) {
+ File[] filesarray = dir.listFiles();
+ if (filesarray != null) {
+ for(File f : filesarray) {
+ if (f.isDirectory()) {
+ collectFiles(startdir, f, files);
+ } else {
+ Path relative = startdir.relativize(f.toPath());
+ files.add(relative);
+ }
+ }
+ }
+ }
+
+ public String getSamplefile() {
+ return samplefile;
+ }
+
+ public void setSamplefile(String samplefile) {
+ this.samplefile = samplefile;
+ }
+
+ public String getOutputsubjectname() {
+ return outputsubjectname;
+ }
+
+ public void setOutputsubjectname(String outputsubjectname) {
+ this.outputsubjectname = outputsubjectname;
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ public static void copyToActivate(Path rootdir, String subjectname, String path) throws IOException {
+ String inactivedir = getActivePath(false);
+ String activedir = getActivePath(true);
+ Path sourcepath = rootdir.resolve(subjectname).resolve(inactivedir).resolve(path);
+ Path targetdir = rootdir.resolve(subjectname).resolve(activedir);
+ Path targetpath = targetdir.resolve(path);
+ targetpath.getParent().toFile().mkdirs(); // the filename can have directories too
+ Files.copy(sourcepath, targetpath, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ @Override
+ public String toString() {
+ return getName() + ": RuleFile";
+ }
+
+ public void update(RuleFileDefinition empty) {
+ if (schema.compareTo(empty.getSchema()) != 0) {
+ if (rulesteps != null) {
+ for (RuleStep s : rulesteps) {
+ if (empty.getRulesteps() != null && empty.getRulesteps().size() > 0) {
+ s.update(empty.getRulesteps().get(0));
+ }
+ }
+ }
+ }
+ setSchema(empty.getSchema());
+ }
+
+ public static RuleFileDefinition createEmptyRuleFileDefinition(String subjectname, RulesService service) throws IOException, RestClientException {
+ org.apache.avro.Schema schema = service.getLatestSchema(subjectname);
+ RuleFileDefinition rg = new RuleFileDefinition(schema);
+ rg.setInputsubjectname(subjectname);
+ RuleStep step = new RuleStep("next step", schema);
+ rg.addRuleStep(step);
+ RuleUtils.addRules(step, schema);
+ return rg;
+ }
+
+ public Schema schema() {
+ if (schemadef == null) {
+ Schema.Parser p = new Schema.Parser();
+ schemadef = p.parse(schema);
+ }
+ return schemadef;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleFileName.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleFileName.java
new file mode 100644
index 0000000..b60a026
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleFileName.java
@@ -0,0 +1,46 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+import java.nio.file.attribute.FileTime;
+
+public class RuleFileName {
+ private String name;
+ private String filetimeinactive;
+ private String filetimeactive;
+ private String subjectname;
+ private Boolean active;
+
+ public RuleFileName(String name, String subjectname, FileTime inactivetime, FileTime activetime) {
+ this.name = name;
+ this.subjectname = subjectname;
+ if (inactivetime != null) {
+ this.filetimeinactive = inactivetime.toString();
+ }
+ if (activetime != null) {
+ this.filetimeactive = activetime.toString();
+ this.active = activetime.compareTo(inactivetime) >= 0;
+ } else {
+ this.active = Boolean.FALSE;
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getFiletimeinactive() {
+ return filetimeinactive;
+ }
+
+ public String getFiletimeactive() {
+ return filetimeactive;
+ }
+
+ public String getSubjectname() {
+ return subjectname;
+ }
+
+ public Boolean getActive() {
+ return active;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleStep.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleStep.java
new file mode 100644
index 0000000..5d7c8ed
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/RuleStep.java
@@ -0,0 +1,62 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+import java.io.IOException;
+
+import org.apache.avro.Schema;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.rtdi.bigdata.kafka.avro.RuleResult;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+import io.rtdi.bigdata.rulesservice.rules.GenericRules;
+import io.rtdi.bigdata.rulesservice.rules.RecordRule;
+import io.rtdi.bigdata.rulesservice.rules.Rule;
+
+public class RuleStep extends RecordRule {
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ mapper.setSerializationInclusion(Include.NON_NULL);
+ }
+
+ public RuleStep(String name, Schema schema) {
+ super(name, schema);
+ }
+
+ public RuleStep() {
+ }
+
+ @Override
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
+ if (getRules() != null) {
+ for ( Rule rule : getRules()) {
+ if (rule.getFieldname() != null && !(rule instanceof GenericRules)) {
+ rule.apply(container.get(rule.getFieldname()), container, test);
+ } else {
+ /*
+ * GenericRules are not bound to a field, they operate on the entire record
+ */
+ rule.apply(container, container, test);
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return getFieldname() + ": RuleStep";
+ }
+
+ @Override
+ public void updateSampleOutput(JexlRecord valuerecord) {
+ if (getRules() != null) {
+ for ( Rule rule : getRules()) {
+ rule.updateSampleOutput(valuerecord);
+ }
+ }
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/SampleFileName.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/SampleFileName.java
new file mode 100644
index 0000000..a99d3eb
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/SampleFileName.java
@@ -0,0 +1,19 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+public class SampleFileName {
+ private String filename;
+
+ public SampleFileName() {
+ }
+
+ public SampleFileName(String filename) {
+ this.filename = filename;
+ }
+
+ public String getFilename() {
+ return filename;
+ }
+ public void setFilename(String filename) {
+ this.filename = filename;
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceSettings.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceSettings.java
new file mode 100644
index 0000000..31a4b6d
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceSettings.java
@@ -0,0 +1,69 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ServiceSettings {
+ private String properties;
+ private boolean schemaregconnected;
+ private int schemaregsubjects;
+ private boolean kafkaconnected;
+ private boolean servicerunning;
+ private String errormessage;
+ private boolean adminuser;
+ private static final String propertieshelp = "The properties must contain info to connect to the schema registry (read/write) and create a Kafka KStream";
+
+ public String getProperties() {
+ if (properties != null) {
+ return properties;
+ } else {
+ return "basic.auth.credentials.source=USER_INFO\n"
+ + "schema.registry.basic.auth.user.info=:\n"
+ + "schema.registry.url=https://,https://\n";
+ }
+ }
+ public void setProperties(String properties) {
+ this.properties = properties;
+ }
+
+ public boolean isSchemaregconnected() {
+ return schemaregconnected;
+ }
+ public void setSchemaregconnected(boolean schemaregconnected) {
+ this.schemaregconnected = schemaregconnected;
+ }
+ public boolean isKafkaconnected() {
+ return kafkaconnected;
+ }
+ public void setKafkaconnected(boolean kafkaconnected) {
+ this.kafkaconnected = kafkaconnected;
+ }
+ public boolean isServicerunning() {
+ return servicerunning;
+ }
+ public void setServicerunning(boolean servicerunning) {
+ this.servicerunning = servicerunning;
+ }
+ public String getErrormessage() {
+ return errormessage;
+ }
+ public void setErrormessage(String errormessage) {
+ this.errormessage = errormessage;
+ }
+ public boolean isAdminuser() {
+ return adminuser;
+ }
+ public void setAdminuser(boolean adminuser) {
+ this.adminuser = adminuser;
+ }
+ public int getSchemaregsubjects() {
+ return schemaregsubjects;
+ }
+ public void setSchemaregsubjects(int schemaregsubjects) {
+ this.schemaregsubjects = schemaregsubjects;
+ }
+ public String getPropertieshelp() {
+ return propertieshelp;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceStatus.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceStatus.java
new file mode 100644
index 0000000..0043be0
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceStatus.java
@@ -0,0 +1,43 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import io.rtdi.bigdata.rulesservice.RuleFileKStream;
+import io.rtdi.bigdata.rulesservice.RulesService;
+
+public class ServiceStatus {
+ private int activetopics = 0;
+ private List topicstatus;
+ private ServiceSettings config;
+
+ public ServiceStatus(RulesService service) {
+ Map services = service.getServices();
+ config = service.getConfig(false);
+ if (services != null) {
+ topicstatus = new ArrayList<>();
+ for (Entry entity : services.entrySet()) {
+ ServiceStatusTopic s = new ServiceStatusTopic(entity.getValue());
+ if (s.getStatus() == Boolean.TRUE) {
+ activetopics++;
+ }
+ topicstatus.add(s);
+ }
+ }
+ }
+
+ public List getTopicstatus() {
+ return topicstatus;
+ }
+
+ public int getActivetopics() {
+ return activetopics;
+ }
+
+ public ServiceSettings getConfig() {
+ return config;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceStatusTopic.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceStatusTopic.java
new file mode 100644
index 0000000..b401a16
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/ServiceStatusTopic.java
@@ -0,0 +1,67 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import io.rtdi.bigdata.rulesservice.RuleFileKStream;
+
+public class ServiceStatusTopic {
+ private Boolean status;
+ private List activerules;
+ private long rowsprocessed = 0L;
+ private long lastprocessedtimestamp = 0L;
+ private Long avgtime = null;
+ private String topicname;
+ private String name;
+ private Integer rulecount = 0;
+
+ public ServiceStatusTopic(RuleFileKStream stream) {
+ topicname = stream.getInputtopicname();
+ name = stream.getName();
+ if (stream.isAlive()) {
+ status = Boolean.TRUE;
+ rowsprocessed = stream.getRowsprocessed();
+ lastprocessedtimestamp = stream.getLastprocessedtimestamp();
+ avgtime = stream.getAvgProcessingtime();
+ activerules = new ArrayList<>();
+ for (RuleFileDefinition r : stream.getRulefiledefinitions().values()) {
+ activerules.add(r.getName());
+ }
+ rulecount = activerules.size();
+ } else {
+ status = Boolean.FALSE;
+ }
+ }
+
+ public Boolean getStatus() {
+ return status;
+ }
+
+ public List getActiverules() {
+ return activerules;
+ }
+
+ public long getRowsprocessed() {
+ return rowsprocessed;
+ }
+
+ public long getLastprocessedtimestamp() {
+ return lastprocessedtimestamp;
+ }
+
+ public Long getAvgtime() {
+ return avgtime;
+ }
+
+ public String getTopicname() {
+ return topicname;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Integer getRulecount() {
+ return rulecount;
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/SubjectName.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/SubjectName.java
new file mode 100644
index 0000000..cac3680
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/SubjectName.java
@@ -0,0 +1,21 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+public class SubjectName {
+ private String name;
+
+ public SubjectName() {
+ super();
+ }
+
+ public SubjectName(String name) {
+ this.name = name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/TopicName.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/TopicName.java
new file mode 100644
index 0000000..57d17f3
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/TopicName.java
@@ -0,0 +1,13 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+public class TopicName {
+ private String name;
+
+ public TopicName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/config/TopicRule.java b/src/main/java/io/rtdi/bigdata/rulesservice/config/TopicRule.java
new file mode 100644
index 0000000..e0f3cd0
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/config/TopicRule.java
@@ -0,0 +1,134 @@
+package io.rtdi.bigdata.rulesservice.config;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.exc.StreamWriteException;
+import com.fasterxml.jackson.databind.DatabindException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class TopicRule implements Comparable {
+ private String inputtopicname;
+ private String outputtopicname;
+ private Boolean modified;
+ private Boolean activate;
+ private String info;
+ private Integer instances;
+ /**
+ * A List of rulefiles to be used for this topic in the format of /
+ */
+ private List rulefiles;
+
+ public TopicRule(String topicname) {
+ this.inputtopicname = topicname;
+ }
+
+ public TopicRule() {
+ }
+
+ public TopicRule(TopicRule source) {
+ this.inputtopicname = source.inputtopicname;
+ this.outputtopicname = source.outputtopicname;
+ this.rulefiles = source.rulefiles;
+ }
+
+ public String getInputtopicname() {
+ return inputtopicname;
+ }
+
+ public void setInputtopicname(String topicname) {
+ this.inputtopicname = topicname;
+ }
+
+ public List getRulefiles() {
+ return rulefiles;
+ }
+
+ public void setRulefiles(List rulefiles) {
+ this.rulefiles = rulefiles;
+ }
+
+ public String getOutputtopicname() {
+ return outputtopicname;
+ }
+
+ public void setOutputtopicname(String outputtopicname) {
+ this.outputtopicname = outputtopicname;
+ }
+
+ public Boolean getModified() {
+ return modified;
+ }
+
+ public void setModified(Boolean modified) {
+ this.modified = modified;
+ }
+
+ public void save(Path topicruledir) throws StreamWriteException, DatabindException, IOException {
+ ObjectMapper om = new ObjectMapper();
+ File file = topicruledir.resolve(inputtopicname + ".json").toFile();
+ TopicRule clone = new TopicRule(this);
+ om.writer()
+ .withDefaultPrettyPrinter()
+ .writeValue(file, clone);
+ if (activate != null && activate) {
+ for (String rulefile : rulefiles) {
+ int pos = rulefile.indexOf('/');
+ if (pos > 0) {
+ String subject = rulefile.substring(0, pos);
+ String filename = rulefile.substring(pos+1);
+ RuleFileDefinition.copyToActivate(topicruledir.getParent(), subject, filename);
+ } else {
+ throw new IOException("The selected key does not follow the path convention /");
+ }
+ }
+ }
+ }
+
+ @Override
+ public int compareTo(TopicRule o) {
+ if (o == null) {
+ return -1;
+ } else {
+ return o.inputtopicname.compareTo(inputtopicname);
+ }
+ }
+
+ public void delete(Path topicruledir) {
+ File file = topicruledir.resolve(inputtopicname + ".json").toFile();
+ if (file.exists()) {
+ file.delete();
+ }
+ }
+
+ public String getInfo() {
+ return info;
+ }
+
+ public void setInfo(String info) {
+ this.info = info;
+ }
+
+ public Boolean getActivate() {
+ return activate;
+ }
+
+ public void setActivate(Boolean activate) {
+ this.activate = activate;
+ }
+
+ public Integer getInstances() {
+ return instances;
+ }
+
+ public void setInstances(Integer instances) {
+ this.instances = instances;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/AvroContainer.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/AvroContainer.java
new file mode 100644
index 0000000..270a3b8
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/AvroContainer.java
@@ -0,0 +1,40 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+import org.apache.commons.jexl3.JexlContext;
+
+public interface AvroContainer extends JexlContext {
+
+ /**
+ * Move the collected change values into the record.
+ * After that call the changed value list is empty.
+ */
+ void mergeReplacementValues();
+
+ void addChangedvalue(String fieldname, Object substitutevalue);
+
+ String getPath();
+
+ AvroContainer getParent();
+
+ void setParent(AvroContainer parent);
+
+ Field getParentField();
+
+ List getRuleresults();
+
+ void setRuleresults(ArrayList list) throws IOException;
+
+ void addRuleresult(JexlRecord r) throws IOException;
+
+ Schema getSchema();
+
+ Map toMap();
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/AvroRuleUtils.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/AvroRuleUtils.java
new file mode 100644
index 0000000..39379aa
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/AvroRuleUtils.java
@@ -0,0 +1,133 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilderException;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.io.JsonDecoder;
+import org.apache.avro.io.JsonEncoder;
+import org.apache.commons.text.StringEscapeUtils;
+
+import io.rtdi.bigdata.kafka.avro.RowType;
+import io.rtdi.bigdata.kafka.avro.RuleResult;
+import io.rtdi.bigdata.kafka.avro.SchemaConstants;
+import io.rtdi.bigdata.kafka.avro.recordbuilders.ValueSchema;
+
+public class AvroRuleUtils {
+
+ public static String convertRecordToJson(GenericRecord record) throws IOException {
+ return record.toString();
+ }
+
+ public static String encodeJson(String text) {
+ /*
+ * Backspace is replaced with \b
+ * Form feed is replaced with \f
+ * Newline is replaced with \n
+ * Carriage return is replaced with \r
+ * Tab is replaced with \t
+ * Double quote is replaced with \"
+ * Backslash is replaced with \\
+ */
+ return StringEscapeUtils.escapeJson(text);
+ }
+
+ public static void mergeResults(JexlRecord valuerecord) throws IOException {
+ try {
+ List auditdetails = valuerecord.getRuleresults();
+ JexlRecord auditrecord = (JexlRecord) valuerecord.get(ValueSchema.AUDIT);
+ RuleResult globalresult;
+ if (auditrecord == null) {
+ auditrecord = new JexlRecord(ValueSchema.audit.getSchema(), valuerecord);
+ globalresult = RuleResult.PASS;
+ valuerecord.put(ValueSchema.AUDIT, auditrecord);
+ } else {
+ globalresult = getRuleResult(auditrecord);
+ }
+ @SuppressWarnings("unchecked")
+ JexlArray details = (JexlArray) auditrecord.get(ValueSchema.AUDITDETAILS);
+ if (details == null) {
+ details = new JexlArray(100, ValueSchema.auditdetails_array_schema, valuerecord);
+ auditrecord.put(ValueSchema.AUDITDETAILS, details);
+ }
+ for ( JexlRecord d : auditdetails) {
+ details.add(d);
+ RuleResult ruleresult = getRuleResult(d);
+ /*
+ * The individual rule result pulls down the global result PASS -> WARN -> FAIL
+ * but never _corrects_ a result.
+ * Examples:
+ * GLOBAL + INDIVIDUAL = NEW GLOBAL
+ * PASS + PASS = PASS
+ * PASS + WARN = WARN
+ * PASS + FAIL = FAIL
+ * WARN + PASS = WARN
+ * WARN + WARN = WARN
+ * WARN + FAIL = FAIL
+ * FAIL + xxxx = FAIL
+ */
+ if (ruleresult == RuleResult.FAIL) {
+ globalresult = RuleResult.FAIL;
+ } else if (ruleresult == RuleResult.WARN && globalresult == RuleResult.PASS) {
+ globalresult = RuleResult.WARN;
+ }
+ }
+ auditrecord.put(ValueSchema.TRANSFORMRESULT, globalresult.name());
+ } catch (SchemaBuilderException e) {
+ throw new IOException("Cannot merge audit data", e);
+ }
+ }
+
+ public static JexlRecord createAuditDetails() {
+ return new JexlRecord(ValueSchema.auditdetails_array_schema.getElementType(), null);
+ }
+
+ public static RowType getChangeType(JexlRecord valuerecord) {
+ if (valuerecord.hasField(SchemaConstants.SCHEMA_COLUMN_CHANGE_TYPE)) {
+ Object changetype = valuerecord.get(SchemaConstants.SCHEMA_COLUMN_CHANGE_TYPE);
+ if (changetype != null) {
+ return RowType.getByIdentifier(changetype.toString());
+ }
+ }
+ return null;
+ }
+
+ public static RuleResult getRuleResult(JexlRecord record) {
+ Object g = record.get(ValueSchema.TRANSFORMRESULT);
+ RuleResult ruleresult;
+ if (g == null) {
+ ruleresult = RuleResult.PASS;
+ } else {
+ String s = g.toString();
+ ruleresult = RuleResult.valueOf(s);
+ }
+ return ruleresult;
+ }
+
+ public static JexlRecord convertFromJson(String json, Schema schema) throws IOException {
+ DatumReader reader = new JexlGenericDatumReader<>(schema);
+ JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, json);
+ return reader.read(null, decoder);
+ }
+
+ public static String convertToJson(JexlRecord record, Schema schema) throws IOException {
+ DatumWriter writer = new GenericDatumWriter<>(schema);
+ try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {
+ JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, stream);
+ writer.write(record, jsonEncoder);
+ jsonEncoder.flush();
+ return stream.toString();
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlArray.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlArray.java
new file mode 100644
index 0000000..cc76003
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlArray.java
@@ -0,0 +1,250 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.generic.GenericData.Array;
+import org.apache.commons.jexl3.annotations.NoJexl;
+
+public class JexlArray extends Array implements AvroContainer {
+
+ private AvroContainer parent = null;
+ Field parentfield;
+ private int parentarrayindex = -1;
+ private String path;
+ private Map changedvalues = null;
+ private List ruleresults = null;
+ private AvroContainer root = null;
+ private Map map = null;
+
+
+ @NoJexl
+ public JexlArray(Schema schema, Collection c) {
+ super(schema, c);
+ }
+
+ @NoJexl
+ public JexlArray(int capacity, Schema schema, JexlRecord parent) {
+ super(capacity, schema);
+ setParent(parent);
+ }
+
+ @Override
+ @NoJexl
+ public void setParent(AvroContainer parent) {
+ this.parent = parent;
+ }
+
+ @Override
+ @NoJexl
+ public void mergeReplacementValues() {
+ if (changedvalues != null) {
+ for( Entry entry : changedvalues.entrySet()) {
+ Object value = entry.getValue();
+ this.set(entry.getKey(), value);
+ }
+ changedvalues.clear();
+ }
+ if (!this.isEmpty()) {
+ Iterator iter = this.iterator();
+ while (iter.hasNext()) {
+ T c = iter.next();
+ if (c instanceof AvroContainer r) {
+ r.mergeReplacementValues();
+ }
+ }
+ }
+ }
+
+ @Override
+ public List getRuleresults() {
+ return ruleresults;
+ }
+
+ public Map getChangedvalues() {
+ return changedvalues;
+ }
+
+ @Override
+ public void addChangedvalue(String fieldname, Object value) {
+ if (changedvalues == null) {
+ changedvalues = new HashMap<>();
+ }
+ changedvalues.put(fieldname, value);
+ }
+
+ @Override
+ public AvroContainer getParent() {
+ return parent;
+ }
+
+ @Override
+ @NoJexl
+ public boolean add(T o) {
+ if (o instanceof JexlRecord r) {
+ r.setParent(this);
+ r.parentarrayindex = super.size();
+ }
+ return super.add(o);
+ }
+
+ @Override
+ @NoJexl
+ public void add(int location, T o) {
+ if (o instanceof JexlRecord r) {
+ r.setParent(this);
+ r.parentarrayindex = location;
+ }
+ super.add(location, o);
+ }
+
+ @Override
+ @NoJexl
+ public T set(int i, T o) {
+ if (o instanceof JexlRecord r) {
+ r.setParent(this);
+ r.parentarrayindex = i;
+ }
+ return super.set(i, o);
+ }
+
+ @Override
+ public String getPath() {
+ if (path == null) {
+ if (parent == null) {
+ path = null;
+ } else if (parentfield != null) {
+ String p = parent.getPath();
+ if (p != null) {
+ path = p + "." + parentfield.name();
+ } else {
+ path = parentfield.name();
+ }
+ } else {
+ path = parent.getPath() + "[" + parentarrayindex + "]";
+ }
+ }
+ return path;
+ }
+
+ @Override
+ public Field getParentField() {
+ return parentfield;
+ }
+
+ @Override
+ public Object get(String name) {
+ if ("parent".equals(name)) {
+ return parent;
+ } else {
+ try {
+ int index = Integer.parseInt(name);
+ if (index > 0 && index < this.size()) {
+ return this.get(index);
+ }
+ } catch (NumberFormatException e) {
+ }
+ }
+ return null;
+ }
+
+ @Override
+ @NoJexl
+ public void set(String name, Object value) {
+ throw new IllegalAccessError("A array is a read-only object");
+ }
+
+ @Override
+ @NoJexl
+ public void addRuleresult(JexlRecord r) throws IOException {
+ AvroContainer root = getRoot();
+ if (root.getRuleresults() == null) {
+ root.setRuleresults(new ArrayList<>());
+ }
+ root.getRuleresults().add(r);
+ }
+
+ @Override
+ @NoJexl
+ public void setRuleresults(ArrayList list) throws IOException {
+ AvroContainer root = getRoot();
+ if (root == this) {
+ ruleresults = list;
+ } else {
+ throw new IOException("rule results can only be set at the root level");
+ }
+ }
+
+ public AvroContainer getRoot() {
+ if (root != null) {
+ return root;
+ } else {
+ AvroContainer root = this;
+ while (root.getParent() != null) {
+ root = root.getParent();
+ }
+ return root;
+ }
+ }
+
+ public JexlRecord find(String field, Object value) {
+ if (field != null && value != null) {
+ for (Object o : this) {
+ if (o instanceof JexlRecord r) {
+ Object v = r.get(field);
+ if (value.equals(v)) {
+ return r;
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Map toMap() {
+ if (map == null) {
+ map = new HashMap<>();
+ for (int i = 0; i < size(); i++) {
+ Object value = super.get(i);
+ if (value instanceof AvroContainer c) {
+ map.put(String.valueOf(i), c.toMap());
+ } else {
+ map.put(String.valueOf(i), value);
+ }
+ }
+ AvroContainer p = this.getParent();
+ if (p != null) {
+ map.put("parent", p.toMap());
+ }
+ }
+ return map;
+ }
+
+ @Override
+ public boolean has(String name) {
+ if ("parent".equals(name)) {
+ return true;
+ } else {
+ try {
+ int index = Integer.parseInt(name);
+ if (index > 0 && index < this.size()) {
+ return true;
+ } else {
+ return false;
+ }
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlAvroDeserializer.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlAvroDeserializer.java
new file mode 100644
index 0000000..cc9316c
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlAvroDeserializer.java
@@ -0,0 +1,80 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import org.apache.avro.AvroRuntimeException;
+import org.apache.avro.Schema;
+import org.apache.avro.io.BinaryDecoder;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.ResolvingDecoder;
+
+import io.rtdi.bigdata.kafka.avro.AvroDeserializer;
+import io.rtdi.bigdata.kafka.avro.AvroUtils;
+
+/**
+ * This is class does deserialize Avro Jexl records. It uses the same format as Kafka itself, hence
+ * even if data is serialized by others, e.g. Kafka Connect, it can be consumed.
+ *
+ */
+public class JexlAvroDeserializer extends AvroDeserializer {
+
+ private static final DecoderFactory decoderFactory = DecoderFactory.get();
+
+ /**
+ * Converts a byte[] into an Avro JexlRecord using the supplied schema.
+ * The schema must be read from the schema registry using the message's schema id, see {@link #getSchemaId(byte[])}
+ *
+ * @param data with the binary Avro representation
+ * @param schema used for the deserialization
+ * @return JexlRecord in Jexl abstraction
+ * @throws IOException in case this is not a valid Avro Kafka message
+ */
+ public static JexlRecord deserialize(byte[] data, Schema schema) throws IOException {
+ if (data != null) {
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data); ) {
+ int b = in.read();
+ if (b != AvroUtils.MAGIC_BYTE) {
+ throw new AvroRuntimeException("Not a valid Kafka Avro message frame");
+ } else {
+ in.skip(Integer.BYTES);
+ BinaryDecoder decoder = decoderFactory.directBinaryDecoder(in, null);
+ DatumReader reader = new JexlGenericDatumReader<>(schema);
+ return reader.read(null, decoder);
+ }
+ }
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Converts a byte[] into an Avro JexlRecord using the supplied schema.
+ * The schema must be read from the schema registry using the message's schema id, see {@link #getSchemaId(byte[])}
+ *
+ * @param data with the binary Avro representation
+ * @param schemawriter used for the deserialization
+ * @param schemareader the evolved schema
+ * @return JexlRecord in Jexl abstraction
+ * @throws IOException in case this is not a valid Avro Kafka message
+ */
+ public static JexlRecord deserialize(byte[] data, Schema schemawriter, Schema schemareader) throws IOException {
+ if (data != null) {
+ try (ByteArrayInputStream in = new ByteArrayInputStream(data); ) {
+ int b = in.read();
+ if (b != AvroUtils.MAGIC_BYTE) {
+ throw new AvroRuntimeException("Not a valid Kafka Avro message frame");
+ } else {
+ in.skip(Integer.BYTES);
+ ResolvingDecoder decoder = decoderFactory.resolvingDecoder(schemawriter, schemareader, decoderFactory.directBinaryDecoder(in, null));
+ DatumReader reader = new JexlGenericDatumReader<>(schemareader);
+ return reader.read(null, decoder);
+ }
+ }
+ } else {
+ return null;
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlDefaultFunction.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlDefaultFunction.java
new file mode 100644
index 0000000..2ca6dd7
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlDefaultFunction.java
@@ -0,0 +1,5 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+public class JexlDefaultFunction {
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlGenericData.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlGenericData.java
new file mode 100644
index 0000000..b0f5fb1
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlGenericData.java
@@ -0,0 +1,31 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+
+public class JexlGenericData extends GenericData {
+ private static JexlGenericData INSTANCE = new JexlGenericData();
+
+ public JexlGenericData() {
+ super();
+ }
+
+ public JexlGenericData(ClassLoader classLoader) {
+ super(classLoader);
+ }
+
+ public static GenericData get() { return INSTANCE; }
+
+ @Override
+ public Object newRecord(Object old, Schema schema) {
+ if (old instanceof JexlRecord) {
+ JexlRecord record = (JexlRecord) old;
+ if (record.getSchema() == schema) {
+ return record;
+ }
+ }
+ return new JexlRecord(schema, null);
+ }
+
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlGenericDatumReader.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlGenericDatumReader.java
new file mode 100644
index 0000000..ce15f36
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlGenericDatumReader.java
@@ -0,0 +1,42 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+import java.util.Collection;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+
+public class JexlGenericDatumReader extends GenericDatumReader {
+
+ public JexlGenericDatumReader() {
+ super(null, null, JexlGenericData.get());
+ }
+
+ public JexlGenericDatumReader(GenericData data) {
+ super(data);
+ }
+
+ public JexlGenericDatumReader(Schema writer, Schema reader, GenericData data) {
+ super(writer, reader, data);
+ }
+
+ public JexlGenericDatumReader(Schema writer, Schema reader) {
+ super(writer, reader, JexlGenericData.get());
+ }
+
+ public JexlGenericDatumReader(Schema schema) {
+ super(schema, schema, JexlGenericData.get());
+ }
+
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ protected Object newArray(Object old, int size, Schema schema) {
+ if (old instanceof Collection) {
+ return super.newArray(old, size, schema);
+ } else {
+ return new JexlArray(size, schema, null);
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlRecord.java b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlRecord.java
new file mode 100644
index 0000000..a760198
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/jexl/JexlRecord.java
@@ -0,0 +1,300 @@
+package io.rtdi.bigdata.rulesservice.jexl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.avro.AvroRuntimeException;
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.generic.GenericData.Record;
+import org.apache.commons.jexl3.annotations.NoJexl;
+
+import io.rtdi.bigdata.kafka.avro.AvroUtils;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroRecord;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroType;
+import io.rtdi.bigdata.kafka.avro.datatypes.IAvroDatatype;
+
+public class JexlRecord extends Record implements AvroContainer {
+ private AvroContainer parent = null;
+ private int schemaid;
+ private Field parentfield;
+ int parentarrayindex = -1;
+ private String path = null;
+ private Map changedvalues = null;
+ private List ruleresults = null;
+ private AvroContainer root = null;
+ private Map map = null;
+
+ @NoJexl
+ public JexlRecord(Record other, boolean deepCopy) {
+ super(other, deepCopy);
+ }
+
+ @NoJexl
+ public JexlRecord(Schema schema, JexlRecord parent) {
+ super(schema);
+ setParent(parent);
+ }
+
+ @NoJexl
+ public JexlRecord(Schema schema, int schemaid) {
+ super(schema);
+ this.schemaid = schemaid;
+ ruleresults = new ArrayList<>();
+ }
+
+ @Override
+ @NoJexl
+ public void setParent(AvroContainer parent) {
+ this.parent = parent;
+ }
+
+ public AvroContainer getRoot() {
+ if (root != null) {
+ return root;
+ } else {
+ AvroContainer root = this;
+ while (root.getParent() != null) {
+ root = root.getParent();
+ }
+ return root;
+ }
+ }
+
+ @Override
+ public List getRuleresults() {
+ return ruleresults;
+ }
+
+ @Override
+ @NoJexl
+ public void mergeReplacementValues() {
+ if (changedvalues != null) {
+ for( Entry entry : changedvalues.entrySet()) {
+ this.put(entry.getKey(), entry.getValue());
+ }
+ changedvalues.clear();
+ }
+ for (Field f : getSchema().getFields()) {
+ Type t = AvroUtils.getBaseSchema(f.schema()).getType();
+ if (t == Type.RECORD || t == Type.ARRAY) {
+ Object value = this.get(f.name());
+ if (value instanceof AvroContainer r) {
+ r.mergeReplacementValues();
+ }
+ }
+ }
+ }
+
+ @Override
+ public AvroContainer getParent() {
+ return parent;
+ }
+
+ @Override
+ @NoJexl
+ public void put(String key, Object value) {
+ Field field = getSchema().getField(key);
+ if (field == null) {
+ throw new AvroRuntimeException("Not a valid schema field: " + key);
+ }
+ if (value instanceof JexlRecord r) {
+ r.setParent(this);
+ r.parentfield = field;
+ } else if (value instanceof JexlArray a) {
+ a.setParent(this);
+ a.parentfield = field;
+ } else {
+ IAvroDatatype t = AvroType.getAvroDataType(field.schema());
+ if (t != null) {
+ value = t.convertToInternal(value);
+ } else {
+ throw new AvroRuntimeException("Not a valid schema for field: " + field.name());
+ }
+ }
+ super.put(field.pos(), value);
+ }
+
+ @Override
+ @NoJexl
+ public void put(int i, Object v) {
+ Field field = getSchema().getFields().get(i);
+ if (v instanceof JexlRecord r) {
+ r.setParent(this);
+ r.parentfield = field;
+ } else if (v instanceof JexlArray a) {
+ a.setParent(this);
+ a.parentfield = field;
+ } else {
+ IAvroDatatype t = AvroType.getAvroDataType(field.schema());
+ if (t != null) {
+ v = t.convertToInternal(v);
+ } else {
+ throw new AvroRuntimeException("Not a valid schema for field: " + field.name());
+ }
+ }
+ super.put(i, v);
+ }
+
+ @NoJexl
+ public JexlRecord addChild(String key) {
+ Object f = super.get(key);
+ if (f == null) {
+ Field field = this.getSchema().getField(key);
+ if (field != null) {
+ Schema s = AvroUtils.getBaseSchema(field.schema());
+ if (s.getType() == Type.ARRAY) {
+ JexlArray a = new JexlArray<>(50, s, this);
+ put(key, a);
+ JexlRecord r = new JexlRecord(s.getElementType(), this);
+ a.add(r);
+ return r;
+ } else if (s.getType() == Type.RECORD) {
+ JexlRecord r = new JexlRecord(s, this);
+ put(key, r);
+ return r;
+ } else {
+ throw new AvroRuntimeException("Column \"" + key + "\" is not a sub record or array of records");
+ }
+ } else {
+ throw new AvroRuntimeException("Field \"" + key + "\" does not exist in schema");
+ }
+ } else if (f instanceof JexlArray) {
+ @SuppressWarnings("unchecked")
+ JexlArray a = (JexlArray) f;
+ JexlRecord r = new JexlRecord(a.getSchema().getElementType(), this);
+ a.add(r);
+ return r;
+ } else if (f instanceof JexlRecord) {
+ throw new AvroRuntimeException("Field \"" + key + "\" is a record and was added already");
+ } else {
+ throw new AvroRuntimeException("Field \"" + key + "\" is not a sub record or array of records");
+ }
+
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer b = new StringBuffer();
+ AvroRecord.create().toString(b, this);
+ return b.toString();
+ }
+
+ @Override
+ public Object get(String key) {
+ if ("parent".equals(key)) {
+ return parent;
+ } else if (getSchema().getField(key) != null) {
+ return super.get(key);
+ } else {
+ return null;
+ }
+ }
+
+ public int getSchemaId() {
+ return schemaid;
+ }
+
+ @NoJexl
+ public void setSchemaId(int schemaid) {
+ this.schemaid = schemaid;
+ }
+
+ @Override
+ public String getPath() {
+ if (path == null) {
+ if (parent == null) {
+ path = null;
+ } else if (parentfield != null) {
+ String p = parent.getPath();
+ if (p != null) {
+ path = p + "." + parentfield.name();
+ } else {
+ path = parentfield.name();
+ }
+ } else {
+ path = parent.getPath() + "[" + parentarrayindex + "]";
+ }
+ }
+ return path;
+ }
+
+ @Override
+ public Field getParentField() {
+ return parentfield;
+ }
+
+ public Map getChangedvalues() {
+ return changedvalues;
+ }
+
+ @Override
+ public void addChangedvalue(String fieldname, Object value) {
+ if (changedvalues == null) {
+ changedvalues = new HashMap<>();
+ }
+ changedvalues.put(fieldname, value);
+ }
+
+ @Override
+ @NoJexl
+ public void addRuleresult(JexlRecord r) throws IOException {
+ AvroContainer root = getRoot();
+ if (root.getRuleresults() == null) {
+ root.setRuleresults(new ArrayList<>());
+ }
+ root.getRuleresults().add(r);
+ }
+
+ @Override
+ @NoJexl
+ public void setRuleresults(ArrayList list) throws IOException {
+ AvroContainer root = getRoot();
+ if (root == this) {
+ ruleresults = list;
+ } else {
+ throw new IOException("rule results can only be set at the root level");
+ }
+ }
+
+ @Override
+ public Map toMap() {
+ if (map == null) {
+ map = new HashMap<>();
+ for (Field f : super.getSchema().getFields()) {
+ Object value = super.get(f.name());
+ if (value instanceof AvroContainer c) {
+ map.put(f.name(), c.toMap());
+ } else {
+ map.put(f.name(), value);
+ }
+ }
+ AvroContainer p = this.getParent();
+ if (p != null) {
+ map.put("parent", p.toMap());
+ }
+ }
+ return map;
+ }
+
+ @Override
+ public boolean has(String key) {
+ if ("parent".equals(key)) {
+ return true;
+ } else {
+ return getSchema().getField(key) != null;
+ }
+ }
+
+ @Override
+ @NoJexl
+ public void set(String name, Object value) {
+ throw new IllegalAccessError("A record is a read-only object");
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/ErrorResponse.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/ErrorResponse.java
new file mode 100644
index 0000000..7ddfe8c
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/ErrorResponse.java
@@ -0,0 +1,30 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.Response.Status;
+
+public class ErrorResponse {
+ private String errormessage;
+
+ public ErrorResponse(Exception e) {
+ this.errormessage = e.getClass().getSimpleName() + ": " + e.getMessage();
+ }
+
+ public ErrorResponse(String message) {
+ this.errormessage = message;
+ }
+
+ public String getErrormessage() {
+ return errormessage;
+ }
+
+ public static Response createErrorResponse(Exception e) {
+ return Response.serverError().entity(new ErrorResponse(e)).build();
+ }
+
+ public static Response createErrorResponse(Exception e, Status statuscode) {
+ return Response.status(statuscode).entity(new ErrorResponse(e)).build();
+ }
+
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceCalculate.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceCalculate.java
new file mode 100644
index 0000000..5db5edf
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceCalculate.java
@@ -0,0 +1,131 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.rtdi.bigdata.rulesservice.LoggingUtil;
+import io.rtdi.bigdata.rulesservice.RulesService;
+import io.rtdi.bigdata.rulesservice.config.RuleFileDefinition;
+import io.rtdi.bigdata.rulesservice.config.RuleStep;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.servlet.ServletContext;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Configuration;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+@Path("/")
+public class RestServiceCalculate {
+ protected static final int SAMPLE_MAX_ROWS = 100;
+
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ @Context
+ private Configuration configuration;
+
+ @Context
+ private ServletContext servletContext;
+
+ @Context
+ HttpServletRequest request;
+
+
+ @POST
+ @Path("/calculate/{stepindex}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "apply the provided sample data for this step",
+ description = "post the entire rule file but process just a single step using its sample value",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Success message",
+ content = {
+ @Content(
+ schema = @Schema(implementation = RuleFileDefinition.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response processStep(
+ @PathParam("stepindex")
+ @Parameter(
+ description = "index of the step to process",
+ example = "0"
+ )
+ Integer stepindex,
+ @RequestBody RuleFileDefinition input) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ RuleStep ret = service.applySampleData(input, stepindex);
+ return LoggingUtil.requestEnd(log, request, ret);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @POST
+ @Path("/calculate")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "Apply the selected sample file to the rule file",
+ description = "Calculate all rules using the specified rule file",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "The rule file with the new sample values",
+ content = {
+ @Content(
+ schema = @Schema(implementation = RuleFileDefinition.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response processAll(
+ @RequestBody RuleFileDefinition input) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ RuleFileDefinition ret = service.applySampleFile(input);
+ return LoggingUtil.requestEnd(log, request, ret);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceConfig.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceConfig.java
new file mode 100644
index 0000000..fd03635
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceConfig.java
@@ -0,0 +1,237 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.rtdi.bigdata.rulesservice.LoggingUtil;
+import io.rtdi.bigdata.rulesservice.RulesService;
+import io.rtdi.bigdata.rulesservice.config.ServiceSettings;
+import io.rtdi.bigdata.rulesservice.config.ServiceStatus;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.servlet.ServletContext;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Configuration;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+@Path("/")
+public class RestServiceConfig {
+ protected static final int SAMPLE_MAX_ROWS = 100;
+
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ @Context
+ private Configuration configuration;
+
+ @Context
+ private ServletContext servletContext;
+
+ @Context
+ HttpServletRequest request;
+
+ @GET
+ @Path("/config")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "Return the service status",
+ description = "Returns the status of the rules service",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Configuration data",
+ content = {
+ @Content(
+ schema = @Schema(implementation = RestServiceConfig.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getConfig() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ boolean admin = request.isUserInRole(ServletSecurityConstants.ROLE_ADMIN);
+ ServiceSettings ret = service.getConfig(admin);
+ return LoggingUtil.requestEnd(log, request, ret);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @POST
+ @Path("/config")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_ADMIN)
+ @Operation(
+ summary = "Save the config",
+ description = "save the config file and re-establish all connections",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Success message",
+ content = {
+ @Content(
+ schema = @Schema(implementation = SuccessResponse.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response postConfig(@RequestBody ServiceSettings input) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ service.saveConfig(input);
+ return LoggingUtil.requestEnd(log, request, new SuccessResponse("Saved"));
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @POST
+ @Path("/config/service")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_ADMIN)
+ @Operation(
+ summary = "Start all services",
+ description = "Reads the topic rule config directory and starts all services with at least one active rule",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Success message",
+ content = {
+ @Content(
+ schema = @Schema(implementation = SuccessResponse.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response startService() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ service.startService();
+ return LoggingUtil.requestEnd(log, request, new SuccessResponse("Started"));
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @DELETE
+ @Path("/config/service")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_ADMIN)
+ @Operation(
+ summary = "Stops all services",
+ description = "Goes through the list of running services and stops them",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Success message",
+ content = {
+ @Content(
+ schema = @Schema(implementation = SuccessResponse.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response stopService() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ service.stopService();
+ return LoggingUtil.requestEnd(log, request, new SuccessResponse("Started"));
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @GET
+ @Path("/config/service")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Operation(
+ summary = "Return the detailed service status",
+ description = "Returns the status of the rules services",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Configuration data",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ServiceStatus.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getServiceStatus() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ ServiceStatus ret = service.getServiceStatus();
+ return LoggingUtil.requestEnd(log, request, ret);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceRules.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceRules.java
new file mode 100644
index 0000000..7a4cfa2
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceRules.java
@@ -0,0 +1,333 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.rtdi.bigdata.rulesservice.LoggingUtil;
+import io.rtdi.bigdata.rulesservice.RulesService;
+import io.rtdi.bigdata.rulesservice.config.RuleFileDefinition;
+import io.rtdi.bigdata.rulesservice.config.RuleFileName;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.ArraySchema;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.servlet.ServletContext;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.PATCH;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Configuration;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+@Path("/")
+public class RestServiceRules {
+ protected static final int SAMPLE_MAX_ROWS = 100;
+
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ @Context
+ private Configuration configuration;
+
+ @Context
+ private ServletContext servletContext;
+
+ @Context
+ HttpServletRequest request;
+
+
+ @GET
+ @Path("/subjects/{subject}/rules")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "All rules for a subject",
+ description = "Returns a list of all rule groups using that subject is using as input from the file system",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "List of all rule group names",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = RuleFileName.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getRuleFiles(
+ @PathParam("subject")
+ @Parameter(
+ description = "name of subject",
+ example = "order_data-value"
+ )
+ String subjectname) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ List rulegroups = service.getRuleFiles(subjectname);
+ return LoggingUtil.requestEnd(log, request, rulegroups);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @GET
+ @Path("/rules")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "All rules",
+ description = "Returns a list of all active rule groups from the file system",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "List of all rule group names",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = RuleFileName.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getAllRuleFiles() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ List rulegroups = service.getAllRuleFiles();
+ return LoggingUtil.requestEnd(log, request, rulegroups);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @GET
+ @Path("/subjects/{subject}/rules/{path:.*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "Get a rule group definition",
+ description = "Returns the rule group definition from the file system",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "List of all rule group names",
+ content = {
+ @Content(
+ schema = @Schema(implementation = RuleFileDefinition.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getRuleFile(
+ @PathParam("path")
+ @Parameter(
+ description = "name of the rule group file",
+ example = "ruleset1/address_rules.json"
+ )
+ String path,
+ @PathParam("subject")
+ @Parameter(
+ description = "name of subject",
+ example = "order_data-value"
+ )
+ String subjectname) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ java.nio.file.Path name = java.nio.file.Path.of(path);
+ RuleFileDefinition rg = RuleFileDefinition.load(service.getRuleFileRootDir(), subjectname, name, false);
+ return LoggingUtil.requestEnd(log, request, rg);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @POST
+ @Path("/subjects/{subject}/rules/{path:.*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "Save a rule file definition",
+ description = "Saves the posted rule file, a pure file system operation",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Success message",
+ content = {
+ @Content(
+ schema = @Schema(implementation = SuccessResponse.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response postRuleFile(
+ @PathParam("path")
+ @Parameter(
+ description = "name of the rule group file to save; note that this can be different from the name inside the payload",
+ example = "ruleset1/address_rules.json"
+ )
+ String path,
+ @PathParam("subject")
+ @Parameter(
+ description = "name of value subject",
+ example = "order_data"
+ )
+ String subjectname,
+ @RequestBody RuleFileDefinition input) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ java.nio.file.Path name = java.nio.file.Path.of(path); // The name of the file to be saved, which is the old name
+ input.setActive(false);
+ input.setInputsubjectname(subjectname);
+ input.save(service.getRuleFileRootDir(), name);
+ return LoggingUtil.requestEnd(log, request, SuccessResponse.SUCCESS);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @PATCH
+ @Path("/subjects/{subject}/rules/{path:.*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "Active a rule file",
+ description = "Returns a list of all rule groups using that subject is using as input, a pure file system operation",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "List of all rule group names",
+ content = {
+ @Content(
+ schema = @Schema(implementation = SuccessResponse.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response activateRuleFile(
+ @PathParam("path")
+ @Parameter(
+ description = "name of the rule group file to save; note that this can be different from the name inside the payload",
+ example = "ruleset1/address_rules.json"
+ )
+ String path,
+ @PathParam("subject")
+ @Parameter(
+ description = "name of value subject",
+ example = "order_data"
+ )
+ String subjectname) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ RuleFileDefinition.copyToActivate(service.getRuleFileRootDir(), subjectname, path);
+ return LoggingUtil.requestEnd(log, request, SuccessResponse.SUCCESS);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @GET
+ @Path("/subjects/{subject}/defaultrule")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "Create an empty rule group definition",
+ description = "Create a new rule group for that subject",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "List of all rule group names",
+ content = {
+ @Content(
+ schema = @Schema(implementation = RuleFileDefinition.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response createEmptyRuleFile(
+ @PathParam("subject")
+ @Parameter(
+ description = "name of subject",
+ example = "order_data-value"
+ )
+ String subjectname) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ RuleFileDefinition rg = RuleFileDefinition.createEmptyRuleFileDefinition(subjectname, service);
+ return LoggingUtil.requestEnd(log, request, rg);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceSample.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceSample.java
new file mode 100644
index 0000000..a1ba149
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceSample.java
@@ -0,0 +1,284 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.BytesDeserializer;
+import org.apache.kafka.common.utils.Bytes;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.glassfish.jersey.server.ChunkedOutput;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
+import io.rtdi.bigdata.rulesservice.LoggingUtil;
+import io.rtdi.bigdata.rulesservice.RulesService;
+import io.rtdi.bigdata.rulesservice.config.SampleFileName;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.ArraySchema;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.servlet.ServletContext;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Configuration;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.Response.Status;
+
+@Path("/")
+public class RestServiceSample {
+ protected static final int SAMPLE_MAX_ROWS = 100;
+
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ @Context
+ private Configuration configuration;
+
+ @Context
+ private ServletContext servletContext;
+
+ @Context
+ HttpServletRequest request;
+
+
+ @POST
+ @Path("/sample")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "Retrieve sample data",
+ description = "Query the list of provided topics",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Sample payloads",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = SampleData.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response sample(@RequestBody List topics) {
+ ObjectMapper om = new ObjectMapper();
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ ChunkedOutput output = new ChunkedOutput(String.class);
+ if (topics == null || topics.size() == 0) {
+ return ErrorResponse.createErrorResponse(new IOException("No topics to read from have been provided"), Status.BAD_REQUEST);
+ } else {
+ RulesService service = RulesService.getRulesService(servletContext);
+
+ new Thread() {
+
+ @Override
+ public void run() {
+ Properties consumerProperties = new Properties();
+ consumerProperties.putAll(service.getKafkaProperties());
+ consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
+ consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
+ consumerProperties.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
+ consumerProperties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+ consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // not actually used
+
+ try (KafkaConsumer consumer = new KafkaConsumer<>(consumerProperties);) { // closing the consumer takes a while, close the output first hence
+ try {
+ /*
+ * get the latest offsets for each partition
+ */
+ Collection partitions = new ArrayList<>();
+ for (String topic : topics) {
+ List partitioninfo = consumer.partitionsFor(topic, Duration.ofSeconds(20));
+ for (PartitionInfo partition : partitioninfo) {
+ TopicPartition t = new TopicPartition(partition.topic(), partition.partition());
+ partitions.add(t);
+ }
+ }
+ Map endoffsets = consumer.endOffsets(partitions);
+ /*
+ * Assign all partitions
+ */
+ consumer.assign(partitions);
+ /*
+ * Configure the consumer offset to the current offset, which is the offset of the next future record, minus 10 to
+ * read a few recent record, hopefully one with the proper subject
+ */
+ for (Entry partition : endoffsets.entrySet()) {
+ long endoffset = partition.getValue() >= 10 ? partition.getValue()-10L : 0L;
+ consumer.seek(partition.getKey(), endoffset);
+ }
+ long endtime = System.currentTimeMillis() + 20000L;
+ int rows = 0;
+ while (System.currentTimeMillis() < endtime && rows < SAMPLE_MAX_ROWS && endoffsets.size() > 0) {
+ ConsumerRecords records = consumer.poll(Duration.ofSeconds(2));
+
+ Iterator> iter = records.iterator();
+ while (iter.hasNext() && rows < SAMPLE_MAX_ROWS && endoffsets.size() > 0) {
+ ConsumerRecord recordset = iter.next();
+ SampleData sampledata = new SampleData(recordset, service.getSchemaclient());
+ String json = om.writeValueAsString(sampledata);
+ output.write(json + "\n");
+ rows++;
+ /*
+ * If the endoffset was reached for a topic partition, then remove it so we can end early
+ */
+ TopicPartition p = new TopicPartition(recordset.topic(), recordset.partition());
+ Long endoffset = endoffsets.get(p);
+ if (endoffset != null && recordset.offset() >= endoffset-1) {
+ endoffsets.remove(p);
+ }
+ }
+ }
+ LoggingUtil.logRequestEnd(log, request);
+ } catch (IOException | RestClientException e) {
+ LoggingUtil.logRequestEndInputError(log, request, e);
+ ErrorResponse response = new ErrorResponse(e);
+ try {
+ output.write(om.writeValueAsString(response));
+ } catch (IOException e1) {
+ }
+ } finally {
+ try {
+ output.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+ }
+ }.start();
+ return Response.ok(output).build();
+ }
+ } catch (Exception e) {
+ LoggingUtil.logRequestEndInputError(log, request, e);
+ return ErrorResponse.createErrorResponse(e);
+ }
+ }
+
+ @POST
+ @Path("/sample/{subject}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "Save one sample record",
+ description = "Saves the sample record as file",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Success message",
+ content = {
+ @Content(
+ schema = @Schema(implementation = SampleFileName.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response savesample(
+ @PathParam("subject")
+ @Parameter(
+ description = "subject name ",
+ example = "order-value"
+ )
+ String subject,
+ @RequestBody SampleData data) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ String filename = data.save(service.getRuleFileRootDir());
+ return LoggingUtil.requestEnd(log, request, new SampleFileName(filename));
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @GET
+ @Path("/sample/{subject}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "List of all sample data",
+ description = "Returns a list of all sample data files for the subject",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "sample files",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = SampleFileName.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getSampleFiles(
+ @PathParam("subject")
+ @Parameter(
+ description = "subject name ",
+ example = "order-value"
+ )
+ String subject
+ ) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ List samplefiles = SampleData.getFiles(service.getRuleFileRootDir(), subject);
+ return LoggingUtil.requestEnd(log, request, samplefiles);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceSubjects.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceSubjects.java
new file mode 100644
index 0000000..a0c0298
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceSubjects.java
@@ -0,0 +1,193 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
+import io.rtdi.bigdata.rulesservice.LoggingUtil;
+import io.rtdi.bigdata.rulesservice.RulesService;
+import io.rtdi.bigdata.rulesservice.config.RuleFileDefinition;
+import io.rtdi.bigdata.rulesservice.config.SubjectName;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.ArraySchema;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.servlet.ServletContext;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Configuration;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.Response.Status;
+
+@Path("/")
+public class RestServiceSubjects {
+ protected static final int SAMPLE_MAX_ROWS = 100;
+
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ @Context
+ private Configuration configuration;
+
+ @Context
+ private ServletContext servletContext;
+
+ @Context
+ HttpServletRequest request;
+
+ @GET
+ @Path("/subjects")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "All subjects",
+ description = "Returns a list of all subjects from the schema registry",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "List of subjects",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = SubjectName.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getSubjects() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ Collection subjects = service.getSubjects();
+ List ret = new ArrayList<>();
+ for (String s : subjects) {
+ ret.add(new SubjectName(s));
+ }
+ return LoggingUtil.requestEnd(log, request, ret);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @POST
+ @Path("/subjects/{subject}/update")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "Update a rule file to the latest schema",
+ description = "Takes the current rule file and updates it to the latest schema version",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "New rule file",
+ content = {
+ @Content(
+ schema = @Schema(implementation = RuleFileDefinition.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response updateRuleFile(
+ @PathParam("subject")
+ @Parameter(
+ description = "name of subject",
+ example = "order_data-value"
+ )
+ String subjectname,
+ @RequestBody RuleFileDefinition input) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ RuleFileDefinition empty = RuleFileDefinition.createEmptyRuleFileDefinition(subjectname, service);
+ input.update(empty);
+ return LoggingUtil.requestEnd(log, request, input);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+
+ @GET
+ @Path("/subjects/{subject}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "Get the schema definition",
+ description = "Returns the schema definition from the schema registry",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "schema metadata",
+ content = {
+ @Content(
+ schema = @Schema(implementation = Schema.class)
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getSchemaDefiniton(
+ @PathParam("subject")
+ @Parameter(
+ description = "name of value subject",
+ example = "order_data"
+ )
+ String subjectname) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ org.apache.avro.Schema schema = service.getLatestSchema(subjectname);
+ return LoggingUtil.requestEnd(log, request, schema);
+ } catch (RestClientException e) {
+ if (e.getStatus() == 404) {
+ // invalid schema
+ return LoggingUtil.requestEndInputError(log, request, e, Status.NOT_FOUND);
+ } else {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceTopics.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceTopics.java
new file mode 100644
index 0000000..3df6516
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RestServiceTopics.java
@@ -0,0 +1,168 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.rtdi.bigdata.rulesservice.LoggingUtil;
+import io.rtdi.bigdata.rulesservice.RulesService;
+import io.rtdi.bigdata.rulesservice.config.TopicName;
+import io.rtdi.bigdata.rulesservice.config.TopicRule;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.ArraySchema;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.servlet.ServletContext;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Configuration;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+
+@Path("/")
+public class RestServiceTopics {
+ protected static final int SAMPLE_MAX_ROWS = 100;
+
+ protected final Logger log = LogManager.getLogger(this.getClass().getName());
+
+ @Context
+ private Configuration configuration;
+
+ @Context
+ private ServletContext servletContext;
+
+ @Context
+ HttpServletRequest request;
+
+
+ @GET
+ @Path("/topicrules")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "List of topics and their rules",
+ description = "Returns a list of all topics and the rules assigned, if there are any",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "rules",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = TopicRule.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getTopicRules() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ Collection topicrules = service.getTopicsAndRules();
+ return LoggingUtil.requestEnd(log, request, topicrules);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @POST
+ @Path("/topicrules")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_EDIT)
+ @Operation(
+ summary = "Save the modified rules",
+ description = "Go through the input object and save all rules with a modified=true flag",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "Success message",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = TopicRule.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response postRules(@RequestBody List input) {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ List ret = service.saveTopicRules(input);
+ return LoggingUtil.requestEnd(log, request, ret);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+ @GET
+ @Path("/topics")
+ @Produces(MediaType.APPLICATION_JSON)
+ @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
+ @Operation(
+ summary = "List of topics",
+ description = "Returns a list of all topics",
+ responses = {
+ @ApiResponse(
+ responseCode = "200",
+ description = "rules",
+ content = {
+ @Content(
+ array = @ArraySchema(schema = @Schema(implementation = TopicName.class))
+ )
+ }
+ ),
+ @ApiResponse(
+ responseCode = "500",
+ description = "Any exception thrown",
+ content = {
+ @Content(
+ schema = @Schema(implementation = ErrorResponse.class)
+ )
+ }
+ )
+ })
+ public Response getTopics() {
+ try {
+ LoggingUtil.logRequestBegin(log, request);
+ RulesService service = RulesService.getRulesService(servletContext);
+ Collection topics = service.getTopics();
+ List topicnames = new ArrayList<>(topics.size());
+ for (String t : topics) {
+ topicnames.add(new TopicName(t));
+ }
+ return LoggingUtil.requestEnd(log, request, topicnames);
+ } catch (Exception e) {
+ return LoggingUtil.requestEndTechnicalError(log, request, e);
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RulesEndpoint.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/RulesEndpoint.java
deleted file mode 100644
index 6ee3003..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rest/RulesEndpoint.java
+++ /dev/null
@@ -1,290 +0,0 @@
-package io.rtdi.bigdata.rulesservice.rest;
-
-import java.io.File;
-import java.io.IOException;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import jakarta.annotation.security.RolesAllowed;
-import jakarta.servlet.ServletContext;
-import jakarta.servlet.http.HttpServletRequest;
-import jakarta.servlet.http.HttpSession;
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.DELETE;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.core.Configuration;
-import jakarta.ws.rs.core.Context;
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.Response;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-
-import io.rtdi.bigdata.connector.connectorframework.WebAppController;
-import io.rtdi.bigdata.connector.connectorframework.controller.ConnectorController;
-import io.rtdi.bigdata.connector.connectorframework.controller.ServiceController;
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.connectorframework.rest.JAXBErrorResponseBuilder;
-import io.rtdi.bigdata.connector.connectorframework.rest.JAXBSuccessResponseBuilder;
-import io.rtdi.bigdata.connector.connectorframework.servlet.ServletSecurityConstants;
-import io.rtdi.bigdata.connector.pipeline.foundation.IPipelineAPI;
-import io.rtdi.bigdata.connector.pipeline.foundation.MicroServiceTransformation;
-import io.rtdi.bigdata.connector.pipeline.foundation.SchemaHandler;
-import io.rtdi.bigdata.connector.pipeline.foundation.SchemaRegistryName;
-import io.rtdi.bigdata.connector.pipeline.foundation.TopicName;
-import io.rtdi.bigdata.connector.pipeline.foundation.TopicPayload;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
-import io.rtdi.bigdata.connector.pipeline.foundation.utils.FileNameEncoder;
-import io.rtdi.bigdata.rulesservice.RuleStep;
-import io.rtdi.bigdata.rulesservice.SchemaRuleSet;
-
-
-@Path("/")
-public class RulesEndpoint {
-
- private static final String SAMPLEDATACACHE = "sample.data.cache";
-
- @Context
- private Configuration configuration;
-
- @Context
- private ServletContext servletContext;
-
- @Context
- HttpServletRequest request;
-
- @GET
- @Path("/rules/{servicename}")
- @Produces(MediaType.APPLICATION_JSON)
- @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
- public Response getSchemas(
- @PathParam("servicename") String servicename) {
- try {
- ConnectorController connector = WebAppController.getConnectorOrFail(servletContext);
- ServiceController service = connector.getServiceOrFail(servicename);
- return Response.ok(new MicroserviceList(service.getMicroservices())).build();
- } catch (Exception e) {
- return JAXBErrorResponseBuilder.getJAXBResponse(e);
- }
- }
-
- @GET
- @Path("/rules/{servicename}/{microservicename}")
- @Produces(MediaType.APPLICATION_JSON)
- @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
- public Response getRuleSchemas(
- @PathParam("servicename") String servicename,
- @PathParam("microservicename") String microservicename) {
- try {
- ConnectorController connector = WebAppController.getConnectorOrFail(servletContext);
- ServiceController service = connector.getServiceOrFail(servicename);
- /*
- * In case this is a brand new microservice, it is not attached to the step yet.
- */
- MicroServiceTransformation m = service.getMicroservice(microservicename);
- Collection existingschemas = null;
- if (m != null) {
- RuleStep step = (RuleStep) m;
- Map data = step.getSchemaRules();
- existingschemas = data.values();
- }
- SchemaListNameEntity ret = new SchemaListNameEntity(connector.getPipelineAPI(), existingschemas);
- return Response.ok(ret).build();
- } catch (Exception e) {
- return JAXBErrorResponseBuilder.getJAXBResponse(e);
- }
- }
-
- @GET
- @Path("/rules/{servicename}/{microservicename}/{schemaname}")
- @Produces(MediaType.APPLICATION_JSON)
- @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
- public Response getRules(
- @PathParam("servicename") String servicename,
- @PathParam("microservicename") String microservicename,
- @PathParam("schemaname") String schemaname) {
- try {
- ConnectorController connector = WebAppController.getConnectorOrFail(servletContext);
- ServiceController service = connector.getServiceOrFail(servicename);
- MicroServiceTransformation m = service.getMicroservice(microservicename);
- SchemaRuleSet data = null;
- if (m != null) {
- RuleStep step = (RuleStep) m;
- data = step.getSchemaRule(SchemaRegistryName.create(schemaname));
- }
- SchemaHandler handler = connector.getPipelineAPI().getSchema(SchemaRegistryName.create(schemaname));
- if (handler == null) {
- throw new ConnectorCallerException("No schema with that name exists", null, null, schemaname);
- } else {
- if (data == null) {
- data = SchemaRuleSet.createUIRuleTree(SchemaRegistryName.create(schemaname), schemaname, handler.getValueSchema(), null);
- } else {
- data.updateSchema(handler.getValueSchema());
- }
- JexlRecord sampledata = getSampleData(
- service.getServiceProperties().getSourceTopic(),
- schemaname,
- connector.getPipelineAPI(),
- request);
- data.assignSamplevalue(sampledata);
- data.validateRule(sampledata);
- return Response.ok(data).build();
- }
- } catch (Exception e) {
- return JAXBErrorResponseBuilder.getJAXBResponse(e);
- }
- }
-
- @POST
- @Path("/rules/{servicename}/{microservicename}/{schemaname}/validate")
- @Produces(MediaType.APPLICATION_JSON)
- @RolesAllowed(ServletSecurityConstants.ROLE_VIEW)
- public Response getValidation(@PathParam("servicename") String servicename,
- @PathParam("microservicename") String microservicename,
- @PathParam("schemaname") String schemaname,
- SchemaRuleSet data) {
- try {
- ConnectorController connector = WebAppController.getConnectorOrFail(servletContext);
- ServiceController service = connector.getServiceOrFail(servicename);
- JexlRecord sampledata = getSampleData(
- service.getServiceProperties().getSourceTopic(),
- schemaname,
- connector.getPipelineAPI(),
- request);
- data.assignSamplevalue(sampledata);
- data.validateRule(sampledata);
- return Response.ok(data).build();
- } catch (Exception e) {
- return JAXBErrorResponseBuilder.getJAXBResponse(e);
- }
- }
-
- private static JexlRecord getSampleData(String topicname, String schemaname, IPipelineAPI api, HttpServletRequest req) throws IOException {
- HttpSession session = req.getSession(false);
- if (session != null) {
- @SuppressWarnings("unchecked")
- Cache cache = (Cache) session.getAttribute(SAMPLEDATACACHE);
- if (cache == null) {
- cache = Caffeine.newBuilder().expireAfterAccess(Duration.ofMinutes(30)).maximumSize(1000).build();
- session.setAttribute(SAMPLEDATACACHE, cache);
- }
- String cachekey = topicname + "_" + schemaname;
- JexlRecord sampledata = cache.getIfPresent(cachekey);
- if (sampledata == null) {
- List l = api.getAllRecordsSince(TopicName.create(topicname), System.currentTimeMillis()-3600000L, 1, SchemaRegistryName.create(schemaname));
- if (l != null && l.size() != 0) { // might be that the schema has no data in this topic
- sampledata = l.get(0).getValueRecord();
- cache.put(cachekey, sampledata);
- }
- }
-
- return sampledata;
- } else {
- return null;
- }
- }
-
- @POST
- @Path("/rules/{servicename}/{microservicename}/{schemaname}")
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- @RolesAllowed(ServletSecurityConstants.ROLE_CONFIG)
- public Response setRules(@PathParam("servicename") String servicename,
- @PathParam("microservicename") String microservicename,
- @PathParam("schemaname") String schemaname,
- SchemaRuleSet data) {
- try {
- ConnectorController connector = WebAppController.getConnectorOrFail(servletContext);
- ServiceController service = connector.getService(servicename);
- MicroServiceTransformation m = service.getMicroservice(microservicename);
- RuleStep step;
- if (m == null) {
- m = new RuleStep(microservicename);
- service.getServiceProperties().addMicroService(m);
- }
- step = (RuleStep) m;
- java.nio.file.Path p = service.getDirectory().toPath();
- File directory = p.resolve(FileNameEncoder.encodeName(m.getName())).toFile();
- if (!directory.exists()) {
- if (!directory.mkdirs()) {
- throw new ConnectorCallerException("Cannot create directory for the microservice schema", null, null, directory.getAbsolutePath());
- }
- } else if (!directory.isDirectory()) {
- throw new ConnectorCallerException("There is a file of that name already", null, null, directory.getAbsolutePath());
- }
- if (data != null) {
- SchemaRegistryName name = SchemaRegistryName.create(schemaname);
- data.setSchemaName(name);
- data.write(directory);
- step.addSchemaRuleSet(data);
- }
- return JAXBSuccessResponseBuilder.getJAXBResponse("created");
- } catch (Exception e) {
- return JAXBErrorResponseBuilder.getJAXBResponse(e);
- }
- }
-
- @DELETE
- @Path("/rules/{servicename}/{microservicename}/{schemaname}")
- @Produces(MediaType.APPLICATION_JSON)
- @RolesAllowed(ServletSecurityConstants.ROLE_CONFIG)
- public Response deleteRules(@PathParam("servicename") String servicename,
- @PathParam("microservicename") String microservicename,
- @PathParam("schemaname") String schemaname) {
- try {
- ConnectorController connector = WebAppController.getConnectorOrFail(servletContext);
- ServiceController service = connector.getServiceOrFail(servicename);
- RuleStep microservice = (RuleStep) service.getMicroserviceOrFail(microservicename);
- java.nio.file.Path p = service.getDirectory().toPath();
- File directory = p.resolve(schemaname).resolve(microservice.getName()).toFile();
- microservice.deleteSchemaRuleSetDefinition(SchemaRegistryName.create(schemaname), directory);
- return JAXBSuccessResponseBuilder.getJAXBResponse("deleted");
- } catch (Exception e) {
- return JAXBErrorResponseBuilder.getJAXBResponse(e);
- }
- }
-
- public static class MicroserviceList {
-
- private List schemas;
-
- public MicroserviceList() {
- }
-
- public MicroserviceList(Set set) {
- if (set != null) {
- this.schemas = new ArrayList<>();
- for (MicroServiceTransformation s : set) {
- this.schemas.add(new MicroserviceNameEntity(s.getName()));
- }
- }
- }
-
- public List getSchemas() {
- return schemas;
- }
-
- }
-
- public static class MicroserviceNameEntity {
-
- private String name;
-
- public MicroserviceNameEntity(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
- }
-
-}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/SampleData.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/SampleData.java
new file mode 100644
index 0000000..b32046a
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/SampleData.java
@@ -0,0 +1,191 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Parser;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.utils.Bytes;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.confluent.kafka.schemaregistry.ParsedSchema;
+import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient;
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
+import io.rtdi.bigdata.kafka.avro.AvroDeserializer;
+import io.rtdi.bigdata.rulesservice.config.SampleFileName;
+import io.rtdi.bigdata.rulesservice.config.SubjectName;
+import io.rtdi.bigdata.rulesservice.jexl.AvroRuleUtils;
+import io.rtdi.bigdata.rulesservice.jexl.JexlAvroDeserializer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SampleData {
+
+ private long offset;
+ private String topic;
+ private int partition;
+ private String filename;
+ private String payload;
+ private String subjectname;
+ private List subjects;
+ private String schema;
+
+ public SampleData() {
+ super();
+ }
+
+ public SampleData(ConsumerRecord recordset, CachedSchemaRegistryClient schemaclient) throws IOException, RestClientException {
+ Bytes value = recordset.value();
+ byte[] data = value.get();
+ int schemaid = AvroDeserializer.getSchemaId(data);
+ ParsedSchema writerparsedschema = schemaclient.getSchemaById(schemaid);
+ Collection subjects = schemaclient.getAllSubjectsById(schemaid);
+ Schema writerschema = (Schema) writerparsedschema.rawSchema();
+ JexlRecord foundrecord = JexlAvroDeserializer.deserialize(data, writerschema);
+ this.offset = recordset.offset();
+ this.topic = recordset.topic();
+ this.partition = recordset.partition();
+ this.filename = null;
+ this.payload = foundrecord.toString(); // convert to a simple json without data types
+ // this.payload = AvroRuleUtils.convertToJson(foundrecord, writerschema);
+ this.setSchema(writerschema.toString());
+ if (subjects != null && subjects.size() > 0) {
+ Iterator iter = subjects.iterator();
+ this.subjects = new ArrayList<>();
+ while (iter.hasNext()) {
+ String subject = iter.next();
+ this.subjectname = subject;
+ this.subjects.add(new SubjectName(subject));
+ }
+ }
+ }
+
+ public String save(Path rootdir) throws IOException {
+ if (filename == null || filename.length() == 0) {
+ filename = "partition_" + partition + "_offset_" + offset + ".json";
+ } else if (filename.contains("..")) {
+ throw new IOException("filename can only contain subdirectories, not <..>");
+ }
+
+ ObjectMapper om = new ObjectMapper();
+ Path dirpath = rootdir.resolve(subjectname).resolve("sampledata");
+ dirpath.toFile().mkdirs();
+ Path filepath = dirpath.resolve(filename);
+ om.writer()
+ .withDefaultPrettyPrinter()
+ .writeValue(filepath.toFile(), this);
+ return filename;
+ }
+
+ public static List getFiles(Path rootdir, String subjectname) {
+ Path dirpath = rootdir.resolve(subjectname).resolve("sampledata");
+ List ret = new ArrayList<>();
+ collectSampleFiles(dirpath, ret, null);
+ return ret;
+ }
+
+ private static void collectSampleFiles(Path dirpath, List ret, String relativepath) {
+ File[] files = dirpath.toFile().listFiles();
+ if (files != null) {
+ for (File f : files) {
+ String filename = (relativepath == null ? "" : relativepath + "/") + f.getName();
+ if (f.isDirectory()) {
+ collectSampleFiles(f.toPath(), ret, filename);
+ } else {
+ ret.add(new SampleFileName(filename));
+ }
+ }
+ }
+ }
+
+ public static SampleData load(Path rootdir, String subjectname, Path filename) throws IOException {
+ Path filepath = rootdir.resolve(subjectname).resolve("sampledata").resolve(filename);
+ ObjectMapper om = new ObjectMapper();
+ return om.readValue(filepath.toFile(), SampleData.class);
+ }
+
+ public String getSubjectname() {
+ return subjectname;
+ }
+
+ public void setSubjectname(String subjectname) {
+ this.subjectname = subjectname;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public void setOffset(long offset) {
+ this.offset = offset;
+ }
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public int getPartition() {
+ return partition;
+ }
+
+ public void setPartition(int partition) {
+ this.partition = partition;
+ }
+
+ public String getFilename() {
+ return filename;
+ }
+
+ public void setFilename(String filename) {
+ this.filename = filename;
+ }
+
+ public String getPayload() {
+ return payload;
+ }
+
+ public void setPayload(String payload) {
+ this.payload = payload;
+ }
+
+ public List getSubjects() {
+ return subjects;
+ }
+
+ public void setSubjects(List subjects) {
+ this.subjects = subjects;
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ public JexlRecord createRecord(Schema targetschema) throws IOException {
+ Parser parser = new Parser();
+ Schema sourceschema;
+ if (schema != null) {
+ sourceschema = parser.parse(schema);
+ } else {
+ sourceschema = targetschema;
+ }
+ JexlRecord jexlrecord = AvroRuleUtils.convertFromJson(payload, sourceschema);
+ return jexlrecord;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/SchemaListNameEntity.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/SchemaListNameEntity.java
deleted file mode 100644
index 39b637a..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rest/SchemaListNameEntity.java
+++ /dev/null
@@ -1,72 +0,0 @@
-package io.rtdi.bigdata.rulesservice.rest;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import io.rtdi.bigdata.connector.pipeline.foundation.IPipelineAPI;
-import io.rtdi.bigdata.connector.pipeline.foundation.SchemaRegistryName;
-import io.rtdi.bigdata.rulesservice.SchemaRuleSet;
-
-public class SchemaListNameEntity {
-
- private List schemas;
-
- public SchemaListNameEntity() {
- }
-
- public SchemaListNameEntity(IPipelineAPI api, Collection rules) throws IOException {
- schemas = new ArrayList<>();
- List allschemas = api.getSchemas();
- Set configuredschemas = new HashSet<>();
- if (rules != null) {
- for (SchemaRuleSet r : rules) {
- String name = r.getSchemaName().getName();
- configuredschemas.add(name);
- }
- }
- for (SchemaRegistryName schemaname : allschemas) {
- schemas.add(new SchemaName(schemaname.getName(), configuredschemas.contains(schemaname.getName())));
- }
- }
-
- public List getSchemas() {
- return schemas;
- }
-
- public void setSchemas(List schemas) {
- this.schemas = schemas;
- }
-
- public void addSchema(String name, boolean contained) {
- if (schemas == null) {
- schemas = new ArrayList<>();
- }
- schemas.add(new SchemaName(name, contained));
- }
-
- public static class SchemaName {
- String schemaname;
- private boolean configured = false;
-
- public SchemaName(String name, boolean configured) {
- this.schemaname = name;
- this.configured = configured;
- }
-
- public SchemaName() {
- }
-
- public String getSchemaname() {
- return schemaname;
- }
-
- public boolean isConfigured() {
- return configured;
- }
-
- }
-}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/ServletSecurityConstants.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/ServletSecurityConstants.java
new file mode 100644
index 0000000..e1fc381
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/ServletSecurityConstants.java
@@ -0,0 +1,8 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+public class ServletSecurityConstants {
+ public static final String ROLE_VIEW = "rulesview";
+ public static final String ROLE_ADMIN = "rulesadmin";
+ public static final String ROLE_EDIT = "rulesedit";
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rest/SuccessResponse.java b/src/main/java/io/rtdi/bigdata/rulesservice/rest/SuccessResponse.java
new file mode 100644
index 0000000..eaa5a11
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rest/SuccessResponse.java
@@ -0,0 +1,25 @@
+package io.rtdi.bigdata.rulesservice.rest;
+
+import jakarta.ws.rs.core.Response;
+
+public class SuccessResponse {
+ private String message;
+ public static final SuccessResponse SUCCESS = new SuccessResponse("Success");
+ public static final Response SUCCESS_RESPONSE = Response.ok(SUCCESS).build();
+
+ public SuccessResponse(String message) {
+ this.message = message;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public static Response createSuccessResponse(String message) {
+ return Response.ok(new SuccessResponse(message)).build();
+ }
+
+ public static Response createSuccessResponse() {
+ return SUCCESS_RESPONSE;
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/ArrayRule.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/ArrayRule.java
index 07ef2bd..9813a18 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/ArrayRule.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/ArrayRule.java
@@ -1,39 +1,45 @@
package io.rtdi.bigdata.rulesservice.rules;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.avro.Schema;
-import org.apache.avro.Schema.Type;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlArray;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroArray;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlArray;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
@JsonIgnoreProperties(ignoreUnknown=true)
-public class ArrayRule extends RecordRule {
+public class ArrayRule extends Rule implements IContainerRule {
- public ArrayRule(String fieldname) {
- super(fieldname);
+ public ArrayRule(String fieldname, Schema schema) {
+ super(fieldname, schema != null ? schema.getFullName() : null);
+ setDataType(AvroArray.create());
}
-
+
public ArrayRule() {
super();
}
@Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException {
- Object o = valuerecord.get(getFieldname());
- if (o != null && o instanceof JexlArray) {
- JexlArray l = (JexlArray) o;
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
+ if (value != null && value instanceof JexlArray l) {
for ( Object r : l) {
- if (r instanceof JexlRecord) {
+ if (getRules() != null) {
for ( Rule rule : getRules()) {
- rule.apply((JexlRecord) r, ruleresults);
+ if (r instanceof JexlRecord rec) {
+ /*
+ * For an array of records, the container is the record itself
+ */
+ rule.apply(rec, rec, test);
+ } else {
+ rule.apply(r, container, test);
+ }
}
}
}
@@ -43,59 +49,36 @@ public RuleResult apply(JexlRecord valuerecord, List ruleresults) th
@Override
public String toString() {
- return "ArrayRule for field \"" + getFieldname() + "\" tests all records";
- }
-
- public static ArrayRule createUIRuleTree(String fieldname, Schema schema, List originalrules) throws PropertiesException {
- if (schema.getType() == Type.ARRAY) {
- ArrayRule a = new ArrayRule(fieldname);
- addFields(a, schema.getElementType(), originalrules);
- return a;
- } else {
- throw new PropertiesException("Provided Schema is not a record schema", (String) null, schema.getName());
- }
+ return getFieldname() + ": ArrayRule";
}
-
- @Override
- public ArrayRule createUIRuleTree(Schema schema) throws PropertiesException {
- return createUIRuleTree(getFieldname(), schema, getRules());
- }
-
@Override
- public void assignSamplevalue(JexlRecord sampledata) {
- Object data = sampledata.get(getFieldname());
- if (data instanceof List) {
- List l = (List) data;
- Object s = l.get(0);
- if (s instanceof JexlRecord && this.getRules() != null) {
- JexlRecord samplerecord = (JexlRecord) s;
- for ( Rule r : this.getRules()) {
- r.assignSamplevalue(samplerecord);
+ public void update(IContainerRule empty) {
+ if (empty instanceof ArrayRule && getRules() != null && getRules().size() > 0) {
+ Rule rule = getRules().get(0);
+ if (empty instanceof ArrayRule && empty.getRules() != null && empty.getRules().size() > 0) {
+ Rule er = empty.getRules().get(0);
+ if (rule instanceof IContainerRule rr && er instanceof IContainerRule cr) {
+ rr.update(cr);
}
}
}
}
@Override
- public RuleResult validateRule(JexlRecord valuerecord) {
- Object o = valuerecord.get(getFieldname());
- if (o != null && o instanceof JexlArray) {
- JexlArray l = (JexlArray) o;
- Object s = l.get(0);
- if (s instanceof JexlRecord && this.getRules() != null) {
- JexlRecord samplerecord = (JexlRecord) s;
- for ( Rule r : this.getRules()) {
- r.validateRule(samplerecord);
- }
+ public Rule clone() {
+ ArrayRule ret = new ArrayRule();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setSchemaname(getSchemaname());
+ if (getRules() != null) {
+ List a = new ArrayList<>(getRules().size());
+ ret.setRules(a);
+ for (Rule r : getRules()) {
+ a.add(r.clone());
}
}
- return null;
- }
-
- @Override
- protected Rule createNewInstance() throws ConnectorCallerException {
- return new ArrayRule(getFieldname());
+ return ret;
}
}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/EmptyRule.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/EmptyRule.java
index 3019c55..d6c33a2 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/EmptyRule.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/EmptyRule.java
@@ -1,72 +1,52 @@
package io.rtdi.bigdata.rulesservice.rules;
-import java.util.List;
-
import org.apache.avro.Schema;
-import org.apache.commons.jexl3.JexlException;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PipelineCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
import io.rtdi.bigdata.kafka.avro.RuleResult;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroType;
+import io.rtdi.bigdata.kafka.avro.datatypes.IAvroDatatype;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
@JsonIgnoreProperties(ignoreUnknown=true)
public class EmptyRule extends Rule {
- private Object value;
-
public EmptyRule() {
super();
}
-
- public EmptyRule(String fieldname) {
- super(fieldname);
- }
-
- @Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) {
- return null;
- }
-
- @Override
- protected Rule createUIRuleTree(Schema fieldschema) throws PropertiesException {
- return this;
- }
-
- @Override
- public
- void assignSamplevalue(JexlRecord sampledata) {
- this.value = sampledata.get(getFieldname());
- }
- @Override
- public Object getSamplevalue() throws PipelineCallerException {
- return PrimitiveRule.valueToJavaObject(value, getDataType());
+ public EmptyRule(String fieldname, Schema schema) {
+ super(fieldname, schema != null ? schema.getFullName() : null);
+ if (schema != null) {
+ IAvroDatatype dt = AvroType.getAvroDataType(schema);
+ this.setDataType(dt);
+ }
}
@Override
- public RuleResult getSampleresult() throws PipelineCallerException {
+ public RuleResult apply(Object value, AvroContainer container, boolean test) {
+ setSampleValue(value, test);
return null;
}
@Override
- public RuleResult validateRule(JexlRecord valuerecord) throws JexlException {
+ public RuleResult getSampleresult() {
return null;
}
@Override
- protected Rule createSimplified() {
- /*
- * An EmptyRule does never exist in the simplified version
- */
- return null;
+ public String toString() {
+ return getFieldname() + ": EmptyRule";
}
@Override
- protected Rule createNewInstance() {
- return null;
+ public Rule clone() {
+ EmptyRule ret = new EmptyRule();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setSchemaname(getSchemaname());
+ return ret;
}
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/GenericRules.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/GenericRules.java
new file mode 100644
index 0000000..30d5c4e
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/GenericRules.java
@@ -0,0 +1,68 @@
+package io.rtdi.bigdata.rulesservice.rules;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.avro.Schema;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import io.rtdi.bigdata.kafka.avro.RuleResult;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+
+/**
+ * This is a rule set no tied to a single field
+ */
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class GenericRules extends TestSet {
+
+ public GenericRules() {
+ super("(more)", null, null);
+ }
+
+ public GenericRules(Rule rule, Schema schema) {
+ super("(more)", null, rule, schema);
+ }
+
+ @Override
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
+ RuleResult result = RuleResult.PASS;
+ for ( Rule rule : getRules()) {
+ result = result.aggregate(rule.apply(value, container, test));
+ if (test) {
+ setSampleresult(result);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return getFieldname() + ": Generic rule tests all child rules";
+ }
+
+ @Override
+ public void updateSampleOutput(JexlRecord valuerecord) {
+ // Generic rules do not have fields and hence nothing to output as sample value
+ }
+
+ @Override
+ public Rule clone() {
+ GenericRules ret = new GenericRules();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setRulename(getRulename());
+ ret.setSchemaname(getSchemaname());
+ if (getRules() != null) {
+ List a = new ArrayList<>(getRules().size());
+ ret.setRules(a);
+ for (Rule r : getRules()) {
+ a.add(r.clone());
+ }
+ }
+ return ret;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/IContainerRule.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/IContainerRule.java
new file mode 100644
index 0000000..257b5c8
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/IContainerRule.java
@@ -0,0 +1,14 @@
+package io.rtdi.bigdata.rulesservice.rules;
+
+import java.util.List;
+
+/**
+ * A rule for a record, array or union, any schema that can have complex children
+ */
+public interface IContainerRule {
+
+ List getRules();
+ void addRule(Rule r);
+ void update(IContainerRule empty);
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/Mapping.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/Mapping.java
new file mode 100644
index 0000000..0fc22f5
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/Mapping.java
@@ -0,0 +1,123 @@
+package io.rtdi.bigdata.rulesservice.rules;
+
+import static java.util.Map.entry;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.commons.jexl3.JexlArithmetic;
+import org.apache.commons.jexl3.JexlBuilder;
+import org.apache.commons.jexl3.JexlEngine;
+import org.apache.commons.jexl3.JexlException;
+import org.apache.commons.jexl3.JexlScript;
+import org.apache.commons.jexl3.introspection.JexlPermissions;
+
+import io.rtdi.bigdata.rulesservice.PropertiesException;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlDefaultFunction;
+
+public abstract class Mapping {
+
+ protected static final JexlEngine jexl = new JexlBuilder().permissions(
+ JexlPermissions.parse(
+ "io.rtdi.bigdata.rulesservice.jexl.*",
+ "java.lang.*",
+ "java.math.*",
+ "java.text.*",
+ "java.util.*",
+ "java.time.*",
+ "java.lang { Runtime {} System {} ProcessBuilder {} Class {} }",
+ "org.apache.commons.jexl3 { JexlBuilder {} }"
+ ))
+ .cache(51200)
+ .strict(true)
+ .silent(false)
+ .namespaces(Map.ofEntries(
+ entry("", JexlDefaultFunction.class)
+ ))
+ .create();
+ protected JexlScript expression;
+
+ protected Mapping(JexlScript e) {
+ this.expression = e;
+ }
+
+ protected Mapping(String e) throws IOException {
+ this();
+ setExpression(e);
+ }
+
+ public Mapping() {
+ super();
+ }
+
+ public Object evaluate(AvroContainer context) throws IOException {
+ try {
+ return expression.execute(context);
+ } catch (JexlException e) {
+ StringBuilder msg = createErrorDetails(e);
+ throw new PropertiesException("Cannot evaluate the Expression" + msg.toString(), e, "Validate the syntax", expression.getSourceText());
+ }
+ }
+
+ private StringBuilder createErrorDetails(JexlException e) {
+ StringBuilder msg = new StringBuilder();
+ if (e.getCause() != e) {
+ if (e.getInfo() != null) {
+ msg.append("line: ");
+ msg.append(e.getInfo().getLine());
+ msg.append(", pos: ");
+ msg.append(e.getInfo().getColumn());
+ }
+ msg.append(' ');
+ msg.append(e.getDetail());
+ Throwable cause = e.getCause();
+ if (cause instanceof JexlArithmetic.NullOperand) {
+ msg.append(" caused by null operand");
+ }
+ if (e.getInfo() != null) {
+ String[] parts = expression.getSourceText().split("\n");
+ if (e.getInfo().getLine()-1 < parts.length) {
+ String line = parts[e.getInfo().getLine()-1];
+ msg.append(", code segment: ");
+ int pos = e.getInfo().getColumn();
+ if (pos > 5) {
+ pos = pos - 4;
+ msg.append("....");
+ }
+ int end = pos + 20;
+ if (end < line.length()) {
+ msg.append(line.substring(pos, end));
+ msg.append("...");
+ } else {
+ msg.append(line.substring(pos));
+ }
+ }
+ }
+ }
+ return msg;
+ }
+
+ public void setExpression(String formula) throws IOException {
+ try {
+ expression = jexl.createScript(formula);
+ } catch (JexlException e) {
+ StringBuilder msg = createErrorDetails(e);
+ throw new PropertiesException("Cannot parse the Expression" + msg.toString(), e, "Validate the syntax", formula);
+ }
+ }
+
+ public String getExpression() {
+ return expression.getSourceText();
+ }
+
+ @Override
+ public String toString() {
+ if (expression != null) {
+ return expression.getSourceText();
+ } else {
+ return "Null-Mapping";
+ }
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/PrimitiveMapping.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/PrimitiveMapping.java
new file mode 100644
index 0000000..c5c1a5d
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/PrimitiveMapping.java
@@ -0,0 +1,14 @@
+package io.rtdi.bigdata.rulesservice.rules;
+
+import java.io.IOException;
+
+public class PrimitiveMapping extends Mapping {
+
+ public PrimitiveMapping(String e) throws IOException {
+ super(e);
+ }
+
+ public PrimitiveMapping() {
+ super();
+ }
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/PrimitiveRule.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/PrimitiveRule.java
index 26c2e18..9046d20 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/PrimitiveRule.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/PrimitiveRule.java
@@ -1,61 +1,57 @@
package io.rtdi.bigdata.rulesservice.rules;
import java.io.IOException;
-import java.util.List;
import org.apache.avro.Schema;
import org.apache.commons.jexl3.JexlException;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.AvroUtils;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
-import io.rtdi.bigdata.kafka.avro.datatypes.*;
import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PipelineCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-import io.rtdi.bigdata.connector.pipeline.foundation.mapping.PrimitiveMapping;
import io.rtdi.bigdata.kafka.avro.recordbuilders.ValueSchema;
+import io.rtdi.bigdata.rulesservice.PropertiesException;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.AvroRuleUtils;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
@JsonIgnoreProperties(ignoreUnknown=true)
public class PrimitiveRule extends RuleWithName {
private PrimitiveMapping condition;
private RuleResult iffalse = RuleResult.FAIL;
private PrimitiveMapping substitute;
- private Object value;
- private RuleResult sampleresult;
- private Object substituteresult;
private String conditionerror;
private String substituteerror;
- public PrimitiveRule(String fieldname, String rulename, String condition, RuleResult iffalse, String substitute) throws IOException {
- super(fieldname, (rulename == null ? "Rule" : rulename));
+ public PrimitiveRule(String fieldname, String rulename, String condition, RuleResult iffalse, String substitute, Schema schema) throws IOException {
+ super(fieldname, (rulename == null ? "Rule" : rulename), schema);
if (condition == null) {
- throw new ConnectorCallerException("Condition formula cannot be null", null, null, fieldname);
+ throw new PropertiesException("Condition formula cannot be null", null, null, fieldname);
}
this.condition = new PrimitiveMapping(condition);
- this.iffalse = (iffalse == null?RuleResult.FAIL:iffalse);
- if (substitute != null) {
+ this.iffalse = (iffalse == null ? RuleResult.FAIL : iffalse);
+ if (substitute != null && substitute.length() > 0) {
this.substitute = new PrimitiveMapping(substitute);
}
}
-
+
public PrimitiveRule() {
super();
}
@Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException {
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
try {
- RuleResult result = calculateResult(valuerecord);
+ setSampleValue(value, test);
+ RuleResult result = calculateResult(container, test);
if (result != null) {
- JexlRecord r = AvroUtils.createAuditDetails();
+ JexlRecord r = AvroRuleUtils.createAuditDetails();
r.put(ValueSchema.AUDITTRANSFORMATIONNAME, getRulePath());
r.put(ValueSchema.TRANSFORMRESULT, result.name());
String v;
- String p = valuerecord.getPath();
- if (p == null) {
+ String p = container.getPath();
+ if (getFieldname() == null) {
+ v = p;
+ } else if (p == null) {
v = getFieldname();
} else {
v = p + "." + getFieldname();
@@ -73,54 +69,75 @@ public RuleResult apply(JexlRecord valuerecord, List ruleresults) th
break;
}
r.put(ValueSchema.AUDIT_TRANSFORMRESULT_QUALITY, q);
- ruleresults.add(r);
-
- return result;
+ container.addRuleresult(r);
+
} else {
- return RuleResult.PASS;
+ result = RuleResult.PASS;
+ }
+ if (test) {
+ setSampleresult(result);
+ setConditionerror(null);
}
+ return result;
} catch (JexlException e) {
- throw new ConnectorCallerException("Evaluationg the expression failed", e, "Check syntax and data", condition.toString());
- }
- }
-
- private RuleResult calculateResult(JexlRecord valuerecord) throws IOException {
- RuleResult r = null;
- Object o = condition.evaluate(valuerecord);
- if (o != null && o instanceof Boolean) {
- if (((Boolean) o).booleanValue()) {
- r = RuleResult.PASS;
+ if (test) {
+ setConditionerror(e.getMessage());
+ return RuleResult.FAIL;
} else {
- r = iffalse;
+ throw new PropertiesException("Evaluating the expression failed", e, "Check syntax and data", condition.toString());
}
- } else {
- throw new ConnectorCallerException("This rule expression does not return true/false", null, "A condition must return true/false", condition.getExpression());
}
- return r;
}
-
- @Override
- public RuleResult validateRule(JexlRecord valuerecord) {
- sampleresult = null;
+
+ private RuleResult calculateResult(AvroContainer valuerecord, boolean test) throws IOException {
if (condition != null) {
- try {
- sampleresult = calculateResult(valuerecord);
- conditionerror = null;
- } catch (IOException e) {
- sampleresult = null;
- conditionerror = getResultString(condition.getExpression(), e);
- }
- }
- if (substitute != null) {
- try {
- substituteresult = substitute.evaluate(valuerecord);
- substituteerror = null;
- } catch (IOException e) {
- substituteresult = null;
- substituteerror = getResultString(substitute.getExpression(), e);
+ RuleResult r = null;
+ Object o = condition.evaluate(valuerecord);
+ if (o != null && o instanceof Boolean) {
+ if (((Boolean) o).booleanValue()) {
+ r = RuleResult.PASS;
+ } else {
+ r = iffalse;
+ /*
+ * Calculate the substitute value
+ */
+ if (substitute != null) {
+ try {
+ Object substitutevalue = substitute.evaluate(valuerecord);
+ valuerecord.addChangedvalue(getFieldname(), substitutevalue);
+ if (test) {
+ setSampleoutput(Rule.valueToJavaObject(substitutevalue, getDataType()));
+ setSubstituteerror(null);
+ }
+ } catch (IOException e) {
+ if (test) {
+ setSampleoutput(null);
+ setSubstituteerror(getResultString(substitute.getExpression(), e));
+ } else {
+ throw e;
+ }
+ }
+ }
+ }
+ } else {
+ if (test) {
+ setConditionerror("This rule expression does not return true/false");
+ } else {
+ throw new PropertiesException("This rule expression does not return true/false", null, "A condition must return true/false", condition.getExpression());
+ }
}
+ return r;
+ } else {
+ return null;
}
- return sampleresult;
+ }
+
+ public void setConditionerror(String conditionerror) {
+ this.conditionerror = conditionerror;
+ }
+
+ public void setSubstituteerror(String substituteerror) {
+ this.substituteerror = substituteerror;
}
public String getCondition() {
@@ -136,7 +153,7 @@ public void setCondition(String condition) {
try {
this.condition = new PrimitiveMapping(condition);
} catch (IOException e) {
- sampleresult = null;
+ setSampleresult(null);
conditionerror = getResultString(condition, e);
}
}
@@ -159,16 +176,16 @@ public String getSubstitute() {
}
public void setSubstitute(String substitute) {
- if (substitute != null) {
+ if (substitute != null && substitute.length() > 0) {
try {
this.substitute = new PrimitiveMapping(substitute);
} catch (IOException e) {
- substituteresult = null;
+ setSampleoutput(null);
substituteerror = getResultString(substitute, e);
}
}
}
-
+
protected String getResultString(String formula, IOException e) {
String m = e.getMessage();
if (e.getCause() != null) {
@@ -182,58 +199,12 @@ protected String getResultString(String formula, IOException e) {
@Override
public String toString() {
if (condition != null) {
- return "Rule: if (" + condition.toString() + ") then PASS else " + iffalse.name();
+ return getFieldname() + ": PrimitiveRule: if (" + condition.toString() + ") then PASS else " + iffalse.name();
} else {
- return "Rule: always PASS ";
+ return getFieldname() + ": PrimitiveRule always PASS ";
}
}
- @Override
- protected PrimitiveRule createUIRuleTree(Schema fieldschema) throws PropertiesException {
- return this;
- }
-
- @Override
- public void assignSamplevalue(JexlRecord sampledata) {
- this.value = sampledata.get(getFieldname());
- }
-
- @Override
- public Object getSamplevalue() throws PipelineCallerException {
- return valueToJavaObject(value, getDataType());
- }
-
- static Object valueToJavaObject(Object value, IAvroDatatype datatype) throws PipelineCallerException {
- if (value != null && datatype != null) {
- return datatype.convertToJava(value);
- } else {
- return null;
- }
- }
-
- @Override
- public RuleResult getSampleresult() throws IOException {
- return sampleresult;
- }
-
- @Override
- protected Rule createSimplified() throws IOException {
- if (getCondition() != null) {
- return createNewInstance();
- } else {
- return null;
- }
- }
-
- @Override
- protected Rule createNewInstance() throws IOException {
- return new PrimitiveRule(getFieldname(), getRulename(), getCondition(), getIffalse(), getSubstitute());
- }
-
- public Object getSubstituteresult() {
- return substituteresult;
- }
-
public String getSubstituteerror() {
return substituteerror;
}
@@ -242,4 +213,17 @@ public String getConditionerror() {
return conditionerror;
}
+ @Override
+ public Rule clone() {
+ PrimitiveRule ret = new PrimitiveRule();
+ ret.setCondition(getCondition());
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setIffalse(getIffalse());
+ ret.setRulename(getRulename());
+ ret.setSchemaname(getSchemaname());
+ ret.setSubstitute(getSubstitute());
+ return ret;
+ }
+
}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/RecordRule.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/RecordRule.java
index 40e6629..eef8c09 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/RecordRule.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/RecordRule.java
@@ -2,126 +2,69 @@
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.avro.Schema;
-import org.apache.avro.Schema.Field;
-import org.apache.avro.Schema.Type;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.kafka.avro.datatypes.*;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PipelineCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-import io.rtdi.bigdata.connector.pipeline.foundation.utils.IOUtils;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroRecord;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
@JsonIgnoreProperties(ignoreUnknown=true)
-public class RecordRule extends Rule {
-
+public class RecordRule extends Rule implements IContainerRule {
+
public RecordRule() {
super();
}
-
- public RecordRule(String fieldname) {
- super(fieldname);
- }
- public ArrayRule addNested(String fieldname) {
- ArrayRule arrayrule = new ArrayRule(fieldname);
- addRule(arrayrule);
- return arrayrule;
+ public RecordRule(String fieldname, Schema schema) {
+ super(fieldname, schema != null ? schema.getFullName() : null);
+ setDataType(AvroRecord.create());
}
-
+
@Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException {
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
if (getRules() != null) {
- if (getFieldname() != null) {
- Object o = valuerecord.get(getFieldname());
- if (o instanceof JexlRecord) {
- for ( Rule rule : getRules()) {
- rule.apply((JexlRecord) o, ruleresults);
+ if (value instanceof JexlRecord rec) {
+ for ( Rule rule : getRules()) {
+ if (rule.getFieldname() != null && !(rule instanceof GenericRules)) {
+ rule.apply(rec.get(rule.getFieldname()), container, test);
+ } else {
+ rule.apply(rec, rec, test);
}
}
}
}
return null;
}
-
+
@Override
public String toString() {
- return "RecordRule \"" + getFieldname() + "\" tests this record";
+ return getFieldname() + ": RecordRule tests all fields of this record";
}
@Override
- public RecordRule createUIRuleTree(Schema schema) throws PropertiesException {
- return createUIRuleTree(getFieldname(), schema, getRules());
- }
-
- public static RecordRule createUIRuleTree(String fieldname, Schema schema, List originalrules) throws PropertiesException {
- if (schema.getType() == Type.RECORD) {
- RecordRule r = new RecordRule(fieldname);
- addFields(r, schema, originalrules);
- return r;
- } else {
- throw new PropertiesException("Provided Schema is not a record schema", (String) null, schema.getName());
- }
- }
-
- protected static void addFields(RecordRule r, Schema schema, List originalrules) throws PropertiesException {
- Map fieldnameindex = new HashMap<>();
- if (originalrules != null) {
- for (Rule o : originalrules) {
- String fieldname = o.getFieldname();
- fieldnameindex.put(fieldname, o);
- }
- }
- /*
- * The Rule array is built using the schema as basis to preserve the schema's field order
- */
- List rules = new ArrayList<>();
- if (schema.getType() == Type.RECORD) {
- for (Field field : schema.getFields()) {
- Schema fieldschema = IOUtils.getBaseSchema(field.schema());
- String fieldname = field.name();
- Rule childrule = fieldnameindex.get(fieldname);
- if (childrule == null) {
- switch (fieldschema.getType()) {
- case ARRAY:
- childrule = ArrayRule.createUIRuleTree(fieldname, fieldschema, null);
- break;
- case RECORD:
- childrule = createUIRuleTree(fieldname, fieldschema, null);
- break;
- default:
- childrule = new EmptyRule(fieldname);
- break;
- }
- childrule.setDataType(AvroType.getAvroDataType(fieldschema));
- rules.add(childrule);
- } else {
- Rule c = childrule.createUIRuleTree(fieldschema);
- c.setDataType(AvroType.getAvroDataType(fieldschema));
- rules.add(c);
- }
-
- }
- r.setRules(rules);
- }
+ public RuleResult getSampleresult() {
+ return null;
}
@Override
- public void assignSamplevalue(JexlRecord sampledata) {
- if (sampledata != null && this.getRules() != null) {
- if (getFieldname() != null) {
- Object o = sampledata.get(getFieldname());
- if (o instanceof JexlRecord) {
- for ( Rule r : this.getRules()) {
- r.assignSamplevalue((JexlRecord) o);
+ public void update(IContainerRule empty) {
+ if (empty instanceof RecordRule && empty.getRules() != null) {
+ Map index = getRules().stream().collect(Collectors.toMap(Rule::getFieldname, Function.identity()));
+ for (Rule rule : empty.getRules()) {
+ Rule r = index.get(rule.getFieldname());
+ if (r == null) {
+ addRule(rule.clone());
+ } else if (r instanceof IContainerRule rr) {
+ if (rule instanceof IContainerRule cr) {
+ rr.update(cr);
}
}
}
@@ -129,33 +72,19 @@ public void assignSamplevalue(JexlRecord sampledata) {
}
@Override
- public String getSamplevalue() {
- return null;
- }
-
- @Override
- public RuleResult getSampleresult() throws PipelineCallerException {
- return null;
- }
-
- @Override
- public RuleResult validateRule(JexlRecord valuerecord) {
- if (valuerecord != null && this.getRules() != null) {
- if (getFieldname() != null) {
- Object o = valuerecord.get(getFieldname());
- if (o instanceof JexlRecord) {
- for ( Rule r : this.getRules()) {
- r.validateRule((JexlRecord) o);
- }
- }
+ public Rule clone() {
+ RecordRule ret = new RecordRule();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setSchemaname(getSchemaname());
+ if (getRules() != null) {
+ List a = new ArrayList<>(getRules().size());
+ ret.setRules(a);
+ for (Rule r : getRules()) {
+ a.add(r.clone());
}
}
- return null;
- }
-
- @Override
- protected Rule createNewInstance() throws ConnectorCallerException {
- return new RecordRule(getFieldname());
+ return ret;
}
}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/Rule.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/Rule.java
index b73144a..5075cc5 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/Rule.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/Rule.java
@@ -3,52 +3,72 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
+import io.rtdi.bigdata.kafka.avro.AvroUtils;
import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.kafka.avro.datatypes.*;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroType;
+import io.rtdi.bigdata.kafka.avro.datatypes.IAvroDatatype;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
@JsonTypeInfo(
- use = JsonTypeInfo.Id.NAME,
- include = JsonTypeInfo.As.PROPERTY,
- property = "type")
-@JsonSubTypes({
- @Type(value = EmptyRule.class, name = "EmptyRule"),
- @Type(value = ArrayRule.class, name = "ArrayRule"),
- @Type(value = PrimitiveRule.class, name = "PrimitiveRule"),
- @Type(value = TestSetAll.class, name = "TestSetAll"),
- @Type(value = TestSetFirstPass.class, name = "TestSetFirstPass"),
- @Type(value = TestSetFirstFail.class, name = "TestSetFirstFail"),
- @Type(value = RecordRule.class, name = "RecordRule")
+ use = JsonTypeInfo.Id.NAME,
+ include = JsonTypeInfo.As.PROPERTY,
+ property = "type")
+@JsonSubTypes({
+ @Type(value = EmptyRule.class, name = "EmptyRule"),
+ @Type(value = ArrayRule.class, name = "ArrayRule"),
+ @Type(value = PrimitiveRule.class, name = "PrimitiveRule"),
+ @Type(value = TestSetAll.class, name = "TestSetAll"),
+ @Type(value = TestSetFirstPass.class, name = "TestSetFirstPass"),
+ @Type(value = TestSetFirstFail.class, name = "TestSetFirstFail"),
+ @Type(value = RecordRule.class, name = "RecordRule"),
+ @Type(value = UnionRule.class, name = "UnionRule"),
+ @Type(value = GenericRules.class, name = "GenericRules")
})
@JsonIgnoreProperties(ignoreUnknown=true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
public abstract class Rule {
private String fieldname;
private List rules = null;
private IAvroDatatype fielddatatype;
protected String rulepath;
- private String sampleresulterror = null;
-
+ private Object sampleinput = null;
+ private Object sampleoutput = null;
+ private RuleResult sampleresult;
+ private String schemaname;
+
public Rule() {
super();
}
- public Rule(String fieldname) {
+ public Rule(String fieldname, String schemaname) {
this();
this.fieldname = fieldname;
+ this.schemaname = schemaname;
+ }
+
+ protected void setSampleValue(Object value, boolean test) {
+ if (test) {
+ setSampleinput(Rule.valueToJavaObject(value, getDataType()));
+ setSampleoutput(getSampleinput()); // Might be overwritten later
+ }
}
- public abstract RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException;
+ public abstract RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException;
public List getRules() {
return rules;
@@ -62,11 +82,11 @@ public void postSerialization() {
}
}
}
-
+
public void setRules(List rules) {
this.rules = rules;
}
-
+
public void addAll(Collection m) {
if (rules == null) {
rules = new ArrayList<>();
@@ -76,7 +96,7 @@ public void addAll(Collection m) {
addRule(iter.next());
}
}
-
+
protected String getRulePath() {
return rulepath;
}
@@ -97,8 +117,6 @@ public void setFieldname(String fieldname) {
this.fieldname = fieldname;
}
- protected abstract Rule createUIRuleTree(Schema fieldschema) throws PropertiesException;
-
public String getFielddatatype() {
if (fielddatatype != null) {
return fielddatatype.toString();
@@ -114,7 +132,7 @@ public void setFielddatatype(String fielddatatype) {
protected void setDataType(IAvroDatatype fielddatatype) {
this.fielddatatype = fielddatatype;
}
-
+
protected IAvroDatatype getDataType() {
return fielddatatype;
}
@@ -123,39 +141,83 @@ public void setParentPath(String parentpath) {
rulepath = parentpath;
}
- public abstract void assignSamplevalue(JexlRecord record);
-
- public abstract Object getSamplevalue() throws IOException;
+ public void setSampleoutput(String sampleoutput) {
+ this.sampleoutput = sampleoutput;
+ }
- public abstract RuleResult getSampleresult() throws IOException;
+ public Object getSampleoutput() {
+ return sampleoutput;
+ }
- public abstract RuleResult validateRule(JexlRecord valuerecord);
+ public void setSampleoutput(Object sampleoutput) {
+ if (sampleoutput instanceof GenericRecord) {
+ // ignore
+ } else if (sampleoutput instanceof List) {
+ // ignore
+ } else {
+ this.sampleoutput = sampleoutput;
+ }
+ }
+ public Object getSampleinput() {
+ return sampleinput;
+ }
- public String getSampleresulterror() {
- return sampleresulterror;
+ public void setSampleinput(Object sampleinput) {
+ this.sampleinput = sampleinput;
}
- protected Rule createSimplified() throws IOException {
- Rule n = createNewInstance();
- addAllMandatory(n);
- if (n.getRules() == null || n.getRules().size() == 0) {
- return null;
+ public static Object valueToJavaObject(Object value, IAvroDatatype datatype) {
+ if (value != null && datatype != null) {
+ return datatype.convertToJava(value);
} else {
- return n;
+ return null;
}
}
- protected abstract Rule createNewInstance() throws IOException;
+ public RuleResult getSampleresult() {
+ return sampleresult;
+ }
+
+ public void setSampleresult(RuleResult sampleresult) {
+ this.sampleresult = sampleresult;
+ }
+
+ /**
+ * Add the substitution value to the changedvalues map
+ *
+ * @param valuerecord
+ * @param fieldname
+ * @param substitutevalue
+ * @param changedvalues
+ */
+ public static void addChangedvalue(JexlRecord valuerecord, String fieldname, Object substitutevalue, Map> changedvalues) {
+ Map fieldmap = changedvalues.get(valuerecord);
+ if (fieldmap == null) {
+ fieldmap = new HashMap<>();
+ changedvalues.put(valuerecord, fieldmap);
+ }
+ fieldmap.put(fieldname, substitutevalue);
+ }
- protected void addAllMandatory(Rule target) throws IOException {
- if (getRules() != null) {
- for (Rule r : getRules()) {
- Rule n = r.createSimplified();
- if (n != null) {
- target.addRule(n);
- }
+ public void updateSampleOutput(JexlRecord valuerecord) {
+ Object value = valuerecord.get(fieldname);
+ if (value instanceof List l) {
+ if (l.size() > 0) {
+ Schema c = AvroUtils.getBaseSchema(valuerecord.getSchema().getField(fieldname).schema());
+ setSampleoutput(valueToJavaObject(l.get(0), AvroType.getDataType(AvroType.getType(c.getElementType()), 10, 0)));
}
}
+ setSampleoutput(valueToJavaObject(value, getDataType()));
+ }
+
+ public String getSchemaname() {
+ return schemaname;
+ }
+
+ public void setSchemaname(String schemaname) {
+ this.schemaname = schemaname;
}
+ @Override
+ public abstract Rule clone();
}
\ No newline at end of file
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/RuleUtils.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/RuleUtils.java
new file mode 100644
index 0000000..1a03c41
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/RuleUtils.java
@@ -0,0 +1,150 @@
+package io.rtdi.bigdata.rulesservice.rules;
+
+import java.util.List;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+
+import io.rtdi.bigdata.kafka.avro.AvroUtils;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroType;
+import io.rtdi.bigdata.kafka.avro.datatypes.IAvroDatatype;
+import io.rtdi.bigdata.rulesservice.PropertiesException;
+import io.rtdi.bigdata.rulesservice.jexl.JexlArray;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+
+public class RuleUtils {
+
+ /**
+ * Create the default rule tree for a (sub-)schema
+ * @param schema is the input structure
+ * @param name of the rule
+ * @return one rule matching the schema; the rule might have child rules
+ */
+ public static Rule getRuleForSchema(org.apache.avro.Schema schema, String name) {
+ schema = AvroUtils.getBaseSchema(schema); // remove the optional field, meaning union(null, string) --> string
+ RecordRule recordrule;
+ switch (schema.getType()) {
+ case BOOLEAN:
+ case BYTES:
+ case DOUBLE:
+ case ENUM:
+ case FIXED:
+ case FLOAT:
+ case INT:
+ case LONG:
+ case MAP:
+ case STRING:
+ return new EmptyRule(name, schema);
+ case UNION:
+ /*
+ * Above getBaseSchema() handles UNION of NULL plus something else.
+ * But if the UNION is more complicated, more handling is required
+ */
+ return new UnionRule(name, schema);
+ case NULL:
+ return new EmptyRule(name, schema);
+ case RECORD:
+ recordrule = new RecordRule(name, schema);
+ addRules(recordrule, schema);
+ return recordrule;
+ case ARRAY:
+ ArrayRule arrayrule = new ArrayRule(name, schema);
+ arrayrule.addRule(getRuleForSchema(schema.getElementType(), name));
+ return arrayrule;
+ default:
+ return new EmptyRule(name, schema);
+ }
+ }
+
+ public static void addRules(RecordRule r, org.apache.avro.Schema schema) {
+ for (Field f : schema.getFields()) {
+ Rule rule = getRuleForSchema(f.schema(), f.name());
+ if (rule != null) {
+ r.addRule(rule);
+ }
+ }
+ GenericRules gen = new GenericRules();
+ gen.addRule(new EmptyRule(null, null));
+ r.addRule(gen); // Record rules have rules on fields and unbound generic rules
+ }
+
+ public static Object getSampleValue(Rule rule, Schema schema) throws PropertiesException {
+ if (schema != null) {
+ switch (schema.getType()) {
+ case ARRAY:
+ if (rule instanceof ArrayRule ar) {
+ if (ar.getRules() != null && ar.getRules().size() > 0) {
+ JexlArray a = new JexlArray<>(schema, null);
+ Object value = getSampleValue(rule.getRules().get(0), schema.getElementType());
+ if (value != null) {
+ a.add(value);
+ }
+ return a;
+ }
+ } else {
+ throw new PropertiesException("There is a mismatch between the schema and rule type for the field <" + rule.getFieldname() + ">");
+ }
+ break;
+ case RECORD:
+ if (rule instanceof RecordRule) {
+ if (rule.getRules() != null) {
+ JexlRecord rec = new JexlRecord(schema, null);
+ for ( Rule r : rule.getRules()) {
+ if (r.getFieldname() != null) {
+ Field field = schema.getField(r.getFieldname());
+ if (field != null) {
+ Schema fieldschema = AvroUtils.getBaseSchema(field.schema());
+ rec.put(r.getFieldname(), getSampleValue(r, fieldschema));
+ }
+ }
+ }
+ return rec;
+ }
+ } else {
+ throw new PropertiesException("There is a mismatch between the schema and rule type for the field <" + rule.getFieldname() + ">");
+ }
+ break;
+ case UNION:
+ if (rule instanceof UnionRule) {
+ if (rule.getRules() != null) {
+ for ( Rule r : rule.getRules()) {
+ if (r.getSampleinput() != null) {
+ return getSampleValue(r, RuleUtils.findUnionSchema(rule.getSchemaname(), schema.getTypes()));
+ }
+ }
+ }
+ }
+ break;
+ case BOOLEAN:
+ case BYTES:
+ case DOUBLE:
+ case ENUM:
+ case FIXED:
+ case FLOAT:
+ case INT:
+ case LONG:
+ case MAP:
+ case NULL:
+ case STRING:
+ IAvroDatatype dt = AvroType.getAvroDataType(schema);
+ if (dt != null) {
+ return dt.convertToInternal(rule.getSampleinput());
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ return null;
+ }
+
+ public static Schema findUnionSchema(String schemaname, List list) {
+ for (Schema s : list) {
+ if (s.getFullName().equals(schemaname)) {
+ return s;
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/RuleWithName.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/RuleWithName.java
index 41630a2..73442fd 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/RuleWithName.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/RuleWithName.java
@@ -1,5 +1,7 @@
package io.rtdi.bigdata.rulesservice.rules;
+import org.apache.avro.Schema;
+
public abstract class RuleWithName extends Rule {
private String rulename;
@@ -7,8 +9,8 @@ public RuleWithName() {
super();
}
- public RuleWithName(String fieldname, String rulename) {
- super(fieldname);
+ public RuleWithName(String fieldname, String rulename, Schema schema) {
+ super(fieldname, schema != null ? schema.getFullName() : null);
this.rulename = rulename;
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSet.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSet.java
index 8800ac4..ab611db 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSet.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSet.java
@@ -4,53 +4,20 @@
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
-import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PipelineCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
-
@JsonIgnoreProperties(ignoreUnknown=true)
public abstract class TestSet extends RuleWithName {
- private Object value;
- protected RuleResult sampleresult;
public TestSet() {
super();
}
- public TestSet(String fieldname, String rulename) {
- super(fieldname, rulename);
+ public TestSet(String fieldname, String rulename, Schema schema) {
+ super(fieldname, rulename, schema);
}
- public TestSet(String fieldname, String rulename, Rule rule) {
- super(fieldname, rulename);
+ public TestSet(String fieldname, String rulename, Rule rule, Schema schema) {
+ super(fieldname, rulename, schema);
addRule(rule);
}
- protected void copyTests(TestSet t, Schema fieldschema) throws PropertiesException {
- for (Rule r : getRules()) {
- t.addRule(r.createUIRuleTree(fieldschema));
- }
- }
-
- @Override
- public void assignSamplevalue(JexlRecord sampledata) {
- this.value = sampledata.get(getFieldname());
- if (this.getRules() != null) {
- for ( Rule r : this.getRules()) {
- r.assignSamplevalue(sampledata);
- }
- }
- }
-
- @Override
- public Object getSamplevalue() throws PipelineCallerException {
- return PrimitiveRule.valueToJavaObject(value, getDataType());
- }
-
- @Override
- public RuleResult getSampleresult() throws PipelineCallerException {
- return sampleresult;
- }
-
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetAll.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetAll.java
index 75c7cec..e19ea6e 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetAll.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetAll.java
@@ -1,16 +1,15 @@
package io.rtdi.bigdata.rulesservice.rules;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.avro.Schema;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
@JsonIgnoreProperties(ignoreUnknown=true)
public class TestSetAll extends TestSet {
@@ -19,50 +18,49 @@ public TestSetAll() {
super();
}
- public TestSetAll(String fieldname, String rulename) {
- super(fieldname, rulename);
+ public TestSetAll(String fieldname, String rulename, Schema schema) {
+ super(fieldname, rulename, schema);
}
- public TestSetAll(String fieldname, String rulename, Rule rule) {
- super(fieldname, rulename, rule);
+ public TestSetAll(String fieldname, String rulename, Rule rule, Schema schema) {
+ super(fieldname, rulename, rule, schema);
}
@Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException {
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
+ setSampleValue(value, test);
RuleResult result = RuleResult.PASS;
for ( Rule rule : getRules()) {
- result = result.aggregate(rule.apply(valuerecord, ruleresults));
+ if (!(rule instanceof EmptyRule)) {
+ result = result.aggregate(rule.apply(value, container, test));
+ if (test) {
+ setSampleresult(result);
+ }
+ }
}
return result;
}
- @Override
- public RuleResult validateRule(JexlRecord valuerecord) {
- sampleresult = RuleResult.PASS;
- for ( Rule rule : getRules()) {
- RuleResult r = rule.validateRule(valuerecord);
- if (sampleresult != null) {
- sampleresult = sampleresult.aggregate(r);
- }
- }
- return sampleresult;
- }
-
@Override
public String toString() {
- return "TestSetAll \"" + getRulename() + "\" tests all child rules";
+ return getFieldname() + ": TestSetAll tests all child rules";
}
@Override
- protected TestSetAll createUIRuleTree(Schema fieldschema) throws PropertiesException {
- TestSetAll t = new TestSetAll(getFieldname(), getRulename());
- copyTests(t, fieldschema);
- return t;
- }
-
- @Override
- protected Rule createNewInstance() throws ConnectorCallerException {
- return new TestSetAll(getFieldname(), getRulename());
+ public Rule clone() {
+ TestSetAll ret = new TestSetAll();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setRulename(getRulename());
+ ret.setSchemaname(getSchemaname());
+ if (getRules() != null) {
+ List a = new ArrayList<>(getRules().size());
+ ret.setRules(a);
+ for (Rule r : getRules()) {
+ a.add(r.clone());
+ }
+ }
+ return ret;
}
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstFail.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstFail.java
index 6ffb769..035303e 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstFail.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstFail.java
@@ -1,17 +1,15 @@
package io.rtdi.bigdata.rulesservice.rules;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.avro.Schema;
-import org.apache.commons.jexl3.JexlException;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
@JsonIgnoreProperties(ignoreUnknown=true)
public class TestSetFirstFail extends TestSet {
@@ -20,60 +18,54 @@ public TestSetFirstFail() {
super();
}
- public TestSetFirstFail(String fieldname, String rulename) {
- super(fieldname, rulename);
+ public TestSetFirstFail(String fieldname, String rulename, Schema schema) {
+ super(fieldname, rulename, schema);
}
- public TestSetFirstFail(String fieldname, String rulename, Rule rule) {
- super(fieldname, rulename, rule);
+ public TestSetFirstFail(String fieldname, String rulename, Rule rule, Schema schema) {
+ super(fieldname, rulename, rule, schema);
}
@Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException {
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
+ setSampleValue(value, test);
RuleResult result = RuleResult.PASS;
for ( Rule rule : getRules()) {
- RuleResult currentresult = rule.apply(valuerecord, ruleresults);
- if (currentresult == RuleResult.FAIL) {
- return RuleResult.FAIL;
- } else {
- result = result.aggregate(currentresult);
+ if (!(rule instanceof EmptyRule)) {
+ RuleResult currentresult = rule.apply(value, container, test);
+ if (currentresult == RuleResult.FAIL) {
+ result = RuleResult.FAIL;
+ break;
+ } else {
+ result = result.aggregate(currentresult);
+ }
}
}
- return result;
- }
-
- @Override
- public RuleResult validateRule(JexlRecord valuerecord) throws JexlException {
- /*
- * Needs to evaluate all rules to display their result. A rule aggregation with Failed is always Failed, so we are fine here.
- */
- sampleresult = RuleResult.PASS;
- for ( Rule rule : getRules()) {
- RuleResult currentresult = rule.validateRule(valuerecord);
- if (currentresult == RuleResult.FAIL) {
- sampleresult = RuleResult.FAIL;
- } else if (sampleresult != null) {
- sampleresult = sampleresult.aggregate(currentresult);
- }
+ if (test) {
+ setSampleresult(result);
}
- return sampleresult;
+ return result;
}
@Override
public String toString() {
- return "TestSetFirstFail \"" + getRulename() + "\" tests until first FAILs";
+ return getFieldname() + ": TestSetFirstFail tests until first FAIL";
}
-
- @Override
- protected TestSetFirstFail createUIRuleTree(Schema fieldschema) throws PropertiesException {
- TestSetFirstFail t = new TestSetFirstFail(getFieldname(), getRulename());
- copyTests(t, fieldschema);
- return t;
- }
-
@Override
- protected Rule createNewInstance() throws ConnectorCallerException {
- return new TestSetFirstFail(getFieldname(), getRulename());
+ public Rule clone() {
+ TestSetFirstFail ret = new TestSetFirstFail();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setRulename(getRulename());
+ ret.setSchemaname(getSchemaname());
+ if (getRules() != null) {
+ List a = new ArrayList<>(getRules().size());
+ ret.setRules(a);
+ for (Rule r : getRules()) {
+ a.add(r.clone());
+ }
+ }
+ return ret;
}
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstPass.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstPass.java
index 2320b7d..77a29bf 100644
--- a/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstPass.java
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/TestSetFirstPass.java
@@ -1,17 +1,15 @@
package io.rtdi.bigdata.rulesservice.rules;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.avro.Schema;
-import org.apache.commons.jexl3.JexlException;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import io.rtdi.bigdata.connector.connectorframework.exceptions.ConnectorCallerException;
-import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
-import io.rtdi.bigdata.connector.pipeline.foundation.exceptions.PropertiesException;
import io.rtdi.bigdata.kafka.avro.RuleResult;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
@JsonIgnoreProperties(ignoreUnknown=true)
public class TestSetFirstPass extends TestSet {
@@ -20,57 +18,52 @@ public TestSetFirstPass() {
super();
}
- public TestSetFirstPass(String fieldname, String rulename) {
- super(fieldname, rulename);
+ public TestSetFirstPass(String fieldname, String rulename, Schema schema) {
+ super(fieldname, rulename, schema);
}
- public TestSetFirstPass(String fieldname, String rulename, PrimitiveRule rule) {
- super(fieldname, rulename, rule);
+ public TestSetFirstPass(String fieldname, String rulename, PrimitiveRule rule, Schema schema) {
+ super(fieldname, rulename, rule, schema);
}
@Override
- public RuleResult apply(JexlRecord valuerecord, List ruleresults) throws IOException {
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
+ setSampleValue(value, test);
RuleResult result = RuleResult.FAIL;
for ( Rule rule : getRules()) {
- result = rule.apply(valuerecord, ruleresults);
- if (result == RuleResult.PASS) {
- return result;
+ if (!(rule instanceof EmptyRule)) {
+ result = rule.apply(value, container, test);
+ if (result == RuleResult.PASS) {
+ break;
+ }
}
}
- return result;
- }
-
- @Override
- public RuleResult validateRule(JexlRecord valuerecord) throws JexlException {
- /*
- * Needs to go through all rules instead of exit early, else the other rules are not evaluated and hence their result is not displayed.
- */
- RuleResult r = RuleResult.FAIL;
- for ( Rule rule : getRules()) {
- RuleResult intermediate = rule.validateRule(valuerecord);
- if (intermediate == RuleResult.PASS) {
- r = intermediate;
- }
+ if (test) {
+ setSampleresult(result);
}
- sampleresult = r;
- return r;
+ return result;
}
@Override
public String toString() {
- return "TestSetFirstPass \"" + getRulename() + "\" tests until first PASSes";
+ return getFieldname() + ": TestSetFirstPass tests until first PASS";
}
@Override
- protected TestSetFirstPass createUIRuleTree(Schema fieldschema) throws PropertiesException {
- TestSetFirstPass t = new TestSetFirstPass(getFieldname(), getRulename());
- copyTests(t, fieldschema);
- return t;
- }
-
- @Override
- protected Rule createNewInstance() throws ConnectorCallerException {
- return new TestSetFirstPass(getFieldname(), getRulename());
+ public Rule clone() {
+ TestSetFirstPass ret = new TestSetFirstPass();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setRulename(getRulename());
+ ret.setSchemaname(getSchemaname());
+ if (getRules() != null) {
+ List a = new ArrayList<>(getRules().size());
+ ret.setRules(a);
+ for (Rule r : getRules()) {
+ a.add(r.clone());
+ }
+ }
+ return ret;
}
}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/rules/UnionRule.java b/src/main/java/io/rtdi/bigdata/rulesservice/rules/UnionRule.java
new file mode 100644
index 0000000..80f071d
--- /dev/null
+++ b/src/main/java/io/rtdi/bigdata/rulesservice/rules/UnionRule.java
@@ -0,0 +1,156 @@
+package io.rtdi.bigdata.rulesservice.rules;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Type;
+
+import io.rtdi.bigdata.kafka.avro.RuleResult;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroUnion;
+import io.rtdi.bigdata.rulesservice.jexl.AvroContainer;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+
+public class UnionRule extends Rule implements IContainerRule {
+
+ public UnionRule() {
+ super();
+ }
+
+ public UnionRule(String name, Schema schema) {
+ super(name, schema != null ? schema.getFullName() : null);
+ setDataType(AvroUnion.create());
+ if (schema != null && schema.getType() == Type.UNION) {
+ for (Schema s : schema.getTypes()) {
+ Rule r = RuleUtils.getRuleForSchema(s, name);
+ if (r != null) {
+ addRule(r);
+ }
+ }
+ }
+ }
+
+ @Override
+ public RuleResult apply(Object value, AvroContainer container, boolean test) throws IOException {
+ // List unionschema = container.getSchema().getField(getFieldname()).schema().getTypes();
+ // TODO: Currently it works on the base schema only, but a union could also encompass multiple types as long as they have different names
+
+ for (Rule r : getRules()) {
+ if (r.getDataType() != null) {
+ switch (r.getDataType().getBackingType()) {
+ case ARRAY:
+ if (value instanceof Collection) {
+ r.apply(value, container, test);
+ }
+ break;
+ case BOOLEAN:
+ if (value instanceof Boolean) {
+ r.apply(value, container, test);
+ }
+ break;
+ case BYTES:
+ if (value instanceof byte[]) {
+ r.apply(value, container, test);
+ }
+ break;
+ case DOUBLE:
+ if (value instanceof Double) {
+ r.apply(value, container, test);
+ }
+ break;
+ case ENUM:
+ if (value instanceof CharSequence) {
+ r.apply(value, container, test);
+ }
+ break;
+ case FIXED:
+ if (value instanceof CharSequence) {
+ r.apply(value, container, test);
+ }
+ break;
+ case FLOAT:
+ if (value instanceof Float) {
+ r.apply(value, container, test);
+ }
+ break;
+ case INT:
+ if (value instanceof Integer) {
+ r.apply(value, container, test);
+ }
+ break;
+ case LONG:
+ if (value instanceof Long) {
+ r.apply(value, container, test);
+ }
+ break;
+ case MAP:
+ if (value instanceof Map) {
+ r.apply(value, container, test);
+ }
+ break;
+ case NULL:
+ if (value == null) {
+ r.apply(value, container, test);
+ }
+ break;
+ case RECORD:
+ if (value instanceof JexlRecord) {
+ r.apply(value, container, test);
+ }
+ break;
+ case STRING:
+ if (value instanceof String) {
+ r.apply(value, container, test);
+ }
+ break;
+ case UNION:
+ // union of union is not allowed according to the Avro specification
+ break;
+ default:
+ break;
+
+ }
+ } else {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void update(IContainerRule empty) {
+ if (empty instanceof UnionRule) {
+ if (getRules() != null && empty.getRules() != null) {
+ for ( int i=0; i i) {
+ Rule emptyrule = getRules().get(i);
+ if (rule instanceof IContainerRule rr && emptyrule instanceof IContainerRule er) {
+ rr.update(er);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public Rule clone() {
+ UnionRule ret = new UnionRule();
+ ret.setDataType(getDataType());
+ ret.setFieldname(getFieldname());
+ ret.setSchemaname(getSchemaname());
+ if (getRules() != null) {
+ List a = new ArrayList<>(getRules().size());
+ ret.setRules(a);
+ for (Rule r : getRules()) {
+ a.add(r.clone());
+ }
+ }
+ return ret;
+ }
+
+}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/servlet/MicroservicePage.java b/src/main/java/io/rtdi/bigdata/rulesservice/servlet/MicroservicePage.java
deleted file mode 100644
index fa1c49c..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/servlet/MicroservicePage.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package io.rtdi.bigdata.rulesservice.servlet;
-
-import jakarta.servlet.annotation.WebServlet;
-import jakarta.servlet.http.HttpServlet;
-
-import io.rtdi.bigdata.connector.connectorframework.servlet.UI5ServletAbstract;
-
-@WebServlet("/ui5/Microservice")
-public class MicroservicePage extends UI5ServletAbstract {
- private static final long serialVersionUID = 684367432L;
-
- /**
- * @see HttpServlet#HttpServlet()
- */
- public MicroservicePage() {
- super("Rule Microservice", "Microservice");
- }
-
-}
diff --git a/src/main/java/io/rtdi/bigdata/rulesservice/servlet/ServiceDetailsPage.java b/src/main/java/io/rtdi/bigdata/rulesservice/servlet/ServiceDetailsPage.java
deleted file mode 100644
index e98f7d5..0000000
--- a/src/main/java/io/rtdi/bigdata/rulesservice/servlet/ServiceDetailsPage.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package io.rtdi.bigdata.rulesservice.servlet;
-
-import jakarta.servlet.annotation.WebServlet;
-import jakarta.servlet.http.HttpServlet;
-
-import io.rtdi.bigdata.connector.connectorframework.servlet.UI5ServletAbstract;
-
-@WebServlet("/ui5/ServiceDetails")
-public class ServiceDetailsPage extends UI5ServletAbstract {
- private static final long serialVersionUID = 68436742432L;
-
- /**
- * @see HttpServlet#HttpServlet()
- */
- public ServiceDetailsPage() {
- super("Rule Configuration", "Ruleset");
- }
-
-}
diff --git a/src/main/resources/META-INF/services/io.rtdi.bigdata.connector.connectorframework.IConnectorFactory b/src/main/resources/META-INF/services/io.rtdi.bigdata.connector.connectorframework.IConnectorFactory
deleted file mode 100644
index dceb2be..0000000
--- a/src/main/resources/META-INF/services/io.rtdi.bigdata.connector.connectorframework.IConnectorFactory
+++ /dev/null
@@ -1 +0,0 @@
-io.rtdi.bigdata.rulesservice.RulesServiceFactory
\ No newline at end of file
diff --git a/src/main/resources/log4j2.properties b/src/main/resources/log4j2.properties
new file mode 100644
index 0000000..8401eae
--- /dev/null
+++ b/src/main/resources/log4j2.properties
@@ -0,0 +1,34 @@
+name=PropertiesConfig
+property.filename = logs
+appenders = console, file
+
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %m%n
+
+# File configuration
+appender.file.type = File
+appender.file.name = LOGFILE
+appender.file.fileName = ${filename}/${web:servletContextName:-webapp}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %m%n
+
+
+rootLogger.level = INFO
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = STDOUT
+rootLogger.appenderRef.file.ref = LOGFILE
+
+logger.kafka.name = org.apache.kafka
+logger.kafka.level = INFO
+
+logger.kafka-consumer.name = org.apache.kafka.clients.consumer
+logger.kafka-consumer.level = INFO
+
+logger.kafka-streams.name = org.apache.kafka.streams
+logger.kafka-streams.level = INFO
+
+logger.jexl.name = org.apache.commons.jexl3
+logger.jexl.level = DEBUG
+
diff --git a/src/main/resources/rulesservice.properties b/src/main/resources/rulesservice.properties
deleted file mode 100644
index 4272d14..0000000
--- a/src/main/resources/rulesservice.properties
+++ /dev/null
@@ -1 +0,0 @@
-basedir=https://github.com/rtdi/RTDIRulesService/blob/main/
diff --git a/src/main/resources/ui5/controller/Ruleset.controller b/src/main/resources/ui5/controller/Ruleset.controller
deleted file mode 100644
index b3a8bc9..0000000
--- a/src/main/resources/ui5/controller/Ruleset.controller
+++ /dev/null
@@ -1,228 +0,0 @@
-showFooter : function() {
- return true;
-},
-init : function() {
- var oModel = new JSONModel();
- var that = this;
- var sServicename = jQuery.sap.getUriParameters().get("servicename");
- var sMicroservicename = jQuery.sap.getUriParameters().get("microservice");
- oModel.attachRequestFailed(function(oEvent) {
- that.getView().byId("TreeTable").setBusy(false);
- that.displayError(JSON.parse(oEvent.getParameter("responseText")));
- });
- oModel.attachRequestCompleted(function(oEvent) {
- that.getView().byId("TreeTable").setBusy(false);
- var aRules = oModel.getProperty("/rules");
- if (oEvent.getParameters("success")) {
- var fAddNew = function(aEntries, sPath) {
- if (aEntries) {
- for (var i = 0; i < aEntries.length; i++) {
- fAddNew(aEntries[i].rules, sPath + "/" + i + "/rules");
- if (aEntries[i].type.startsWith("TestSet")) {
- if (aEntries[i].rules) {
- aEntries[i].rules.push({fieldname: aEntries[i].fieldname, type: "EmptyRule"});
- } else {
- aEntries[i].rules = [{fieldname: aEntries[i].fieldname, type: "EmptyRule"}];
- }
- }
- }
- oModel.setProperty(sPath, aEntries);
- }
- };
- fAddNew(aRules, "/rules");
- }
- });
- this.getView().setModel(oModel);
-
- var oStateModel = this.getView().getModel("state");
- oStateModel.setProperty("/locked", false);
- oStateModel.setProperty("/title", "Define Schema rules");
- oStateModel.setProperty("/breadcrumbs", [
- {"text" : "Homepage", "link" : "./Home"},
- {"text" : "Services", "link" : "./Services"},
- {"text" : "Service", "link" : "./Service?servicename=" + encodeURI(sServicename) }
- ] );
-
- var oModelConfiguredSchemas = new JSONModel();
- oModelConfiguredSchemas.attachRequestFailed(function(oEvent) {
- that.displayError(JSON.parse(oEvent.getParameter("responseText")));
- });
- oModelConfiguredSchemas.attachRequestCompleted(function(oEvent) {
- if (oEvent.getParameters("success")) {
- if (oModelConfiguredSchemas.getData() && oModelConfiguredSchemas.getData().schemas && oModelConfiguredSchemas.getData().schemas.length > 0) {
- var sFirstSchema;
- for ( var i = 0; i < oModelConfiguredSchemas.getData().schemas.length; i++) {
- var oSchema = oModelConfiguredSchemas.getData().schemas[i];
- if (oSchema.configured) {
- sFirstSchema = oSchema.schemaname;
- break;
- }
- }
- if (sFirstSchema) {
- var oSchemaControl = that.getView().byId("schemaselector");
- oSchemaControl.setSelectedKey(sFirstSchema);
- that._loadSchemaData(sFirstSchema);
- }
- }
- }
- });
- oModelConfiguredSchemas.setSizeLimit(1000);
- oModelConfiguredSchemas.loadData("../rest/rules/" + encodeURI(sServicename) + "/" + encodeURI(sMicroservicename));
- this.getView().setModel(oModelConfiguredSchemas, "schemarules");
-
-},
-edit : function(oEvent) {
-},
-save : function(oEvent) {
- var oModel = this.getView().getModel();
- var sServicename = jQuery.sap.getUriParameters().get("servicename");
- var sMicroservicename = jQuery.sap.getUriParameters().get("microservice");
- var oSchemaControl = this.getView().byId("schemaselector");
- var sSchemaname = oSchemaControl.getSelectedKey();
- if (!!sSchemaname) {
- var xmlhttp = new XMLHttpRequest();
- xmlhttp.open("POST", "../rest/rules/" + encodeURI(sServicename) + "/" + encodeURI(sMicroservicename) + "/" + encodeURI(sSchemaname));
- xmlhttp.setRequestHeader("Content-Type", "application/json");
- var that = this;
- xmlhttp.onreadystatechange = function() {
- if (xmlhttp.readyState == 4) {
- if (xmlhttp.status == 200) {
- that.displaySuccess(JSON.parse(xmlhttp.responseText));
- var oStateModel = that.getView().getModel("state");
- oStateModel.setProperty("/locked", false);
- that.displaySuccess( { text: "Saved" } );
- } else {
- that.displayError(JSON.parse(xmlhttp.responseText));
- }
- }
- }
- xmlhttp.send(oModel.getJSON());
- } else {
- this.displayError({ text: "No name of the schema entered" } );
- }
-},
-cancel : function(oEvent) {
- var oSchemaControl = this.getView().byId("schemaselector");
- var sSchemaname = oSchemaControl.getSelectedKey();
- var oStateModel = this.getView().getModel("state");
- oStateModel.setProperty("/locked", false);
- this._loadSchemaData(sSchemaname);
-},
-onSchemaChange : function(oEvent) {
- var sSchemaname = oEvent.getParameter("selectedItem").getKey();
- this._loadSchemaData(sSchemaname);
-},
-_loadSchemaData : function(sSchemaname) {
- var oModel = this.getView().getModel();
- var sServicename = jQuery.sap.getUriParameters().get("servicename");
- var sMicroservicename = jQuery.sap.getUriParameters().get("microservice");
- if (!!sSchemaname) {
- oModel.loadData("../rest/rules/" + encodeURI(sServicename) + "/" + encodeURI(sMicroservicename) + "/" + encodeURI(sSchemaname));
- oModel.refresh();
- this.getView().byId("TreeTable").setBusy(true);
- }
-},
-onChangeRuleType: function(oEvent) {
- var oTargetModel = this.getView().getModel();
- var oRulePath = oEvent.getParameter("selectedItem").getBindingContext().getPath();
- var oTargetRow = oTargetModel.getProperty(oRulePath);
- var sRuleType = oTargetRow.type;
- var oSourceControl = oEvent.getSource();
- var sParentPath = oRulePath.substring(0, oRulePath.lastIndexOf("/rules/"));
- if (sParentPath && sParentPath.length > 0) {
- /*
- * Within a TestSet, there should be always at least on additional item to enter a new rule
- */
- var oParentControl = oTargetModel.getProperty(sParentPath);
- if (oParentControl.type.startsWith("TestSet")) {
- if (oParentControl.rules && oParentControl.rules.length > 0) {
- var oLastEntity = oParentControl.rules[oParentControl.rules.length-1];
- if (oLastEntity.type !== "EmptyRule") {
- oTargetModel.setProperty(sParentPath + "/rules/" + oParentControl.rules.length,
- {fieldname: oParentControl.fieldname, type: "EmptyRule", iffalse: "FAIL", sampleresult: null});
- }
- }
- }
- }
- if (sRuleType === 'EmptyRule') {
- oTargetModel.setProperty(oRulePath + "/rules", null);
- } else if (sRuleType === 'PrimitiveRule') {
- oTargetModel.setProperty(oRulePath + "/rules", null);
- if (!!!oTargetRow.iffalse) {
- oTargetModel.setProperty(oRulePath + "/iffalse", "FAIL");
- }
- oTargetModel.setProperty(oRulePath + "/sampleresult", null);
- } else {
- var oChildRules = oTargetModel.getProperty(oRulePath + "/rules");
- var oTableRow = oSourceControl.getParent();
- var oTreeTable = this.getView().byId("TreeTable");
- var rowindex = oTreeTable.indexOfRow(oTableRow);
- if (!oChildRules || oChildRules.length === 0) {
- oTargetModel.setProperty(oRulePath + "/rules",
- [ { fieldname: oTargetRow.fieldname, type: 'PrimitiveRule', iffalse: "FAIL", sampleresult: null },
- { fieldname: oTargetRow.fieldname, type: 'EmptyRule', iffalse: "FAIL", sampleresult: null } ] );
- }
- oTreeTable.expand(rowindex);
- }
- var oStateModel = this.getView().getModel("state");
- oStateModel.setProperty("/locked", true);
-},
-onValidate: function(oEvent) {
- var oModel = this.getView().getModel();
- var sServicename = jQuery.sap.getUriParameters().get("servicename");
- var sMicroservicename = jQuery.sap.getUriParameters().get("microservice");
- var oSchemaControl = this.getView().byId("schemaselector");
- var sSchemaname = oSchemaControl.getSelectedKey();
- if (!!sSchemaname) {
- var xmlhttp = new XMLHttpRequest();
- xmlhttp.open("POST", "../rest/rules/" + encodeURI(sServicename) + "/" + encodeURI(sMicroservicename) + "/" + encodeURI(sSchemaname) + "/validate");
- xmlhttp.setRequestHeader("Content-Type", "application/json");
- var that = this;
- xmlhttp.onreadystatechange = function() {
- if (xmlhttp.readyState == 4) {
- if (xmlhttp.status == 200) {
- oModel.setJSON(xmlhttp.responseText);
- that.displaySuccess( { text: "Tests completed" } );
- } else {
- that.displayError(JSON.parse(xmlhttp.responseText));
- }
- }
- }
- xmlhttp.send(oModel.getJSON());
- } else {
- that.displayError( { message: "No name of the schema entered" } );
- }
-},
-resulttypeFormatter : function(sResulttype) {
- if (sResulttype === "PASS") {
- return "sap-icon://message-success";
- } else if (sResulttype === "FAIL") {
- return "sap-icon://message-warning";
- } else if (sResulttype === "WARN") {
- return "sap-icon://question-mark";
- } else {
- return "sap-icon://message-error";
- }
-},
-schemaIconFormatter : function(bConfigured) {
- if (bConfigured) {
- return "sap-icon://accept";
- } else {
- return null;
- }
-},
-onFormulaPress: function (oEvent) { // works for both formula controls
- var oButton = oEvent.getSource();
- var oPopOver = oButton.getParent().getItems()[2];
- if (oPopOver.isOpen()) {
- oPopOver.close();
- } else {
- oPopOver.openBy(oButton);
- }
-},
-onRuleFormulaClose: function (oEvent) {
- var oPopOver = oEvent.getSource();
- oPopOver.close();
-}
-
-
diff --git a/src/main/resources/ui5/fragment/xml/RuleFormula.xml b/src/main/resources/ui5/fragment/xml/RuleFormula.xml
deleted file mode 100644
index 9c7e941..0000000
--- a/src/main/resources/ui5/fragment/xml/RuleFormula.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
-
diff --git a/src/main/resources/ui5/fragment/xml/SubstitutionFormula.xml b/src/main/resources/ui5/fragment/xml/SubstitutionFormula.xml
deleted file mode 100644
index 406923a..0000000
--- a/src/main/resources/ui5/fragment/xml/SubstitutionFormula.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
-
diff --git a/src/main/resources/ui5/view/Ruleset.view b/src/main/resources/ui5/view/Ruleset.view
deleted file mode 100644
index a432367..0000000
--- a/src/main/resources/ui5/view/Ruleset.view
+++ /dev/null
@@ -1,117 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/src/test/java/io/rtdi/bigdata/rulesservice/GenerateDataTest.java b/src/test/java/io/rtdi/bigdata/rulesservice/GenerateDataTest.java
new file mode 100644
index 0000000..63e03d5
--- /dev/null
+++ b/src/test/java/io/rtdi/bigdata/rulesservice/GenerateDataTest.java
@@ -0,0 +1,131 @@
+package io.rtdi.bigdata.rulesservice;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient;
+import io.confluent.kafka.schemaregistry.client.SchemaMetadata;
+import io.confluent.kafka.serializers.KafkaAvroSerializer;
+
+class GenerateDataTest {
+
+ private static Map propertiesmap;
+ private static CachedSchemaRegistryClient schemaclient;
+ private static Admin admin;
+
+ @BeforeAll
+ static void setUpBeforeClass() throws Exception {
+ Path settingsdir = Path.of("/apps/rulesservice/settings");
+ Path.of("/apps/rulesservice/definitions");
+ File propertiesfile = new File(settingsdir.toFile(), "kafka.properties");
+ if (!propertiesfile.isFile()) {
+ throw new IOException("The mandatory kafka.properties file does not exist at <" + propertiesfile.toString() + ">");
+ }
+ Properties kafkaproperties = new Properties();
+ try (InputStream is = new FileInputStream(propertiesfile)) {
+ kafkaproperties.load(is);
+ }
+ String schemaurls = kafkaproperties.getProperty("schema.registry.url");
+ if (schemaurls == null) {
+ throw new IOException("The kafka.properties file does not contain a ");
+ }
+ String[] urls = schemaurls.split(",");
+ List schemaRegistryUrls = new ArrayList<>();
+ for (String url : urls) {
+ schemaRegistryUrls.add(url.trim());
+ }
+ propertiesmap = kafkaproperties.entrySet().stream().collect(
+ Collectors.toMap(
+ e -> e.getKey().toString(),
+ e -> e.getValue().toString()
+ )
+ );
+ propertiesmap.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
+ propertiesmap.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class);
+
+ schemaclient = new CachedSchemaRegistryClient(schemaRegistryUrls, 100, propertiesmap);
+ schemaclient.getMode(); // invoke schema registry to validate connection
+ admin = Admin.create(kafkaproperties);
+ admin.describeCluster();
+ }
+
+ @AfterAll
+ static void tearDownAfterClass() throws Exception {
+ if (schemaclient != null) {
+ schemaclient.close();
+ }
+ if (admin != null) {
+ admin.close();
+ }
+ }
+
+ @Test
+ void test() {
+ try (Producer producer = new KafkaProducer(propertiesmap);) {
+ SchemaMetadata schemametadata = schemaclient.getLatestSchemaMetadata("adresses-value");
+ String schemastring = schemametadata.getSchema();
+ Schema schema = new Schema.Parser().parse(schemastring);
+ int id = 1;
+ GenericRecord record = new GenericData.Record(schema);
+ /*
+ {
+ "id": 1,
+ "firstname": "Werner",
+ "lastname": "rtdi",
+ "addresses": [
+ {
+ "addresstype": "HOME",
+ "addressline1": "whatsoever 34",
+ "city": "Montreal",
+ "state": null,
+ "zipcode": "H1A 0A1",
+ "country": "CA"
+ }
+ ]
+ }
+ */
+ record.put("id", id);
+ record.put("firstname", "Werner");
+ record.put("lastname", "rtdi");
+ Schema addressschema = record.getSchema().getField("addresses").schema().getTypes().get(1).getElementType();
+ List addresses = new ArrayList<>();
+ GenericRecord homeaddress = new GenericData.Record(addressschema);
+ homeaddress.put("addresstype", "HOME");
+ homeaddress.put("addressline1", "whatsoever 34");
+ homeaddress.put("city", "Montreal");
+ homeaddress.put("state", null);
+ homeaddress.put("zipcode", "H1A 0A1");
+ homeaddress.put("country", "CA");
+ addresses.add(homeaddress);
+ record.put("addresses", addresses);
+ producer.send(new ProducerRecord("adresses", id, record));
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("Exception");
+ }
+ }
+
+}
diff --git a/src/test/java/io/rtdi/bigdata/rulesservice/RulesServiceIT.java b/src/test/java/io/rtdi/bigdata/rulesservice/RulesServiceIT.java
deleted file mode 100644
index 644254c..0000000
--- a/src/test/java/io/rtdi/bigdata/rulesservice/RulesServiceIT.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package io.rtdi.bigdata.rulesservice;
-
-import static org.junit.Assert.fail;
-
-import java.io.File;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import io.rtdi.bigdata.connector.connectorframework.controller.ConnectorController;
-import io.rtdi.bigdata.connector.connectorframework.controller.ServiceController;
-import io.rtdi.bigdata.connector.pipeline.foundation.PipelineAbstract;
-import io.rtdi.bigdata.connector.pipeline.foundation.SchemaRegistryName;
-import io.rtdi.bigdata.kafka.avro.RuleResult;
-import io.rtdi.bigdata.connectors.pipeline.kafkadirect.KafkaAPIdirect;
-import io.rtdi.bigdata.connectors.pipeline.kafkadirect.KafkaConnectionProperties;
-import io.rtdi.bigdata.rulesservice.rules.ArrayRule;
-import io.rtdi.bigdata.rulesservice.rules.PrimitiveRule;
-import io.rtdi.bigdata.rulesservice.rules.TestSetFirstPass;
-
-public class RulesServiceIT {
- private PipelineAbstract api;
-
- @Before
- public void setUp() throws Exception {
- File dir = new File("./src/test/resources/tmp");
- KafkaConnectionProperties kafkaprops = new KafkaConnectionProperties();
- kafkaprops.read(dir);
- api = new KafkaAPIdirect(kafkaprops);
- api.open();
- }
-
- @After
- public void tearDown() throws Exception {
- api.close();
- }
-
- @Test
- public void testRulesService() {
- try {
- ConnectorController connectorcontroller = new ConnectorController(new RulesServiceFactory(), "./src/test/resources/tmp", null);
- connectorcontroller.setAPI(api);
- RulesServiceProperties properties = new RulesServiceProperties("RulesTest1");
- properties.setSourceTopic("SALES");
- properties.setTargetTopic("SALES_CLEANSED");
- ServiceController servicecontroller = new ServiceController(properties , connectorcontroller);
- connectorcontroller.addChild(properties.getName(), servicecontroller);
-
- RuleStep rulestep = new RuleStep("Step1");
- properties.addMicroService(rulestep);
- SchemaRuleSet ruleset = rulestep.getSchemaRule(SchemaRegistryName.create("schema1"));
-
- ruleset.addRule(new TestSetFirstPass("SoldTo", "Test SoldTo column", new PrimitiveRule("Test1", "SoldTo is not null", "SoldTo != null", RuleResult.FAIL, null)));
- ruleset.addRule(new TestSetFirstPass("BillTo", "Test BillTo column", new PrimitiveRule("Test1", "BillTo is not null", "BillTo != null", RuleResult.FAIL, null)));
- ArrayRule c = ruleset.addNested("SalesItems");
- c.addRule(new TestSetFirstPass("MaterialNumber", "Check material not null", new PrimitiveRule("Test1", "MaterialNumber is not null", "MaterialNumber != null", RuleResult.FAIL, null)));
- c.addRule(new TestSetFirstPass("Quantity", "Check qty not null", new PrimitiveRule("Test1", "QTY is not null", "Quantity != null", RuleResult.FAIL, null)));
-
- File dir = new File("./src/test/resources/tmp/services");
- if (!dir.exists()) {
- dir.mkdir();
- }
- properties.write(dir, null);
-
- RulesServiceProperties properties2 = new RulesServiceProperties("RulesTest1");
- properties2.read(dir);
- System.out.println(properties2.toString());
- // connectorcontroller.startController(false);
-
- // Thread.sleep(240000);
-
- // connectorcontroller.stopController(ControllerExitType.ENDBATCH);
-
- // System.out.println("Got " + data.size() + " records");
- // assertTrue(data.size() <= 10);
- } catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
-}
diff --git a/src/test/java/io/rtdi/bigdata/rulesservice/TransformationTest.java b/src/test/java/io/rtdi/bigdata/rulesservice/TransformationTest.java
new file mode 100644
index 0000000..27672fe
--- /dev/null
+++ b/src/test/java/io/rtdi/bigdata/rulesservice/TransformationTest.java
@@ -0,0 +1,61 @@
+package io.rtdi.bigdata.rulesservice;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import io.rtdi.bigdata.kafka.avro.RuleResult;
+import io.rtdi.bigdata.kafka.avro.datatypes.AvroNVarchar;
+import io.rtdi.bigdata.kafka.avro.recordbuilders.SchemaBuilder;
+import io.rtdi.bigdata.kafka.avro.recordbuilders.ValueSchema;
+import io.rtdi.bigdata.rulesservice.config.RuleFileDefinition;
+import io.rtdi.bigdata.rulesservice.config.RuleStep;
+import io.rtdi.bigdata.rulesservice.jexl.JexlRecord;
+import io.rtdi.bigdata.rulesservice.rules.PrimitiveRule;
+
+class TransformationTest {
+
+ @BeforeAll
+ static void setUpBeforeClass() throws Exception {
+ }
+
+ @AfterAll
+ static void tearDownAfterClass() throws Exception {
+ }
+
+ @Test
+ void test() {
+ try {
+ SchemaBuilder schemabuilder = new ValueSchema("ADDRESS", "Address table");
+ schemabuilder.add("POSTALCODE", AvroNVarchar.getSchema(5), null, false, null);
+ schemabuilder.add("COUNTRY", AvroNVarchar.getSchema(40), null, false, null);
+ schemabuilder.build();
+
+ RuleFileDefinition rg = new RuleFileDefinition(Path.of("Address validation.json"), "CUSTOMER");
+ RuleStep step1 = new RuleStep("01_CLEANSE", schemabuilder.getSchema());
+ rg.addRuleStep(step1);
+ PrimitiveRule rule1 = new PrimitiveRule("POSTALCODE", "Validate Format", "POSTALCODE != null && POSTALCODE.length() == 5", RuleResult.WARN, null, schemabuilder.getSchema().getField("POSTALCODE").schema());
+ PrimitiveRule rule2 = new PrimitiveRule("COUNTRY", "Validate country not empty", "COUNTRY != null", RuleResult.FAIL, null, schemabuilder.getSchema().getField("COUNTRY").schema());
+ step1.addRule(rule1);
+ step1.addRule(rule2);
+
+ JexlRecord record1 = new JexlRecord(schemabuilder.getSchema(), null);
+ record1.put("POSTALCODE", "81959");
+ record1.put("COUNTRY", "GERMANY");
+ rg.setSchema(schemabuilder.getSchema().toString());
+ rg.apply(record1, false);
+ System.out.println(record1);
+
+ rg.save(Path.of("src/test/resources/tmp/rules"), null);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("Exception thrown");
+ }
+ }
+
+}
diff --git a/src/test/resources/tmp/Kafka.json b/src/test/resources/tmp/Kafka.json
deleted file mode 100644
index 692a9f6..0000000
--- a/src/test/resources/tmp/Kafka.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "values": [
- {
- "type": "PropertyString",
- "name": "tenantid",
- "value": "TENANT1"
- },
- {
- "type": "PropertyString",
- "name": "kafka.bootstrapservers",
- "value": "kafka.rtdi.io:9092"
- },
- {
- "type": "PropertyString",
- "name": "kafka.schemaregistry.url",
- "value": "http://kafka.rtdi.io:8081"
- }
- ]
-}
\ No newline at end of file
diff --git a/src/test/resources/tmp/rules/CUSTOMER/inactive/Address validation.json b/src/test/resources/tmp/rules/CUSTOMER/inactive/Address validation.json
new file mode 100644
index 0000000..41f8050
--- /dev/null
+++ b/src/test/resources/tmp/rules/CUSTOMER/inactive/Address validation.json
@@ -0,0 +1,25 @@
+{
+ "name" : "Address validation.json",
+ "inputsubjectname" : "CUSTOMER",
+ "outputsubjectname" : null,
+ "sampletopic" : null,
+ "samplerecord" : null,
+ "schema" : "{\"type\":\"record\",\"name\":\"ADDRESS\",\"doc\":\"Address table\",\"fields\":[{\"name\":\"__change_type\",\"type\":{\"type\":\"string\",\"logicalType\":\"VARCHAR\",\"length\":1},\"doc\":\"Indicates how the row is to be processed: Insert, Update, Delete, upsert/Autocorrect, eXterminate, Truncate\",\"default\":\"UPSERT\",\"__originalname\":\"__change_type\",\"__internal\":true,\"__technical\":true},{\"name\":\"__change_time\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"},\"doc\":\"Timestamp of the transaction. All rows of the transaction have the same value.\",\"default\":0,\"__originalname\":\"__change_time\",\"__internal\":true,\"__technical\":true},{\"name\":\"__source_rowid\",\"type\":[\"null\",{\"type\":\"string\",\"logicalType\":\"VARCHAR\",\"length\":30}],\"doc\":\"Optional unqiue and static pointer to the row, e.g. Oracle rowid\",\"__originalname\":\"__source_rowid\",\"__internal\":true,\"__technical\":true},{\"name\":\"__source_transaction\",\"type\":[\"null\",{\"type\":\"string\",\"logicalType\":\"VARCHAR\",\"length\":30}],\"doc\":\"Optional source transaction information for auditing\",\"__originalname\":\"__source_transaction\",\"__internal\":true,\"__technical\":true},{\"name\":\"__source_system\",\"type\":[\"null\",{\"type\":\"string\",\"logicalType\":\"VARCHAR\",\"length\":30}],\"doc\":\"Optional source system information for auditing\",\"__originalname\":\"__source_system\",\"__internal\":true,\"__technical\":true},{\"name\":\"__extension\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"__extension\",\"doc\":\"Extension point to add custom values to each record\",\"fields\":[{\"name\":\"__path\",\"type\":{\"type\":\"string\",\"logicalType\":\"STRING\"},\"doc\":\"An unique identifier, e.g. \\\"street\\\".\\\"house number component\\\"\",\"__originalname\":\"__path\"},{\"name\":\"__value\",\"type\":[\"null\",{\"type\":\"boolean\",\"logicalType\":\"BOOLEAN\"},{\"type\":\"bytes\",\"logicalType\":\"BYTES\"},{\"type\":\"double\",\"logicalType\":\"DOUBLE\"},{\"type\":\"float\",\"logicalType\":\"FLOAT\"},{\"type\":\"int\",\"logicalType\":\"INT\"},{\"type\":\"long\",\"logicalType\":\"LONG\"},{\"type\":\"string\",\"logicalType\":\"STRING\"}],\"doc\":\"The value of any primitive datatype of Avro\",\"__originalname\":\"__value\"}],\"__originalname\":\"__extension\"}}],\"doc\":\"Add more columns beyond the official logical data model\",\"default\":null,\"__originalname\":\"__extension\",\"__internal\":true},{\"name\":\"__audit\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"__audit\",\"doc\":\"If data is transformed this information is recorded here\",\"fields\":[{\"name\":\"__transformresult\",\"type\":{\"type\":\"string\",\"logicalType\":\"VARCHAR\",\"length\":4},\"doc\":\"Is the record PASS, FAILED or WARN?\",\"__originalname\":\"__transformresult\"},{\"name\":\"__details\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"__audit_details\",\"doc\":\"Details of all transformations\",\"fields\":[{\"name\":\"__transformationname\",\"type\":{\"type\":\"string\",\"logicalType\":\"NVARCHAR\",\"length\":1024},\"doc\":\"A name identifying the applied transformation\",\"__originalname\":\"__transformationname\"},{\"name\":\"__transformresult\",\"type\":{\"type\":\"string\",\"logicalType\":\"VARCHAR\",\"length\":4},\"doc\":\"Is the record PASS, FAIL or WARN?\",\"__originalname\":\"__transformresult\"},{\"name\":\"__transformresult_text\",\"type\":[\"null\",{\"type\":\"string\",\"logicalType\":\"NVARCHAR\",\"length\":1024}],\"doc\":\"Transforms can optionally describe what they did\",\"__originalname\":\"__transformresult_text\"},{\"name\":\"__transformresult_quality\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"BYTE\"}],\"doc\":\"Transforms can optionally return a percent value from 0 (FAIL) to 100 (PASS)\",\"__originalname\":\"__transformresult_quality\"}],\"__originalname\":\"__audit_details\"}}],\"doc\":\"Details of all transformations\",\"default\":null,\"__originalname\":\"__details\"}],\"__originalname\":\"__audit\"}],\"doc\":\"If data is transformed this information is recorded here\",\"default\":null,\"__originalname\":\"__audit\",\"__internal\":true},{\"name\":\"POSTALCODE\",\"type\":{\"type\":\"string\",\"logicalType\":\"NVARCHAR\",\"length\":5},\"__originalname\":\"POSTALCODE\"},{\"name\":\"COUNTRY\",\"type\":{\"type\":\"string\",\"logicalType\":\"NVARCHAR\",\"length\":40},\"__originalname\":\"COUNTRY\"}],\"__originalname\":\"ADDRESS\"}",
+ "rulesteps" : [ {
+ "type" : "RuleStep",
+ "fieldname" : "01_CLEANSE",
+ "rules" : [ {
+ "type" : "PrimitiveRule",
+ "fieldname" : "POSTALCODE",
+ "rulename" : "Validate Format",
+ "condition" : "POSTALCODE != null && POSTALCODE.length() == 5",
+ "iffalse" : "WARN"
+ }, {
+ "type" : "PrimitiveRule",
+ "fieldname" : "COUNTRY",
+ "rulename" : "Validate country not empty",
+ "condition" : "COUNTRY != null",
+ "iffalse" : "FAIL"
+ } ]
+ } ]
+}
\ No newline at end of file
diff --git a/src/test/resources/tmp/rules/inactive/sample_order-value.json b/src/test/resources/tmp/rules/inactive/sample_order-value.json
new file mode 100644
index 0000000..5f15dcd
--- /dev/null
+++ b/src/test/resources/tmp/rules/inactive/sample_order-value.json
@@ -0,0 +1,107 @@
+{
+ "rulesteps": [
+ {
+ "type": "RuleStep",
+ "fieldname": "next step",
+ "rules": [
+ {
+ "type": "EmptyRule",
+ "fieldname": "ordertime",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "EmptyRule",
+ "fieldname": "orderid",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "EmptyRule",
+ "fieldname": "itemid",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "EmptyRule",
+ "fieldname": "orderunits",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "RecordRule",
+ "fieldname": "address",
+ "rules": [
+ {
+ "type": "EmptyRule",
+ "fieldname": "city",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "EmptyRule",
+ "fieldname": "state",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "EmptyRule",
+ "fieldname": "zipcode",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "GenericRules",
+ "fieldname": "(more)",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "rulename": null,
+ "sampleresult": null,
+ "samplevalue": null
+ }
+ ],
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ },
+ {
+ "type": "GenericRules",
+ "fieldname": "(more)",
+ "rules": null,
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "rulename": null,
+ "sampleresult": null,
+ "samplevalue": null
+ }
+ ],
+ "fielddatatype": null,
+ "sampleresulterror": null,
+ "sampleresult": null,
+ "samplevalue": null
+ }
+ ]
+}
\ No newline at end of file
diff --git a/src/test/resources/tmp/services/RulesTest1.json b/src/test/resources/tmp/services/RulesTest1.json
deleted file mode 100644
index dc063e1..0000000
--- a/src/test/resources/tmp/services/RulesTest1.json
+++ /dev/null
@@ -1 +0,0 @@
-{"values":[{"type":"PropertyText","name":"text"},{"type":"PropertyString","name":"service.source","value":"SALES"},{"type":"PropertyString","name":"service.target","value":"SALES_CLEANSED"}]}
\ No newline at end of file
diff --git a/src/test/resources/tmp/services/Step1/SalesOrder/SalesOrder.json b/src/test/resources/tmp/services/Step1/SalesOrder/SalesOrder.json
deleted file mode 100644
index 7d6f875..0000000
--- a/src/test/resources/tmp/services/Step1/SalesOrder/SalesOrder.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
- "type": "SchemaRuleSet",
- "fieldname": null,
- "rules": [
- {
- "type": "TestSetFirstPass",
- "fieldname": "SoldTo",
- "rules": [
- {
- "type": "PrimitiveRule",
- "fieldname": "SoldTo",
- "rules": null,
- "rulename": "Test1",
- "condition": "SoldTo != null",
- "iffalse": "FAIL",
- "columnreplacementvalue": null
- }
- ],
- "rulename": "Test SoldTo column"
- },
- {
- "type": "TestSetFirstPass",
- "fieldname": "BillTo",
- "rules": [
- {
- "type": "PrimitiveRule",
- "fieldname": "BillTo",
- "rules": null,
- "rulename": "Test1",
- "condition": "BillTo != null",
- "iffalse": "FAIL",
- "columnreplacementvalue": null
- }
- ],
- "rulename": "Test BillTo column"
- },
- {
- "type": "ArrayRule",
- "fieldname": "SalesItems",
- "rules": [
- {
- "type": "TestSetFirstPass",
- "fieldname": "MaterialNumber",
- "rules": [
- {
- "type": "PrimitiveRule",
- "fieldname": "MaterialNumber",
- "rules": null,
- "rulename": "Test1",
- "condition": "MaterialNumber != null",
- "iffalse": "FAIL",
- "columnreplacementvalue": null
- }
- ],
- "rulename": "Check material not null"
- },
- {
- "type": "TestSetFirstPass",
- "fieldname": "Quantity",
- "rules": [
- {
- "type": "PrimitiveRule",
- "fieldname": "Quantity",
- "rules": null,
- "rulename": "Test1",
- "condition": "Quantity != null",
- "iffalse": "FAIL",
- "columnreplacementvalue": null
- }
- ],
- "rulename": "Check qty not null"
- }
- ]
- }
- ],
- "schemaname": "SalesOrder"
-}
\ No newline at end of file
+
+
+```
+basic.auth.credentials.source=USER_INFO
+schema.registry.basic.auth.user.info=XXXXXXXXXXXXXXXX:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+schema.registry.url=https://XXXXXXXXXX.eu-central-1.aws.confluent.cloud
+
+bootstrap.servers=XXXXXXXXX.eu-central-1.aws.confluent.cloud:9092
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule \
+required username="XXXXXXXXXXXXXXXX" password="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
+security.protocol=SASL_SSL
+sasl.mechanism=PLAIN
+```
+
+
+### Step 2: Define Rules
+
+A rule file applies logic to messages of a given schema/subject. It consumes the input message, applies the rules and creates an output message with a derived schema - a schema that contains the rule results as additional field. What the input and output topic is, will be configured in the next step.
+
+Rule files are saved by default in the directory `/apps/rulesservice/definitions/
+
+
+### Step 3: Topics
+
+An input topic can contain messages of different subjects, hence the dialog asks what input topic should cater which rule files (remember, a rule file work on a specific subject) and what the output topic should be.
+Scaling is achieved by increasing the number of KStream instances used for this topic or by spinning up more containers with the same settings.
+
+The screen also allows to copy the rule files being used into the active folder to simplify activating each from the rule file dialog.
-The first step is to connect the application to a Kafka server, in this example Confluent Cloud.
+
-
+### Result
+If the input schema has an `_audit` field, it is assumed the schema contains the structure for the rule results already. This would be a preferred case, because input schema = output schema.
+In all other cases the input schema's latest version is read from the schema registry and the additional `_audit` structure is being created. This will be the output schema.
-### Define Services
+The reason for using the latest is because of schema evolution scenarios. It might happen that schema id 2 has an additional field `NAME` compared to schema id 1. So the subject evolved from schema 1 to 2. The KStream does receive a message with schema 2 first, adds the `_audit` field and it is saved in the schema registry. The next message has an input schema 1 and if that would get registered as output schema next, it would fail due to the missing `NAME` field. Hence both must be outputted with the latest schema version always. This also explains why adding the `_audit` on the original input schema already is preferred.
+The overall rule result is stored (did it pass all tests?), a list of all rules executed and their individual results.
+Querying this data allows detailed reporting which records were processed by what rule and the results.
-Each Service is a Kafka KStream, a distributed process listening on a topic and validating the data. Hence the first setting of each service are the input and output topic names to use.
+The exact Avro schema field definition can be found [here](docs/audit-schema.md)
-Within one rule service for each schema multiple steps can be performed, a microservice transformation so to speak. These transformation steps happen within the service. For example the first microservice step might check missing data, the next standardize on the different spellings. The result of this step is then put into a third step, validating if the STATUS is valid and consistent with other data.
+
-
-Once the structure is saved, by clicking on the individual steps the rules themselves are defined.
+### Sample files
-Because a topic can contain data for multiple schemas, the rules are defined for each schema individually. If the message schema has no rules defined, it is passed through unchanged.
+To create sample files, one or multiple topics are selected in the screen and executed. All topics and all their partitions are read to find the most recent 100 messages.
+The found messages are streamed in chunks into the screen and can be saved, either individually or by selecting some/all in the table and clicking the `Save selected` button.
-
+The files are stored in the directory `/apps/rulesservice/definitions/
-An example would be a rule on the OrderStatus column, which must contain the values C (..Completed) or N (..New). One way would be a a single rule on the column with a OR condition. The other option would be five rules testing for a single status value only and the test type is "Test until first Passes".
+## Rules
-
+### Rule Formula
-### Data content
+The Single Test rule type applied to a field is used to specify conditions and optionally to modify a value.
+The Condition is a formula like `firstname != null` and specifies what the expected rule is - here it is expected that the firstname field has a value. This formula must return true or false.
-The result of all rule transformations is stored in the audit array of the record. Querying this data allows detailed reporting which records were processed by what rule and the results.
-If the input schema does not have such __audit array, a new version of the schema will be created automatically.
-
+ and if false, the `if test failed...` setting tells the severity. In most cases a false means the test `=Fail`, but it can be set to `=Warn` or even `=Pass`.
-### Rule types
+Formulas can get more complicated obviously, e.g. `lastname != null && lastname.length() > 10`.
-Rules can be applied on fields only and this field acts as the trigger and is used when providing a substitution value.
-For example, the SalesOrderHeader has a field SoldTo at root level, hence all rules will be executed once per message.
+Formulas can also access other fields as well, e.g. for the field `lastname` the condition is `lastname != null || firstname != null` meaning, either lastname or firstname must be set.
-A rule on a field of the SalesItems array will be triggered for each item. Thus the field a rule belongs to controls primarily that.
+To access fields of a deeper level, e.g. when lastname is set, the first record in the addresses array must have addresstype HOME, many cases must be considered. What if the addresses array is empty? What if the addresses array is an empty array? What if the addresstype field is null for some records? This would lead to a very long condition but there are shortcuts: `lastname != null && 'HOME' == addresses?[0]?addresstype`. The `?` tells that it is okay if the field is null and it should return null then.
-For each field either a single rule "Test a column" can be defined or a Test-Set with multiple rules.
+Formulas at a deeper level, e.g. a formula for `addresstype`, can access fields from the higher level via `parent`. Example: `('HOME' == addresstype && parent.parent.lastname != null) || 'HOME' != addresstype` tells that for addresstype HOME the lastname must be set as well, for all other addresstype values there is no such requirement.
-- Test for all conditions: Every single rule is applied, thus acting as a AND combination of rules. This is convenient for cases where a field needs to be checked for different conditions, e.g. the OrderDate has to be within three months or raise a warning and the OrderDate cannot be later than the expected delivery date. The rule set will return the lowest individual rule result. If all passed the result is pass. If at least one said warning, the result is a warning. And if one is failed, the result is failed.
-- Test until one failed: Each rule will be tested and stopped once the first rule violation is found. It is a AND combination of the rules as well but while in above case all rules are tested, here all other rules are not added to the test result. The rule set will return failed when at least one is failed and pass only if all rules passed.
-- Test until one passes: This is an OR condition. If condition1 is not met, maybe condition2 is. As soon as the first positive test is found, no further processing is needed, a valid value has been found. The rule set will return failed only if all tests failed. One positive test makes the entire rule set positive.
+Note: In this example the customer record has an addresses array of address record. When within an address record, the parent is the array and its parent is the customer record with the lastname field.
-For each individual rule the rule result can be specified if the condition is met. This way the rule can specify the severity, e.g. a SoldTo == null shall be treated as failed, SoldTo.length() < 5 as well but SoldTo.length() < 10 shall be a warning only.
-Other tests might not impact the rule result at all, they return passed always. For those the audit array will show that the rule has been tested but the data is of no harm. For example in a gender column the test could be if the value is either M or F and in all other cases a substitution value of X is used. As the gender does return the values M,F or X only, it is to be considered valid.
+These are just very basic examples, more below.
+
+If a condition returns false, maybe the correct value can be derived and then the rule as such has not been violated. The optional formula entered in `..change value to...` is executed only if the condition returns false and it overwrites the field value. Example: The rule for the field `addresstype` is `addresstype.upper() != addresstype` and the change-value formula is `addresstype.upper()`. This does change all values to upper case. In such a case the rule is considered to have been passed instead of failed, and that is accomplished via the `if test failed...` setting `=Pass`.
+Each test and its rule result is available in the audit structure and hence we can see that this test was executed and passed.
+
+
+### Rule Sets
+
+In above examples there was the test `('HOME' == addresstype && parent.parent.lastname != null) || 'HOME' != addresstype`. A more realistic formula would say: if HOME then the lastname must be set, if COMPANY the companyname field must be set and other allowed values are SHIPPING and BILLING. This would get quite a long formula.
+
+To simplify that, the Test-Set rule allows to specify multiple individual rules
+
+ - Test for all conditions (`Test all`): Every single rule is applied, thus acting as a AND combination of rules. This is convenient for cases where a field needs to be checked for different conditions, e.g. the OrderDate has to be within three months or raise a warning and the OrderDate cannot be later than the expected delivery date. The rule set will return the lowest individual rule result. If all passed the result is pass. If at least one said warning, the result is a warning. And if one is failed, the result is failed.
+ - Test until one failed (`Until failed`): Each rule will be tested and stopped once the first rule violation is found. It is a AND combination of the rules as well but while in above case all rules are tested, here all other rules are not added to the test result. The rule set will return failed when at least one is failed and pass only if all rules passed.
+ - Test until one passes (`Until passes`): This is an OR condition. If condition1 is not met, maybe condition2 is. As soon as the first positive test is found, no further processing is needed, a valid value has been found. The rule set will return failed only if all tests failed. One positive test makes the entire rule set positive and the remaining tests are not executed.
+
+For the addresstype example, the `Until-passes` test set is best suited with the individual rules
+
+ - `'HOME' == addresstype && parent.parent.lastname != null`
+ - `'COMPANY' == addresstype && parent.parent.companyname != null`
+ - `'SHIPPING' == addresstype`
+ - `'BILLING' == addresstype`
+
+If the addresstype is `SHIPPING`, then the first test returns false, hence the second is executed also returning false and the third condition returns true --> no more tests being made and the test-set is Pass.
+
+If the addresstype is `ABCD`, none of the conditions will return true --> the test-set is Fail.
+
+This simplifies creating difficult rules, especially in combination with the `if test failed...` setting, e.g. a `SoldTo == null` shall be treated as failed, `SoldTo.length() < 5` as well but `SoldTo.length() < 10` shall be a warning only.
+
+
+### Generic rules
+
+Each record also has a `(more)` node to enter rules that do not belong to a single field. Such generic rules cannot have a change-value formula as they are not bound to a field.
+
+
+### Rule Steps
+
+Another typical scenario is to standardize the values first, e.g. gender should be `M`, `F`, `X`, `?` only and then create rules based on the standardized values. In other words, rules build on each other. To enable that, the rule file consists of multiple tabs - the rule steps - and each tab is executed one after the other.
-A more extreme case would be to assign a column with a fixed value. In that case the condition is the forumla "true", the rules result will be pass and the substitution formula the constant to assign.
### Rule syntax
-When entering formulas, the columns of the current level can be used directly. For example on the SalesItems level a formula might be "MaterialNumber != null" if the SalesItems structure consists of records that have a MaterialNumber column. The keyword "parent" refers to the parent element, in this example the array of SalesItems.
-Thus a formula might be "LineNumber <= parent.length()" if the assumption is, 3 line items are numbered as items 1,2 and 3.
+For more examples [see](docs/rule-syntax.md)
+
+
+### FAQs
+
+ * Can a new output column be created via a formula? No, the output schema is always derived from the input schema, for two reasons. First, if adding fields would be possible, it might collide when the input subject is evolved to a new version. The other reason is performance. It would require to create a new output message from scratch, copying the majority of the data even if nothing has changed. That would be too expensive. So the only option is to add the column to the input schema first.
-Note: The library used here is [Apache JEXL](https://commons.apache.org/proper/commons-jexl/reference/syntax.html).
## Licensing
-This application is provided as dual license. For all users with less than 100'000 messages created per month, the application can be used free of charge and the code falls under a Gnu Public License. Users with more than 100'000 messages are asked to get a commercial license to support further development of this solution. The commercial license is on a monthly pay-per-use basis.
+This application is provided as dual license. For all users with less than 100'000 messages processed per month, the application can be used free of charge and the code falls under a Gnu Public License. Users with more than 100'000 messages per month are asked to get a  to support further development of this solution. The commercial license is on a monthly pay-per-use basis.
## Data protection and privacy
Every ten minutes the application does send the message statistics via a http call to a central server where the data is stored for information along with the public IP address (usually the IP address of the router). It is just a count which service was invoked how often, no information about endpoints, users, data or URL parameters. This information is collected to get an idea about the adoption.
-To disable that, set the environment variable HANAAPPCONTAINERSTATISTICS=FALSE.
\ No newline at end of file
+To disable that, set the environment variable STATISTICS=FALSE.
\ No newline at end of file
diff --git a/WebContent/Controller.controller.js b/WebContent/Controller.controller.js
new file mode 100644
index 0000000..99dd016
--- /dev/null
+++ b/WebContent/Controller.controller.js
@@ -0,0 +1,14 @@
+sap.ui.define([ "sap/ui/core/mvc/Controller"], function(Controller) {
+ "use strict";
+
+ return Controller.extend("io.rtdi.bigdata.rulesservice.Controller", {
+
+ onInit : function() {
+ var oModel = new sap.ui.model.json.JSONModel();
+ oModel.loadData("./rest/config/service");
+ this.getView().setModel(oModel);
+ }
+
+ });
+
+});
diff --git a/WebContent/View.view.xml b/WebContent/View.view.xml
new file mode 100644
index 0000000..e04fce2
--- /dev/null
+++ b/WebContent/View.view.xml
@@ -0,0 +1,116 @@
+