Skip to content

Commit

Permalink
feat(support-querying): Add support to querying schemas (#51)
Browse files Browse the repository at this point in the history
* feat(container-is-queryable): send in the api if a container cluster can search or index

* feat(support-querying): Add support to querying schemas

---------

Co-authored-by: Broknloop <broknloop@gmail.com>
  • Loading branch information
broknloop and joaomcclain authored Nov 20, 2023
1 parent 8f1d10a commit c713a62
Show file tree
Hide file tree
Showing 44 changed files with 1,333 additions and 525 deletions.
8 changes: 7 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,22 @@
},
"homepage": "https://github.com/vispana/vispana",
"dependencies": {
"ace-builds": "^1.31.1",
"ace-linters": "^0.13.3",
"daisyui": "^3.9.3",
"install": "^0.13.0",
"npm": "^10.2.1",
"react": "^18.2.0",
"react-ace": "^10.1.0",
"react-data-table-component": "^7.5.4",
"react-dom": "^18.2.0",
"react-router-dom": "^6.17.0",
"react-syntax-highlighter": "^15.5.0",
"react-tabs": "^6.0.2",
"react-tooltip": "^5.21.6",
"rest": "^1.3.1"
"rest": "^1.3.1",
"styled-components": "^5.3.3",
"uuid": "^9.0.1"
},
"scripts": {
"watch": "webpack --watch"
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,11 @@
<artifactId>jsonschema2pojo-maven-plugin</artifactId>
<version>1.2.1</version>
<configuration>
<sourceDirectory>${basedir}/src/main/resources/json/schema</sourceDirectory>
<sourceDirectory>${basedir}/src/main/resources/json/schema/</sourceDirectory>
<targetPackage>com.vispana.client.vespa.model</targetPackage>
<useLongIntegers>true</useLongIntegers>
<includes>
<include>*.schema.json</include>
<include>**/*.schema.json</include>
</includes>
</configuration>
<executions>
Expand Down
24 changes: 19 additions & 5 deletions src/main/java/com/vispana/api/MainController.java
Original file line number Diff line number Diff line change
@@ -1,28 +1,42 @@
package com.vispana.api;

import com.vispana.api.model.VispanaRoot;
import com.vispana.client.vespa.VespaClient;
import com.vispana.vespa.query.VespaQueryClient;
import com.vispana.vespa.state.VespaStateClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;

@RestController
public class MainController {

private final VespaClient vespaClient;
private final VespaStateClient vespaStateClient;
private final VespaQueryClient vespaQueryClient;

@Autowired
public MainController(VespaClient vespaClient) {
this.vespaClient = vespaClient;
public MainController(VespaStateClient vespaStateClient, VespaQueryClient vespaQueryClient) {
this.vespaStateClient = vespaStateClient;
this.vespaQueryClient = vespaQueryClient;
}

@GetMapping(
value = "/api/overview",
produces = {"application/json"})
@ResponseBody
public VispanaRoot root(@RequestParam(name = "config_host") String configHost) {
return vespaClient.vespaState(configHost);
return vespaStateClient.vespaState(configHost);
}

@PostMapping(
value = "/api/query",
produces = {"application/json"})
@ResponseBody
public String query(
@RequestParam(name = "container_host") String containerHost, @RequestBody String query) {
return vespaQueryClient.query(containerHost, query);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@

import java.util.List;

public record ContainerCluster(String name, List<ContainerNode> nodes) {}
public record ContainerCluster(
String name, List<ContainerNode> nodes, boolean canIndex, boolean canSearch) {}
7 changes: 0 additions & 7 deletions src/main/java/com/vispana/client/vespa/VespaClient.java

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
public class JSONSchemaGenerator {
public static void main(String[] args) throws IOException {
var jsonMapper = new JSONSchemaGenerator();
var application = jsonMapper.getResource("application.json");
var application = jsonMapper.getResource("container-components.json");
jsonMapper.toSchema(application);
}

Expand Down
44 changes: 44 additions & 0 deletions src/main/java/com/vispana/vespa/query/VespaQueryClient.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package com.vispana.vespa.query;

import static org.springframework.http.MediaType.APPLICATION_JSON;

import java.nio.channels.UnresolvedAddressException;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.springframework.stereotype.Component;
import org.springframework.web.client.ResourceAccessException;
import org.springframework.web.client.RestClient;

@Component
public class VespaQueryClient {

private static final RestClient restClient = RestClient.create();

public String query(String vespaContainerHost, String query) {
try {
return restClient
.post()
.uri(vespaContainerHost + "/search/")
.contentType(APPLICATION_JSON)
.body(query)
.retrieve()
.body(String.class);
} catch (ResourceAccessException e) {
var exception = ExceptionUtils.getRootCause(e);
if (exception instanceof UnresolvedAddressException) {
var message =
"Failed to reach to Vespa container for host: '"
+ vespaContainerHost
+ "'.\n"
+ "Vespa clusters may have internal access to this address, please check if "
+ "the host is reachable from Vispana. If not, you may configure a routing "
+ "address in Vispana's configuration pointing to a reachable address (e.g"
+ "., a load balancer or a k8s service).";
throw new RuntimeException(message);
} else {
throw e;
}
} catch (Exception e) {
throw new RuntimeException("Error querying Vespa." + e.getMessage(), e);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,19 +1,18 @@
package com.vispana.client.vespa;
package com.vispana.vespa.state;

import com.vispana.api.model.VispanaRoot;
import com.vispana.client.vespa.assemblers.AppPackageAssembler;
import com.vispana.client.vespa.assemblers.ConfigNodesAssembler;
import com.vispana.client.vespa.assemblers.ContainerAssembler;
import com.vispana.client.vespa.assemblers.ContentAssembler;
import com.vispana.client.vespa.helpers.ApplicationUrlFetcher;
import com.vispana.client.vespa.helpers.MetricsFetcher;
import com.vispana.client.vespa.helpers.VespaVersionFetcher;
import com.vispana.vespa.state.assemblers.AppPackageAssembler;
import com.vispana.vespa.state.assemblers.ConfigNodesAssembler;
import com.vispana.vespa.state.assemblers.ContainerAssembler;
import com.vispana.vespa.state.assemblers.ContentAssembler;
import com.vispana.vespa.state.helpers.ApplicationUrlFetcher;
import com.vispana.vespa.state.helpers.MetricsFetcher;
import com.vispana.vespa.state.helpers.VespaVersionFetcher;
import java.util.concurrent.StructuredTaskScope;
import org.springframework.stereotype.Component;

@Component
public class DefaultVespaClient implements VespaClient {
@Override
public class VespaStateClient {
public VispanaRoot vespaState(String configHost) {

try (var scope = new StructuredTaskScope.ShutdownOnFailure()) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.vispana.vespa.state.assemblers;

import static com.vispana.vespa.state.helpers.Request.requestGet;
import static com.vispana.vespa.state.helpers.Request.requestGetWithDefaultValue;

import com.vispana.api.model.apppackage.ApplicationPackage;
import com.vispana.client.vespa.model.ApplicationSchema;

public class AppPackageAssembler {

public static ApplicationPackage assemble(String appUrl) {
var appSchema = requestGet(appUrl, ApplicationSchema.class);
var hostContent = requestGetWithDefaultValue(appUrl + "/content/hosts.xml", String.class, "");
var servicesContent = requestGet(appUrl + "/content/services.xml", String.class);

return new ApplicationPackage(
appSchema.getGeneration().toString(), servicesContent, hostContent);
}
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package com.vispana.client.vespa.assemblers;
package com.vispana.vespa.state.assemblers;

import static com.vispana.client.vespa.helpers.ProcessStatus.processStatus;
import static com.vispana.client.vespa.helpers.Request.request;
import static com.vispana.client.vespa.helpers.SystemMetrics.systemMetrics;
import static com.vispana.vespa.state.helpers.ProcessStatus.processStatus;
import static com.vispana.vespa.state.helpers.Request.requestGet;
import static com.vispana.vespa.state.helpers.SystemMetrics.systemMetrics;

import com.vispana.api.model.Host;
import com.vispana.api.model.config.ConfigCluster;
Expand All @@ -19,17 +19,24 @@ public static ConfigNodes assemble(String configHost, Map<String, MetricsNode> v
var clusterControllerUrl =
configHost + "/config/v1/cloud.config.cluster-info/admin/cluster-controllers";

var clusterControllers = request(clusterControllerUrl, ClusterControllersSchema.class);
var clusterControllers = requestGet(clusterControllerUrl, ClusterControllersSchema.class);
var configNodes =
clusterControllers.getServices().stream()
.map(
service -> {
var hostname = service.getHostname();
var queryPort =
service.getPorts().stream()
.filter(port -> port.getTags().contains("query"))
.map(port -> port.getNumber())
.findFirst()
.orElse(-1L);

var processStatus = processStatus(hostname, vespaMetrics);
var systemMetrics = systemMetrics(vespaMetrics.get(hostname));
return new ConfigNode(
service.getIndex().toString(),
new Host(hostname, -1),
new Host(hostname, queryPort.intValue()),
processStatus,
systemMetrics);
})
Expand Down
Loading

0 comments on commit c713a62

Please sign in to comment.