diff --git a/cli/src/main/java/dev/enola/cli/CommandWithEntityID.java b/cli/src/main/java/dev/enola/cli/CommandWithEntityID.java index 00f57fdb3..fccda459f 100644 --- a/cli/src/main/java/dev/enola/cli/CommandWithEntityID.java +++ b/cli/src/main/java/dev/enola/cli/CommandWithEntityID.java @@ -17,12 +17,11 @@ */ package dev.enola.cli; -import com.google.protobuf.TypeRegistry; - import dev.enola.common.io.resource.WriterResource; import dev.enola.common.protobuf.ProtoIO; import dev.enola.core.IDs; import dev.enola.core.meta.EntityKindRepository; +import dev.enola.core.meta.TypeRegistryWrapper; import dev.enola.core.meta.proto.EntityKind; import dev.enola.core.proto.EnolaServiceGrpc.EnolaServiceBlockingStub; import dev.enola.core.proto.Entity; @@ -48,12 +47,12 @@ public abstract class CommandWithEntityID extends CommandWithModel { String idString; private WriterResource resource; - private TypeRegistry typeRegistry; + private TypeRegistryWrapper typeRegistryWrapper; @Override protected final void run(EntityKindRepository ekr, EnolaServiceBlockingStub service) throws Exception { - typeRegistry = esp.getTypeRegistry(); + typeRegistryWrapper = esp.getTypeRegistryWrapper(); ID id = IDs.parse(idString); // TODO replace with ITypeConverter // TODO Validate id; here it must have ns+name+path! @@ -69,6 +68,6 @@ protected abstract void run( throws Exception; protected void write(Entity entity) throws IOException { - new ProtoIO(typeRegistry).write(entity, resource); + new ProtoIO(typeRegistryWrapper.get()).write(entity, resource); } } diff --git a/common/protobuf/src/main/java/dev/enola/common/protobuf/ProtoIO.java b/common/protobuf/src/main/java/dev/enola/common/protobuf/ProtoIO.java index 6cd74304c..8d7c38d3f 100644 --- a/common/protobuf/src/main/java/dev/enola/common/protobuf/ProtoIO.java +++ b/common/protobuf/src/main/java/dev/enola/common/protobuf/ProtoIO.java @@ -131,7 +131,6 @@ public B read(ReadableResource resource, B builder) builder.mergeFrom(is, extensionRegistry); } } else { - // TODO Use resource.mediaType().charset().or(UTF_8) try (Reader reader = resource.charSource(UTF_8).openBufferedStream()) { if (normalizedNoParamsEquals(mediaType, PROTOBUF_TEXTPROTO_UTF_8)) { textFormatParser.merge(reader, extensionRegistry, builder); diff --git a/connectors/demo/src/test/java/dev/enola/demo/ServerTest.java b/connectors/demo/src/test/java/dev/enola/demo/ServerTest.java index 0bc311bdd..c2408e578 100644 --- a/connectors/demo/src/test/java/dev/enola/demo/ServerTest.java +++ b/connectors/demo/src/test/java/dev/enola/demo/ServerTest.java @@ -21,8 +21,6 @@ import static java.util.concurrent.TimeUnit.SECONDS; -import com.google.protobuf.TypeRegistry; - import dev.enola.common.io.resource.ClasspathResource; import dev.enola.common.io.resource.MemoryResource; import dev.enola.common.io.resource.ReplacingResource; @@ -36,6 +34,7 @@ import dev.enola.core.connector.proto.ConnectorServiceGrpc; import dev.enola.core.connector.proto.ConnectorServiceListRequest; import dev.enola.core.meta.EntityKindRepository; +import dev.enola.core.meta.TypeRegistryWrapper; import dev.enola.core.proto.Entity; import dev.enola.core.proto.GetEntityRequest; import dev.enola.core.proto.ID; @@ -54,7 +53,7 @@ public class ServerTest { private EntityKindRepository ekr; private EnolaService enola; - private TypeRegistry typeRegistry; + private TypeRegistryWrapper typeRegistryWrapper; @Test public void bothConnectorDirectlyAndViaServer() @@ -110,7 +109,7 @@ private void createEnolaService(int port) var esp = new EnolaServiceProvider(); enola = esp.get(ekr); - typeRegistry = esp.getTypeRegistry(); + typeRegistryWrapper = esp.getTypeRegistryWrapper(); } private void checkEnolaGet(EnolaService enola) throws EnolaException, IOException { @@ -125,7 +124,7 @@ private void checkEnolaGet(EnolaService enola) throws EnolaException, IOExceptio assertThat(something.getText()).isEqualTo("hello, world"); assertThat(something.getNumber()).isEqualTo(123); - var io = new ProtoIO(typeRegistry); + var io = new ProtoIO(typeRegistryWrapper.get()); var resource = new MemoryResource(ProtobufMediaTypes.PROTOBUF_YAML_UTF_8); var entityKind = ekr.get(ID.newBuilder().setNs("demo").setEntity("foo").build()); io.write(entity, resource); diff --git a/core/impl/BUILD b/core/impl/BUILD index 93afda2d3..95616be9b 100644 --- a/core/impl/BUILD +++ b/core/impl/BUILD @@ -69,6 +69,7 @@ java_library( "//core/lib:core_java_proto", "//core/lib:lib_java", "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", "@maven//:com_google_truth_truth", "@maven//:io_grpc_grpc_api", "@maven//:junit_junit", diff --git a/core/impl/src/main/java/dev/enola/core/EnolaServiceProvider.java b/core/impl/src/main/java/dev/enola/core/EnolaServiceProvider.java index 421826c98..e7aff6836 100644 --- a/core/impl/src/main/java/dev/enola/core/EnolaServiceProvider.java +++ b/core/impl/src/main/java/dev/enola/core/EnolaServiceProvider.java @@ -18,23 +18,24 @@ package dev.enola.core; import com.google.common.collect.ImmutableList; -import com.google.protobuf.TypeRegistry; import dev.enola.common.protobuf.ValidationException; import dev.enola.core.aspects.*; import dev.enola.core.meta.EntityAspectWithRepository; import dev.enola.core.meta.EntityKindRepository; +import dev.enola.core.meta.SchemaAspect; +import dev.enola.core.meta.TypeRegistryWrapper; import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; public class EnolaServiceProvider { - private TypeRegistry typeRegistry; + private TypeRegistryWrapper typeRegistry; // TODO rename to getService public EnolaService get(EntityKindRepository ekr) throws ValidationException, EnolaException { - var trb = TypeRegistry.newBuilder(); + var trb = TypeRegistryWrapper.newBuilder(); var sr = new EnolaServiceRegistry(); for (var ek : ekr.list()) { var aspectsBuilder = ImmutableList.builder(); @@ -59,6 +60,9 @@ public EnolaService get(EntityKindRepository ekr) throws ValidationException, En ((EntityAspectWithRepository) connector) .setEntityKindRepository(ekr); } + if (connector instanceof SchemaAspect) { + ((SchemaAspect) connector).setESP(this); + } aspectsBuilder.add(connector); break; @@ -101,23 +105,18 @@ public EnolaService get(EntityKindRepository ekr) throws ValidationException, En var s = new EntityAspectService(ek, aspects); sr.register(ek.getId(), s); - populateTypeRegistry(trb, aspects); + for (var aspect : aspects) { + trb.add(aspect.getDescriptors()); + } } this.typeRegistry = trb.build(); return sr; } - public TypeRegistry getTypeRegistry() { + public TypeRegistryWrapper getTypeRegistryWrapper() { if (typeRegistry == null) { throw new IllegalStateException("getTypeRegistry() must be called after get()"); } return typeRegistry; } - - private void populateTypeRegistry(TypeRegistry.Builder trb, ImmutableList aspects) - throws EnolaException { - for (var aspect : aspects) { - trb.add(aspect.getDescriptors()); - } - } } diff --git a/core/impl/src/main/java/dev/enola/core/meta/EntityKindRepository.java b/core/impl/src/main/java/dev/enola/core/meta/EntityKindRepository.java index 961b763da..a954746c9 100644 --- a/core/impl/src/main/java/dev/enola/core/meta/EntityKindRepository.java +++ b/core/impl/src/main/java/dev/enola/core/meta/EntityKindRepository.java @@ -17,6 +17,9 @@ */ package dev.enola.core.meta; +import static dev.enola.common.protobuf.ProtobufMediaTypes.PROTOBUF_TEXTPROTO_UTF_8; + +import dev.enola.common.io.resource.ClasspathResource; import dev.enola.common.io.resource.ErrorResource; import dev.enola.common.io.resource.ReadableResource; import dev.enola.common.protobuf.MessageValidators; @@ -42,8 +45,13 @@ public class EntityKindRepository { public EntityKindRepository() { try { put(EntityKindAspect.ENTITY_KIND_ENTITY_KIND); + try { + load(new ClasspathResource("schema.textproto", PROTOBUF_TEXTPROTO_UTF_8)); + } catch (IOException e) { + throw new IllegalStateException("Built-in ClasspathResource missing?!", e); + } } catch (ValidationException e) { - // This cannot happen, because ENTITY_KIND_ENTITY_KIND is valid. + // This cannot happen, because the built-in kinds are valid. throw new IllegalStateException("BUG!", e); } } diff --git a/core/impl/src/main/java/dev/enola/core/meta/SchemaAspect.java b/core/impl/src/main/java/dev/enola/core/meta/SchemaAspect.java new file mode 100644 index 000000000..36e2bbc54 --- /dev/null +++ b/core/impl/src/main/java/dev/enola/core/meta/SchemaAspect.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2023 The Enola Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.enola.core.meta; + +import static com.google.protobuf.Any.pack; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; + +import dev.enola.core.EnolaException; +import dev.enola.core.EnolaServiceProvider; +import dev.enola.core.EntityAspect; +import dev.enola.core.connector.proto.ConnectorServiceListRequest; +import dev.enola.core.meta.proto.EntityKind; +import dev.enola.core.proto.Entity; +import dev.enola.core.proto.ID; + +import java.util.List; + +public class SchemaAspect implements EntityAspect { + + // Matching schema.textproto + static final ID.Builder idBuilderTemplate = ID.newBuilder().setNs("enola").setEntity("schema"); + + private EnolaServiceProvider esp; + + public void setESP(EnolaServiceProvider enolaServiceProvider) { + this.esp = enolaServiceProvider; + } + + @Override + public void augment(Entity.Builder entity, EntityKind entityKind) throws EnolaException { + var name = entity.getId().getPaths(0); + var descriptor = esp.getTypeRegistryWrapper().get().find(name).toProto(); + var any = pack(descriptor, "type.googleapis.com/"); + entity.putData("proto", any); + } + + @Override + public void list( + ConnectorServiceListRequest request, + EntityKind entityKind, + List entities) + throws EnolaException { + // TODO if (!IDs.withoutPath(request.getId()).equals(idBuilderTemplate)) return + for (var name : esp.getTypeRegistryWrapper().names()) { + var id = idBuilderTemplate.clone().addPaths(name); + var newSchemaEntity = Entity.newBuilder().setId(id); + augment(newSchemaEntity, null); + entities.add(newSchemaEntity); + } + } + + public List getDescriptors() throws EnolaException { + return ImmutableList.of(DescriptorProtos.DescriptorProto.getDescriptor()); + } +} diff --git a/core/impl/src/main/java/dev/enola/core/meta/TypeRegistryWrapper.java b/core/impl/src/main/java/dev/enola/core/meta/TypeRegistryWrapper.java new file mode 100644 index 000000000..f0a6b2f18 --- /dev/null +++ b/core/impl/src/main/java/dev/enola/core/meta/TypeRegistryWrapper.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2023 The Enola Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.enola.core.meta; + +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Descriptors; +import com.google.protobuf.TypeRegistry; + +import java.util.List; + +public class TypeRegistryWrapper { + private final TypeRegistry originalTypeRegistry; + private final ImmutableSet names; + + private TypeRegistryWrapper(TypeRegistry typeRegistry, ImmutableSet names) { + this.originalTypeRegistry = typeRegistry; + this.names = names; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public TypeRegistry get() { + return originalTypeRegistry; + } + + public ImmutableSet names() { + return names; + } + + public static final class Builder { + private final TypeRegistry.Builder originalBuilder = TypeRegistry.newBuilder(); + private final ImmutableSet.Builder names = ImmutableSet.builder(); + + private Builder() {} + + public Builder add(List descriptors) { + originalBuilder.add(descriptors); + for (Descriptors.Descriptor type : descriptors) { + addFile(type.getFile()); + } + return this; + } + + private void addFile(Descriptors.FileDescriptor file) { + for (Descriptors.FileDescriptor dependency : file.getDependencies()) { + addFile(dependency); + } + for (Descriptors.Descriptor message : file.getMessageTypes()) { + addMessage(message); + } + } + + private void addMessage(Descriptors.Descriptor message) { + for (Descriptors.Descriptor nestedType : message.getNestedTypes()) { + addMessage(nestedType); + } + names.add(message.getFullName()); + } + + public TypeRegistryWrapper build() { + return new TypeRegistryWrapper(originalBuilder.build(), names.build()); + } + } +} diff --git a/core/impl/src/main/resources/schema.textproto b/core/impl/src/main/resources/schema.textproto new file mode 100644 index 000000000..2a4c4d188 --- /dev/null +++ b/core/impl/src/main/resources/schema.textproto @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright 2023 The Enola Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://protobuf.dev/reference/protobuf/textformat-spec/ +# proto-file: dev/enola/core/meta/enola_meta.proto +# proto-message: EntityKinds + +kinds { + id: { ns: "enola" entity: "schema" paths: "fqn" } + label: "Schema (Proto) used in Enola Entity Data" + emoji: "💠" + doc_url: "https://docs.enola.dev/use/connector/#grpc" + connectors: { java_class: "dev.enola.core.meta.SchemaAspect" } + data: { + key: "proto" + value: { + label: "Protocol Buffer Descriptor (Proto)" + description: "This is the Proto ('Schema') for the 'Any' fields in the 'data' of a Connector." + type_url: "type.googleapis.com/google.protobuf.DescriptorProto" + } + } + # TODO Add a "source" kind of data entry, which links to the Connector? +} diff --git a/core/impl/src/test/java/dev/enola/core/EntityServiceProviderTest.java b/core/impl/src/test/java/dev/enola/core/EntityServiceProviderTest.java index 6adff6ef5..d0c650561 100644 --- a/core/impl/src/test/java/dev/enola/core/EntityServiceProviderTest.java +++ b/core/impl/src/test/java/dev/enola/core/EntityServiceProviderTest.java @@ -154,9 +154,10 @@ public void testGrpcConnector() throws IOException, ValidationException, EnolaEx @Test public void testEntityKindInception() throws ValidationException, EnolaException { var kid = ID.newBuilder().setNs("enola").setEntity("entity_kind").addPaths("name").build(); + var sid = ID.newBuilder().setNs("enola").setEntity("schema").addPaths("fqn").build(); var ekr = new EntityKindRepository(); - assertThat(ekr.listID()).containsExactly(kid); + assertThat(ekr.listID()).containsExactly(kid, sid); var service = new EnolaServiceProvider().get(ekr); var eid = ID.newBuilder(kid).clearPaths().addPaths("enola.entity_kind").build(); @@ -169,6 +170,6 @@ public void testEntityKindInception() throws ValidationException, EnolaException var listRequest = ListEntitiesRequest.newBuilder().setId(kid).build(); var listResponse = service.listEntities(listRequest); - assertThat(listResponse.getEntitiesList()).hasSize(1); + assertThat(listResponse.getEntitiesList()).hasSize(2); } } diff --git a/core/impl/src/test/java/dev/enola/core/meta/EntityKindRepositoryTest.java b/core/impl/src/test/java/dev/enola/core/meta/EntityKindRepositoryTest.java index efa36db65..d7029cc6e 100644 --- a/core/impl/src/test/java/dev/enola/core/meta/EntityKindRepositoryTest.java +++ b/core/impl/src/test/java/dev/enola/core/meta/EntityKindRepositoryTest.java @@ -43,8 +43,8 @@ public class EntityKindRepositoryTest { @Test public void testEmptyRepository() throws ValidationException { // Even an empty repository always has the built-in enola.entity_kind - assertThat(r.list()).hasSize(1); - assertThat(r.listID()).hasSize(1); + assertThat(r.list()).hasSize(2); + assertThat(r.listID()).hasSize(2); var id = ID.newBuilder().setEntity("non-existant").build(); assertThrows(IllegalArgumentException.class, () -> r.get(id)); @@ -90,7 +90,7 @@ public void testLoadTextproto() throws ValidationException, IOException { IDs.parse("demo.bar/foo/name"), IDs.parse("demo.baz/uuid")); // The 4th one is the built-in enola.entity_kind - assertThat(r.list()).hasSize(4); + assertThat(r.list()).hasSize(5); } @Test @@ -99,7 +99,7 @@ public void testLoadYAML() throws ValidationException, IOException { assertThat(r.listID()) .containsAtLeast(IDs.parse("demo.foo/name"), IDs.parse("demo.bar/foo/name")); // The 3rd one is the built-in enola.entity_kind - assertThat(r.list()).hasSize(3); + assertThat(r.list()).hasSize(4); } @Test diff --git a/core/impl/src/test/java/dev/enola/core/meta/SchemaAspectTest.java b/core/impl/src/test/java/dev/enola/core/meta/SchemaAspectTest.java new file mode 100644 index 000000000..408c1bfd7 --- /dev/null +++ b/core/impl/src/test/java/dev/enola/core/meta/SchemaAspectTest.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2023 The Enola Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.enola.core.meta; + +import static com.google.common.truth.Truth.assertThat; + +import dev.enola.common.protobuf.ValidationException; +import dev.enola.core.EnolaException; +import dev.enola.core.EnolaService; +import dev.enola.core.EnolaServiceProvider; +import dev.enola.core.proto.GetEntityRequest; +import dev.enola.core.proto.ID; +import dev.enola.core.proto.ListEntitiesRequest; + +import org.junit.Test; + +public class SchemaAspectTest { + + EntityKindRepository ekr = new EntityKindRepository(); + EnolaService service = new EnolaServiceProvider().get(ekr); + ID.Builder schemaKindID = SchemaAspect.idBuilderTemplate; + + public SchemaAspectTest() throws ValidationException, EnolaException {} + + @Test + public void list() throws ValidationException, EnolaException { + var request = ListEntitiesRequest.newBuilder().setId(schemaKindID).build(); + var response = service.listEntities(request); + + assertThat(response.getEntitiesList()).hasSize(54); + } + + @Test + public void get() throws ValidationException, EnolaException { + var id = schemaKindID.clone().addPaths("google.protobuf.Timestamp"); + var request = GetEntityRequest.newBuilder().setId(id).build(); + var response = service.getEntity(request); + + assertThat(response.getEntity().getDataOrThrow("proto")).isNotNull(); + } +} diff --git a/core/impl/src/test/java/dev/enola/core/meta/TypeRegistryWrapperTest.java b/core/impl/src/test/java/dev/enola/core/meta/TypeRegistryWrapperTest.java new file mode 100644 index 000000000..9df9d2914 --- /dev/null +++ b/core/impl/src/test/java/dev/enola/core/meta/TypeRegistryWrapperTest.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2023 The Enola Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.enola.core.meta; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.protobuf.Timestamp; + +import org.junit.Test; + +import java.util.List; + +public class TypeRegistryWrapperTest { + @Test + public void empty() { + assertThat(TypeRegistryWrapper.newBuilder().build().names()).isEmpty(); + } + + @Test + public void one() { + var wrapper = + TypeRegistryWrapper.newBuilder().add(List.of(Timestamp.getDescriptor())).build(); + assertThat(wrapper.names()).containsExactly("google.protobuf.Timestamp"); + } +} diff --git a/core/impl/src/test/resources/demo-model-docgen.md b/core/impl/src/test/resources/demo-model-docgen.md index 46301f060..622e2602f 100644 --- a/core/impl/src/test/resources/demo-model-docgen.md +++ b/core/impl/src/test/resources/demo-model-docgen.md @@ -25,6 +25,10 @@ classDiagram 🆔 name } link Entity_kind "#enola.entity_kind" + class Schema{ + 🆔 fqn + } + link Schema "#enola.schema" ``` ## 👩‍🎤 `demo.bar` (FUBAR?) @@ -61,5 +65,11 @@ classDiagram [See documentation...](https://docs.enola.dev/concepts/core-arch/) +## 💠 `enola.schema` (Schema (Proto) used in Enola Entity Data) + +* fqn + +[See documentation...](https://docs.enola.dev/use/connector/#grpc) + --- _This model documentation was generated with ❤️ by [Enola.dev](https://www.enola.dev)_