feat (core): Expose Descriptor Protos as a new EntityKind

cli/src/main/java/dev/enola/cli/CommandWithEntityID.java

@@ -17,12 +17,11 @@
  */
 package dev.enola.cli;
 
-import com.google.protobuf.TypeRegistry;
-
 import dev.enola.common.io.resource.WriterResource;
 import dev.enola.common.protobuf.ProtoIO;
 import dev.enola.core.IDs;
 import dev.enola.core.meta.EntityKindRepository;
+import dev.enola.core.meta.TypeRegistryWrapper;
 import dev.enola.core.meta.proto.EntityKind;
 import dev.enola.core.proto.EnolaServiceGrpc.EnolaServiceBlockingStub;
 import dev.enola.core.proto.Entity;
@@ -48,12 +47,12 @@ public abstract class CommandWithEntityID extends CommandWithModel {
     String idString;
 
     private WriterResource resource;
-    private TypeRegistry typeRegistry;
+    private TypeRegistryWrapper typeRegistryWrapper;
 
     @Override
     protected final void run(EntityKindRepository ekr, EnolaServiceBlockingStub service)
             throws Exception {
-        typeRegistry = esp.getTypeRegistry();
+        typeRegistryWrapper = esp.getTypeRegistryWrapper();
 
         ID id = IDs.parse(idString); // TODO replace with ITypeConverter
         // TODO Validate id; here it must have ns+name+path!
@@ -69,6 +68,6 @@ protected abstract void run(
             throws Exception;
 
     protected void write(Entity entity) throws IOException {
-        new ProtoIO(typeRegistry).write(entity, resource);
+        new ProtoIO(typeRegistryWrapper.get()).write(entity, resource);
     }
 }

common/protobuf/src/main/java/dev/enola/common/protobuf/ProtoIO.java

@@ -131,7 +131,6 @@ public <B extends Message.Builder> B read(ReadableResource resource, B builder)
                 builder.mergeFrom(is, extensionRegistry);
             }
         } else {
-            // TODO Use resource.mediaType().charset().or(UTF_8)
             try (Reader reader = resource.charSource(UTF_8).openBufferedStream()) {
                 if (normalizedNoParamsEquals(mediaType, PROTOBUF_TEXTPROTO_UTF_8)) {
                     textFormatParser.merge(reader, extensionRegistry, builder);

connectors/demo/src/test/java/dev/enola/demo/ServerTest.java

@@ -21,8 +21,6 @@
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 
-import com.google.protobuf.TypeRegistry;
-
 import dev.enola.common.io.resource.ClasspathResource;
 import dev.enola.common.io.resource.MemoryResource;
 import dev.enola.common.io.resource.ReplacingResource;
@@ -36,6 +34,7 @@
 import dev.enola.core.connector.proto.ConnectorServiceGrpc;
 import dev.enola.core.connector.proto.ConnectorServiceListRequest;
 import dev.enola.core.meta.EntityKindRepository;
+import dev.enola.core.meta.TypeRegistryWrapper;
 import dev.enola.core.proto.Entity;
 import dev.enola.core.proto.GetEntityRequest;
 import dev.enola.core.proto.ID;
@@ -54,7 +53,7 @@ public class ServerTest {
 
     private EntityKindRepository ekr;
     private EnolaService enola;
-    private TypeRegistry typeRegistry;
+    private TypeRegistryWrapper typeRegistryWrapper;
 
     @Test
     public void bothConnectorDirectlyAndViaServer()
@@ -110,7 +109,7 @@ private void createEnolaService(int port)
 
         var esp = new EnolaServiceProvider();
         enola = esp.get(ekr);
-        typeRegistry = esp.getTypeRegistry();
+        typeRegistryWrapper = esp.getTypeRegistryWrapper();
     }
 
     private void checkEnolaGet(EnolaService enola) throws EnolaException, IOException {
@@ -125,7 +124,7 @@ private void checkEnolaGet(EnolaService enola) throws EnolaException, IOExceptio
         assertThat(something.getText()).isEqualTo("hello, world");
         assertThat(something.getNumber()).isEqualTo(123);
 
-        var io = new ProtoIO(typeRegistry);
+        var io = new ProtoIO(typeRegistryWrapper.get());
         var resource = new MemoryResource(ProtobufMediaTypes.PROTOBUF_YAML_UTF_8);
         var entityKind = ekr.get(ID.newBuilder().setNs("demo").setEntity("foo").build());
         io.write(entity, resource);

core/impl/BUILD

@@ -69,6 +69,7 @@ java_library(
         "//core/lib:core_java_proto",
         "//core/lib:lib_java",
         "@maven//:com_google_guava_guava",
+        "@maven//:com_google_protobuf_protobuf_java",
         "@maven//:com_google_truth_truth",
         "@maven//:io_grpc_grpc_api",
         "@maven//:junit_junit",

core/impl/src/main/java/dev/enola/core/EnolaServiceProvider.java

@@ -18,23 +18,24 @@
 package dev.enola.core;
 
 import com.google.common.collect.ImmutableList;
-import com.google.protobuf.TypeRegistry;
 
 import dev.enola.common.protobuf.ValidationException;
 import dev.enola.core.aspects.*;
 import dev.enola.core.meta.EntityAspectWithRepository;
 import dev.enola.core.meta.EntityKindRepository;
+import dev.enola.core.meta.SchemaAspect;
+import dev.enola.core.meta.TypeRegistryWrapper;
 
 import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Path;
 
 public class EnolaServiceProvider {
 
-    private TypeRegistry typeRegistry;
+    private TypeRegistryWrapper typeRegistry;
 
     // TODO rename to getService
     public EnolaService get(EntityKindRepository ekr) throws ValidationException, EnolaException {
-        var trb = TypeRegistry.newBuilder();
+        var trb = TypeRegistryWrapper.newBuilder();
         var sr = new EnolaServiceRegistry();
         for (var ek : ekr.list()) {
             var aspectsBuilder = ImmutableList.<EntityAspect>builder();
@@ -59,6 +60,9 @@ public EnolaService get(EntityKindRepository ekr) throws ValidationException, En
                                 ((EntityAspectWithRepository) connector)
                                         .setEntityKindRepository(ekr);
                             }
+                            if (connector instanceof SchemaAspect) {
+                                ((SchemaAspect) connector).setESP(this);
+                            }
                             aspectsBuilder.add(connector);
                             break;
 
@@ -101,23 +105,18 @@ public EnolaService get(EntityKindRepository ekr) throws ValidationException, En
             var s = new EntityAspectService(ek, aspects);
             sr.register(ek.getId(), s);
 
-            populateTypeRegistry(trb, aspects);
+            for (var aspect : aspects) {
+                trb.add(aspect.getDescriptors());
+            }
         }
         this.typeRegistry = trb.build();
         return sr;
     }
 
-    public TypeRegistry getTypeRegistry() {
+    public TypeRegistryWrapper getTypeRegistryWrapper() {
         if (typeRegistry == null) {
             throw new IllegalStateException("getTypeRegistry() must be called after get()");
         }
         return typeRegistry;
     }
-
-    private void populateTypeRegistry(TypeRegistry.Builder trb, ImmutableList<EntityAspect> aspects)
-            throws EnolaException {
-        for (var aspect : aspects) {
-            trb.add(aspect.getDescriptors());
-        }
-    }
 }

core/impl/src/main/java/dev/enola/core/meta/EntityKindRepository.java

@@ -17,6 +17,9 @@
  */
 package dev.enola.core.meta;
 
+import static dev.enola.common.protobuf.ProtobufMediaTypes.PROTOBUF_TEXTPROTO_UTF_8;
+
+import dev.enola.common.io.resource.ClasspathResource;
 import dev.enola.common.io.resource.ErrorResource;
 import dev.enola.common.io.resource.ReadableResource;
 import dev.enola.common.protobuf.MessageValidators;
@@ -42,8 +45,13 @@ public class EntityKindRepository {
     public EntityKindRepository() {
         try {
             put(EntityKindAspect.ENTITY_KIND_ENTITY_KIND);
+            try {
+                load(new ClasspathResource("schema.textproto", PROTOBUF_TEXTPROTO_UTF_8));
+            } catch (IOException e) {
+                throw new IllegalStateException("Built-in ClasspathResource missing?!", e);
+            }
         } catch (ValidationException e) {
-            // This cannot happen, because ENTITY_KIND_ENTITY_KIND is valid.
+            // This cannot happen, because the built-in kinds are valid.
             throw new IllegalStateException("BUG!", e);
         }
     }

core/impl/src/main/java/dev/enola/core/meta/SchemaAspect.java

@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2023 The Enola <https://enola.dev> Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.enola.core.meta;
+
+import static com.google.protobuf.Any.pack;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.DescriptorProtos;
+import com.google.protobuf.Descriptors;
+
+import dev.enola.core.EnolaException;
+import dev.enola.core.EnolaServiceProvider;
+import dev.enola.core.EntityAspect;
+import dev.enola.core.connector.proto.ConnectorServiceListRequest;
+import dev.enola.core.meta.proto.EntityKind;
+import dev.enola.core.proto.Entity;
+import dev.enola.core.proto.ID;
+
+import java.util.List;
+
+public class SchemaAspect implements EntityAspect {
+
+    // Matching schema.textproto
+    static final ID.Builder idBuilderTemplate = ID.newBuilder().setNs("enola").setEntity("schema");
+
+    private EnolaServiceProvider esp;
+
+    public void setESP(EnolaServiceProvider enolaServiceProvider) {
+        this.esp = enolaServiceProvider;
+    }
+
+    @Override
+    public void augment(Entity.Builder entity, EntityKind entityKind) throws EnolaException {
+        var name = entity.getId().getPaths(0);
+        var descriptor = esp.getTypeRegistryWrapper().get().find(name).toProto();
+        var any = pack(descriptor, "type.googleapis.com/");
+        entity.putData("proto", any);
+    }
+
+    @Override
+    public void list(
+            ConnectorServiceListRequest request,
+            EntityKind entityKind,
+            List<Entity.Builder> entities)
+            throws EnolaException {
+        // TODO if (!IDs.withoutPath(request.getId()).equals(idBuilderTemplate)) return
+        for (var name : esp.getTypeRegistryWrapper().names()) {
+            var id = idBuilderTemplate.clone().addPaths(name);
+            var newSchemaEntity = Entity.newBuilder().setId(id);
+            augment(newSchemaEntity, null);
+            entities.add(newSchemaEntity);
+        }
+    }
+
+    public List<Descriptors.Descriptor> getDescriptors() throws EnolaException {
+        return ImmutableList.of(DescriptorProtos.DescriptorProto.getDescriptor());
+    }
+}

core/impl/src/main/java/dev/enola/core/meta/TypeRegistryWrapper.java

@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2023 The Enola <https://enola.dev> Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package dev.enola.core.meta;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.TypeRegistry;
+
+import java.util.List;
+
+public class TypeRegistryWrapper {
+    private final TypeRegistry originalTypeRegistry;
+    private final ImmutableSet<String> names;
+
+    private TypeRegistryWrapper(TypeRegistry typeRegistry, ImmutableSet<String> names) {
+        this.originalTypeRegistry = typeRegistry;
+        this.names = names;
+    }
+
+    public static Builder newBuilder() {
+        return new Builder();
+    }
+
+    public TypeRegistry get() {
+        return originalTypeRegistry;
+    }
+
+    public ImmutableSet<String> names() {
+        return names;
+    }
+
+    public static final class Builder {
+        private final TypeRegistry.Builder originalBuilder = TypeRegistry.newBuilder();
+        private final ImmutableSet.Builder<String> names = ImmutableSet.builder();
+
+        private Builder() {}
+
+        public Builder add(List<Descriptors.Descriptor> descriptors) {
+            originalBuilder.add(descriptors);
+            for (Descriptors.Descriptor type : descriptors) {
+                addFile(type.getFile());
+            }
+            return this;
+        }
+
+        private void addFile(Descriptors.FileDescriptor file) {
+            for (Descriptors.FileDescriptor dependency : file.getDependencies()) {
+                addFile(dependency);
+            }
+            for (Descriptors.Descriptor message : file.getMessageTypes()) {
+                addMessage(message);
+            }
+        }
+
+        private void addMessage(Descriptors.Descriptor message) {
+            for (Descriptors.Descriptor nestedType : message.getNestedTypes()) {
+                addMessage(nestedType);
+            }
+            names.add(message.getFullName());
+        }
+
+        public TypeRegistryWrapper build() {
+            return new TypeRegistryWrapper(originalBuilder.build(), names.build());
+        }
+    }
+}

core/impl/src/main/resources/schema.textproto

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2023 The Enola <https://enola.dev> Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# https://protobuf.dev/reference/protobuf/textformat-spec/
+# proto-file: dev/enola/core/meta/enola_meta.proto
+# proto-message: EntityKinds
+
+kinds {
+  id: { ns: "enola" entity: "schema" paths: "fqn" }
+  label: "Schema (Proto) used in Enola Entity Data"
+  emoji: "💠"
+  doc_url: "https://docs.enola.dev/use/connector/#grpc"
+  connectors: { java_class: "dev.enola.core.meta.SchemaAspect" }
+  data: {
+    key: "proto"
+    value: {
+      label: "Protocol Buffer Descriptor (Proto)"
+      description: "This is the Proto ('Schema') for the 'Any' fields in the 'data' of a Connector."
+      type_url: "type.googleapis.com/google.protobuf.DescriptorProto"
+    }
+  }
+  # TODO Add a "source" kind of data entry, which links to the Connector?
+}