[wip] Introduce MaybeSharedPtr for Mutable classes

- Mutable objects either start reference counted or managed by a
collection depending on how they are created.
- Default objects can only be created by collections and are hence
always managed by the collection.
- Make constructors from Obj* (or MaybeSharedPtr<Obj>) private and
introduce a public static Object::makeEmpty function to create empty
handles that are necessary to model relations that have not been
persisted.

python/templates/Collection.cc.jinja2

@@ -48,11 +48,11 @@
 }
 
 Mutable{{ class.bare_type }} {{ collection_type }}::operator[](std::size_t index) {
-  return Mutable{{ class.bare_type }}(m_storage.entries[index]);
+  return Mutable{{ class.bare_type }}(podio::utils::MaybeSharedPtr(m_storage.entries[index]));
 }
 
 Mutable{{ class.bare_type }} {{ collection_type }}::at(std::size_t index) {
-  return Mutable{{ class.bare_type }}(m_storage.entries.at(index));
+  return Mutable{{ class.bare_type }}(podio::utils::MaybeSharedPtr(m_storage.entries.at(index)));
 }
 
 std::size_t {{ collection_type }}::size() const {
@@ -85,7 +85,7 @@ Mutable{{ class.bare_type }} {{ collection_type }}::create() {
 {% endif %}
 
   obj->id = {int(m_storage.entries.size() - 1), m_collectionID};
-  return Mutable{{ class.bare_type }}(obj);
+  return Mutable{{ class.bare_type }}(podio::utils::MaybeSharedPtr(obj));
 }
 
 void {{ collection_type }}::clear() {
@@ -133,7 +133,7 @@ bool {{ collection_type }}::setReferences(const podio::ICollectionProvider* coll
   return m_storage.setReferences(collectionProvider, m_isSubsetColl);
 }
 
-void {{ collection_type }}::push_back({{ class.bare_type }} object) {
+void {{ collection_type }}::push_back(Mutable{{ class.bare_type }} object) {
   // We have to do different things here depending on whether this is a
   // subset collection or not. A normal collection cannot collect objects
   // that are already part of another collection, while a subset collection
@@ -143,22 +143,29 @@ void {{ collection_type }}::push_back({{ class.bare_type }} object) {
     if (obj->id.index == podio::ObjectID::untracked) {
       const auto size = m_storage.entries.size();
       obj->id = {(int)size, m_collectionID};
-      m_storage.entries.push_back(obj);
+      m_storage.entries.push_back(obj.release());
 {% if OneToManyRelations or VectorMembers %}
-      m_storage.createRelations(obj);
+      m_storage.createRelations(obj.get());
 {% endif %}
     } else {
       throw std::invalid_argument("Object already in a collection. Cannot add it to a second collection");
     }
   } else {
-    const auto obj = object.m_obj;
-    if (obj->id.index < 0) {
-      throw std::invalid_argument("Objects can only be stored in a subset collection if they are already elements of a collection");
-    }
-    m_storage.entries.push_back(obj);
-    // No need to handle any relations here, since this is already done by the
-    // "owning" collection
+
+    push_back({{ class.bare_type }}(object));
+  }
+}
+
+void {{ collection_type }}::push_back({{ class.bare_type }} object) {
+  if (!m_isSubsetColl) {
+    throw std::invalid_argument("Can only add immutable objects to subset collections");
+  }
+  auto obj = object.m_obj;
+  if (obj->id.index < 0) {
+    // This path is only possible if we arrive here from an untracked Mutable object
+    throw std::invalid_argument("Object needs to be tracked by another collection in order for it to be storable in a subset collection");
   }
+  m_storage.entries.push_back(obj);
 }
 
 podio::CollectionWriteBuffers {{ collection_type }}::getBuffers() {

python/templates/Collection.h.jinja2

@@ -115,7 +115,9 @@ public:
 
 
   /// Append object to the collection
-  void push_back({{class.bare_type}} object);
+  void push_back(Mutable{{class.bare_type}} object);
+  /// Append an object to the (subset) collection
+  void push_back({{ class.bare_type }} object);
 
   void prepareForWrite() const final;
   void prepareAfterRead() final;
@@ -194,7 +196,7 @@ Mutable{{ class.bare_type }} {{ class.bare_type }}Collection::create(Args&&... a
 {% endfor %}
   m_storage.createRelations(obj);
 {% endif %}
-  return Mutable{{ class.bare_type }}(obj);
+  return Mutable{{ class.bare_type }}(podio::utils::MaybeSharedPtr(obj));
 }
 
 #if defined(PODIO_JSON_OUTPUT) && !defined(__CLING__)

python/templates/MutableObject.cc.jinja2

@@ -1,3 +1,4 @@
+#include "podio/utilities/MaybeSharedPtr.h"
 {% import "macros/utils.jinja2" as utils %}
 {% import "macros/implementations.jinja2" as macros %}
 // AUTOMATICALLY GENERATED FILE - DO NOT EDIT
@@ -16,8 +17,26 @@
 
 {{ utils.namespace_open(class.namespace) }}
 
-{{ macros.constructors_destructors(class.bare_type, Members, prefix='Mutable') }}
-Mutable{{ class.bare_type }}::operator {{ class.bare_type }}() const { return {{ class.bare_type }}(m_obj); }
+{{ macros.common_constructors_destructors(class.bare_type, Members, prefix='Mutable') }}
+
+{% with full_type = 'Mutable' + class.bare_type %}
+{{ full_type }}::{{ full_type }}() :
+  m_obj(new {{ class.bare_type }}Obj{}, podio::utils::MarkOwned) {}
+
+{% if Members %}
+{{ full_type }}::{{ full_type }}({{ Members | map(attribute='signature') | join(', ') }}) :
+  m_obj(new {{ class.bare_type }}Obj{}, podio::utils::MarkOwned) {
+{% for member in Members %}
+  m_obj->data.{{ member.name }} = {{ member.name }};
+{% endfor %}
+}
+{% endif %}
+
+{{ full_type }}::{{ full_type }}(podio::utils::MaybeSharedPtr<{{ class.bare_type }}Obj> obj) : m_obj(obj) {}
+
+{{ full_type }}::operator {{ class.bare_type }}() const { return {{ class.bare_type }}(m_obj.get()); }
+
+{% endwith %}
 
 {{ macros.member_getters(class, Members, use_get_syntax, prefix='Mutable') }}
 {{ macros.single_relation_getters(class, OneToOneRelations, use_get_syntax, prefix='Mutable') }}

python/templates/MutableObject.h.jinja2

@@ -1,3 +1,4 @@
+#include "podio/utilities/MaybeSharedPtr.h"
 {% import "macros/declarations.jinja2" as macros %}
 {% import "macros/utils.jinja2" as utils %}
 // AUTOMATICALLY GENERATED FILE - DO NOT EDIT
@@ -12,7 +13,9 @@
 {% for include in includes %}
 {{ include }}
 {% endfor %}
-#include "podio/ObjectID.h"
+
+#include "podio/utilities/MaybeSharedPtr.h"
+
 #include <ostream>
 #include <cstddef>
 
@@ -35,7 +38,16 @@ public:
   using object_type = {{ class.bare_type }};
   using collection_type = {{ class.bare_type }}Collection;
 
-{{ macros.constructors_destructors(class.bare_type, Members, prefix='Mutable') }}
+  /// Default constructor
+  Mutable{{ class.bare_type }}();
+
+{{ macros.common_constructors_destructors(class.bare_type, Members, prefix='Mutable') }}
+{% if Members %}
+
+  /// Constructor initializing all members
+  Mutable{{ class.bare_type }}({{ Members | map(attribute='signature') | join(', ')  }});
+{% endif %}
+
   /// conversion to const object
   operator {{ class.bare_type }}() const;
 
@@ -50,8 +62,14 @@ public:
 {{ utils.if_present(MutableExtraCode, "declaration") }}
 {{ macros.common_object_funcs(class.bare_type, prefix='Mutable') }}
 
+  /// disconnect from {{ class.bare_type }}Obj instance
+  void unlink() { m_obj = podio::utils::MaybeSharedPtr<{{ class.bare_type }}Obj>{nullptr}; }
+
 private:
-  {{ class.bare_type }}Obj* m_obj;
+  /// constructor from existing {{ class.bare_type }}Obj
+  explicit Mutable{{ class.bare_type }}(podio::utils::MaybeSharedPtr<{{ class.bare_type }}Obj> obj);
+
+  podio::utils::MaybeSharedPtr<{{ class.bare_type }}Obj> m_obj{nullptr};
 };
 
 {{ macros.json_output(class.bare_type, prefix='Mutable') }}

python/templates/Obj.cc.jinja2

@@ -1,3 +1,4 @@
+#include "podio/ObjectID.h"
 {% import "macros/utils.jinja2" as utils %}
 // AUTOMATICALLY GENERATED FILE - DO NOT EDIT
 
@@ -16,7 +17,7 @@
 {{ utils.namespace_open(class.namespace) }}
 {% with obj_type = class.bare_type + 'Obj' %}
 {{ obj_type }}::{{ obj_type }}() :
-{% raw %}  ObjBase{{}, 0}{% endraw %},
+  id(),
   data(){{ single_relations_initialize(OneToOneRelations) }}
 {%- for relation in OneToManyRelations + VectorMembers %},
   m_{{ relation.name }}(new std::vector<{{ relation.full_type }}>())
@@ -25,11 +26,11 @@
 {  }
 
 {{ obj_type }}::{{ obj_type }}(const podio::ObjectID id_, {{ class.bare_type }}Data data_) :
-  ObjBase{id_, 0}, data(data_)
+  id(id_), data(data_)
 {  }
 
 {{ obj_type }}::{{ obj_type }}(const {{ obj_type }}& other) :
-{% raw %}  ObjBase{{}, 0}{% endraw %},
+  id(),
   data(other.data){{ single_relations_initialize(OneToOneRelations) }}
 {%- for relation in OneToManyRelations + VectorMembers %},
   m_{{ relation.name }}(new std::vector<{{ relation.full_type }}>(*(other.m_{{ relation.name }})))

python/templates/Obj.h.jinja2

@@ -10,7 +10,7 @@
 {{ include }}
 {% endfor %}
 
-#include "podio/ObjBase.h"
+#include "podio/ObjectID.h"
 {% if OneToManyRelations or VectorMembers %}
 #include <vector>
 {%- endif %}
@@ -21,7 +21,7 @@
 class {{ class.bare_type }};
 
 {% with obj_type = class.bare_type + 'Obj' %}
-class {{ class.bare_type }}Obj : public podio::ObjBase {
+class {{ class.bare_type }}Obj {
 public:
   /// constructor
   {{ obj_type }}();
@@ -39,6 +39,7 @@ public:
 {% endif %}
 
 public:
+  podio::ObjectID id;
   {{ class.bare_type }}Data data;
 {% for relation in OneToOneRelations %}
   {{ relation.full_type }}* m_{{ relation.name }}{nullptr};

python/templates/Object.cc.jinja2

@@ -16,7 +16,14 @@
 
 {{ utils.namespace_open(class.namespace) }}
 
-{{ macros.constructors_destructors(class.bare_type, Members) }}
+{{ macros.common_constructors_destructors(class.bare_type, Members) }}
+
+{{ class.bare_type }}::{{ class.bare_type }}({{ class.bare_type }}Obj* obj) : m_obj(obj) {}
+
+{{ class.bare_type }} {{ class.bare_type }}::makeEmpty() {
+  return {nullptr};
+}
+
 
 {{ macros.member_getters(class, Members, use_get_syntax) }}
 {{ macros.single_relation_getters(class, OneToOneRelations, use_get_syntax) }}

python/templates/Object.h.jinja2

@@ -10,7 +10,7 @@
 {% for include in includes %}
 {{ include }}
 {% endfor %}
-#include "podio/ObjectID.h"
+
 #include <ostream>
 #include <cstddef>
 
@@ -23,19 +23,23 @@
 {{ utils.namespace_open(class.namespace) }}
 class Mutable{{ class.bare_type }};
 class {{ class.bare_type }}Collection;
+class {{ class.bare_type }}CollectionData;
 
 {{ macros.class_description(class.bare_type, Description, Author) }}
 class {{ class.bare_type }} {
 
   friend class Mutable{{ class.bare_type }};
   friend class {{ class.bare_type }}Collection;
+  friend class {{ class.full_type }}CollectionData;
   friend class {{ class.bare_type }}CollectionIterator;
 
 public:
   using mutable_type = Mutable{{ class.bare_type }};
   using collection_type = {{ class.bare_type }}Collection;
 
-{{ macros.constructors_destructors(class.bare_type, Members) }}
+{{ macros.common_constructors_destructors(class.bare_type, Members) }}
+
+  static {{ class.bare_type }} makeEmpty();
 
 public:
 
@@ -45,8 +49,14 @@ public:
 {{ utils.if_present(ExtraCode, "declaration") }}
 {{ macros.common_object_funcs(class.bare_type) }}
 
+  /// disconnect from the {{ class.bare_type }}Obj
+  void unlink() { m_obj = nullptr; }
+
 private:
-  {{ class.bare_type }}Obj* m_obj;
+  /// constructor from existing {{ class.bare_type }}Obj
+  {{ class.bare_type}}({{ class.bare_type }}Obj* obj);
+
+  {{ class.bare_type }}Obj* m_obj{nullptr};
 };
 
 std::ostream& operator<<(std::ostream& o, const {{ class.bare_type }}& value);

python/templates/SIOBlock.cc.jinja2

@@ -89,7 +89,7 @@ namespace {
   // library actually needs linking to the core library. Otherwise it is
   // possible that the registry is not populated when the SioBlock library is
   // loaded, e.g. when using the python bindings.
-  const auto elem = {{ class.full_type }}{};
+  const auto elem = {{ class.full_type }}::makeEmpty();
 }
 
 {% endwith %}

python/templates/macros/collections.jinja2

@@ -68,14 +68,14 @@ std::vector<{{ member.full_type }}> {{ class.bare_type }}Collection::{{ member.n
     if (id.index != podio::ObjectID::invalid) {
       podio::CollectionBase* coll = nullptr;
       if (!collectionProvider->get(id.collectionID, coll)) {
-        m_rel_{{ relation.name }}->emplace_back(nullptr);
+        m_rel_{{ relation.name }}->emplace_back({{ relation.full_type }}::makeEmpty());
         continue;
       }
       {{ relation.full_type }}Collection* tmp_coll = static_cast<{{ relation.full_type }}Collection*>(coll);
       const auto tmp = (*tmp_coll)[id.index];
       m_rel_{{ relation.name }}->emplace_back(tmp);
     } else {
-      m_rel_{{ relation.name }}->emplace_back(nullptr);
+      m_rel_{{ relation.name }}->emplace_back({{ relation.full_type }}::makeEmpty());
     }
   }
 {% endmacro %}

python/templates/macros/declarations.jinja2

@@ -54,19 +54,10 @@
 {% endmacro %}
 
 
-{% macro constructors_destructors(type, members, prefix='') %}
+{% macro common_constructors_destructors(type, members, prefix='') %}
 {% set full_type = prefix + type %}
-  /// default constructor
-  {{  full_type }}();
-{% if members %}
-  {{  full_type }}({{ members | map(attribute='signature') | join(', ')  }});
-{% endif %}
-
-  /// constructor from existing {{ type }}Obj
-  {{  full_type }}({{ type }}Obj* obj);
-
   /// copy constructor
-  {{  full_type }}(const {{  full_type }}& other);
+  {{  full_type }}(const {{  full_type }}& other) = default;
 
   /// copy-assignment operator
   {{  full_type }}& operator=({{ full_type }} other);
@@ -75,16 +66,14 @@
   Mutable{{ type }} clone() const;
 
   /// destructor
-  ~{{  full_type }}();
+  ~{{  full_type }}() = default;
 {% endmacro %}
 
 
 {% macro common_object_funcs(type, prefix='') %}
 {% set full_type = prefix + type %}
   /// check whether the object is actually available
   bool isAvailable() const;
-  /// disconnect from {{ type }}Obj instance
-  void unlink() { m_obj = nullptr; }
 
 {% set inverse_type = type if prefix else 'Mutable' + type %}
   bool operator==(const {{ full_type }}& other) const { return m_obj == other.m_obj; }