Custom hierarchies – other API concept, for now without extended meshes and particles path

CMakeLists.txt

@@ -394,6 +394,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake)
 set(CORE_SOURCE
         src/config.cpp
         src/ChunkInfo.cpp
+        src/CustomHierarchy.cpp
         src/Dataset.cpp
         src/Datatype.cpp
         src/Error.cpp
@@ -832,6 +833,7 @@ if(openPMD_BUILD_TESTING)
             list(APPEND ${out_list}
                 test/Files_Core/automatic_variable_encoding.cpp
                 test/Files_Core/read_nonexistent_attribute.cpp
+                test/Files_Core/custom_hierarchy.cpp
             )
         endif()
     endmacro()

examples/10_streaming_write.cpp

@@ -82,6 +82,8 @@ int main()
             pos.resetDataset(dataset);
             pos.storeChunk(local_data, Offset{0}, global_extent);
         }
+        auto ch = iteration.customHierarchies();
+        ch["rabimmel"].setAttribute("rabammel", "rabumm");
         iteration.close();
     }
 

examples/2_read_serial.cpp

@@ -31,7 +31,8 @@ using namespace openPMD;
 int main()
 {
     Series series = Series(
-        "../samples/git-sample/data%T.h5",
+        // "../samples/git-sample/data%T.h5",
+        "data.h5",
         Access::READ_ONLY,
         R"({"defer_iteration_parsing": true})");
     cout << "Read a Series with openPMD standard version " << series.openPMD()
@@ -99,9 +100,24 @@ int main()
 
     auto all_data = E_x.loadChunk<double>();
 
+    auto ch = series.customHierarchies();
+    ch.printRecursively();
+    std::cout << "READING 200/fields" << std::endl;
+    ch["data"]["200"]["fields"].read();
+    std::cout << "READING 200/particles" << std::endl;
+    ch["data"]["200"]["particles"].read();
+    std::cout << "READING 300" << std::endl;
+    ch["data"]["300"].read(0);
+    ch.printRecursively();
+
     // The iteration can be closed in order to help free up resources.
     // The iteration's content will be flushed automatically.
     i.close();
+    std::cout << "OPENING 200" << std::endl;
+    i = series.snapshots()[200].open();
+    ch.printRecursively();
+    series.snapshots()[300].open();
+
     cout << "Full E/x starts with:\n\t{";
     for (size_t col = 0; col < extent[1] && col < 5; ++col)
         cout << all_data.get()[col] << ", ";

include/openPMD/CustomHierarchy.hpp

@@ -0,0 +1,206 @@
+/* Copyright 2023 Franz Poeschel
+ *
+ * This file is part of openPMD-api.
+ *
+ * openPMD-api is free software: you can redistribute it and/or modify
+ * it under the terms of of either the GNU General Public License or
+ * the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * openPMD-api is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License and the GNU Lesser General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * and the GNU Lesser General Public License along with openPMD-api.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "openPMD/IO/AbstractIOHandler.hpp"
+#include "openPMD/Mesh.hpp"
+#include "openPMD/ParticleSpecies.hpp"
+#include "openPMD/RecordComponent.hpp"
+#include "openPMD/backend/Attributable.hpp"
+#include "openPMD/backend/Container.hpp"
+
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+namespace openPMD
+{
+class CustomHierarchy;
+namespace internal
+{
+    using CustomHierarchyData = ContainerData<Attributable>;
+} // namespace internal
+
+class CustomHierarchy;
+
+/*
+ * This is its own class, so the return value of asContainerOf() is also
+ * convsersible again.
+ */
+template <typename MappedType>
+class ConvertibleContainer : public Container<MappedType>
+{
+    template <typename>
+    friend class ConversibleContainer;
+    friend class CustomHierarchy;
+
+protected:
+    using Container_t = Container<MappedType>;
+    using Data_t = internal::ContainerData<MappedType>;
+    static_assert(
+        std::is_base_of_v<typename Container_t::ContainerData, Data_t>);
+
+    using Container_t::Container_t;
+
+private:
+    explicit ConvertibleContainer() = default;
+
+public:
+    template <typename TargetType>
+    auto asContainerOf() -> ConvertibleContainer<TargetType>
+    {
+        if constexpr (
+            std::is_same_v<TargetType, CustomHierarchy> ||
+            std::is_same_v<TargetType, Mesh> ||
+            std::is_same_v<TargetType, ParticleSpecies> ||
+            std::is_same_v<TargetType, RecordComponent>)
+        {
+            // TODO: If Mesh or ParticleSpecies, create Container on the fly by
+            // evaluating the meshes/particles path. If RecordComponent, maybe
+            // use dynamic casting to check if children are datasets?
+            throw std::runtime_error("UNIMPLEMENTED");
+        }
+        else
+        {
+            static_assert(
+                auxiliary::dependent_false_v<TargetType>,
+                "[CustomHierarchy::asContainerOf] Type parameter must be "
+                "one of: CustomHierarchy, RecordComponent, Mesh, "
+                "ParticleSpecies.");
+        }
+    }
+};
+
+namespace traits
+{
+    template <>
+    struct DeferredInitPolicy<Container<CustomHierarchy>>
+    {
+        template <typename Container_const_or_not>
+        static void call(Container_const_or_not &);
+    };
+
+    template <>
+    struct GenerationPolicy<CustomHierarchy>
+    {
+        constexpr static bool is_noop = false;
+        template <typename Container, typename Iterator>
+        void operator()(Container &cont, Iterator &it)
+        {
+            auto &writable = it->second.writable();
+
+            // These should be different
+            auto child_shared_data = &it->second.Attributable::get();
+            auto parent_shared_data = &cont.Attributable::get();
+            if (child_shared_data == parent_shared_data)
+            {
+                throw std::runtime_error(
+                    "Trying to emplace object as its own child");
+            }
+
+            // These might be different, but might also be the same
+            //
+            // For an explanation, ref. the documentation of
+            // Writable::attributable: This is a pointer back to the first
+            // created Attributable instance linking this Writable. There might
+            // be multiple Attributable objects linking the same backend
+            // Writable object when opening multiple "views" on the same backend
+            // object, e.g. when a scalar Record is at the same time a
+            // RecordComponent, or when reopening an object as a
+            // CustomHierarchy.
+            //
+            // Since CustomHierarchy performs no memory management by default,
+            // we must ensure that the backpointer in Writable::attributable
+            // remains valid when the frontend instance pointed by
+            // Writable::attributable *is* the CustomHierarchy instance (happens
+            // when it is the first frontend object created for that backend
+            // object).
+            auto backpointer = writable.attributable;
+            auto emplaced_pointer = it->second.m_attri.get();
+            if (backpointer == emplaced_pointer)
+            {
+                (**cont.m_attri)
+                    .m_children_managed_as_custom_hierarchy[it->first] =
+                    it->second;
+            }
+        }
+    };
+} // namespace traits
+
+// TODO: Use Container<Attributable> internally, but otherwise override members
+// such that we have:
+//
+// operator[](key) -> CustomHierarchy
+//
+// Or find a better solution for having this automatically..
+class CustomHierarchy : public ConvertibleContainer<CustomHierarchy>
+{
+    friend class Iteration;
+    friend class Container<CustomHierarchy>;
+    friend class Attributable;
+    friend struct traits::DeferredInitPolicy<Container<CustomHierarchy>>;
+
+private:
+    using Parent_t = ConvertibleContainer<CustomHierarchy>;
+    using Container_t = typename Parent_t::Container_t;
+    using Data_t = typename Parent_t::Data_t;
+
+protected:
+    CustomHierarchy(NoInit);
+    CustomHierarchy(std::shared_ptr<internal::SharedAttributableData> other);
+    CustomHierarchy(Attributable const &other);
+
+    void read(std::vector<std::string> &currentPath);
+
+    void flush_internal(
+        internal::FlushParams const &, std::vector<std::string> currentPath);
+    void flush(std::string const &path, internal::FlushParams const &) override;
+
+    /**
+     * @brief Link with parent.
+     *
+     * @param w The Writable representing the parent.
+     */
+    void linkHierarchy(Writable &w) override;
+
+public:
+    CustomHierarchy();
+
+    CustomHierarchy(CustomHierarchy const &other) = default;
+    CustomHierarchy(CustomHierarchy &&other) = default;
+
+    CustomHierarchy &operator=(CustomHierarchy const &) = default;
+    CustomHierarchy &operator=(CustomHierarchy &&) = default;
+
+    // TODO maybe make this automatic somehow
+    // set max_recursion_depth = 0 for infinite cycling
+    // recursion depth includes the current object
+    // recursion will not continue expanding into regions that are already known
+    // (hence not transitively expand into unknown subregions of known regions)
+    void read(size_t max_recursion_depth = 1);
+
+    void printRecursively();
+
+private:
+    void printRecursively(std::string indent);
+};
+} // namespace openPMD

include/openPMD/IO/AbstractIOHandler.hpp

@@ -26,6 +26,7 @@
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/config.hpp"
 #include "openPMD/version.hpp"
+#include <ostream>
 
 #if openPMD_HAVE_MPI
 #include <mpi.h>
@@ -81,6 +82,66 @@ enum class FlushLevel
     CreateOrOpenFiles
 };
 
+std::ostream &operator<<(std::ostream &, FlushLevel);
+
+namespace flush_level
+{
+    inline constexpr auto global_flushpoint(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    // same as global_flushpoint for now, but we will soon introduce
+    // immediate_flush
+    inline constexpr auto write_datasets(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto write_attributes(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+            return true;
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto flush_hierarchy(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+            return true;
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+} // namespace flush_level
+
 enum class OpenpmdStandard
 {
     v_1_0_0,
@@ -121,6 +182,7 @@ namespace internal
      * To be used for reading
      */
     FlushParams const defaultFlushParams{};
+    FlushParams const publicFlush{FlushLevel::UserFlush};
 
     struct ParsedFlushParams;
 

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -39,7 +39,7 @@ class AbstractIOHandlerImpl
 
     virtual ~AbstractIOHandlerImpl() = default;
 
-    std::future<void> flush();
+    std::future<void> flush(FlushLevel);
 
     /**
      * Close the file corresponding with the writable and release file handles.

include/openPMD/IO/IOTask.hpp

@@ -35,6 +35,7 @@
 #include <cstddef>
 #include <memory>
 #include <optional>
+#include <ostream>
 #include <string>
 #include <utility>
 #include <variant>
@@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){
 }; // note: if you change the enum members here, please update
    // docs/source/dev/design.rst
 
+std::ostream &operator<<(std::ostream &os, Operation op);
+
 namespace internal
 {
     /*

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
 
-    std::future<void> flush();
+    std::future<void> flush(internal::ParsedFlushParams &params);
 
 private:
 #if openPMD_HAVE_MPI