Python/allen catalogue (#1074)

* Add import method for mechanism catalogues, allowing mechanisms from a second catalogue to be incorporated with a prefix, together with Python interface.
* Expose `global_allen_catalogue`.
* Extend unit tests to cover mechanism catalogue import and check for name collision error cases.

arbor/include/arbor/mechcat.hpp

@@ -106,7 +106,10 @@ public:
         register_impl(std::type_index(typeid(B)), name, std::move(generic_proto));
     }
 
-    ~mechanism_catalogue();
+   // Copy over another catalogue's mechanism and attach a -- possibly empty -- prefix
+   void import(const mechanism_catalogue& other, const std::string& prefix);
+
+   ~mechanism_catalogue();
 
 private:
     std::unique_ptr<catalogue_state> state_;
@@ -119,5 +122,6 @@ private:
 // Reference to global default mechanism catalogue.
 
 const mechanism_catalogue& global_default_catalogue();
+const mechanism_catalogue& global_allen_catalogue();
 
 } // namespace arb

arbor/mechcat.cpp

@@ -123,25 +123,44 @@ struct catalogue_state {
     catalogue_state() = default;
 
     catalogue_state(const catalogue_state& other) {
-        info_map_.clear();
+        import(other, "");
+    }
+
+    void import(const catalogue_state& other, const std::string& prefix) {
+        // Do all checks before adding anything, otherwise we might get inconsistent state.
+        auto assert_undefined = [&](const std::string& key) {
+            auto pkey = prefix+key;
+            if (defined(pkey)) {
+                throw duplicate_mechanism(pkey);
+            }
+        };
+
+        for (const auto& kv: other.info_map_) {
+            assert_undefined(kv.first);
+        }
+
+        for (const auto& kv: other.derived_map_) {
+            assert_undefined(kv.first);
+        }
+
         for (const auto& kv: other.info_map_) {
-            info_map_[kv.first] = make_unique<mechanism_info>(*kv.second);
+            auto key = prefix + kv.first;
+            info_map_[key] = make_unique<mechanism_info>(*kv.second);
         }
 
-        derived_map_.clear();
         for (const auto& kv: other.derived_map_) {
+            auto key = prefix + kv.first;
             const derivation& v = kv.second;
-            derived_map_[kv.first] = {v.parent, v.globals, v.ion_remap, make_unique<mechanism_info>(*v.derived_info)};
+            derived_map_[key] = {prefix + v.parent, v.globals, v.ion_remap, make_unique<mechanism_info>(*v.derived_info)};
         }
 
-        impl_map_.clear();
         for (const auto& name_impls: other.impl_map_) {
             std::unordered_map<std::type_index, std::unique_ptr<mechanism>> impls;
             for (const auto& tidx_mptr: name_impls.second) {
                 impls[tidx_mptr.first] = tidx_mptr.second->clone();
             }
-
-            impl_map_[name_impls.first] = std::move(impls);
+            auto key = prefix + name_impls.first;
+            impl_map_[key] = std::move(impls);
         }
     }
 
@@ -540,6 +559,10 @@ void mechanism_catalogue::derive(const std::string& name, const std::string& par
     state_->bind(name, value(state_->derive(parent)));
 }
 
+void mechanism_catalogue::import(const mechanism_catalogue& other, const std::string& prefix) {
+    state_->import(*other.state_, prefix);
+}
+
 void mechanism_catalogue::remove(const std::string& name) {
     if (!has(name)) {
         throw no_such_mechanism(name);

mechanisms/CMakeLists.txt

 include(BuildModules.cmake)
 
-set(mech_dir "${CMAKE_CURRENT_BINARY_DIR}/generated")
-file(MAKE_DIRECTORY "${mech_dir}")
-
 set(external_modcc)
 if(ARB_WITH_EXTERNAL_MODCC)
     set(external_modcc MODCC ${modcc})
@@ -15,6 +12,8 @@ set(mech_sources "")
 # ALLEN
 set(allen_mechanisms CaDynamics Ca_HVA Ca_LVA Ih Im Im_v2 K_P K_T Kd Kv2like Kv3_1 NaTa NaTs NaV Nap SK)
 set(allen_mod_srcdir "${CMAKE_CURRENT_SOURCE_DIR}/allen")
+set(mech_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/allen")
+file(MAKE_DIRECTORY "${mech_dir}")
 
 build_modules(
     ${allen_mechanisms}
@@ -53,6 +52,8 @@ endforeach()
 # DEFAULT
 set(default_mechanisms exp2syn expsyn hh kamt kdrmt nax nernst pas)
 set(default_mod_srcdir "${CMAKE_CURRENT_SOURCE_DIR}/default")
+set(mech_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/default")
+file(MAKE_DIRECTORY "${mech_dir}")
 
 build_modules(
     ${default_mechanisms}

python/mechanism.cpp

@@ -118,6 +118,10 @@ void register_mechanisms(pybind11::module& m) {
                     throw std::runtime_error(util::pprintf("\nKeyError: '{}'", name));
                 }
             })
+        .def("import", &arb::mechanism_catalogue::import,
+             "other"_a, "Catalogue to import into self",
+             "prefix"_a, "Prefix for names in other",
+             "Import another catalogue, possibly with a prefix. Will overwrite in case of name collisions.")
         .def("derive", &apply_derive,
                 "name"_a, "parent"_a,
                 "globals"_a=std::unordered_map<std::string, double>{},
@@ -130,6 +134,7 @@ void register_mechanisms(pybind11::module& m) {
                     return util::pprintf("<arbor.mechanism_catalogue>"); });
 
     m.def("default_catalogue", [](){return arb::global_default_catalogue();});
+    m.def("allen_catalogue", [](){return arb::global_allen_catalogue();});
 
     // arb::mechanism_desc
     // For specifying a mechanism in the cable_cell interface.

test/unit/test_mechcat.cpp

@@ -443,4 +443,95 @@ TEST(mechcat, copy) {
     EXPECT_EQ(typeid(*fleeb2_inst.mech.get()), typeid(*fleeb2_inst2.mech.get()));
 }
 
+TEST(mechcat, import) {
+    auto cat = build_fake_catalogue();
+    mechanism_catalogue cat2;
+    cat2.import(cat, "fake_");
+
+    EXPECT_TRUE(cat.has("fleeb2"));
+    EXPECT_FALSE(cat.has("fake_fleeb2"));
+
+    EXPECT_TRUE(cat2.has("fake_fleeb2"));
+    EXPECT_FALSE(cat2.has("fleeb2"));
+
+    EXPECT_EQ(cat["fleeb2"], cat2["fake_fleeb2"]);
+
+    auto fleeb2_inst  = cat.instance<foo_backend>("fleeb2");
+    auto fleeb2_inst2 = cat2.instance<foo_backend>("fake_fleeb2");
+
+    EXPECT_EQ(typeid(*fleeb2_inst.mech.get()), typeid(*fleeb2_inst2.mech.get()));
+}
+
+TEST(mechcat, import_collisions) {
+    {
+        auto cat = build_fake_catalogue();
+
+        mechanism_catalogue cat2;
+        EXPECT_NO_THROW(cat2.import(cat, "prefix:")); // Should have no collisions.
+        EXPECT_NO_THROW(cat.import(cat2, "prefix:")); // Should have no collisions here either.
+
+        // cat should have both original entries and copies with 'prefix:prefix:' prefixed.
+        ASSERT_TRUE(cat.has("fleeb2"));
+        ASSERT_TRUE(cat.has("prefix:prefix:fleeb2"));
+    }
+
+    // We should throw if there any collisions between base or derived mechanism
+    // names between the catalogues. If the import fails, the catalogue should
+    // remain unchanged.
+    {
+        // Collision between two base mechanisms.
+        {
+            auto cat = build_fake_catalogue();
 
+            mechanism_catalogue other;
+            other.add("fleeb", burble_info); // Note different mechanism info!
+
+            EXPECT_THROW(cat.import(other, ""), arb::duplicate_mechanism);
+            ASSERT_EQ(cat["fleeb"], fleeb_info);
+        }
+
+        // Collision derived vs base.
+        {
+            auto cat = build_fake_catalogue();
+
+            mechanism_catalogue other;
+            other.add("fleeb2", burble_info);
+
+            auto fleeb2_info = cat["fleeb2"];
+            EXPECT_THROW(cat.import(other, ""), arb::duplicate_mechanism);
+            EXPECT_EQ(cat["fleeb2"], fleeb2_info);
+        }
+
+        // Collision base vs derived.
+        {
+            auto cat = build_fake_catalogue();
+
+            mechanism_catalogue other;
+            other.add("zonkers", fleeb_info);
+            other.derive("fleeb", "zonkers", {{"plugh", 8.}});
+            ASSERT_FALSE(other["fleeb"]==fleeb_info);
+
+            ASSERT_FALSE(cat.has("zonkers"));
+            EXPECT_THROW(cat.import(other, ""), arb::duplicate_mechanism);
+            EXPECT_EQ(cat["fleeb"], fleeb_info);
+            EXPECT_FALSE(cat.has("zonkers"));
+        }
+
+        // Collision derived vs derived.
+        {
+            auto cat = build_fake_catalogue();
+
+            mechanism_catalogue other;
+            other.add("zonkers", fleeb_info);
+            other.derive("fleeb2", "zonkers", {{"plugh", 8.}});
+
+            auto fleeb2_info = cat["fleeb2"];
+            ASSERT_FALSE(other["fleeb2"]==fleeb2_info);
+
+            ASSERT_FALSE(cat.has("zonkers"));
+            EXPECT_THROW(cat.import(other, ""), arb::duplicate_mechanism);
+            EXPECT_EQ(cat["fleeb2"], fleeb2_info);
+            EXPECT_FALSE(cat.has("zonkers"));
+        }
+    }
+}