From 7a85f92d73c8e944f886132296b615b0b6de06ad Mon Sep 17 00:00:00 2001
From: "w.klijn" <nonoice@gmail.com>
Date: Tue, 9 Aug 2016 09:27:20 +0200
Subject: [PATCH] Add basic implementation of the file exporter. Add unique
_ptr to the model class (will be refactored later to have a manager)
---
src/communication/export_manager.hpp | 27 ++++
src/communication/exporter_interface.hpp | 42 ++++++
src/communication/exporter_spike_file.hpp | 123 +++++++++++++++
.../exporter_spike_single_file.hpp | 140 ++++++++++++++++++
src/model.hpp | 4 +
tests/global_communication/CMakeLists.txt | 33 ++++-
tests/global_communication/test.cpp | 19 +++
.../test_exporter_spike_file.cpp | 118 +++++++++++++++
.../test_exporter_spike_single_file.cpp | 118 +++++++++++++++
9 files changed, 623 insertions(+), 1 deletion(-)
create mode 100644 src/communication/export_manager.hpp
create mode 100644 src/communication/exporter_interface.hpp
create mode 100644 src/communication/exporter_spike_file.hpp
create mode 100644 src/communication/exporter_spike_single_file.hpp
create mode 100644 tests/global_communication/test.cpp
create mode 100644 tests/global_communication/test_exporter_spike_file.cpp
create mode 100644 tests/global_communication/test_exporter_spike_single_file.cpp
diff --git a/src/communication/export_manager.hpp b/src/communication/export_manager.hpp
new file mode 100644
index 00000000..ccef9c1c
--- /dev/null
+++ b/src/communication/export_manager.hpp
@@ -0,0 +1,27 @@
+//#pragma once
+//
+//#include <algorithm>
+//#include <iostream>
+//#include <fstream>
+//#include <vector>
+//#include <random>
+//
+//#include <spike.hpp>
+//
+//#include "exporter_interface.hpp"
+//
+//namespace nest {
+//namespace mc {
+//namespace communication {
+//
+//class export_manager {
+//
+//
+//
+//};
+//
+//
+//
+//} //communication
+//} // namespace mc
+//} // namespace nest
\ No newline at end of file
diff --git a/src/communication/exporter_interface.hpp b/src/communication/exporter_interface.hpp
new file mode 100644
index 00000000..ecb2f7e8
--- /dev/null
+++ b/src/communication/exporter_interface.hpp
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <random>
+#include <string>
+
+#include <spike.hpp>
+#include <common_types.hpp>
+
+namespace nest {
+namespace mc {
+namespace communication {
+
+template <typename Time, typename CommunicationPolicy> // TODO: Templating on data type, for now only spike_type
+class exporter_interface {
+
+public:
+ using time_type = Time;
+ using spike_type = spike<cell_member_type, time_type>;
+
+ // Performs the export of the data, thread safe buffered
+ virtual void do_export() = 0;
+
+ // Add data to the internal storage to be exported
+ // Does not do the actual export
+ virtual void add_data(std::vector<spike_type>) = 0;
+
+ // Internal state is ok
+ // Export might encounter problems in a separate thread.
+ virtual bool ok() const = 0;
+
+ // TODO: Enum with status strings (might be added to the implemenation)
+ // returns a textual explanation of the current error state
+ //
+ // virtual string status_description() = 0;
+ //virtual string status_id() = 0;
+};
+
+
+
+} //communication
+} // namespace mc
+} // namespace nest
\ No newline at end of file
diff --git a/src/communication/exporter_spike_file.hpp b/src/communication/exporter_spike_file.hpp
new file mode 100644
index 00000000..68b2f3c4
--- /dev/null
+++ b/src/communication/exporter_spike_file.hpp
@@ -0,0 +1,123 @@
+#pragma once
+
+#include <cstdio>
+#include <fstream>
+#include <iomanip>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <vector>
+
+#include <common_types.hpp>
+#include <util.hpp>
+#include <spike.hpp>
+
+
+#include "exporter_interface.hpp"
+
+namespace nest {
+namespace mc {
+namespace communication {
+
+template <typename Time, typename CommunicationPolicy> // TODO: Templating on data type, for now only spike_type
+class exporter_spike_file : public exporter_interface<Time, CommunicationPolicy> {
+
+public:
+ using time_type = Time;
+ using spike_type = spike<cell_member_type, time_type>;
+ using communication_policy_type = CommunicationPolicy;
+
+ //
+ exporter_spike_file(std::string file_name, std::string path,
+ std::string file_extention, bool over_write=true)
+ :
+ ok_(false)
+ {
+ std::string file_path(create_output_file_path(
+ file_name, path, file_extention, communication_policy_.id()));
+
+ //test if the file exist and depending on over_write throw or delete
+ std::ifstream f(file_path);
+ if (f.good()) {
+ if (!over_write) {
+ std::string error_string("Tried opening file for writing but it exists and over_write is false:\n" +
+ file_path);
+
+ throw std::runtime_error(error_string);
+ }
+
+ std::remove(file_path.c_str());
+ }
+
+ file_handle_ = nest::mc::util::make_unique<std::ofstream>(file_path,
+ std::fstream::app);
+
+ if (file_handle_->good()) {
+ ok_ = true;
+ }
+
+ // Force output of the spike times with precision
+ // TODO: We need to make this selectable
+ file_handle_->precision(4);
+ file_handle_->setf(std::ios::fixed, std::ios::floatfield);
+ }
+
+
+ // Performs the export of the data,
+ // Does not throw
+ void do_export() override
+ {
+ for (auto spike : spikes_) {
+ *file_handle_ << spike.source.gid << " " << spike.time << std::endl;
+ }
+ if (!file_handle_->good()){
+ ok_ = false;
+ }
+
+ spikes_.clear();
+ }
+
+ // Add data to the internal storage to be exported
+ // Does not do the actual export
+ void add_data(std::vector<spike_type>spikes) override
+ {
+ spikes_.insert(std::end(spikes_),
+ std::begin(spikes), std::end(spikes));
+ }
+
+ // Internal state is ok
+ // We are working with fstreams possibly on a seperate thread
+ // We need a way to assertain the status of the stream
+ bool ok() const override
+ {
+ return ok_ && file_handle_->good();
+ }
+
+ // Creates an indexed filename
+ static std::string create_output_file_path(std::string file_name, std::string path,
+ std::string file_extention, unsigned index)
+ {
+ std::string file_path = path + file_name + "_" + std::to_string(index) +
+ "." + file_extention;
+ // TODO: Nest does not produce the indexing for nrank == 0
+ // I have the feeling this disrupts consistent output. Id rather
+ // always put the zero in.
+ return file_path;
+ }
+
+private:
+ // Are we in a valid state?
+ bool ok_;
+
+ // Handle to our owned opened file handle
+ std::unique_ptr<std::ofstream> file_handle_;
+
+ // local storage for sending spikes
+ std::vector<spike_type> spikes_;
+
+ communication_policy_type communication_policy_;
+};
+
+} //communication
+} // namespace mc
+} // namespace nest
diff --git a/src/communication/exporter_spike_single_file.hpp b/src/communication/exporter_spike_single_file.hpp
new file mode 100644
index 00000000..a10205fe
--- /dev/null
+++ b/src/communication/exporter_spike_single_file.hpp
@@ -0,0 +1,140 @@
+#pragma once
+
+#include <cstdio>
+#include <fstream>
+#include <iomanip>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <vector>
+
+#include <common_types.hpp>
+#include <util.hpp>
+#include <spike.hpp>
+
+
+#include "exporter_interface.hpp"
+
+namespace nest {
+namespace mc {
+namespace communication {
+
+template <typename Time, typename CommunicationPolicy> // TODO: Templating on data type, for now only spike_type
+class exporter_spike_single_file : public exporter_interface<Time, CommunicationPolicy> {
+
+public:
+ using time_type = Time;
+ using spike_type = spike<cell_member_type, time_type>;
+ using communication_policy_type = CommunicationPolicy;
+
+ //
+ exporter_spike_single_file(std::string file_name, std::string path,
+ std::string file_extention, bool over_write=true)
+ :
+ ok_(false)
+ {
+ if (!communication_policy_.id() == 0) {
+ ok_ = true;
+ return;
+ }
+
+ std::string file_path(create_output_file_path(
+ file_name, path, file_extention, communication_policy_.id()));
+
+ //test if the file exist and depending on over_write throw or delete
+ std::ifstream f(file_path);
+ if (f.good()) {
+ if (!over_write) {
+ std::string error_string("Tried opening file for writing but it exists and over_write is false:\n" +
+ file_path);
+
+ throw std::runtime_error(error_string);
+ }
+
+ std::remove(file_path.c_str());
+ }
+
+ file_handle_ = nest::mc::util::make_unique<std::ofstream>(file_path,
+ std::fstream::app);
+
+ if (file_handle_->good()) {
+ ok_ = true;
+ }
+
+ // Force output of the spike times with precision
+ // TODO: We need to make this selectable
+ file_handle_->precision(4);
+ file_handle_->setf(std::ios::fixed, std::ios::floatfield);
+ }
+
+
+ // Performs the export of the data,
+ // Does not throw
+ void do_export() override
+ {
+ if (!communication_policy_.id() == 0) {
+ return;
+ }
+
+ for (auto spike : spikes_) {
+ *file_handle_ << spike.source.gid << " " << spike.time << std::endl;
+ }
+ if (!file_handle_->good()){
+ ok_ = false;
+ }
+
+ spikes_.clear();
+ }
+
+ // Add data to the internal storage to be exported
+ // Does not do the actual export
+ void add_data(std::vector<spike_type>spikes) override
+ {
+ if (!communication_policy_.id() == 0) {
+ return;
+ }
+
+ spikes_.insert(std::end(spikes_),
+ std::begin(spikes), std::end(spikes));
+ }
+
+ // Internal state is ok
+ // We are working with fstreams possibly on a seperate thread
+ // We need a way to assertain the status of the stream
+ bool ok() const override
+ {
+ if (!communication_policy_.id() == 0) {
+ return true;
+ }
+
+ return ok_ && file_handle_->good();
+ }
+
+ // Creates an indexed filename
+ static std::string create_output_file_path(std::string file_name, std::string path,
+ std::string file_extention, unsigned index)
+ {
+ std::string file_path = path + file_name + "_" + std::to_string(index) +
+ "." + file_extention;
+ // TODO: Nest does not produce the indexing for nrank == 0
+ // I have the feeling this disrupts consistent output. Id rather
+ // always put the zero in.
+ return file_path;
+ }
+
+private:
+ // Are we in a valid state?
+ bool ok_;
+
+ // Handle to our owned opened file handle
+ std::unique_ptr<std::ofstream> file_handle_;
+
+ // local storage for sending spikes
+ std::vector<spike_type> spikes_;
+
+ communication_policy_type communication_policy_;
+};
+
+} //communication
+} // namespace mc
+} // namespace nest
diff --git a/src/model.hpp b/src/model.hpp
index 84dd8429..b92ca4a8 100644
--- a/src/model.hpp
+++ b/src/model.hpp
@@ -7,10 +7,12 @@
#include <cell.hpp>
#include <cell_group.hpp>
#include <fvm_cell.hpp>
+#include <memory>
#include <recipe.hpp>
#include <thread_private_spike_store.hpp>
#include <communication/communicator.hpp>
#include <communication/global_policy.hpp>
+#include <communication/exporter_interface.hpp>
#include <profiling/profiler.hpp>
#include "trace_sampler.hpp"
@@ -182,6 +184,8 @@ private:
using local_spike_store_type = thread_private_spike_store<time_type>;
util::double_buffer< local_spike_store_type > local_spikes_;
+ using exporter_interface_type = nest::mc::communication::exporter_interface<time_type, communicator_type>;
+ std::unique_ptr<exporter_interface_type> exporter_;
// Convenience functions that map the spike buffers and event queues onto
// the appropriate integration interval.
//
diff --git a/tests/global_communication/CMakeLists.txt b/tests/global_communication/CMakeLists.txt
index f7d00015..dff31ad7 100644
--- a/tests/global_communication/CMakeLists.txt
+++ b/tests/global_communication/CMakeLists.txt
@@ -1 +1,32 @@
-# Nothing to be done yet
+set(HEADERS
+ ${PROJECT_SOURCE_DIR}/src/swcio.hpp
+)
+set(COMMUNICATION_SOURCES
+ test_exporter_spike_file.cpp
+ test_exporter_spike_single_file.cpp
+ # unit test driver
+ test.cpp
+)
+
+add_executable(global_communication.exe ${COMMUNICATION_SOURCES} ${HEADERS})
+
+set(TARGETS global_communication.exe)
+
+foreach(target ${TARGETS})
+ target_link_libraries(${target} LINK_PUBLIC cellalgo gtest)
+
+ if(WITH_TBB)
+ target_link_libraries(${target} LINK_PUBLIC ${TBB_LIBRARIES})
+ endif()
+
+ if(WITH_MPI)
+ target_link_libraries(${target} LINK_PUBLIC ${MPI_C_LIBRARIES})
+ set_property(TARGET ${target} APPEND_STRING PROPERTY LINK_FLAGS "${MPI_C_LINK_FLAGS}")
+ endif()
+
+ set_target_properties(${target}
+ PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests"
+ )
+endforeach()
+
diff --git a/tests/global_communication/test.cpp b/tests/global_communication/test.cpp
new file mode 100644
index 00000000..c67a065b
--- /dev/null
+++ b/tests/global_communication/test.cpp
@@ -0,0 +1,19 @@
+#include <iostream>
+#include <fstream>
+#include <numeric>
+#include <vector>
+
+#include "gtest.h"
+
+#include "../../src/communication/communicator.hpp"
+#include "../../src/communication/global_policy.hpp"
+
+int main(int argc, char **argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+
+ // We need to set the communicator policy at the top level
+ // this allows us to build multiple communicators in the tests
+ nest::mc::communication::global_policy_guard global_guard(argc, argv);
+
+ return RUN_ALL_TESTS();
+}
diff --git a/tests/global_communication/test_exporter_spike_file.cpp b/tests/global_communication/test_exporter_spike_file.cpp
new file mode 100644
index 00000000..7d30877c
--- /dev/null
+++ b/tests/global_communication/test_exporter_spike_file.cpp
@@ -0,0 +1,118 @@
+#include "gtest.h"
+
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include <communication/communicator.hpp>
+#include <communication/global_policy.hpp>
+#include <communication/exporter_spike_file.hpp>
+
+class exporter_spike_file_fixture : public ::testing::Test {
+
+protected:
+ using time_type = float;
+ using communicator_type = nest::mc::communication::global_policy;
+
+ using spike_type = nest::mc::communication::exporter_spike_file<time_type,
+ communicator_type>::spike_type;
+
+ using exporter_type = nest::mc::communication::exporter_spike_file<time_type, communicator_type>;
+ std::string file_name;
+ std::string path;
+ std::string extention;
+ unsigned index;
+
+ exporter_spike_file_fixture()
+ :
+ file_name("spikes"),
+ path("./"),
+ extention("gdf"),
+ index(0)
+ {}
+
+ std::string get_standard_file_name()
+ {
+ return exporter_type::create_output_file_path(
+ file_name, path, extention, index);
+ }
+
+ void SetUp() {
+ // code here will execute just before the test ensues
+ }
+
+ void TearDown() {
+ // delete the start create file
+ std::remove(get_standard_file_name().c_str());
+ }
+
+ ~exporter_spike_file_fixture() {
+
+ }
+};
+
+TEST_F(exporter_spike_file_fixture, constructor)
+{
+ exporter_type exporter(file_name,
+ path, extention, index);
+
+ // after construction the state of the exporter should be valid
+ EXPECT_TRUE(exporter.ok());
+
+
+ //test if the file exist and depending on over_write throw or delete
+ std::ifstream f(get_standard_file_name());
+ EXPECT_TRUE(f.good());
+}
+
+TEST_F(exporter_spike_file_fixture, create_output_file_path)
+{
+ // Create some random paths, no need for fancy tests here
+ std::string produced_filename =
+ exporter_type::create_output_file_path(
+ "spikes", "./", "gdf", 0);
+ EXPECT_STREQ(produced_filename.c_str(), "./spikes_0.gdf");
+
+ produced_filename =
+ exporter_type::create_output_file_path(
+ "a_name", "../../", "txt", 5);
+ EXPECT_STREQ(produced_filename.c_str(), "../../a_name_5.txt");
+}
+
+TEST_F(exporter_spike_file_fixture, do_export)
+{
+
+
+ exporter_type exporter(file_name,
+ path, extention, index);
+
+ // Create some spikes
+ std::vector<spike_type> spikes;
+ spikes.push_back({ { 0, 0 }, 0.0});
+ spikes.push_back({ { 0, 0 }, 0.1 });
+ spikes.push_back({ { 1, 0 }, 1.0 });
+ spikes.push_back({ { 1, 0 }, 1.1 });
+
+ // add to the exporter
+ exporter.add_data(spikes);
+
+ // now do the export
+ exporter.do_export();
+
+ // Test if we have spikes in the file
+ std::ifstream f(get_standard_file_name());
+ EXPECT_TRUE(f.good());
+
+ std::string line;
+
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "0 0.0000");
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "0 0.1000");
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "1 1.0000");
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "1 1.1000");
+}
diff --git a/tests/global_communication/test_exporter_spike_single_file.cpp b/tests/global_communication/test_exporter_spike_single_file.cpp
new file mode 100644
index 00000000..aee5bb9c
--- /dev/null
+++ b/tests/global_communication/test_exporter_spike_single_file.cpp
@@ -0,0 +1,118 @@
+#include "gtest.h"
+
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include <communication/communicator.hpp>
+#include <communication/global_policy.hpp>
+#include <communication/exporter_spike_single_file.hpp>
+
+class exporter_spike_single_file_fixture : public ::testing::Test {
+
+protected:
+ using time_type = float;
+ using communicator_type = nest::mc::communication::global_policy;
+
+ using spike_type = nest::mc::communication::exporter_spike_single_file<time_type,
+ communicator_type>::spike_type;
+
+ using exporter_type = nest::mc::communication::exporter_spike_single_file<time_type, communicator_type>;
+ std::string file_name;
+ std::string path;
+ std::string extention;
+ unsigned index;
+
+ exporter_spike_single_file_fixture()
+ :
+ file_name("spikes"),
+ path("./"),
+ extention("gdf"),
+ index(0)
+ {}
+
+ std::string get_standard_file_name()
+ {
+ return exporter_type::create_output_file_path(
+ file_name, path, extention, index);
+ }
+
+ void SetUp() {
+ // code here will execute just before the test ensues
+ }
+
+ void TearDown() {
+ // delete the start create file
+ std::remove(get_standard_file_name().c_str());
+ }
+
+ ~exporter_spike_single_file_fixture() {
+
+ }
+};
+
+TEST_F(exporter_spike_single_file_fixture, constructor)
+{
+ exporter_type exporter(file_name,
+ path, extention, index);
+
+ // after construction the state of the exporter should be valid
+ EXPECT_TRUE(exporter.ok());
+
+
+ //test if the file exist and depending on over_write throw or delete
+ std::ifstream f(get_standard_file_name());
+ EXPECT_TRUE(f.good());
+}
+
+TEST_F(exporter_spike_single_file_fixture, create_output_file_path)
+{
+ // Create some random paths, no need for fancy tests here
+ std::string produced_filename =
+ exporter_type::create_output_file_path(
+ "spikes", "./", "gdf", 0);
+ EXPECT_STREQ(produced_filename.c_str(), "./spikes_0.gdf");
+
+ produced_filename =
+ exporter_type::create_output_file_path(
+ "a_name", "../../", "txt", 5);
+ EXPECT_STREQ(produced_filename.c_str(), "../../a_name_5.txt");
+}
+
+TEST_F(exporter_spike_single_file_fixture, do_export)
+{
+
+
+ exporter_type exporter(file_name,
+ path, extention, index);
+
+ // Create some spikes
+ std::vector<spike_type> spikes;
+ spikes.push_back({ { 0, 0 }, 0.0});
+ spikes.push_back({ { 0, 0 }, 0.1 });
+ spikes.push_back({ { 1, 0 }, 1.0 });
+ spikes.push_back({ { 1, 0 }, 1.1 });
+
+ // add to the exporter
+ exporter.add_data(spikes);
+
+ // now do the export
+ exporter.do_export();
+
+ // Test if we have spikes in the file
+ std::ifstream f(get_standard_file_name());
+ EXPECT_TRUE(f.good());
+
+ std::string line;
+
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "0 0.0000");
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "0 0.1000");
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "1 1.0000");
+ EXPECT_TRUE(std::getline(f, line));
+ EXPECT_STREQ(line.c_str(), "1 1.1000");
+}
--
GitLab