From c793e8c5fe813f17fbc05c99ab01d3c5e9e98fd4 Mon Sep 17 00:00:00 2001
From: "w.klijn" <nonoice@gmail.com>
Date: Thu, 11 Aug 2016 13:01:29 +0200
Subject: [PATCH] File export can now be controlled with json parameters.
Default is false No output via command line parameters
---
mechanisms/generate.sh | 2 +
miniapp/io.cpp | 26 ++++++-
miniapp/io.hpp | 8 +++
miniapp/miniapp.cpp | 11 ++-
nrn/generate_validation.sh | 10 +--
src/communication/export_manager.hpp | 17 +++--
src/communication/exporter_spike_file.hpp | 69 ++++++++++++++++++-
src/model.hpp | 40 ++++++++++-
.../test_exporter_spike_file.cpp | 16 +++--
.../test_exporter_spike_single_file.cpp | 6 +-
tests/performance/io/disk_io.cpp | 53 ++++++++++++--
11 files changed, 225 insertions(+), 33 deletions(-)
diff --git a/mechanisms/generate.sh b/mechanisms/generate.sh
index 1fcd49be..8771b55c 100755
--- a/mechanisms/generate.sh
+++ b/mechanisms/generate.sh
@@ -1,3 +1,5 @@
+#!/etc/bash
+
#flags="-t cpu -O"
flags="-t cpu"
diff --git a/miniapp/io.cpp b/miniapp/io.cpp
index 46498fd2..8c72f463 100644
--- a/miniapp/io.cpp
+++ b/miniapp/io.cpp
@@ -43,8 +43,19 @@ namespace io {
cl_options read_options(int argc, char** argv) {
// set default options
+ // TODO: the declaration of this defopts is realistic if we have allot
+ // more options. We should use a name scheme.
const cl_options defopts{"", 1000, 500, "expsyn", 100, 100., 0.025, false,
- false, 1.0, "trace_", util::nothing};
+ false, 1.0, "trace_", util::nothing,
+
+ // spike_output_parameters:
+ false, // no spike output
+ false, // single_file_per_simulation
+ true, // Overwrite outputfile if exists
+ "./", // output path
+ "spikes", // file name
+ "gdf" // file extention
+ };
cl_options options;
// parse command line arguments
@@ -120,6 +131,17 @@ cl_options read_options(int argc, char** argv) {
options.dt = fopts["dt"];
options.tfinal = fopts["tfinal"];
options.all_to_all = fopts["all_to_all"];
+
+
+ // Parameters for spike output
+ options.spike_file_output = fopts["spike_file_output"];
+ if (options.spike_file_output) {
+ options.single_file_per_rank = fopts["single_file_per_rank"];
+ options.over_write = fopts["over_write"];
+ options.output_path = fopts["output_path"].get<std::string>();;
+ options.file_name = fopts["file_name"].get<std::string>();;
+ options.file_extention = fopts["file_extention"].get<std::string>();;
+ }
}
catch (std::exception& e) {
throw model_description_error(
@@ -135,7 +157,7 @@ cl_options read_options(int argc, char** argv) {
}
std::ostream& operator<<(std::ostream& o, const cl_options& options) {
- o << "simultion options:\n";
+ o << "simulation options:\n";
o << " cells : " << options.cells << "\n";
o << " compartments/segment : " << options.compartments_per_segment << "\n";
o << " synapses/cell : " << options.synapses_per_cell << "\n";
diff --git a/miniapp/io.hpp b/miniapp/io.hpp
index 71af6181..52cca594 100644
--- a/miniapp/io.hpp
+++ b/miniapp/io.hpp
@@ -26,6 +26,14 @@ struct cl_options {
double probe_ratio;
std::string trace_prefix;
util::optional<unsigned> trace_max_gid;
+
+ // Parameters for spike output
+ bool spike_file_output;
+ bool single_file_per_rank;
+ bool over_write;
+ std::string output_path;
+ std::string file_name;
+ std::string file_extention;
};
class usage_error: public std::runtime_error {
diff --git a/miniapp/miniapp.cpp b/miniapp/miniapp.cpp
index 3e9d4dd2..9433ffad 100644
--- a/miniapp/miniapp.cpp
+++ b/miniapp/miniapp.cpp
@@ -62,7 +62,16 @@ int main(int argc, char** argv) {
auto cell_range = distribute_cells(recipe->num_cells());
// build model from recipe
- model_type m(*recipe, cell_range.first, cell_range.second);
+
+ model_type::file_output_parameters model_params(
+ options.spike_file_output,
+ options.single_file_per_rank,
+ options.over_write,
+ options.output_path,
+ options.file_name,
+ options.file_extention
+ );
+ model_type m(*recipe, cell_range.first, cell_range.second, model_params);
// inject some artificial spikes, 1 per 20 neurons.
cell_gid_type spike_cell = 20*((cell_range.first+19)/20);
diff --git a/nrn/generate_validation.sh b/nrn/generate_validation.sh
index 532a7cd7..5f6cbf95 100755
--- a/nrn/generate_validation.sh
+++ b/nrn/generate_validation.sh
@@ -1,5 +1,5 @@
-python2.7 ./soma.py
-python2.7 ./ball_and_stick.py
-python2.7 ./ball_and_3stick.py
-python2.7 ./simple_synapse.py --synapse exp2
-python2.7 ./simple_synapse.py --synapse exp
+python ./soma.py
+python ./ball_and_stick.py
+python ./ball_and_3stick.py
+python ./simple_synapse.py --synapse exp2
+python ./simple_synapse.py --synapse exp
diff --git a/src/communication/export_manager.hpp b/src/communication/export_manager.hpp
index 4211b747..52e73071 100644
--- a/src/communication/export_manager.hpp
+++ b/src/communication/export_manager.hpp
@@ -6,6 +6,7 @@
#include <vector>
#include <memory>
#include <utility>
+#include <string>
#include <spike.hpp>
#include <util.hpp>
@@ -23,23 +24,27 @@ class export_manager {
public:
using time_type = Time;
using spike_type = spike<cell_member_type, time_type>;
- export_manager(bool file_per_rank)
+ export_manager(bool spike_file_output, bool single_file_per_rank, bool over_write,
+ std::string output_path, std::string file_name, std::string file_extention)
{
-
- if (file_per_rank) { // single file per rank
+ if (!spike_file_output)
+ {
+ return;
+ }
+ if (single_file_per_rank) { // single file per rank
rank_exporters_.push_back(
nest::mc::util::make_unique<
nest::mc::communication::exporter_spike_file<Time, CommunicationPolicy> >(
- "rank", "./", "gdf"));
+ file_name, output_path, file_extention, over_write));
}
- if (!file_per_rank) { // single file per simulation
+ if (!single_file_per_rank) { // single file per simulation
single_exporters_.push_back(
nest::mc::util::make_unique<
nest::mc::communication::exporter_spike_single_file<Time, CommunicationPolicy> >(
- "single", "./", "gdf"));
+ file_name, output_path, file_extention, over_write));
}
}
diff --git a/src/communication/exporter_spike_file.hpp b/src/communication/exporter_spike_file.hpp
index 580efa56..033fa97d 100644
--- a/src/communication/exporter_spike_file.hpp
+++ b/src/communication/exporter_spike_file.hpp
@@ -8,6 +8,8 @@
#include <stdexcept>
#include <vector>
+#include <cstring>
+
#include <common_types.hpp>
#include <util.hpp>
#include <spike.hpp>
@@ -40,7 +42,7 @@ public:
std::ifstream f(file_path);
if (f.good()) {
if (!over_write) {
- std::string error_string("Tried opening file for writing but it exists and over_write is false:\n" +
+ std::string error_string("Tried opening file for writing but it exists and over_write is false: " +
file_path);
throw std::runtime_error(error_string);
@@ -48,6 +50,17 @@ public:
std::remove(file_path.c_str());
}
+
+ buffer = new char[length];
+ /*
+ // Manually setting buffer did not help (version 1)
+ //http://stackoverflow.com/questions/12997131/stdfstream-buffering-vs-manual-buffering-why-10x-gain-with-manual-buffering
+
+
+ file_handle_ = nest::mc::util::make_unique<std::ofstream>();
+ file_handle_->rdbuf()->pubsetbuf(buffer, length);
+ file_handle_ ->open(file_path, std::fstream::app);
+ */
file_handle_ = nest::mc::util::make_unique<std::ofstream>(file_path,
std::fstream::app);
@@ -67,9 +80,55 @@ public:
// Does not throw
void do_export() override
{
- for (auto spike : spikes_) {
- *file_handle_ << spike.source.gid << " " << spike.time << std::endl;
+ unsigned current_loc_in_buffer = 0;
+ unsigned nr_chars_written = 0;
+ char single_value_buffer[20]; // Much to big
+
+ // Some constants needed for printing
+ const char * space = " ";
+ const char * endline = "\n";
+
+ for (auto spike : spikes_)
+ {
+ // First the id as output
+ nr_chars_written = std::snprintf(single_value_buffer, 20, "%u",
+ spike.source.gid);
+ std::memcpy(buffer + current_loc_in_buffer, single_value_buffer,
+ nr_chars_written);
+ current_loc_in_buffer += nr_chars_written;
+
+ // The a space
+ std::memcpy(buffer + current_loc_in_buffer, space, 1);
+ current_loc_in_buffer += 1;
+
+ // Then the float
+ nr_chars_written = std::snprintf(single_value_buffer, 20, "%.4f",
+ spike.time);
+ std::memcpy(buffer + current_loc_in_buffer, single_value_buffer,
+ nr_chars_written);
+ current_loc_in_buffer += nr_chars_written;
+
+ // Then the endline
+ std::memcpy(buffer + current_loc_in_buffer, endline, 2);
+ current_loc_in_buffer += 1; // Only a single char in the actual file!!
+
+ // Check if we are nearing the end of our buffer
+ if (current_loc_in_buffer > length - 45)
+ {
+ file_handle_->write(buffer, current_loc_in_buffer);
+ current_loc_in_buffer = 0;
+ }
+ }
+ // also write to buffer at end of the spikes processing
+ if (current_loc_in_buffer != 0)
+ {
+ file_handle_->write(buffer, current_loc_in_buffer);
+ current_loc_in_buffer = 0; // not needed
}
+
+ file_handle_->flush();
+
+
if (!file_handle_->good()){
ok_ = false;
}
@@ -124,6 +183,10 @@ private:
std::vector<spike_type> spikes_;
communication_policy_type communication_policy_;
+
+ char *buffer;
+
+ const unsigned int length = 4096;
};
} //communication
diff --git a/src/model.hpp b/src/model.hpp
index f7516c5c..816b3261 100644
--- a/src/model.hpp
+++ b/src/model.hpp
@@ -23,6 +23,35 @@ namespace mc {
template <typename Cell>
class model {
public:
+
+ // TODO:We need to think how to transport parameters accros the different classes
+ // Move the io to src directory would make it allot easier!!
+ struct file_output_parameters
+ {
+ bool spike_file_output;
+ bool single_file_per_rank;
+ bool over_write;
+ std::string output_path;
+ std::string file_name;
+ std::string file_extention;
+
+ file_output_parameters(bool spike_file_output_i,
+ bool single_file_per_rank_i,
+ bool over_write_i,
+ std::string output_path_i,
+ std::string file_name_i,
+ std::string file_extention_i)
+ :
+ spike_file_output(spike_file_output_i),
+ single_file_per_rank(single_file_per_rank_i),
+ over_write(over_write_i),
+ output_path(output_path_i),
+ file_name(file_name_i),
+ file_extention(file_extention_i)
+ {}
+ };
+
+
using cell_group_type = cell_group<Cell>;
using time_type = typename cell_group_type::time_type;
using value_type = typename cell_group_type::value_type;
@@ -34,7 +63,8 @@ public:
probe_spec probe;
};
- model(const recipe& rec, cell_gid_type cell_from, cell_gid_type cell_to):
+ model(const recipe& rec, cell_gid_type cell_from, cell_gid_type cell_to,
+ file_output_parameters file_output_parameters_i):
cell_from_(cell_from),
cell_to_(cell_to),
communicator_(cell_from, cell_to)
@@ -72,7 +102,13 @@ public:
bool single_file = true;
if (single_file == true) {
- exporter_ = nest::mc::util::make_unique<exporter_manager_type>(false);
+ exporter_ = nest::mc::util::make_unique<exporter_manager_type>(
+ file_output_parameters_i.spike_file_output,
+ file_output_parameters_i.single_file_per_rank,
+ file_output_parameters_i.over_write,
+ file_output_parameters_i.output_path,
+ file_output_parameters_i.file_name,
+ file_output_parameters_i.file_extention);
}
// Allocate an empty queue buffer for each cell group
diff --git a/tests/global_communication/test_exporter_spike_file.cpp b/tests/global_communication/test_exporter_spike_file.cpp
index 7d30877c..12f33001 100644
--- a/tests/global_communication/test_exporter_spike_file.cpp
+++ b/tests/global_communication/test_exporter_spike_file.cpp
@@ -27,7 +27,7 @@ protected:
exporter_spike_file_fixture()
:
- file_name("spikes"),
+ file_name("spikes_exporter_spike_file_fixture"),
path("./"),
extention("gdf"),
index(0)
@@ -36,7 +36,7 @@ protected:
std::string get_standard_file_name()
{
return exporter_type::create_output_file_path(
- file_name, path, extention, index);
+ file_name, path, extention, 0);
}
void SetUp() {
@@ -55,8 +55,8 @@ protected:
TEST_F(exporter_spike_file_fixture, constructor)
{
- exporter_type exporter(file_name,
- path, extention, index);
+
+ exporter_type exporter(file_name, path, extention, true);
// after construction the state of the exporter should be valid
EXPECT_TRUE(exporter.ok());
@@ -64,6 +64,8 @@ TEST_F(exporter_spike_file_fixture, constructor)
//test if the file exist and depending on over_write throw or delete
std::ifstream f(get_standard_file_name());
+
+
EXPECT_TRUE(f.good());
}
@@ -84,9 +86,9 @@ TEST_F(exporter_spike_file_fixture, create_output_file_path)
TEST_F(exporter_spike_file_fixture, do_export)
{
-
+
exporter_type exporter(file_name,
- path, extention, index);
+ path, extention);
// Create some spikes
std::vector<spike_type> spikes;
@@ -100,7 +102,7 @@ TEST_F(exporter_spike_file_fixture, do_export)
// now do the export
exporter.do_export();
-
+
// Test if we have spikes in the file
std::ifstream f(get_standard_file_name());
EXPECT_TRUE(f.good());
diff --git a/tests/global_communication/test_exporter_spike_single_file.cpp b/tests/global_communication/test_exporter_spike_single_file.cpp
index aee5bb9c..8f78aa7c 100644
--- a/tests/global_communication/test_exporter_spike_single_file.cpp
+++ b/tests/global_communication/test_exporter_spike_single_file.cpp
@@ -27,7 +27,7 @@ protected:
exporter_spike_single_file_fixture()
:
- file_name("spikes"),
+ file_name("spikes_exporter_spike_single_file_fixture"),
path("./"),
extention("gdf"),
index(0)
@@ -56,7 +56,7 @@ protected:
TEST_F(exporter_spike_single_file_fixture, constructor)
{
exporter_type exporter(file_name,
- path, extention, index);
+ path, extention);
// after construction the state of the exporter should be valid
EXPECT_TRUE(exporter.ok());
@@ -86,7 +86,7 @@ TEST_F(exporter_spike_single_file_fixture, do_export)
exporter_type exporter(file_name,
- path, extention, index);
+ path, extention);
// Create some spikes
std::vector<spike_type> spikes;
diff --git a/tests/performance/io/disk_io.cpp b/tests/performance/io/disk_io.cpp
index 7cd78737..98868f2b 100644
--- a/tests/performance/io/disk_io.cpp
+++ b/tests/performance/io/disk_io.cpp
@@ -3,7 +3,8 @@
#include <ctime>
#include <fstream>
#include <numeric>
-
+#include <stdio.h>
+#include <cstring>
#include <common_types.hpp>
#include <fvm_cell.hpp>
@@ -26,7 +27,8 @@ using spike_type = communication::exporter_spike_file<time_type,
global_policy>::spike_type;
int main(int argc, char** argv) {
- // Setup the possible mpi environment
+
+ //Setup the possible mpi environment
nest::mc::communication::global_policy_guard global_guard(argc, argv);
// very simple command line parsing
@@ -78,7 +80,8 @@ int main(int argc, char** argv) {
}
// Create the sut
- nest::mc::communication::export_manager<time_type, global_policy> manager(file_per_rank);
+ nest::mc::communication::export_manager<time_type, global_policy> manager(
+ true, file_per_rank, true, "./", "spikes", "gdf");
// We need the nr of ranks to calculate the nr of spikes to produce per
// rank
@@ -98,7 +101,7 @@ int main(int argc, char** argv) {
unsigned simulated_neurons = spikes_per_rank / 20;
for (unsigned idx = 0; idx < spikes_per_rank; ++idx)
{
- spikes.push_back({ { idx % simulated_neurons, 0 }, 0.0f + 1 / ( 0.05 +idx % 20)});
+ spikes.push_back({ { idx % simulated_neurons, 0 }, 0.0f + 1 / ( 0.05f +idx % 20)});
}
std::vector<int> timings;
@@ -160,3 +163,45 @@ int main(int argc, char** argv) {
return 0;
}
+
+
+/*
+
+float time = 1234.56789123455f;
+int id = 123456;
+
+char float_as_char[20]; // absurdly big!!
+char int_as_char[20];
+const char * space = " ";
+const char * endline = "\n";
+
+unsigned nr_chars_float = std::snprintf(float_as_char, 20, "%.4f", time);
+unsigned nr_chars_int = std::snprintf(int_as_char, 20, "%u", id);
+
+
+std::cout << nr_chars_float << "," << float_as_char << std::endl;
+std::cout << nr_chars_int << "," << int_as_char << std::endl;
+
+
+const unsigned int length = 4096;
+char buffer[length];
+unsigned current_loc_in_buffer = 0;
+std::ofstream file("test.txt", std::fstream::app);
+
+std::memcpy(buffer+ current_loc_in_buffer, int_as_char, nr_chars_int);
+current_loc_in_buffer += nr_chars_int;
+
+std::memcpy(buffer + current_loc_in_buffer, space, 1);
+current_loc_in_buffer += 1;
+
+std::memcpy(buffer + current_loc_in_buffer, float_as_char, nr_chars_float);
+current_loc_in_buffer += nr_chars_float;
+
+std::memcpy(buffer + current_loc_in_buffer, endline, 2);
+current_loc_in_buffer += 1; // Only a single char in the actual file!!
+
+
+
+file.write(buffer, current_loc_in_buffer);
+file.close();
+*/
\ No newline at end of file
--
GitLab