File export can now be controlled with json parameters.

Default is false
No output via command line parameters

mechanisms/generate.sh

+#!/etc/bash
+
 #flags="-t cpu -O"
 flags="-t cpu"
 

miniapp/io.cpp

@@ -43,8 +43,19 @@ namespace io {
 cl_options read_options(int argc, char** argv) {
 
     // set default options
+    // TODO: the declaration of this defopts is realistic if we have allot
+    //       more options. We should use a name scheme.
     const cl_options defopts{"", 1000, 500, "expsyn", 100, 100., 0.025, false,
-                             false, 1.0, "trace_", util::nothing};
+                             false, 1.0, "trace_", util::nothing, 
+
+        // spike_output_parameters:
+        false,      // no spike output
+        false,      // single_file_per_simulation
+        true,       // Overwrite outputfile if exists
+        "./",       // output path
+        "spikes",   // file name
+        "gdf"       // file extention                            
+    };
 
     cl_options options;
     // parse command line arguments
@@ -120,6 +131,17 @@ cl_options read_options(int argc, char** argv) {
                 options.dt = fopts["dt"];
                 options.tfinal = fopts["tfinal"];
                 options.all_to_all = fopts["all_to_all"];
+
+
+                // Parameters for spike output
+                options.spike_file_output = fopts["spike_file_output"];
+                if (options.spike_file_output) {
+                    options.single_file_per_rank = fopts["single_file_per_rank"];
+                    options.over_write = fopts["over_write"];
+                    options.output_path = fopts["output_path"].get<std::string>();;
+                    options.file_name = fopts["file_name"].get<std::string>();;
+                    options.file_extention = fopts["file_extention"].get<std::string>();;
+                }
             }
             catch (std::exception& e) {
                 throw model_description_error(
@@ -135,7 +157,7 @@ cl_options read_options(int argc, char** argv) {
 }
 
 std::ostream& operator<<(std::ostream& o, const cl_options& options) {
-    o << "simultion options:\n";
+    o << "simulation options:\n";
     o << "  cells                : " << options.cells << "\n";
     o << "  compartments/segment : " << options.compartments_per_segment << "\n";
     o << "  synapses/cell        : " << options.synapses_per_cell << "\n";

miniapp/io.hpp

@@ -26,6 +26,14 @@ struct cl_options {
     double probe_ratio;
     std::string trace_prefix;
     util::optional<unsigned> trace_max_gid;
+
+    // Parameters for spike output
+    bool spike_file_output;
+    bool single_file_per_rank;
+    bool over_write;
+    std::string output_path;
+    std::string file_name;
+    std::string file_extention;
 };
 
 class usage_error: public std::runtime_error {

miniapp/miniapp.cpp

@@ -62,7 +62,16 @@ int main(int argc, char** argv) {
         auto cell_range = distribute_cells(recipe->num_cells());
 
         // build model from recipe
-        model_type m(*recipe, cell_range.first, cell_range.second);
+
+        model_type::file_output_parameters model_params(
+            options.spike_file_output,
+            options.single_file_per_rank,
+            options.over_write,
+            options.output_path,
+            options.file_name,
+            options.file_extention
+        );
+        model_type m(*recipe, cell_range.first, cell_range.second, model_params);
 
         // inject some artificial spikes, 1 per 20 neurons.
         cell_gid_type spike_cell = 20*((cell_range.first+19)/20);

nrn/generate_validation.sh

-python2.7 ./soma.py
-python2.7 ./ball_and_stick.py
-python2.7 ./ball_and_3stick.py
-python2.7 ./simple_synapse.py --synapse exp2
-python2.7 ./simple_synapse.py --synapse exp
+python ./soma.py
+python ./ball_and_stick.py
+python ./ball_and_3stick.py
+python ./simple_synapse.py --synapse exp2
+python ./simple_synapse.py --synapse exp

src/communication/export_manager.hpp

@@ -6,6 +6,7 @@
 #include <vector>
 #include <memory>
 #include <utility>
+#include <string>
 
 #include <spike.hpp>
 #include <util.hpp>
@@ -23,23 +24,27 @@ class export_manager {
 public:
     using time_type = Time;
     using spike_type = spike<cell_member_type, time_type>;
-    export_manager(bool file_per_rank) 
+    export_manager(bool spike_file_output, bool single_file_per_rank, bool over_write,
+        std::string output_path, std::string file_name, std::string file_extention)
     {
-
-        if (file_per_rank) { // single file per rank
+        if (!spike_file_output)
+        {
+            return;
+        }
+        if (single_file_per_rank) { // single file per rank
 
             rank_exporters_.push_back(
                 nest::mc::util::make_unique<
                     nest::mc::communication::exporter_spike_file<Time, CommunicationPolicy> >(
-                        "rank", "./", "gdf"));
+                        file_name, output_path, file_extention, over_write));
         }
 
 
-        if (!file_per_rank) { // single file per simulation
+        if (!single_file_per_rank) { // single file per simulation
             single_exporters_.push_back(
                 nest::mc::util::make_unique<
                     nest::mc::communication::exporter_spike_single_file<Time, CommunicationPolicy> >(
-                    "single", "./", "gdf"));
+                        file_name, output_path, file_extention, over_write));
         }
     }
 

src/communication/exporter_spike_file.hpp

@@ -8,6 +8,8 @@
 #include <stdexcept>
 #include <vector>
 
+#include <cstring>
+
 #include <common_types.hpp>
 #include <util.hpp>
 #include <spike.hpp>
@@ -40,7 +42,7 @@ public:
         std::ifstream f(file_path);
         if (f.good()) {
             if (!over_write) {
-                std::string error_string("Tried opening file for writing but it exists and over_write is false:\n" +
+                std::string error_string("Tried opening file for writing but it exists and over_write is false: " +
                     file_path);
 
                 throw std::runtime_error(error_string);
@@ -48,6 +50,17 @@ public:
 
             std::remove(file_path.c_str());
         }
+        
+        buffer = new char[length];
+        /*
+        // Manually setting buffer did not help (version 1)
+        //http://stackoverflow.com/questions/12997131/stdfstream-buffering-vs-manual-buffering-why-10x-gain-with-manual-buffering
+
+
+        file_handle_ = nest::mc::util::make_unique<std::ofstream>();
+        file_handle_->rdbuf()->pubsetbuf(buffer, length);
+        file_handle_ ->open(file_path, std::fstream::app);
+        */
 
         file_handle_ = nest::mc::util::make_unique<std::ofstream>(file_path,
                                                        std::fstream::app);
@@ -67,9 +80,55 @@ public:
     // Does not throw
     void do_export() override
     {
-        for (auto spike : spikes_) {
-            *file_handle_ << spike.source.gid << " " << spike.time << std::endl;
+        unsigned current_loc_in_buffer = 0;
+        unsigned nr_chars_written = 0;
+        char single_value_buffer[20]; // Much to big 
+
+        // Some constants needed for printing
+        const char * space = " ";
+        const char * endline = "\n";
+
+        for (auto spike : spikes_) 
+        {
+            // First the id as output
+            nr_chars_written = std::snprintf(single_value_buffer, 20, "%u", 
+                                             spike.source.gid);
+            std::memcpy(buffer + current_loc_in_buffer, single_value_buffer,
+                        nr_chars_written);
+            current_loc_in_buffer += nr_chars_written;
+
+            // The a space
+            std::memcpy(buffer + current_loc_in_buffer, space, 1);
+            current_loc_in_buffer += 1;
+
+            // Then the float
+            nr_chars_written = std::snprintf(single_value_buffer, 20, "%.4f",
+                spike.time);
+            std::memcpy(buffer + current_loc_in_buffer, single_value_buffer,
+                nr_chars_written);
+            current_loc_in_buffer += nr_chars_written;
+
+            // Then the endline
+            std::memcpy(buffer + current_loc_in_buffer, endline, 2);
+            current_loc_in_buffer += 1;  // Only a single char in the actual file!!
+
+            // Check if we are nearing the end of our buffer
+            if (current_loc_in_buffer > length - 45)
+            {
+                file_handle_->write(buffer, current_loc_in_buffer);
+                current_loc_in_buffer = 0;
+            }
+        }
+        // also write to buffer at end of the spikes processing
+        if (current_loc_in_buffer != 0)
+        {
+            file_handle_->write(buffer, current_loc_in_buffer);
+            current_loc_in_buffer = 0; // not needed
         }
+
+        file_handle_->flush();
+
+
         if (!file_handle_->good()){
             ok_ = false;
         }
@@ -124,6 +183,10 @@ private:
     std::vector<spike_type> spikes_;
 
     communication_policy_type communication_policy_;
+
+    char *buffer;
+
+    const unsigned int length = 4096;
 };
 
 } //communication

src/model.hpp

@@ -23,6 +23,35 @@ namespace mc {
 template <typename Cell>
 class model {
 public:
+
+    // TODO:We need to think how to transport parameters accros the different classes
+    // Move the io to src directory would make it allot easier!!
+    struct file_output_parameters
+    {
+        bool spike_file_output;
+        bool single_file_per_rank;
+        bool over_write;
+        std::string output_path;
+        std::string file_name;
+        std::string file_extention;
+
+        file_output_parameters(bool spike_file_output_i,
+            bool single_file_per_rank_i,
+            bool over_write_i,
+            std::string output_path_i,
+            std::string file_name_i,
+            std::string file_extention_i)
+            :
+            spike_file_output(spike_file_output_i),
+            single_file_per_rank(single_file_per_rank_i),
+            over_write(over_write_i),
+            output_path(output_path_i),
+            file_name(file_name_i),
+            file_extention(file_extention_i)
+        {}
+    };
+
+
     using cell_group_type = cell_group<Cell>;
     using time_type = typename cell_group_type::time_type;
     using value_type = typename cell_group_type::value_type;
@@ -34,7 +63,8 @@ public:
         probe_spec probe;
     };
 
-    model(const recipe& rec, cell_gid_type cell_from, cell_gid_type cell_to):
+    model(const recipe& rec, cell_gid_type cell_from, cell_gid_type cell_to,
+        file_output_parameters file_output_parameters_i):
         cell_from_(cell_from),
         cell_to_(cell_to),
         communicator_(cell_from, cell_to)
@@ -72,7 +102,13 @@ public:
 
         bool single_file = true;
         if (single_file == true) {
-            exporter_ = nest::mc::util::make_unique<exporter_manager_type>(false);
+            exporter_ = nest::mc::util::make_unique<exporter_manager_type>(
+                file_output_parameters_i.spike_file_output,
+            file_output_parameters_i.single_file_per_rank,
+            file_output_parameters_i.over_write,
+            file_output_parameters_i.output_path,
+            file_output_parameters_i.file_name,
+            file_output_parameters_i.file_extention);
         }
 
         // Allocate an empty queue buffer for each cell group

tests/global_communication/test_exporter_spike_file.cpp

@@ -27,7 +27,7 @@ protected:
 
     exporter_spike_file_fixture()
         :
-        file_name("spikes"),
+        file_name("spikes_exporter_spike_file_fixture"),
         path("./"),
         extention("gdf"),
         index(0)
@@ -36,7 +36,7 @@ protected:
     std::string get_standard_file_name()
     {
         return exporter_type::create_output_file_path(
-            file_name, path, extention, index);
+            file_name, path, extention, 0);
     }
 
     void SetUp() {
@@ -55,8 +55,8 @@ protected:
 
 TEST_F(exporter_spike_file_fixture, constructor)
 {
-    exporter_type exporter(file_name,
-        path, extention, index);
+    
+    exporter_type exporter(file_name, path, extention, true);
 
     // after construction the state of the exporter should be valid
     EXPECT_TRUE(exporter.ok());
@@ -64,6 +64,8 @@ TEST_F(exporter_spike_file_fixture, constructor)
 
     //test if the file exist and depending on over_write throw or delete
     std::ifstream f(get_standard_file_name());
+
+    
     EXPECT_TRUE(f.good());
 }
 
@@ -84,9 +86,9 @@ TEST_F(exporter_spike_file_fixture, create_output_file_path)
 TEST_F(exporter_spike_file_fixture, do_export)
 {
     
-
+    
     exporter_type exporter(file_name,
-        path, extention, index);
+        path, extention);
 
     // Create some spikes
     std::vector<spike_type> spikes;
@@ -100,7 +102,7 @@ TEST_F(exporter_spike_file_fixture, do_export)
 
     // now do the export
     exporter.do_export();
-
+    
     // Test if we have spikes in the file
     std::ifstream f(get_standard_file_name());
     EXPECT_TRUE(f.good());

tests/global_communication/test_exporter_spike_single_file.cpp

@@ -27,7 +27,7 @@ protected:
 
     exporter_spike_single_file_fixture()
         :
-        file_name("spikes"),
+        file_name("spikes_exporter_spike_single_file_fixture"),
         path("./"),
         extention("gdf"),
         index(0)
@@ -56,7 +56,7 @@ protected:
 TEST_F(exporter_spike_single_file_fixture, constructor)
 {
     exporter_type exporter(file_name,
-        path, extention, index);
+        path, extention);
 
     // after construction the state of the exporter should be valid
     EXPECT_TRUE(exporter.ok());
@@ -86,7 +86,7 @@ TEST_F(exporter_spike_single_file_fixture, do_export)
     
 
     exporter_type exporter(file_name,
-        path, extention, index);
+        path, extention);
 
     // Create some spikes
     std::vector<spike_type> spikes;

tests/performance/io/disk_io.cpp

@@ -3,7 +3,8 @@
 #include <ctime>
 #include <fstream>
 #include <numeric>
-
+#include <stdio.h>
+#include <cstring>
 
 #include <common_types.hpp>
 #include <fvm_cell.hpp>
@@ -26,7 +27,8 @@ using spike_type = communication::exporter_spike_file<time_type,
     global_policy>::spike_type;
 
 int main(int argc, char** argv) {
-    // Setup the possible mpi environment
+
+    //Setup the possible mpi environment
     nest::mc::communication::global_policy_guard global_guard(argc, argv);
 
     // very simple command line parsing
@@ -78,7 +80,8 @@ int main(int argc, char** argv) {
     }
 
     // Create the sut  
-    nest::mc::communication::export_manager<time_type, global_policy> manager(file_per_rank);
+    nest::mc::communication::export_manager<time_type, global_policy> manager(
+    true, file_per_rank, true, "./", "spikes", "gdf");
 
     // We need the nr of ranks to calculate the nr of spikes to produce per
     // rank
@@ -98,7 +101,7 @@ int main(int argc, char** argv) {
     unsigned simulated_neurons = spikes_per_rank / 20;
     for (unsigned idx = 0; idx < spikes_per_rank; ++idx) 
     {
-        spikes.push_back({ { idx % simulated_neurons, 0 }, 0.0f + 1 / ( 0.05 +idx % 20)});
+        spikes.push_back({ { idx % simulated_neurons, 0 }, 0.0f + 1 / ( 0.05f +idx % 20)});
     }
 
     std::vector<int> timings;
@@ -160,3 +163,45 @@ int main(int argc, char** argv) {
 
     return 0;
 }
+
+
+/*
+
+float time = 1234.56789123455f;
+int id = 123456;
+
+char float_as_char[20];  // absurdly big!!
+char int_as_char[20];
+const char * space = " ";
+const char * endline = "\n";
+
+unsigned nr_chars_float = std::snprintf(float_as_char, 20, "%.4f", time);
+unsigned nr_chars_int = std::snprintf(int_as_char, 20, "%u", id);
+
+
+std::cout << nr_chars_float << "," << float_as_char << std::endl;
+std::cout << nr_chars_int << "," << int_as_char << std::endl;
+
+
+const unsigned int length = 4096;
+char  buffer[length];
+unsigned current_loc_in_buffer = 0;
+std::ofstream file("test.txt", std::fstream::app);
+
+std::memcpy(buffer+ current_loc_in_buffer, int_as_char, nr_chars_int);
+current_loc_in_buffer += nr_chars_int;
+
+std::memcpy(buffer + current_loc_in_buffer, space, 1);
+current_loc_in_buffer += 1;
+
+std::memcpy(buffer + current_loc_in_buffer, float_as_char, nr_chars_float);
+current_loc_in_buffer += nr_chars_float;
+
+std::memcpy(buffer + current_loc_in_buffer, endline, 2);
+current_loc_in_buffer += 1;  // Only a single char in the actual file!!
+
+
+
+file.write(buffer, current_loc_in_buffer);
+file.close();
+*/
\ No newline at end of file