From b53112e5e70fdecbc1ceb391388048dee22704ef Mon Sep 17 00:00:00 2001
From: "w.klijn" <nonoice@gmail.com>
Date: Thu, 11 Aug 2016 16:38:44 +0200
Subject: [PATCH] Cleanup of the performance test
---
src/communication/exporter_spike_file.hpp | 2 +-
src/model.hpp | 11 +-
.../test_exporter_spike_file.cpp | 4 +-
tests/performance/io/CMakeLists.txt | 2 +-
tests/performance/io/disk_io.cpp | 125 +++++-------------
tests/performance/io/disk_io.py | 6 +-
6 files changed, 38 insertions(+), 112 deletions(-)
diff --git a/src/communication/exporter_spike_file.hpp b/src/communication/exporter_spike_file.hpp
index 5c0297cf..7b94c1b6 100644
--- a/src/communication/exporter_spike_file.hpp
+++ b/src/communication/exporter_spike_file.hpp
@@ -149,7 +149,7 @@ private:
// Buffer (and size) for raw output of spikes
char *buffer;
- const unsigned int length = 4096;
+ const unsigned int length = 32768;
};
} //communication
diff --git a/src/model.hpp b/src/model.hpp
index 5cf4a71f..9ad1387f 100644
--- a/src/model.hpp
+++ b/src/model.hpp
@@ -51,7 +51,6 @@ public:
{}
};
-
using cell_group_type = cell_group<Cell>;
using time_type = typename cell_group_type::time_type;
using value_type = typename cell_group_type::value_type;
@@ -100,16 +99,13 @@ public:
}
communicator_.construct();
- bool single_file = true;
- if (single_file == true) {
- exporter_ = nest::mc::util::make_unique<exporter_manager_type>(
- file_output_parameters_i.spike_file_output,
+ exporter_ = nest::mc::util::make_unique<exporter_manager_type>(
+ file_output_parameters_i.spike_file_output,
file_output_parameters_i.single_file_per_rank,
file_output_parameters_i.over_write,
file_output_parameters_i.output_path,
file_output_parameters_i.file_name,
file_output_parameters_i.file_extention);
- }
// Allocate an empty queue buffer for each cell group
// These must be set initially to ensure that a queue is available for each
@@ -172,12 +168,12 @@ public:
PE("stepping", "exchange");
auto local_spikes = previous_spikes().gather();
future_events() = communicator_.exchange(local_spikes,
+ // send the exporter function as pointers to export
[&](const std::vector<spike_type>& spikes) { exporter_->do_export_local(spikes); },
[&] (const std::vector<spike_type>& spikes){ exporter_->do_export_global(spikes); });
PL(2);
};
-
// run the tasks, overlapping if the threading model and number of
// available threads permits it.
threading::task_group g;
@@ -229,7 +225,6 @@ private:
util::double_buffer< local_spike_store_type > local_spikes_;
using exporter_manager_type = nest::mc::communication::export_manager<time_type, communication::global_policy>;
-
std::unique_ptr<exporter_manager_type> exporter_;
// Convenience functions that map the spike buffers and event queues onto
// the appropriate integration interval.
diff --git a/tests/global_communication/test_exporter_spike_file.cpp b/tests/global_communication/test_exporter_spike_file.cpp
index b7d9b893..f9702926 100644
--- a/tests/global_communication/test_exporter_spike_file.cpp
+++ b/tests/global_communication/test_exporter_spike_file.cpp
@@ -51,9 +51,7 @@ protected:
}
~exporter_spike_file_fixture()
- {
-
- }
+ {}
};
TEST_F(exporter_spike_file_fixture, constructor)
diff --git a/tests/performance/io/CMakeLists.txt b/tests/performance/io/CMakeLists.txt
index 861b4e41..5d614939 100644
--- a/tests/performance/io/CMakeLists.txt
+++ b/tests/performance/io/CMakeLists.txt
@@ -15,4 +15,4 @@ if(WITH_MPI)
endif()
# Copy the python file that drives the performance tests and produces the output
-file(COPY disk_io.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
\ No newline at end of file
+file(COPY disk_io.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/tests/performance/io/disk_io.cpp b/tests/performance/io/disk_io.cpp
index e0a2c7d7..f023374f 100644
--- a/tests/performance/io/disk_io.cpp
+++ b/tests/performance/io/disk_io.cpp
@@ -1,22 +1,20 @@
-#include <iostream>
#include <cstdlib>
#include <ctime>
#include <fstream>
+#include <iostream>
#include <numeric>
#include <stdio.h>
#include <cstring>
-#include <common_types.hpp>
-#include <fvm_cell.hpp>
#include <cell.hpp>
#include <cell_group.hpp>
+#include <common_types.hpp>
+#include <fvm_cell.hpp>
#include <communication/communicator.hpp>
#include <communication/global_policy.hpp>
#include <communication/export_manager.hpp>
-
-
using namespace nest::mc;
using global_policy = communication::global_policy;
@@ -26,15 +24,14 @@ using time_type = typename cell_group_type::time_type;
using spike_type = communication::exporter_spike_file<time_type,
global_policy>::spike_type;
-int main(int argc, char** argv) {
-
+int main(int argc, char** argv)
+{
//Setup the possible mpi environment
nest::mc::communication::global_policy_guard global_guard(argc, argv);
// very simple command line parsing
- if (argc < 3)
- {
- std::cout << "disk_io <int nrspikes> <int nr_repeats> <file_per_rank (true|false)> [simple_output (false|true)]"
+ if (argc < 3) {
+ std::cout << "disk_io <int nrspikes> <int nr_repeats> <file_per_rank (true|false)> [simple_output (false|true)]"
<< " Simple performance test runner for the exporter manager"
<< " It exports nrspikes nr_repeats using the export_manager and will produce"
<< " the total, mean and std of the time needed to perform the output to disk"
@@ -47,33 +44,29 @@ int main(int argc, char** argv) {
}
int nr_spikes = atoi(argv[1]);
- if (nr_spikes == 0)
- {
+ if (nr_spikes == 0) {
std::cout << "disk_io <nrspikes>" << std::endl;
std::cout << " nrspikes should be a valid integer higher then zero" << std::endl;
exit(1);
}
int nr_repeats = atoi(argv[2]);
- if (nr_repeats == 0)
- {
+ if (nr_repeats == 0) {
std::cout << "disk_io <nrspikes>" << std::endl;
std::cout << " nr_repeats should be a valid integer higher then zero" << std::endl;
exit(1);
}
-
+
bool file_per_rank = false;
std::string single(argv[3]);
- if (single == std::string("true"))
- {
+ if (single == std::string("true")) {
file_per_rank = true;
}
bool simple_stats = false;
- if (argc == 5)
- {
+ if (argc == 5) {
std::string simple(argv[4]);
- if (simple == std::string("true"))
+ if (simple == std::string("true"))
{
simple_stats = true;
}
@@ -81,27 +74,28 @@ int main(int argc, char** argv) {
// Create the sut
nest::mc::communication::export_manager<time_type, global_policy> manager(
- true, file_per_rank, true, "./", "spikes", "gdf");
+ true, file_per_rank, true, "./", "spikes", "gdf");
// We need the nr of ranks to calculate the nr of spikes to produce per
// rank
global_policy communication_policy;
unsigned nr_ranks = communication_policy.size();
- unsigned spikes_per_rank = nr_spikes / nr_ranks;
+ unsigned spikes_per_rank = nr_spikes / nr_ranks;
// Create a set of spikes
std::vector<spike_type> spikes;
// *********************************************************************
- // To have a realworld data set we get the average spikes per number
- // and use that to get the nr of 'simulated' neurons, and create idxs
- // using this value.
- // Also assume that we have only a single second of simulated time
- // All spike times should be between 0.0 and 1.0:
+ // To have a somewhat realworld data set we calculate from the nr of spikes
+ // (assuming 20 hz average) the number of nr of 'simulated' neurons,
+ // and create idxs using this value. The number of chars in the number
+ // influences the size of the output and thus the speed
+ // Also taken that we have only a single second of simulated time
+ // all spike times should be between 0.0 and 1.0:
unsigned simulated_neurons = spikes_per_rank / 20;
- for (unsigned idx = 0; idx < spikes_per_rank; ++idx)
- {
- spikes.push_back({ { idx % simulated_neurons, 0 }, 0.0f + 1 / ( 0.05f +idx % 20)});
+ for (unsigned idx = 0; idx < spikes_per_rank; ++idx) {
+ spikes.push_back({ { idx % simulated_neurons, 0 }, // correct idx
+ 0.0f + 1 / (0.05f + idx % 20) }); // semi random float
}
std::vector<int> timings;
@@ -109,28 +103,16 @@ int main(int argc, char** argv) {
int time_total = 0;
// now output to disk nr_repeats times, while keeping track of the times
- for (int idx = 0; idx < nr_repeats; ++idx)
- {
+ for (int idx = 0; idx < nr_repeats; ++idx) {
int time_start = clock();
manager.do_export_local(spikes);
-
int time_stop = clock();
int run_time = (time_stop - time_start);
time_total += run_time;
timings.push_back(run_time);
}
-
-
- // Autoput the individual timings to a comma seperated file
- std::ofstream file("file_io_results.csv");
- file << *timings.begin() / double(CLOCKS_PER_SEC) * 1000;
- for (auto time_entry = timings.begin()++; time_entry < timings.end(); ++time_entry)
- {
- file << "," << *time_entry / double(CLOCKS_PER_SEC) * 1000 ;
- }
- file << std::endl;
// Calculate some statistics
double sum = std::accumulate(timings.begin(), timings.end(), 0.0);
@@ -142,20 +124,17 @@ int main(int argc, char** argv) {
double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
double stdev = std::sqrt(sq_sum / timings.size());
- if (communication_policy.id() != 0)
- {
+ if (communication_policy.id() != 0) {
return 0;
}
// and output
- if (simple_stats)
- {
- std::cout << time_total / double(CLOCKS_PER_SEC) * 1000 << "," <<
- mean / double(CLOCKS_PER_SEC) * 1000 << "," <<
- stdev / double(CLOCKS_PER_SEC) * 1000;
+ if (simple_stats) {
+ std::cout << time_total / double(CLOCKS_PER_SEC) * 1000 << "," <<
+ mean / double(CLOCKS_PER_SEC) * 1000 << "," <<
+ stdev / double(CLOCKS_PER_SEC) * 1000;
}
- else
- {
+ else {
std::cout << "total time (ms): " << time_total / double(CLOCKS_PER_SEC) * 1000 << std::endl;
std::cout << "mean time (ms): " << mean / double(CLOCKS_PER_SEC) * 1000 << std::endl;
std::cout << "stdev time (ms): " << stdev / double(CLOCKS_PER_SEC) * 1000 << std::endl;
@@ -163,45 +142,3 @@ int main(int argc, char** argv) {
return 0;
}
-
-
-/*
-
-float time = 1234.56789123455f;
-int id = 123456;
-
-char float_as_char[20]; // absurdly big!!
-char int_as_char[20];
-const char * space = " ";
-const char * endline = "\n";
-
-unsigned nr_chars_float = std::snprintf(float_as_char, 20, "%.4f", time);
-unsigned nr_chars_int = std::snprintf(int_as_char, 20, "%u", id);
-
-
-std::cout << nr_chars_float << "," << float_as_char << std::endl;
-std::cout << nr_chars_int << "," << int_as_char << std::endl;
-
-
-const unsigned int length = 4096;
-char buffer[length];
-unsigned current_loc_in_buffer = 0;
-std::ofstream file("test.txt", std::fstream::app);
-
-std::memcpy(buffer+ current_loc_in_buffer, int_as_char, nr_chars_int);
-current_loc_in_buffer += nr_chars_int;
-
-std::memcpy(buffer + current_loc_in_buffer, space, 1);
-current_loc_in_buffer += 1;
-
-std::memcpy(buffer + current_loc_in_buffer, float_as_char, nr_chars_float);
-current_loc_in_buffer += nr_chars_float;
-
-std::memcpy(buffer + current_loc_in_buffer, endline, 2);
-current_loc_in_buffer += 1; // Only a single char in the actual file!!
-
-
-
-file.write(buffer, current_loc_in_buffer);
-file.close();
-*/
\ No newline at end of file
diff --git a/tests/performance/io/disk_io.py b/tests/performance/io/disk_io.py
index 23b41fae..d813c7e3 100644
--- a/tests/performance/io/disk_io.py
+++ b/tests/performance/io/disk_io.py
@@ -1,9 +1,7 @@
+import matplotlib.pyplot as plt
import subprocess
import os
-import matplotlib.pyplot as plt
-
-
current_script_dir = os.path.dirname(os.path.abspath(__file__))
spikes_to_save = 100000
@@ -30,8 +28,6 @@ for n_rank in range_nr_rank:
print ( "performed test for n_rank= " + str(n_rank))
-
-
print ( range_nr_rank )
print ( mean )
print ( std )
--
GitLab