From 68fb568893275d088618c5540fbd858bad83818a Mon Sep 17 00:00:00 2001
From: Dilawar Singh <dilawars@ncbs.res.in>
Date: Mon, 20 May 2019 09:55:55 +0530
Subject: [PATCH] Squashed 'moose-core/' changes from 0f34490..1b38d64
1b38d64 Merge pull request #356 from hrani/master
a5e0e95 Fixes to issue #360. Regression caused by #352 . (#361)
58292d6 Fix for typo in socket implementation. (#359)
d3c2c5f clean up with color conversion to hex values
473e29d function import was missing
a812045 Merge branch 'master' of http://github.com/hrani/moose-core
f723a63 getCordColor is unused function removing
1dfbbeb Merge branch 'master' into master
9c4458f HOTFIX #357 | Cleaning up HDF5 related macros in cmake. (#358)
9b8f038 converting dictionary compatible to python3
bfde5cc travis complaining of same function name
ddcbd4d adding pickle file rainbow2.pkl for mapping colors from genesis
2ea935b syntax cleanup
d9d36b8 cleanedup for python3 as well while loading pickle file
09d5239 readKkit.cpp : create annotator field for kinetic, readSBML.py: validator flag is set to on from true,groupname and groupid is passed, if both are missing then its not a valid sbml file, textcolor is set now, validator.py: added errormsg at validator, writeSBML.py: from chemconnectUtil getcolor is called and also group width and height is written if it comes from GUI, chemConnectUtil.py: getColor,validcolorcheck,getRandcolor moved from gui to core, writeKkit.py: text color might be a rgb which should go as string in kkit file, moose/utils.py: cleaned mooseaddchemsolver to mooseAddChemSolver, moose.py: solver is set if passed with moose.mooseReadSBML(), Annotator.h,cpp: added field height and width for group, rainbow.pkl added file
f227bb0 Travis Fix | Don't run streamer multitab tests on OSX/Travis. (#354)
a1a256b Efficient Network Streamer for MOOSE (#352)
git-subtree-dir: moose-core
git-subtree-split: 1b38d64f451a6b5b0d6caf610376d6ba3ba93cb6
---
.travis.yml | 10 +-
.travis/travis_build_linux.sh | 14 +-
.travis/travis_build_osx.sh | 4 +-
.travis/travis_prepare_linux.sh | 5 +-
CMakeLists.txt | 66 ---
basecode/ObjId.cpp | 5 +-
basecode/header.h | 1 -
builtins/CMakeLists.txt | 49 ++
builtins/MooseSocketInfo.h | 83 +++
builtins/SocketStreamer.cpp | 296 +++++-----
builtins/SocketStreamer.h | 47 +-
builtins/Streamer.cpp | 53 +-
builtins/Streamer.h | 4 +
builtins/StreamerBase.cpp | 8 +-
builtins/Table.cpp | 76 ++-
builtins/Table.h | 12 +-
kinetics/ReadKkit.cpp | 5 +-
pymoose/moosemodule.cpp | 28 +-
pymoose/vec.cpp | 12 +-
python/moose/SBML/readSBML.py | 63 ++-
python/moose/SBML/validation.py | 16 +-
python/moose/SBML/writeSBML.py | 61 ++-
python/moose/chemUtil/chemConnectUtil.py | 99 +++-
python/moose/chemUtil/rainbow2.pkl | 514 ++++++++++++++++++
python/moose/genesis/writeKkit.py | 24 +-
python/moose/helper.py | 83 +++
python/moose/moose.py | 24 +-
python/moose/server.py | 333 ++++++++----
python/moose/streamer_utils.py | 58 ++
python/moose/utils.py | 6 +-
python/setup.cmake.py | 2 +-
scheduling/Clock.cpp | 8 +-
shell/Shell.cpp | 109 ++--
.../testDisabled_socket_streamer_tcp.py | 106 ++++
tests/python/test_socket_streamer_multitab.py | 148 +++++
tests/python/test_socket_streamer_tcp.py | 182 -------
tests/python/test_socket_streamer_uds.py | 112 ++--
tests/python/test_streamer.py | 40 +-
tests/python/test_table_streaming_support.py | 2 +-
utility/Annotator.cpp | 37 +-
utility/Annotator.h | 6 +
utility/CMakeLists.txt | 1 +
utility/fileutils.cpp | 26 +
utility/print_function.hpp | 5 +-
utility/setupenv.cpp | 1 +
utility/simple_assert.hpp | 6 +-
utility/strutil.h | 1 +
utility/testing_macros.hpp | 18 +-
utility/utility.h | 3 +
49 files changed, 2011 insertions(+), 861 deletions(-)
create mode 100644 builtins/MooseSocketInfo.h
create mode 100644 python/moose/chemUtil/rainbow2.pkl
create mode 100644 python/moose/helper.py
create mode 100644 python/moose/streamer_utils.py
create mode 100644 tests/python/testDisabled_socket_streamer_tcp.py
create mode 100644 tests/python/test_socket_streamer_multitab.py
delete mode 100644 tests/python/test_socket_streamer_tcp.py
create mode 100644 utility/fileutils.cpp
diff --git a/.travis.yml b/.travis.yml
index c4aa5e91..2f37437c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,16 +1,12 @@
language: cpp
sudo: required
-group: travis_lts
-dist: xenial
-
-os:
- - linux
- - osx
matrix:
include:
+ - os : linux
+ dist: xenial
- os : osx
- osx_image: xcode10
+ osx_image: xcode10.1
notifications:
email:
diff --git a/.travis/travis_build_linux.sh b/.travis/travis_build_linux.sh
index 9cbc8075..eb36a324 100755
--- a/.travis/travis_build_linux.sh
+++ b/.travis/travis_build_linux.sh
@@ -45,7 +45,7 @@ echo "Currently in `pwd`"
(
mkdir -p _GSL_BUILD && cd _GSL_BUILD
cmake -DDEBUG=ON -DPYTHON_EXECUTABLE="$PYTHON2" ..
- $MAKE && ctest --output-on-failure
+ $MAKE && ctest --output-on-failure -E ".*socket_streamer.*"
sudo make install && cd /tmp
$PYTHON2 -c 'import moose;print(moose.__file__);print(moose.version())'
)
@@ -58,16 +58,16 @@ if type $PYTHON3 > /dev/null; then
sudo apt-get install -qq python3-networkx || echo "Error with apt"
# GSL.
(
- mkdir -p _GSL_BUILD2 && cd _GSL_BUILD2 && \
- cmake -DPYTHON_EXECUTABLE="$PYTHON3" -DDEBUG=ON ..
- $MAKE && ctest --output-on-failure
+ mkdir -p _GSL_BUILD_PY3 && cd _GSL_BUILD_PY3 && \
+ cmake -DWITH_NSDF=ON -DPYTHON_EXECUTABLE="$PYTHON3" -DDEBUG=ON ..
+ $MAKE && ctest --output-on-failure -E ".*socket_streamer.*"
)
# BOOST
(
- mkdir -p _BOOST_BUILD2 && cd _BOOST_BUILD2 && \
- cmake -DWITH_BOOST_ODE=ON -DPYTHON_EXECUTABLE="$PYTHON3" ..
- $MAKE && ctest --output-on-failure
+ mkdir -p _BOOST_BUILD_PY3 && cd _BOOST_BUILD_PY3 && \
+ cmake -DWITH_NSDF=ON -DWITH_BOOST_ODE=ON -DPYTHON_EXECUTABLE="$PYTHON3" ..
+ $MAKE && ctest --output-on-failure -E ".*socket_streamer.*"
)
echo "All done"
else
diff --git a/.travis/travis_build_osx.sh b/.travis/travis_build_osx.sh
index 7e947674..bb1fffe0 100755
--- a/.travis/travis_build_osx.sh
+++ b/.travis/travis_build_osx.sh
@@ -35,14 +35,14 @@ set -e
&& cmake -DDEBUG=ON \
-DPYTHON_EXECUTABLE=`which python` ..
make pylint -j3
- make && ctest --output-on-failure
+ make && ctest --output-on-failure -E ".*socket_streamer.*"
cd .. # Now with boost.
mkdir -p _BOOST_BUILD && cd _BOOST_BUILD \
&& cmake -DWITH_BOOST_ODE=ON -DDEBUG=ON \
-DPYTHON_EXECUTABLE=`which python` ..
- make && ctest --output-on-failure
+ make && ctest --output-on-failure -E ".*socket_streamer.*"
cd ..
set +e
diff --git a/.travis/travis_prepare_linux.sh b/.travis/travis_prepare_linux.sh
index 37f5dd33..49afa941 100755
--- a/.travis/travis_prepare_linux.sh
+++ b/.travis/travis_prepare_linux.sh
@@ -27,11 +27,14 @@ apt-get install -qq python3-lxml python-lxml
apt-get install -qq python3-numpy python3-matplotlib python3-dev
apt-get install -qq python-pip python3-pip
apt-get install -qq python-tk python3-tk
-apt-get install -qq libgraphviz-dev
+apt-get install -qq libgraphviz-dev
# Gsl
apt-get install -qq libgsl0-dev || apt-get install -qq libgsl-dev
+# hdf5
+apt-get install -qq libhdf5-serial-dev
+
# Boost related.
apt-get install -qq liblapack-dev
apt-get install -qq libboost-all-dev
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b30ca55d..0fa33eb6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -181,72 +181,6 @@ if(WITH_BOOST_ODE)
add_definitions(-DUSE_BOOST_ODE -UUSE_GSL)
endif()
-# NSDF5 support. Disabled by default.
-if(WITH_NSDF)
- find_package(HDF5 COMPONENTS CXX HL)
- if(NOT HDF5_FOUND)
- message(
- "==================================================================\n"
- " HDF5 not found. Disabling NSDF support.\n\n"
- " If you need NSDF support, please install hdf5-dev or hdf5-devel\n"
- " package or equivalent.\n\n"
- " $ sudo apt-get install libhdf5-dev \n"
- " $ sudo yum install libhdf5-devel \n"
- " $ brew install hdf5 \n\n"
- " Otherwise, continue with 'make' and 'make install' \n"
- " If you install hdf5 to non-standard path, export environment \n"
- " variable HDF5_ROOT to the location. Rerun cmake \n"
- "================================================================ \n"
- )
- endif(NOT HDF5_FOUND)
-
- if(HDF5_FOUND)
- include_directories( ${HDF5_INCLUDE_DIRS} )
- add_definitions( -DUSE_HDF5 -DENABLE_NSDF )
- if(HDF5_USE_STATIC_LIBRARIES)
- message(STATUS "Finding static HDF5 libraries in $ENV{HDF5_ROOT}")
- find_library(HDF5_CXX_LIBRARIES NAMES libhdf5.a
- PATHS $ENV{HDF5_ROOT}/lib $ENV{HDF5_ROOT}/lib64
- )
- find_library(HDF5_HL_LIBRARIES NAMES libhdf5_hl.a
- PATHS $ENV{HDF5_ROOT}/lib $ENV{HDF5_ROOT}/lib64
- )
- set(HDF5_LIBRARIES ${HDF5_CXX_LIBRARIES} ${HDF5_HL_LIBRARIES})
- endif()
-
-
- # Make sure, HDF5_HL_LIBRARIES are set. The COMPONENTS in find_package may
- # or may not work. See BhallaLab/moose-core#163.
- if(NOT HDF5_HL_LIBRARIES)
- set(HDF5_HL_LIBRARIES ${HDF5_HL_LIBRARIES})
- endif(NOT HDF5_HL_LIBRARIES)
- list(APPEND HDF5_LIBRARIES ${HDF5_HL_LIBRARIES})
-
- # message(STATUS "MOOSE will use following HDF5 ${HDF5_LIBRARIES}" )
- foreach(HDF5_LIB ${HDF5_LIBRARIES})
- if(HDF5_LIB)
- get_filename_component( HDF5_LIB_EXT ${HDF5_LIB} EXT )
- if(HDF5_LIB_EXT)
- if(${HDF5_LIB_EXT} STREQUAL ".a")
- list(APPEND STATIC_LIBRARIES ${HDF5_LIB} )
- else( )
- list(APPEND SYSTEM_SHARED_LIBS ${HDF5_LIB} )
- endif( )
- endif()
- endif( )
- endforeach( )
- else( HDF5_FOUND )
- message(STATUS "HDF5 is not found" )
- endif( HDF5_FOUND )
- # This is a fix for new HDF5 package on Debian/Ubuntu which installs hdf5
- # headers in non-standard path. issue #80.
- if(HDF5_LIBRARY_DIRS)
- set_target_properties( libmoose PROPERTIES LINK_FLAGS "-L${HDF5_LIBRARY_DIRS}" )
- endif()
-else(WITH_NSDF)
- message(STATUS "NSDF support is disabled" )
-endif(WITH_NSDF)
-
# Openmpi
if(WITH_MPI)
find_package(MPI REQUIRED)
diff --git a/basecode/ObjId.cpp b/basecode/ObjId.cpp
index 9c1c8231..540bc4be 100644
--- a/basecode/ObjId.cpp
+++ b/basecode/ObjId.cpp
@@ -112,8 +112,9 @@ bool ObjId::isOffNode() const
char* ObjId::data() const
{
- return id.element()->data( id.element()->rawIndex( dataIndex ),
- fieldIndex );
+ return id.element()->data(
+ id.element()->rawIndex( dataIndex ), fieldIndex
+ );
}
string ObjId::path() const
diff --git a/basecode/header.h b/basecode/header.h
index 22cb6fc2..97a8bdf4 100644
--- a/basecode/header.h
+++ b/basecode/header.h
@@ -99,7 +99,6 @@ class Neutral;
#include "SharedFinfo.h"
#include "FieldElementFinfo.h"
#include "FieldElement.h"
-#include "../builtins/Streamer.h"
#include "../shell/Neutral.h"
diff --git a/builtins/CMakeLists.txt b/builtins/CMakeLists.txt
index 3df4cd96..8b96a98e 100644
--- a/builtins/CMakeLists.txt
+++ b/builtins/CMakeLists.txt
@@ -1,6 +1,52 @@
cmake_minimum_required(VERSION 2.8)
include( ${CMAKE_CURRENT_SOURCE_DIR}/../CheckCXXCompiler.cmake )
+# NSDF5 support. Disabled by default.
+if(WITH_NSDF)
+ find_package(HDF5 COMPONENTS CXX HL)
+ if(NOT HDF5_FOUND)
+ message(
+ "==================================================================\n"
+ " HDF5 not found. Disabling NSDF support.\n\n"
+ " If you need NSDF support, please install hdf5-dev or hdf5-devel\n"
+ " package or equivalent.\n\n"
+ " $ sudo apt-get install libhdf5-dev \n"
+ " $ sudo yum install libhdf5-devel \n"
+ " $ brew install hdf5 \n\n"
+ " Otherwise, continue with 'make' and 'make install' \n"
+ " If you install hdf5 to non-standard path, export environment \n"
+ " variable HDF5_ROOT to the location. Rerun cmake \n"
+ "================================================================ \n"
+ )
+ endif(NOT HDF5_FOUND)
+
+ if(HDF5_FOUND)
+ include_directories(${HDF5_CXX_INCLUDE_DIRS} )
+ add_definitions(-DUSE_HDF5 -DENABLE_NSDF )
+ add_definitions(${HDF5_CXX_DEFINITIONS})
+ if(HDF5_USE_STATIC_LIBRARIES)
+ message(STATUS "Finding static HDF5 libraries in $ENV{HDF5_ROOT}")
+ find_library(HDF5_CXX_LIBRARIES NAMES libhdf5.a
+ PATHS $ENV{HDF5_ROOT}/lib $ENV{HDF5_ROOT}/lib64
+ )
+ find_library(HDF5_HL_LIBRARIES NAMES libhdf5_hl.a
+ PATHS $ENV{HDF5_ROOT}/lib $ENV{HDF5_ROOT}/lib64
+ )
+ set(HDF5_LIBRARIES ${HDF5_CXX_LIBRARIES} ${HDF5_HL_LIBRARIES})
+ endif()
+
+ # Make sure, HDF5_HL_LIBRARIES are set. The COMPONENTS in find_package may
+ # or may not work. See BhallaLab/moose-core#163.
+ if(NOT HDF5_HL_LIBRARIES)
+ set(HDF5_HL_LIBRARIES ${HDF5_HL_LIBRARIES})
+ endif(NOT HDF5_HL_LIBRARIES)
+ list(APPEND HDF5_LIBRARIES ${HDF5_HL_LIBRARIES})
+
+ else(HDF5_FOUND)
+ message(STATUS "HDF5 is not found. Disabling NSDF support." )
+ endif( HDF5_FOUND )
+endif(WITH_NSDF)
+
set(SRCS
Arith.cpp
Group.cpp
@@ -33,3 +79,6 @@ if(WITH_NSDF AND HDF5_FOUND)
endif()
add_library(moose_builtins ${SRCS} )
+if(WITH_NSDF AND HDF5_FOUND)
+ target_link_libraries(moose_builtins ${HDF5_CXX_LIBRARIES} ${HDF5_HL_LIBRARIES})
+endif()
diff --git a/builtins/MooseSocketInfo.h b/builtins/MooseSocketInfo.h
new file mode 100644
index 00000000..9e825eec
--- /dev/null
+++ b/builtins/MooseSocketInfo.h
@@ -0,0 +1,83 @@
+/*
+ * Description: Utility class for handling socket.
+ * Author: Dilawar Singh (), dilawars@ncbs.res.in
+ * Organization: NCBS Bangalore
+ */
+
+#ifndef MOOSESOCKETINFO_H
+#define MOOSESOCKETINFO_H
+
+typedef enum t_socket_type_ {TCP_SOCKET, UNIX_DOMAIN_SOCKET} SocketType; // Type of socket.
+
+class MooseSocketInfo
+{
+public:
+ MooseSocketInfo( const string& addr = "" )
+ : address(addr), valid(false)
+ {
+ if( addr.size() > 0 )
+ init();
+ }
+
+ ~MooseSocketInfo() { ; }
+
+ void setAddress( const string addr )
+ {
+ address = addr;
+ init();
+ }
+
+ void init( void )
+ {
+ if( "file://" == address.substr(0, 7))
+ {
+ type = UNIX_DOMAIN_SOCKET;
+ filepath = address.substr(7);
+ valid = true;
+ }
+ else if( "http://" == address.substr(0, 7))
+ {
+ type = TCP_SOCKET;
+ auto colPos = address.find_last_of(':');
+ if( colPos == string::npos )
+ {
+ port = 0;
+ host = address;
+ }
+ else
+ {
+ host = address.substr(0, colPos);
+ port = std::stoi(address.substr(colPos+1));
+ }
+ valid = true;
+ }
+ else if( '/' == address[0] )
+ {
+ type = UNIX_DOMAIN_SOCKET;
+ filepath = address;
+ valid = true;
+ }
+ }
+
+ // C++ is so stupid: https://stackoverflow.com/q/476272/1805129
+ friend std::ostream& operator<<(ostream& os, const MooseSocketInfo& info)
+ {
+ os << "ADDRESS= " << info.address << " TYPE= " << info.type;
+ if( info.type == UNIX_DOMAIN_SOCKET)
+ os << ". FILEPATH " << info.filepath;
+ else
+ os << ". HOST:PORT=" << info.host << ":" << info.port;
+ return os;
+ }
+
+public:
+ SocketType type;
+ string address;
+ string filepath;
+ string host;
+ bool valid;
+ size_t port;
+};
+
+
+#endif /* end of include guard: MOOSESOCKETINFO_H */
diff --git a/builtins/SocketStreamer.cpp b/builtins/SocketStreamer.cpp
index e3097f08..38cbae5e 100644
--- a/builtins/SocketStreamer.cpp
+++ b/builtins/SocketStreamer.cpp
@@ -66,25 +66,25 @@ const Cinfo* SocketStreamer::initCinfo()
static DestFinfo addTable(
"addTable"
, "Add a table to SocketStreamer"
- , new OpFunc1<SocketStreamer, Id>(&SocketStreamer::addTable)
+ , new OpFunc1<SocketStreamer, ObjId>(&SocketStreamer::addTable)
);
static DestFinfo addTables(
"addTables"
, "Add many tables to SocketStreamer"
- , new OpFunc1<SocketStreamer, vector<Id> >(&SocketStreamer::addTables)
+ , new OpFunc1<SocketStreamer, vector<ObjId> >(&SocketStreamer::addTables)
);
static DestFinfo removeTable(
"removeTable"
, "Remove a table from SocketStreamer"
- , new OpFunc1<SocketStreamer, Id>(&SocketStreamer::removeTable)
+ , new OpFunc1<SocketStreamer, ObjId>(&SocketStreamer::removeTable)
);
static DestFinfo removeTables(
"removeTables"
, "Remove tables -- if found -- from SocketStreamer"
- , new OpFunc1<SocketStreamer, vector<Id>>(&SocketStreamer::removeTables)
+ , new OpFunc1<SocketStreamer, vector<ObjId>>(&SocketStreamer::removeTables)
);
/*-----------------------------------------------------------------------------
@@ -134,12 +134,9 @@ static const Cinfo* tableStreamCinfo = SocketStreamer::initCinfo();
SocketStreamer::SocketStreamer() :
currTime_(0.0)
, numMaxClients_(1)
- , sockType_ ( UNIX_DOMAIN_SOCKET )
, sockfd_(-1)
, clientfd_(-1)
- , ip_( TCP_SOCKET_IP )
- , port_( TCP_SOCKET_PORT )
- , address_ ( "file://MOOSE" )
+ , sockInfo_( MooseSocketInfo( "file://MOOSE" ) )
{
clk_ = reinterpret_cast<Clock*>( Id(1).eref().data() );
@@ -150,7 +147,6 @@ SocketStreamer::SocketStreamer() :
tableIds_.resize(0);
tableTick_.resize(0);
tableDt_.resize(0);
-
}
SocketStreamer& SocketStreamer::operator=( const SocketStreamer& st )
@@ -167,36 +163,29 @@ SocketStreamer::~SocketStreamer()
if(sockfd_ > 0)
{
LOG(moose::debug, "Closing socket " << sockfd_ );
- shutdown(sockfd_, SHUT_RD);
+ shutdown(sockfd_, SHUT_RDWR);
close(sockfd_);
- if( sockType_ == UNIX_DOMAIN_SOCKET )
- ::unlink( unixSocketFilePath_.c_str() );
+ if( sockInfo_.type == UNIX_DOMAIN_SOCKET )
+ ::unlink( sockInfo_.filepath.c_str() );
}
- if( processThread_.joinable() )
+ if( processThread_.joinable())
processThread_.join();
-}
-/* --------------------------------------------------------------------------*/
-/**
- * @Synopsis Stop a thread.
- * See: http://www.bo-yang.net/2017/11/19/cpp-kill-detached-thread
- *
- * @Param tname name of thread.
- */
-/* ----------------------------------------------------------------------------*/
-//void SocketStreamer::stopThread(const std::string& tname)
-//{
-// ThreadMap::const_iterator it = tm_.find(tname);
-// if (it != tm_.end())
-// {
-// it->second.std::thread::~thread(); // thread not killed
-// tm_.erase(tname);
-// LOG(moose::debug, "Thread " << tname << " killed." );
-// }
-//}
+ // Close the client as well.
+ if( clientfd_ > -1 )
+ {
+ shutdown(clientfd_, SHUT_RDWR);
+ close(clientfd_);
+ }
+}
+void SocketStreamer::addStringToDoubleVec(vector<double>&res, const string s)
+{
+ for(char c : s)
+ res.push_back((double)c);
+}
/* --------------------------------------------------------------------------*/
/**
@@ -217,17 +206,16 @@ void SocketStreamer::listenToClients(size_t numMaxClients)
void SocketStreamer::initServer( void )
{
- setSocketType( );
- if( sockType_ == UNIX_DOMAIN_SOCKET )
+ if( sockInfo_.type == UNIX_DOMAIN_SOCKET )
initUDSServer();
else
initTCPServer();
- LOG(moose::debug, "Successfully created SocketStreamer server: " << sockfd_);
+ LOG(moose::info, "Successfully initialized streamer socket: " << sockfd_);
// Listen for incoming clients. This function does nothing if connection is
// already made.
- listenToClients(2);
+ listenToClients(1);
}
void SocketStreamer::configureSocketServer( )
@@ -236,7 +224,7 @@ void SocketStreamer::configureSocketServer( )
// for details. We are making it 'reusable'.
int on = 1;
-#ifdef SO_REUSEADDR
+#ifdef SO_REUSEPORT
if(0 > setsockopt(sockfd_, SOL_SOCKET, SO_REUSEPORT, (const char *)&on, sizeof(on)))
LOG(moose::warning, "Warn: setsockopt() failed");
#endif
@@ -257,30 +245,39 @@ void SocketStreamer::initUDSServer( void )
if( sockfd_ > 0 )
{
- unixSocketFilePath_ = address_.substr(7); bzero(&sockAddrUDS_, sizeof(sockAddrUDS_));
+ bzero(&sockAddrUDS_, sizeof(sockAddrUDS_));
sockAddrUDS_.sun_family = AF_UNIX;
- strncpy(sockAddrUDS_.sun_path, unixSocketFilePath_.c_str(), sizeof(sockAddrUDS_.sun_path)-1);
+ strncpy(sockAddrUDS_.sun_path, sockInfo_.filepath.c_str(), sizeof(sockAddrUDS_.sun_path)-1);
configureSocketServer();
// Bind. Make sure bind is not std::bind
if(0 > ::bind(sockfd_, (struct sockaddr*) &sockAddrUDS_, sizeof(sockAddrUDS_)))
{
isValid_ = false;
- LOG(moose::warning, "Warn: Failed to create socket at " << unixSocketFilePath_
+ LOG(moose::warning, "Warn: Failed to create socket at " << sockInfo_.filepath
<< ". File descriptor: " << sockfd_
<< ". Erorr: " << strerror(errno)
);
}
+
+ if(! moose::filepath_exists(sockInfo_.filepath))
+ {
+ LOG( moose::warning, "No file " << sockInfo_.filepath << " exists." );
+ isValid_ = false;
+ }
}
if( (! isValid_) || (sockfd_ < 0) )
- ::unlink( unixSocketFilePath_.c_str() );
+ {
+ LOG( moose::warning, "Failed to create socket : " << sockInfo_ );
+ ::unlink( sockInfo_.filepath.c_str() );
+ }
}
void SocketStreamer::initTCPServer( void )
{
// Create a blocking socket.
- LOG( moose::debug, "Creating TCP socket on port: " << port_ );
+ LOG( moose::debug, "Creating TCP socket on port: " << sockInfo_.port );
sockfd_ = socket(AF_INET, SOCK_STREAM, 0);
if( 0 > sockfd_ )
{
@@ -293,122 +290,137 @@ void SocketStreamer::initTCPServer( void )
bzero((char*) &sockAddrTCP_, sizeof(sockAddrTCP_));
sockAddrTCP_.sin_family = AF_INET;
sockAddrTCP_.sin_addr.s_addr = INADDR_ANY;
- sockAddrTCP_.sin_port = htons( port_ );
+ sockAddrTCP_.sin_port = htons( sockInfo_.port );
// Bind. Make sure bind is not std::bind
if(0 > ::bind(sockfd_, (struct sockaddr*) &sockAddrTCP_, sizeof(sockAddrTCP_)))
{
isValid_ = false;
- LOG(moose::warning, "Warn: Failed to create server at " << ip_ << ":" << port_
- << ". File descriptor: " << sockfd_
- << ". Erorr: " << strerror(errno)
+ LOG(moose::warning, "Warn: Failed to create server at "
+ << sockInfo_.host << ":" << sockInfo_.port
+ << ". File descriptor: " << sockfd_
+ << ". Erorr: " << strerror(errno)
);
return;
}
}
-
/* --------------------------------------------------------------------------*/
/**
- * @Synopsis Stream data over socket.
+ * @Synopsis Convert data to JSON.
*
- * @Returns True of success, false otherwise. It is callee job to clean up data_
- * on a successful return from this function.
+ * @Returns JSON representation.
*/
/* ----------------------------------------------------------------------------*/
-bool SocketStreamer::streamData( )
+void SocketStreamer::dataToStream(map<string, vector<double>>& data)
{
- if( clientfd_ > 0)
+ for( size_t i = 0; i < tables_.size(); i++)
{
- buffer_ += dataToString();
-
- if( buffer_.size() < frameSize_ )
- buffer_ += string(frameSize_-buffer_.size(), ' ');
-
- string toSend = buffer_.substr(0, frameSize_);
-
- int sent = send(clientfd_, buffer_.substr(0, frameSize_).c_str(), frameSize_, MSG_MORE);
- buffer_ = buffer_.erase(0, sent);
-
- assert( sent == (int)frameSize_);
- LOG(moose::debug, "Sent " << sent << " bytes." );
-
- if(0 > sent)
- {
- LOG(moose::warning, "Failed to send. Error: " << strerror(errno)
- << ". client id: " << clientfd_ );
- return false;
- }
-
- // clear up the tables.
- for( auto t : tables_ )
- t->clearVec();
- return true;
+ vector<double> vec;
+ tables_[i]->collectData(vec, true, false);
+ if( ! vec.empty() )
+ data[columns_[i+1]] = vec;
}
- else
- LOG(moose::warning, "No client found to stream data. ClientFD: " << clientfd_ );
-
- return false;
}
/* --------------------------------------------------------------------------*/
/**
- * @Synopsis Convert table to string (use scientific notation).
+ * @Synopsis Stream data over socket.
*
- * @Returns String in JSON like format.
+ * @Returns True of success, false otherwise. It is callee job to clean up data_
+ * on a successful return from this function.
*/
/* ----------------------------------------------------------------------------*/
-string SocketStreamer::dataToString( )
+int SocketStreamer::streamData( )
{
- stringstream ss;
- // Enabling this would be require quite a lot of characters to be streamed.
- //ss.precision( 7 );
- //ss << std::scientific;
- vector<double> data;
-
- // Else stream the data.
- ss << "{";
- for( size_t i = 0; i < tables_.size(); i++)
+ map<string, vector<double>> data;
+ dataToStream(data);
+
+ if(data.empty())
+ {
+ LOG(moose::debug, "No data in tables.");
+ return 0;
+ }
+
+ // Construct a void* array to send over the socket. Serialize the data.
+ // e.g. H 10 / a / t a b l e / a V 4 0.1 0.2 0.3 0.2
+ // H => Header start (chars)
+ // V => Value starts (double)
+ for(auto v: data)
{
- ss << "\"" << columns_[i+1] << "\":[";
- ss << tables_[i]->toJSON(true);
- ss << "],";
+ // First header.
+ vecToStream_.push_back((double)'H');
+ vecToStream_.push_back(v.first.size());
+ addStringToDoubleVec(vecToStream_, v.first);
+
+ // Now data.
+ vecToStream_.push_back((double)'V');
+ vecToStream_.push_back(v.second.size());
+ vecToStream_.insert(vecToStream_.end(), v.second.begin(), v.second.end());
}
- // remove , at the end else it won't be a valid JSON.
- string res = ss.str();
- if( ',' == res.back())
- res.pop_back();
- res += "}\n";
- return res;
+ size_t dtypeSize = sizeof(double);
+ int sent = send(clientfd_, (void*) &vecToStream_[0], dtypeSize*vecToStream_.size(), MSG_MORE);
+ LOG(moose::debug, "Sent " << sent << " bytes." );
+ if( sent < 0 )
+ return errno;
+
+ vecToStream_.erase(vecToStream_.begin(), vecToStream_.begin()+(sent/dtypeSize));
+ return 0;
}
+
bool SocketStreamer::enoughDataToStream(size_t minsize)
{
for( size_t i = 0; i < tables_.size(); i++)
- if(tables_[i]->getVec().size() >= minsize )
+ if(tables_[i]->getVec().size() >= minsize)
return true;
return false;
}
-void SocketStreamer::connectAndStream( )
+void SocketStreamer::connect( )
{
- currTime_ = clk_->getCurrentTime();
-
// If server was invalid then there is no point.
if( ! isValid_ )
- {
- LOG( moose::error, "Server could not set up." );
return;
+
+ struct sockaddr_storage clientAddr;
+ socklen_t addrLen = sizeof(clientAddr);
+ clientfd_ = ::accept(sockfd_,(struct sockaddr*) &clientAddr, &addrLen);
+ assert( clientfd_ );
+
+ char ipstr[INET6_ADDRSTRLEN];
+ int port = -1;
+ // deal with both IPv4 and IPv6:
+ if (clientAddr.ss_family == AF_INET)
+ {
+ struct sockaddr_in *s = (struct sockaddr_in *)&clientAddr;
+ port = ntohs(s->sin_port);
+ inet_ntop(AF_INET, &s->sin_addr, ipstr, sizeof ipstr);
+ }
+ else
+ { // AF_INET6
+ struct sockaddr_in6 *s = (struct sockaddr_in6 *)&clientAddr;
+ port = ntohs(s->sin6_port);
+ inet_ntop(AF_INET6, &s->sin6_addr, ipstr, sizeof ipstr);
}
- // Now lets get into a loop to stream data.
- while( (! all_done_) )
+ LOG(moose::info, "Connected to " << ipstr << ':' << port);
+ return;
+}
+
+void SocketStreamer::stream( void )
+{
+ if(clientfd_ > 0)
{
- clientfd_ = ::accept(sockfd_, NULL, NULL);
- if( clientfd_ >= 0 )
- streamData();
+ if( EPIPE == streamData() )
+ {
+ LOG( moose::warning, "Broken pipe. Couldn't stream." );
+ return;
+ }
}
+ else
+ LOG( moose::warning, "No client." );
}
/**
@@ -443,7 +455,10 @@ void SocketStreamer::reinit(const Eref& e, ProcPtr p)
// Launch a thread in background which monitors the any client trying to
// make connection to server.
- processThread_ = std::thread(&SocketStreamer::connectAndStream, this);
+ processThread_ = std::thread(&SocketStreamer::connect, this);
+
+ // NOw introduce some delay.
+ timeStamp_ = std::chrono::high_resolution_clock::now();
}
/**
@@ -454,8 +469,10 @@ void SocketStreamer::reinit(const Eref& e, ProcPtr p)
*/
void SocketStreamer::process(const Eref& e, ProcPtr p)
{
- // It does nothing. See the connectAndStream function.
- ;
+ // processTickMicroSec = std::chrono::duration_cast<std::chrono::microseconds>(
+ // std::chrono::high_resolution_clock::now() - timeStamp_).count();
+ // timeStamp_ = std::chrono::high_resolution_clock::now();
+ stream();
}
/**
@@ -463,7 +480,7 @@ void SocketStreamer::process(const Eref& e, ProcPtr p)
*
* @param table Id of table.
*/
-void SocketStreamer::addTable( Id table )
+void SocketStreamer::addTable( ObjId table )
{
// If this table is not already in the vector, add it.
for( size_t i = 0; i < tableIds_.size(); i++)
@@ -488,29 +505,21 @@ void SocketStreamer::addTable( Id table )
*
* @param tables
*/
-void SocketStreamer::addTables( vector<Id> tables )
-{
- if( tables.size() == 0 )
- return;
- for( vector<Id>::const_iterator it = tables.begin(); it != tables.end(); it++)
- addTable( *it );
-}
-
void SocketStreamer::addTables( vector<ObjId> tables )
{
if( tables.size() == 0 )
return;
- for( auto t : tables )
- addTable( Id(t) );
-}
+ for(auto t : tables)
+ addTable(t);
+}
/**
* @brief Remove a table from SocketStreamer.
*
* @param table. Id of table.
*/
-void SocketStreamer::removeTable( Id table )
+void SocketStreamer::removeTable( ObjId table )
{
int matchIndex = -1;
for (size_t i = 0; i < tableIds_.size(); i++)
@@ -530,32 +539,14 @@ void SocketStreamer::removeTable( Id table )
}
}
-/* --------------------------------------------------------------------------*/
-/**
- * @Synopsis Determines socket type from the given address.
- */
-/* ----------------------------------------------------------------------------*/
-void SocketStreamer::setSocketType( )
-{
- LOG( moose::debug, "Socket address is " << address_ );
- if( "file://" == address_.substr(0, 7))
- sockType_ = UNIX_DOMAIN_SOCKET;
- else if( "http://" == address_.substr(0,7))
- sockType_ = TCP_SOCKET;
- else
- sockType_ = UNIX_DOMAIN_SOCKET;
- return;
-}
-
/**
* @brief Remove multiple tables -- if found -- from SocketStreamer.
*
* @param tables
*/
-void SocketStreamer::removeTables( vector<Id> tables )
+void SocketStreamer::removeTables( vector<ObjId> tables )
{
- for( vector<Id>::const_iterator it = tables.begin(); it != tables.end(); it++)
- removeTable( *it );
+ for(auto &t: tables) removeTable(t);
}
/**
@@ -571,21 +562,20 @@ size_t SocketStreamer::getNumTables( void ) const
void SocketStreamer::setPort( const size_t port )
{
- port_ = port;
+ sockInfo_.port = port;
}
size_t SocketStreamer::getPort( void ) const
{
- assert( port_ > 1 );
- return port_;
+ return sockInfo_.port;
}
void SocketStreamer::setAddress( const string addr )
{
- address_ = addr;
+ sockInfo_.setAddress(addr);
}
string SocketStreamer::getAddress( void ) const
{
- return address_;
+ return sockInfo_.address;
}
diff --git a/builtins/SocketStreamer.h b/builtins/SocketStreamer.h
index c52bb7dd..7e811c81 100644
--- a/builtins/SocketStreamer.h
+++ b/builtins/SocketStreamer.h
@@ -5,10 +5,9 @@
#ifndef SocketStreamer_INC
#define SocketStreamer_INC
-#define STRINGSTREAM_DOUBLE_PRECISION 10
-
#include <iostream>
#include <string>
+#include <vector>
#include <map>
#include <fstream>
#include <sstream>
@@ -16,6 +15,7 @@
#include <atomic>
#include "StreamerBase.h"
+#include "MooseSocketInfo.h"
#include "Table.h"
// If cmake does not set it, use the default port.
@@ -37,6 +37,7 @@
#include <sys/ioctl.h>
#include <sys/un.h>
#include <netinet/in.h>
+#include <arpa/inet.h>
// MSG_MORE is not defined in OSX. So stupid!
#ifndef MSG_MORE
@@ -45,10 +46,10 @@
using namespace std;
-typedef enum t_socket_type_ {TCP_SOCKET, UNIX_DOMAIN_SOCKET} SocketType; // Type of socket.
class Clock;
+
class SocketStreamer : public StreamerBase
{
@@ -75,6 +76,9 @@ public:
// Make connection to client
void listenToClients(size_t numMaxClients);
+ // Find minimum number of elements in tables.
+ pair<size_t, size_t> minMaxNumberOfElemementsInTables( );
+
/* Cleaup before quitting */
void cleanUp( void );
@@ -84,28 +88,26 @@ public:
size_t getPort( void ) const;
void setPort( const size_t port );
- SocketType getSocketType( );
- void setSocketType(void);
-
/*-----------------------------------------------------------------------------
* Streaming data.
*-----------------------------------------------------------------------------*/
bool enoughDataToStream(size_t minsize=10);
- bool streamData();
- void connectAndStream( );
+ int streamData();
+ // connect is monitored in a thread.
+ void connect( void );
+ void stream(void);
size_t getNumTables( void ) const;
- void addTable( Id table );
- void addTables( vector<Id> tables);
+ void addTable( ObjId table );
void addTables( vector<ObjId> tables);
- void removeTable( Id table );
- void removeTables( vector<Id> table );
+ void removeTable( ObjId table );
+ void removeTables( vector<ObjId> table );
- string dataToString();
+ void dataToStream(map<string, vector<double>>& vec);
- // void stopThread(const std::string& tname);
+ static void addStringToDoubleVec(vector<double>&res, const string s);
/** Dest functions.
* The process function called by scheduler on every tick
@@ -119,6 +121,7 @@ public:
static const Cinfo * initCinfo();
+
private:
// dt_ and tick number of Table's clock
@@ -133,16 +136,9 @@ private:
/* Socket related */
int numMaxClients_;
- SocketType sockType_ = UNIX_DOMAIN_SOCKET; // Default is UNIX_DOMAIN_SOCKET
int sockfd_; // socket file descriptor.
int clientfd_; // client file descriptor
- // TCP socket.
- string ip_; // ip_ address of server.
- unsigned short port_; // port number if socket is TCP_SOCKET
- string address_; // adress of socket. Specified by user.
- string unixSocketFilePath_;
-
// address holdder for TCP and UDS sockets.
struct sockaddr_in sockAddrTCP_;
struct sockaddr_un sockAddrUDS_;
@@ -152,13 +148,18 @@ private:
bool isValid_ = true;
std::thread processThread_;
string buffer_;
+ vector<double> vecToStream_;
double thisDt_;
- size_t frameSize_ = 2048;
-
// We need clk_ pointer for handling
Clock* clk_ = nullptr;
+ // Socket Info
+ MooseSocketInfo sockInfo_;
+
+ // How long it takes between process calls.
+ size_t processTickMicroSec;
+ std::chrono::high_resolution_clock::time_point timeStamp_;
};
#endif /* ----- #ifndef SocketStreamer_INC ----- */
diff --git a/builtins/Streamer.cpp b/builtins/Streamer.cpp
index c583be1f..aae926bc 100644
--- a/builtins/Streamer.cpp
+++ b/builtins/Streamer.cpp
@@ -1,17 +1,8 @@
/***
- * Filename: Streamer.cpp
- *
* Description: Stream table data.
*
- * Version: 0.0.1
- * Created: 2016-04-26
-
- * Revision: none
- *
* Author: Dilawar Singh <dilawars@ncbs.res.in>
* Organization: NCBS Bangalore
- *
- * License: GNU GPL2
*/
#include <algorithm>
@@ -45,12 +36,18 @@ const Cinfo* Streamer::initCinfo()
, &Streamer::getFormat
);
- static ReadOnlyValueFinfo< Streamer, size_t > numTables (
+ static ReadOnlyValueFinfo<Streamer, size_t> numTables (
"numTables"
, "Number of Tables handled by Streamer "
, &Streamer::getNumTables
);
+ static ReadOnlyValueFinfo<Streamer, size_t> numWriteEvents(
+ "numWriteEvents"
+ , "Number of time streamer was called to write. (For debugging/performance reason only)"
+ , &Streamer::getNumWriteEvents
+ );
+
/*-----------------------------------------------------------------------------
*
*-----------------------------------------------------------------------------*/
@@ -96,7 +93,7 @@ const Cinfo* Streamer::initCinfo()
*-----------------------------------------------------------------------------*/
static Finfo* procShared[] =
{
- &process , &reinit , &addTable, &addTables, &removeTable, &removeTables
+ &process, &reinit, &addTable, &addTables, &removeTable, &removeTables
};
static SharedFinfo proc(
@@ -107,7 +104,7 @@ const Cinfo* Streamer::initCinfo()
static Finfo * tableStreamFinfos[] =
{
- &outfile, &format, &proc, &numTables
+ &outfile, &format, &proc, &numTables, &numWriteEvents
};
static string doc[] =
@@ -140,8 +137,9 @@ Streamer::Streamer()
{
// Not all compilers allow initialization during the declaration of class
// methods.
- format_ = "npy";
- columns_.push_back( "time" ); /* First column is time. */
+ format_ = "csv";
+ numWriteEvents_ = 0;
+ columns_.push_back("time"); /* First column is time. */
tables_.resize(0);
tableIds_.resize(0);
tableTick_.resize(0);
@@ -238,7 +236,7 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
// write now.
currTime_ = 0.0;
zipWithTime( );
- StreamerBase::writeToOutFile( outfilePath_, format_, "w", data_, columns_);
+ StreamerBase::writeToOutFile(outfilePath_, format_, "w", data_, columns_);
data_.clear( );
}
@@ -261,15 +259,11 @@ void Streamer::cleanUp( )
*/
void Streamer::process(const Eref& e, ProcPtr p)
{
- //LOG( moose::debug, "Writing to table" );
+ // LOG( moose::debug, "Writing Streamer data to file." );
zipWithTime( );
-
- // Write only if there are more than 100 entry in first table.
- if( tables_[0]->getVecSize() > 100 )
- {
- StreamerBase::writeToOutFile( outfilePath_, format_, "a", data_, columns_ );
- data_.clear( );
- }
+ StreamerBase::writeToOutFile( outfilePath_, format_, "a", data_, columns_ );
+ data_.clear();
+ numWriteEvents_ += 1;
}
@@ -356,6 +350,19 @@ size_t Streamer::getNumTables( void ) const
return tables_.size();
}
+/* --------------------------------------------------------------------------*/
+/**
+ * @Synopsis Get number of write events in streamer. Useful for debugging and
+ * performance measuerments.
+ *
+ * @Returns
+ */
+/* ----------------------------------------------------------------------------*/
+size_t Streamer::getNumWriteEvents( void ) const
+{
+ return numWriteEvents_;
+}
+
string Streamer::getOutFilepath( void ) const
{
diff --git a/builtins/Streamer.h b/builtins/Streamer.h
index 0a06e547..3136211f 100644
--- a/builtins/Streamer.h
+++ b/builtins/Streamer.h
@@ -49,6 +49,7 @@ public:
void setFormat( string format );
size_t getNumTables( void ) const;
+ size_t getNumWriteEvents( void ) const;
void addTable( Id table );
void addTables( vector<Id> tables);
@@ -74,6 +75,9 @@ private:
string outfilePath_;
string format_;
+
+ size_t numWriteEvents_;
+
bool isOutfilePathSet_;
// dt_ and tick number of Table's clock
diff --git a/builtins/StreamerBase.cpp b/builtins/StreamerBase.cpp
index e6d590be..902c3f24 100644
--- a/builtins/StreamerBase.cpp
+++ b/builtins/StreamerBase.cpp
@@ -91,14 +91,14 @@ void StreamerBase::writeToCSVFile( const string& filepath, const string& openmod
}
// If writing in "w" mode, write the header first.
- if( openmode == "w" )
+ if(openmode == "w")
{
string headerText = "";
for( vector<string>::const_iterator it = columns.begin();
it != columns.end(); it++ )
headerText += ( *it + delimiter_ );
headerText += eol;
- fprintf( fp, "%s", headerText.c_str() );
+ fprintf(fp, "%s", headerText.c_str());
}
string text = "";
@@ -111,8 +111,8 @@ void StreamerBase::writeToCSVFile( const string& filepath, const string& openmod
// At the end of each row, we remove the delimiter_ and append newline_.
*(text.end()-1) = eol;
}
- fprintf( fp, "%s", text.c_str() );
- fclose( fp );
+ fprintf(fp, "%s", text.c_str() );
+ fclose(fp);
}
/* write data to a numpy file */
diff --git a/builtins/Table.cpp b/builtins/Table.cpp
index 4f426b14..be3702b1 100644
--- a/builtins/Table.cpp
+++ b/builtins/Table.cpp
@@ -211,24 +211,25 @@ const Cinfo* Table::initCinfo()
static const Cinfo* tableCinfo = Table::initCinfo();
Table::Table() :
- threshold_( 0.0 ) ,
- lastTime_( 0.0 ) ,
- input_( 0.0 ),
- fired_(false),
- useSpikeMode_(false),
- dt_( 0.0 )
+ threshold_( 0.0 ) ,
+ lastTime_( 0.0 ) ,
+ input_( 0.0 ),
+ fired_(false),
+ useSpikeMode_(false),
+ dt_( 0.0 )
{
// Initialize the directory to which each table should stream.
rootdir_ = "_tables";
- useStreamer_ = false;
+ useFileStreamer_ = false;
format_ = "csv";
outfileIsSet_ = false;
+ lastN_ = 0;
}
Table::~Table( )
{
// Make sure to write to rest of the entries to file before closing down.
- if( useStreamer_ )
+ if( useFileStreamer_ )
{
mergeWithTime( data_ );
StreamerBase::writeToOutFile( outfile_, format_, "a", data_, columns_ );
@@ -267,7 +268,7 @@ void Table::process( const Eref& e, ProcPtr p )
* vector.
* Write at every 5 seconds or whenever size of vector is more than 10k.
*/
- if( useStreamer_ )
+ if( useFileStreamer_ )
{
if( fmod(lastTime_, 5.0) == 0.0 or getVecSize() >= 10000 )
{
@@ -300,7 +301,7 @@ void Table::reinit( const Eref& e, ProcPtr p )
fired_ = false;
/** Create the default filepath for this table. */
- if( useStreamer_ )
+ if( useFileStreamer_ )
{
// The first column is variable time.
columns_.push_back( "time" );
@@ -331,7 +332,7 @@ void Table::reinit( const Eref& e, ProcPtr p )
tvec_.push_back(lastTime_);
- if( useStreamer_ )
+ if( useFileStreamer_ )
{
mergeWithTime( data_ );
StreamerBase::writeToOutFile( outfile_, format_, "w", data_, columns_);
@@ -412,12 +413,12 @@ void Table::setColumnName( const string colname )
/* Enable/disable streamer support. */
void Table::setUseStreamer( bool useStreamer )
{
- useStreamer_ = useStreamer;
+ useFileStreamer_ = useStreamer;
}
bool Table::getUseStreamer( void ) const
{
- return useStreamer_;
+ return useFileStreamer_;
}
/* Enable/disable spike mode. */
@@ -477,25 +478,60 @@ void Table::mergeWithTime( vector<double>& data )
/**
* @Synopsis. Convert table data to JOSN.
*
- * @Param t. Upto this time.
- * @Param withTime. Zip with time.
- *
* @Returns string.
*/
/* ----------------------------------------------------------------------------*/
-string Table::toJSON( bool withTime)
+string Table::toJSON(bool withTime, bool clear)
{
- auto v = vec();
stringstream ss;
- for (size_t i = 0; i < v.size(); i++)
+ auto v = vec();
+ if( clear )
+ lastN_ = 0;
+
+ for (size_t i = lastN_; i < v.size(); i++)
{
if(withTime)
ss << '[' << tvec_[i] << ',' << v[i] << "],";
else
ss << v[i] << ',';
}
+
string res = ss.str();
- if( ',' == res.back())
+ if(',' == res.back())
res.pop_back();
+
+ if( clear )
+ clearAllVecs();
+ else
+ lastN_ += v.size();
+
return res;
}
+
+/* --------------------------------------------------------------------------*/
+/**
+ * @Synopsis Collect data in given vector. Its similar to toJSON function.
+ *
+ * @Param data
+ * @Param withTime
+ * @Param clear
+ */
+/* ----------------------------------------------------------------------------*/
+void Table::collectData(vector<double>& data, bool withTime, bool clear)
+{
+ auto v = vec();
+ if( clear )
+ lastN_ = 0;
+
+ for (size_t i = lastN_; i < v.size(); i++)
+ {
+ if(withTime)
+ data.push_back(tvec_[i]);
+ data.push_back(v[i]);
+ }
+
+ if( clear )
+ clearAllVecs();
+ else
+ lastN_ = v.size();
+}
diff --git a/builtins/Table.h b/builtins/Table.h
index b3a0293b..2d59f67d 100644
--- a/builtins/Table.h
+++ b/builtins/Table.h
@@ -51,7 +51,10 @@ public:
// merge time value among values. e.g. t1, v1, t2, v2, etc.
void mergeWithTime( vector<double>& data );
- string toJSON(bool withTime=true);
+ string toJSON(bool withTime=true, bool clear = false);
+
+ void collectData(vector<double>& data, bool withTime=true, bool clear = false);
+
void clearAllVecs();
@@ -83,8 +86,11 @@ private:
vector<double> tvec_; /* time data */
vector<string> columns_; /* Store the name of tables */
- string tablePath_;
+ // Upto which indices we have read the data. This variable is used when
+ // SocketStreamer is used.
+ size_t lastN_ = 0;
+ string tablePath_;
/**
* @brief Column name of this table. Use it when writing data to a datafile.
@@ -96,7 +102,7 @@ private:
* of outfile_ is table path starting from `pwd`/_tables_ . On table, set
* streamToFile to true.
*/
- bool useStreamer_;
+ bool useFileStreamer_;
/**
* @brief Table directory into which dump the stream data.
diff --git a/kinetics/ReadKkit.cpp b/kinetics/ReadKkit.cpp
index 01254b62..a33986ac 100644
--- a/kinetics/ReadKkit.cpp
+++ b/kinetics/ReadKkit.cpp
@@ -131,9 +131,10 @@ Id makeStandardElements( Id pa, const string& modelname )
mgr = shell->doCreate( "Neutral", pa, modelname, 1, MooseGlobal );
Id kinetics( modelPath + "/kinetics" );
if ( kinetics == Id() ) {
- kinetics =
- shell->doCreate( "CubeMesh", mgr, "kinetics", 1, MooseGlobal );
+ kinetics = shell->doCreate( "CubeMesh", mgr, "kinetics", 1, MooseGlobal );
SetGet2< double, unsigned int >::set( kinetics, "buildDefaultMesh", 1e-15, 1 );
+ Id cInfo = shell->doCreate( "Annotator", kinetics, "info", 1 );
+ assert( cInfo != Id() );
}
assert( kinetics != Id() );
diff --git a/pymoose/moosemodule.cpp b/pymoose/moosemodule.cpp
index 0a71f466..affdc78f 100644
--- a/pymoose/moosemodule.cpp
+++ b/pymoose/moosemodule.cpp
@@ -89,9 +89,6 @@ extern void speedTestMultiNodeIntFireNetwork(
extern void mooseBenchmarks( unsigned int option );
-// global variable.
-bool tcpSocketStreamerEnabled_ = false;
-
/*-----------------------------------------------------------------------------
* Random number generator for this module.
*
@@ -113,7 +110,6 @@ double pymoose_mtrand_( void )
bool setupSocketStreamer(const string addr )
{
LOG(moose::debug, "Setting streamer with addr " << addr );
-
// Find all tables.
vector< ObjId > tables;
wildcardFind( "/##[TYPE=Table2]", tables );
@@ -130,8 +126,9 @@ bool setupSocketStreamer(const string addr )
Id st = SHELLPTR->doCreate("SocketStreamer", stBase, "streamer", 1);
Field<string>::set(st, "address", addr);
+ LOG(moose::debug, "Found " << tables.size() << " tables.");
for( auto &t : tables )
- SetGet1<Id>::set(st, "addTable", t.id );
+ SetGet1<ObjId>::set(st, "addTable", t );
return true;
}
@@ -1773,11 +1770,10 @@ PyObject * moose_reinit(PyObject * dummy, PyObject * args)
string envSocketServer = moose::getEnv( "MOOSE_STREAMER_ADDRESS" );
if(! envSocketServer.empty())
{
- LOG( moose::debug, "Environment variable set of socket" << envSocketServer );
+ LOG( moose::debug, "Environment variable MOOSE_STREAMER_ADDRESS: " << envSocketServer );
if( envSocketServer.size() > 0 )
setupSocketStreamer(envSocketServer);
}
-
SHELLPTR->doReinit();
Py_RETURN_NONE;
}
@@ -1909,7 +1905,7 @@ PyObject * moose_saveModel(PyObject * dummy, PyObject * args)
SHELLPTR->doSaveModel(model, filename);
Py_RETURN_NONE;
}
-#endif
+#endif
PyObject * moose_setCwe(PyObject * dummy, PyObject * args)
{
@@ -2971,17 +2967,17 @@ PyObject * moose_element(PyObject* dummy, PyObject * args)
unsigned nid = 0, did = 0, fidx = 0;
Id id;
unsigned int numData = 0;
- if (PyArg_ParseTuple(args, "s", &path))
+
+ // Parse into str or bytes-like object. Using 's' parses into const char*
+ // which is portable with bytes often returned when working with python3.
+ if (PyArg_ParseTuple(args, "s*", &path))
{
oid = ObjId(path);
- // cout << "Original Path " << path << ", Element Path: " << oid.path() << endl;
if ( oid.bad() )
{
- PyErr_SetString(PyExc_ValueError, ( std::string("moose_element: '")
- + std::string(path)
- + std::string("' does not exist!")
- ).c_str()
- );
+ PyErr_SetString(PyExc_ValueError
+ , (std::string("moose_element: '") + std::string(path) + std::string("' does not exist!")).c_str()
+ );
return NULL;
}
PyObject * new_obj = oid_to_element(oid);
@@ -3272,4 +3268,4 @@ PyMODINIT_FUNC MODINIT(_moose)
#ifdef PY3K
return moose_module;
#endif
-}
+}
diff --git a/pymoose/vec.cpp b/pymoose/vec.cpp
index e5e2ddb3..fc57831c 100644
--- a/pymoose/vec.cpp
+++ b/pymoose/vec.cpp
@@ -48,8 +48,6 @@
#include <Python.h>
#include <structmember.h> // This defines the type id macros like T_STRING
-// #include "numpy/arrayobject.h"
-
#include "../utility/simple_logger.hpp"
#include <iostream>
@@ -1259,20 +1257,12 @@ int moose_Id_setattro(_Id * self, PyObject * attr, PyObject *value)
default:
break;
}
+
// MOOSE Field::set returns 1 for success 0 for
// failure. Python treats return value 0 from setters as
// success, anything else failure.
if (ret && (PyErr_Occurred() == NULL))
- {
return 0;
- }
else
- {
return -1;
- }
-
}
-
-
-//
-// vec.cpp ends here
diff --git a/python/moose/SBML/readSBML.py b/python/moose/SBML/readSBML.py
index 356d2400..093f2085 100644
--- a/python/moose/SBML/readSBML.py
+++ b/python/moose/SBML/readSBML.py
@@ -13,9 +13,13 @@
** copyright (C) 2003-2017 Upinder S. Bhalla. and NCBS
Created : Thu May 13 10:19:00 2016(+0530)
Version
-Last-Updated: Tue Dec 3 17:30:00 2018(+0530)
+Last-Updated: Sat Jan 19 10:30:00 2019(+0530)
By:HarshaRani
**********************************************************************/
+2019:
+Jan 19: - validator flag is set 'on' from True
+ - groupname if missing in the sbml file then groupid is taken,
+ if both are missing then its not a valide sbml file
2018
Dec 3: - reading motor and diffconstant from pool
Nov 30: - groups and subgroups are read from xml to moose
@@ -82,7 +86,7 @@ try:
except ImportError:
pass
-def mooseReadSBML(filepath, loadpath, solver="ee",validate="True"):
+def mooseReadSBML(filepath, loadpath, solver="ee",validate="on"):
"""Load SBML model
"""
global foundLibSBML_
@@ -102,13 +106,15 @@ def mooseReadSBML(filepath, loadpath, solver="ee",validate="True"):
with open(filepath, "r") as filep:
loadpath = loadpath[loadpath.find('/')+1:]
loaderror = None
+ errorFlag = ""
filep = open(filepath, "r")
document = libsbml.readSBML(filepath)
tobecontinue = False
- if validate == "True":
- tobecontinue = validateModel(document)
+ if validate == "on":
+ tobecontinue,errorFlag = validateModel(document)
else:
tobecontinue = True
+
if tobecontinue:
level = document.getLevel()
version = document.getVersion()
@@ -126,6 +132,7 @@ def mooseReadSBML(filepath, loadpath, solver="ee",validate="True"):
loadpath ='/'+loadpath
baseId = moose.Neutral(loadpath)
basePath = baseId
+
# All the model will be created under model as
# a thumbrule
basePath = moose.Neutral(baseId.path)
@@ -150,7 +157,7 @@ def mooseReadSBML(filepath, loadpath, solver="ee",validate="True"):
errorFlag,msgCmpt = createCompartment(
basePath, model, comptSbmlidMooseIdMap)
- groupInfo = checkGroup(basePath,model)
+ groupInfo = checkGroup(basePath,model,comptSbmlidMooseIdMap)
funcDef = checkFuncDef(model)
if errorFlag:
specInfoMap = {}
@@ -207,8 +214,11 @@ def mooseReadSBML(filepath, loadpath, solver="ee",validate="True"):
loaderror = loaderror
return moose.element(loadpath), loaderror
else:
- print("Validation failed while reading the model.")
- return moose.element('/'), "This document is not valid SBML"
+ print("Validation failed while reading the model."+"\n"+errorFlag)
+ if errorFlag != "":
+ return moose.element('/'),errorFlag
+ else:
+ return moose.element('/'), "This document is not valid SBML"
def checkFuncDef(model):
funcDef = {}
@@ -229,7 +239,7 @@ def checkFuncDef(model):
funcDef[f.getName()] = {'bvar':bvar, "MathML": fmath.getRightChild()}
return funcDef
-def checkGroup(basePath,model):
+def checkGroup(basePath,model,comptSbmlidMooseIdMap):
groupInfo = {}
modelAnnotaInfo = {}
if model.getPlugin("groups") != None:
@@ -237,27 +247,33 @@ def checkGroup(basePath,model):
modelgn = mplugin.getNumGroups()
for gindex in range(0, mplugin.getNumGroups()):
p = mplugin.getGroup(gindex)
+ grpName = ""
groupAnnoInfo = {}
groupAnnoInfo = getObjAnnotation(p, modelAnnotaInfo)
+
if groupAnnoInfo != {}:
- if moose.exists(basePath.path+'/'+groupAnnoInfo["Compartment"]):
+ if moose.exists(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name):
+ groupName = p.getName()
+ if groupName == " ":
+ groupName = p.getId()
if "Group" in groupAnnoInfo:
- if moose.exists(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"]):
- if moose.exists(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"]+'/'+p.getName()):
- moosegrp = moose.element(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"]+'/'+p.getName())
+ if moose.exists(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"]):
+ if moose.exists(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"]+'/'+groupName):
+ moosegrp = moose.element(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"]+'/'+groupName)
else:
- moosegrp = moose.Neutral(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"]+'/'+p.getName())
+ moosegrp = moose.Neutral(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"]+'/'+groupName)
else:
- moose.Neutral(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"])
- if moose.exists(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"]+'/'+p.getName()):
- moosegrp = moose.element(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"]+'/'+p.getName())
+ moose.Neutral(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"])
+ if moose.exists(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"]+'/'+groupName):
+ moosegrp = moose.element(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"]+'/'+groupName)
else:
- moosegrp = moose.Neutral(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+groupAnnoInfo["Group"]+'/'+p.getName())
+ moosegrp = moose.Neutral(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupAnnoInfo["Group"]+'/'+groupName)
else:
- if not moose.exists(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+p.getName()):
- moosegrp = moose.Neutral(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+p.getName())
+ if not moose.exists(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupName):
+ moosegrp = moose.Neutral(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupName)
else:
- moosegrp = moose.element(basePath.path+'/'+groupAnnoInfo["Compartment"]+'/'+p.getName())
+ moosegrp = moose.element(basePath.path+'/'+comptSbmlidMooseIdMap[groupAnnoInfo["Compartment"]]["MooseId"].name+'/'+groupName)
+
moosegrpinfo = moose.Annotator(moosegrp.path+'/info')
moosegrpinfo.color = groupAnnoInfo["bgColor"]
else:
@@ -1206,7 +1222,7 @@ def createSpecies(basePath, model, comptSbmlidMooseIdMap,
poolInfo = moose.Annotator(poolId.path + '/info')
else:
poolInfo = moose.element(poolId.path + '/info')
-
+
for k, v in list(specAnnoInfo.items()):
if k == 'xCord':
poolInfo.x = float(v)
@@ -1214,12 +1230,13 @@ def createSpecies(basePath, model, comptSbmlidMooseIdMap,
poolInfo.y = float(v)
elif k == 'bgColor':
poolInfo.color = v
- elif k == 'Color':
+ elif k == 'textColor':
poolInfo.textColor = v
elif k == 'diffConstant':
poolId.diffConst = float(v)
elif k == 'motorConstant':
- poolId.motorConst = float(v)
+ poolId.motorConst = float(v)
+
specInfoMap[sId] = {
"Mpath": poolId,
"const": constant,
diff --git a/python/moose/SBML/validation.py b/python/moose/SBML/validation.py
index 48facdf0..1291c0c5 100644
--- a/python/moose/SBML/validation.py
+++ b/python/moose/SBML/validation.py
@@ -18,7 +18,11 @@ Last-Updated: Fri Jul 28 15:50:00 2017(+0530)
By:
**********************************************************************/
+**********************************************************************/
+2019
+Jan 19: - returned errorMsg
'''
+
foundLibSBML_ = False
try:
from libsbml import *
@@ -29,13 +33,15 @@ except Exception as e:
def validateModel(sbmlDoc):
if sbmlDoc.getNumErrors() > 0:
tobecontinued = False
+ validError = ""
for i in range(0,sbmlDoc.getNumErrors()):
- print (sbmlDoc.getError(i).getMessage())
- return False
+ validError = validError+sbmlDoc.getError(i).getMessage()
+ #print (validError)
+ return False, validError
if (not sbmlDoc):
print("validateModel: given a null SBML Document")
- return False
+ return False, "validateModel: given a null SBML Document"
consistencyMessages = ""
validationMessages = ""
@@ -86,7 +92,7 @@ def validateModel(sbmlDoc):
validationMessages = oss
if (noProblems):
- return True
+ return True,""
else:
if consistencyMessages is None:
consistencyMessages = ""
@@ -107,7 +113,7 @@ def validateModel(sbmlDoc):
if validationMessages:
print(validationMessages)
- return False
+ return False,validationMessages
# return ( numConsistencyErrors == 0 and numValidationErrors == 0,
# consistencyMessages)
diff --git a/python/moose/SBML/writeSBML.py b/python/moose/SBML/writeSBML.py
index d8325ee2..0f021c67 100644
--- a/python/moose/SBML/writeSBML.py
+++ b/python/moose/SBML/writeSBML.py
@@ -13,10 +13,12 @@
** copyright (C) 2003-2017 Upinder S. Bhalla. and NCBS
Created : Friday May 27 12:19:00 2016(+0530)
Version
-Last-Updated: Tue 3 Dec 15:15:10 2018(+0530)
+Last-Updated: Tue 29 Jan 15:15:10 2019(+0530)
By: HarshaRani
**********************************************************************/
/****************************
+2019
+Jan 29: getColor are taken from chemConnectUtil, group's width and height are written
2018
Dec 07: using fixXreac's restoreXreacs function to remove xfer
Dec 03: add diff and motor constants to pool
@@ -47,9 +49,10 @@ import re
import os
import moose
from moose.SBML.validation import validateModel
-from moose.chemUtil.chemConnectUtil import *
+from moose.chemUtil.chemConnectUtil import xyPosition,mooseIsInstance,findCompartment,getColor,setupItem
from moose.chemUtil.graphUtils import *
from moose.fixXreacs import restoreXreacs
+import numpy as np
foundLibSBML_ = False
try:
@@ -98,6 +101,7 @@ def mooseWriteSBML(modelpath, filename, sceneitems={}):
if moose.exists(p.path+'/info'):
xcord.append(moose.element(p.path+'/info').x)
ycord.append(moose.element(p.path+'/info').y)
+ getColor(moose.element(p.path+'/info').path)
recalculatecoordinates(modelpath,mObj,xcord,ycord)
positionInfoexist = False
@@ -158,8 +162,8 @@ def mooseWriteSBML(modelpath, filename, sceneitems={}):
mplugin = cremodel_.getPlugin("groups")
group = mplugin.createGroup()
name = str(idBeginWith(moose.element(key).name))
- moosegrpId = name +"_" + str(moose.element(key).getId().value) + "_" + str(moose.element(key).getDataIndex())
- group.setId(moosegrpId)
+ moosegrpId = name +"_" + str(moose.element(key).getId().value) + "_" + str(moose.element(key).getDataIndex())+"_"
+ group.setId(moosegrpId)
group.setName(name)
group.setKind("collection")
@@ -167,13 +171,32 @@ def mooseWriteSBML(modelpath, filename, sceneitems={}):
ginfo = moose.element(key.path+'/info')
else:
ginfo = moose.Annotator(key.path+'/info')
+
groupCompartment = findCompartment(key)
+
grpAnno = "<moose:GroupAnnotation>"
- grpAnno = grpAnno + "<moose:Compartment>" + groupCompartment.name + "</moose:Compartment>\n"
+ grpAnno = grpAnno + "<moose:Compartment>" + groupCompartment.name +"_"+ str(moose.element(groupCompartment).getId().value) +"_"+ str(moose.element(groupCompartment).getDataIndex())+"_"+"</moose:Compartment>\n"
+ #grpAnno = grpAnno + "<moose:Compartment>" + groupCompartment.name + "_" + str(moose.element(groupCompartment).getId().value) + "_" + str(moose.element(groupCompartment).getDataIndex()) + "_"+ "</moose:Compartment>\n"
if moose.element(key.parent).className == "Neutral":
+
grpAnno = grpAnno + "<moose:Group>" + key.parent.name + "</moose:Group>\n"
- if ginfo.color:
- grpAnno = grpAnno + "<moose:bgColor>" + ginfo.color + "</moose:bgColor>\n"
+ grpparent = key.parent.name + "_" + str(moose.element(key.parent).getId().value) + "_" + str(moose.element(key.parent).getDataIndex()) + "_"
+ grpAnno = grpAnno + "<moose:Parent>" + grpparent + "</moose:Parent>\n"
+ else:
+ grpparent = groupCompartment.name + "_" + str(moose.element(groupCompartment).getId().value) + "_" + str(moose.element(groupCompartment).getDataIndex()) + "_"
+ grpAnno = grpAnno + "<moose:Parent>" + grpparent + "</moose:Parent>\n"
+
+ if moose.exists(key.path+'/info'):
+ ginfo = moose.element(key.path+'/info')
+ if ginfo.height and ginfo.width:
+ grpAnno = grpAnno + "<moose:x>" + str(ginfo.x) + "</moose:x>\n"
+ grpAnno = grpAnno + "<moose:y>" + str(ginfo.y) + "</moose:y>\n"
+ grpAnno = grpAnno + "<moose:width>" + str(ginfo.width) + "</moose:width>\n"
+ grpAnno = grpAnno + "<moose:height>" + str(ginfo.height) + "</moose:height>\n"
+ if ginfo.color:
+ grpAnno = grpAnno + "<moose:bgColor>" + ginfo.color + "</moose:bgColor>\n"
+ if ginfo.notes:
+ grpAnno = grpAnno + "<moose:Notes>" + ginfo.notes + "</moose:Notes>\n"
grpAnno = grpAnno + "</moose:GroupAnnotation>"
group.setAnnotation(grpAnno)
@@ -1048,8 +1071,12 @@ def writeCompt(modelpath, cremodel_):
"<moose:surround>" + \
str(comptID_sbml[compt.surround])+ "</moose:surround>\n" + \
"<moose:isMembraneBound>" + \
- str(compt.isMembraneBound)+ "</moose:isMembraneBound>\n" + \
- "</moose:CompartmentAnnotation>"
+ str(compt.isMembraneBound)+ "</moose:isMembraneBound>\n"
+ if moose.exists(compt.path+'/info'):
+ if moose.element(compt.path+'/info').notes != "":
+ comptAnno = comptAnno + "<moose:Notes>" \
+ + moose.element(compt.path+'/info').notes + "</moose:Notes>"
+ comptAnno = comptAnno+ "</moose:CompartmentAnnotation>"
elif isinstance (compt,moose.CylMesh) :
size = (compt.volume/compt.numDiffCompts)*pow(10,3)
comptAnno = "<moose:CompartmentAnnotation><moose:Mesh>" + \
@@ -1059,14 +1086,22 @@ def writeCompt(modelpath, cremodel_):
"<moose:diffLength>" + \
str(compt.diffLength)+ "</moose:diffLength>\n" + \
"<moose:isMembraneBound>" + \
- str(compt.isMembraneBound)+ "</moose:isMembraneBound>\n" + \
- "</moose:CompartmentAnnotation>"
+ str(compt.isMembraneBound)+ "</moose:isMembraneBound>\n"
+ if moose.exists(compt.path+'/info'):
+ if moose.element(compt.path+'/info').notes != "":
+ comptAnno = comptAnno + "<moose:Notes>" \
+ + moose.element(compt.path+'/info').notes + "</moose:Notes>"
+ comptAnno = comptAnno+ "</moose:CompartmentAnnotation>"
else:
comptAnno = "<moose:CompartmentAnnotation><moose:Mesh>" + \
str(compt.className) + "</moose:Mesh>\n" + \
"<moose:isMembraneBound>" + \
- str(compt.isMembraneBound)+ "</moose:isMembraneBound>\n" + \
- "</moose:CompartmentAnnotation>"
+ str(compt.isMembraneBound)+ "</moose:isMembraneBound>\n"
+ if moose.exists(compt.path+'/info'):
+ if moose.element(compt.path+'/info').notes != "":
+ comptAnno = comptAnno + "<moose:Notes>" \
+ + moose.element(compt.path+'/info').notes + "</moose:Notes>"
+ comptAnno = comptAnno+ "</moose:CompartmentAnnotation>"
if createCompt:
c1 = cremodel_.createCompartment()
c1.setId(csetId)
diff --git a/python/moose/chemUtil/chemConnectUtil.py b/python/moose/chemUtil/chemConnectUtil.py
index 9c92e613..f15708be 100644
--- a/python/moose/chemUtil/chemConnectUtil.py
+++ b/python/moose/chemUtil/chemConnectUtil.py
@@ -9,10 +9,105 @@ __version__ = "1.0.0"
__maintainer__ = "Harsha Rani"
__email__ = "hrani@ncbs.res.in"
__status__ = "Development"
-__updated__ = "Nov 8 2018"
+__updated__ = "Mar 11 2019"
import moose
import numpy as np
+import pickle
+import matplotlib
+import os
+import random
+
+
+direct = os.path.dirname(__file__)
+colormap_file = open(os.path.join(direct, "rainbow2.pkl"), "r")
+colorMap = pickle.loads(colormap_file.read().encode('utf8'))
+colormap_file.close()
+ignoreColor= ["mistyrose","antiquewhite","aliceblue","azure","bisque","black","blanchedalmond","blue","cornsilk","darkolivegreen","darkslategray","dimgray","floralwhite","gainsboro","ghostwhite","honeydew","ivory","lavender","lavenderblush","lemonchiffon","lightcyan","lightgoldenrodyellow","lightgray","lightyellow","linen","mediumblue","mintcream","navy","oldlace","papayawhip","saddlebrown","seashell","snow","wheat","white","whitesmoke","aquamarine","lightsalmon","moccasin","limegreen","snow","sienna","beige","dimgrey","lightsage"]
+matplotcolor = {}
+for name,hexno in matplotlib.colors.cnames.items():
+ matplotcolor[name]=hexno
+
+def getRandColor():
+ k = random.choice(matplotcolor.keys())
+ if k in ignoreColor:
+ return getRandColor()
+ else:
+ #return matplotcolor[k]
+ return str(matplotlib.colors.cnames[k])
+
+def getColor(iteminfo):
+ """ Getting a textcolor and background color for the given mooseObject \
+ If textcolor is empty replaced with green \
+ background color is empty replaced with blue
+ if textcolor and background is same as it happend in kkit files \
+ replacing textcolor with random color\
+ The colors are not valid there are siliently replaced with some values \
+ but while model building can raise an exception
+ """
+ textcolor = moose.element(iteminfo).textColor
+ bgcolor = moose.element(iteminfo).color
+ if(textcolor == ''): textcolor = 'green'
+
+ if(bgcolor == ''): bgcolor = 'orange'
+
+ if(textcolor == bgcolor):
+ textcolor = getRandColor()
+
+ textcolor = colorCheck(textcolor)
+ bgcolor = colorCheck(bgcolor)
+ # if moose.exists(iteminfo):
+ # moose.element(iteminfo).textColor = textcolor
+ # moose.element(iteminfo).color = bgcolor
+ return(textcolor,bgcolor)
+
+def colorCheck(fc_bgcolor):
+ """ textColor or background can be anything like string or tuple or list \
+ if string its taken as colorname further down in validColorcheck checked for valid color, \
+ but for tuple and list its taken as r,g,b value.
+ """
+ if isinstance(fc_bgcolor,str):
+ if fc_bgcolor.startswith("#"):
+ fc_bgcolor = fc_bgcolor
+ elif fc_bgcolor.isdigit():
+ """ color is int a map from int to r,g,b triplets from pickled color map file """
+ tc = (int(fc_bgcolor))*2
+ if tc < len(colorMap):
+ pickledColor = colorMap[tc]
+ else:
+ pickledColor = (255, 0, 0)
+ fc_bgcolor = '#%02x%02x%02x' % (pickledColor)
+
+ elif fc_bgcolor.isalpha() or fc_bgcolor.isalnum():
+ fc_bgcolor = validColorcheck(fc_bgcolor)
+
+ else:
+ for r in ['[',']','(',')']:
+ fc_bgcolor = fc_bgcolor.replace(r,"")
+ fc_bgcolor = fc_bgcolor.split(",")
+ c = 0
+ hexlist ="#"
+ for n in fc_bgcolor:
+ if c < 3:
+ hexlist = hexlist+str("%02x" % int(n))
+ c = c+1;
+ fc_bgcolor = hexlist
+ return(fc_bgcolor)
+
+def validColorcheck(color):
+ '''
+ Both in Qt4.7 and 4.8 if not a valid color it makes it as back but in 4.7 there will be a warning mssg which is taken here
+ checking if textcolor or backgroundcolor is valid color, if 'No' making white color as default
+ where I have not taken care for checking what will be backgroundcolor for textcolor or textcolor for backgroundcolor
+ '''
+ #if QColor(color).isValid():
+ if matplotlib.colors.is_color_like(color):
+ if color == "blue":
+ color = "orange"
+ color = matplotlib.colors.cnames[color.lower()]
+ return color
+ else:
+ return(matplotlib.colors.cnames["orange"])
def xyPosition(objInfo,xory):
try:
@@ -89,7 +184,7 @@ def setupMeshObj(modelRoot):
ymin = min(ycord)
ymax = max(ycord)
return meshEntry,xmin,xmax,ymin,ymax,positionInfoExist,listOfitems
-
+
def getxyCord(xcord,ycord,list1,listOfitems):
for item in list1:
# if isinstance(item,Function):
diff --git a/python/moose/chemUtil/rainbow2.pkl b/python/moose/chemUtil/rainbow2.pkl
new file mode 100644
index 00000000..8097c29c
--- /dev/null
+++ b/python/moose/chemUtil/rainbow2.pkl
@@ -0,0 +1,514 @@
+((I248
+I0
+I255
+tp0
+(I240
+I0
+I255
+tp1
+(I232
+I0
+I255
+tp2
+(I224
+I0
+I255
+tp3
+(I216
+I0
+I255
+tp4
+(I208
+I0
+I255
+tp5
+(I200
+I0
+I255
+tp6
+(I192
+I0
+I255
+tp7
+(I184
+I0
+I255
+tp8
+(I176
+I0
+I255
+tp9
+(I168
+I0
+I255
+tp10
+(I160
+I0
+I255
+tp11
+(I152
+I0
+I255
+tp12
+(I144
+I0
+I255
+tp13
+(I136
+I0
+I255
+tp14
+(I128
+I0
+I255
+tp15
+(I120
+I0
+I255
+tp16
+(I112
+I0
+I255
+tp17
+(I104
+I0
+I255
+tp18
+(I96
+I0
+I255
+tp19
+(I88
+I0
+I255
+tp20
+(I80
+I0
+I255
+tp21
+(I72
+I0
+I255
+tp22
+(I64
+I0
+I255
+tp23
+(I56
+I0
+I255
+tp24
+(I48
+I0
+I255
+tp25
+(I40
+I0
+I255
+tp26
+(I32
+I0
+I255
+tp27
+(I24
+I0
+I255
+tp28
+(I16
+I0
+I255
+tp29
+(I8
+I0
+I255
+tp30
+(I0
+I0
+I255
+tp31
+(I0
+I8
+I248
+tp32
+(I0
+I16
+I240
+tp33
+(I0
+I24
+I232
+tp34
+(I0
+I32
+I224
+tp35
+(I0
+I40
+I216
+tp36
+(I0
+I48
+I208
+tp37
+(I0
+I56
+I200
+tp38
+(I0
+I64
+I192
+tp39
+(I0
+I72
+I184
+tp40
+(I0
+I80
+I176
+tp41
+(I0
+I88
+I168
+tp42
+(I0
+I96
+I160
+tp43
+(I0
+I104
+I152
+tp44
+(I0
+I112
+I144
+tp45
+(I0
+I120
+I136
+tp46
+(I0
+I128
+I128
+tp47
+(I0
+I136
+I120
+tp48
+(I0
+I144
+I112
+tp49
+(I0
+I152
+I104
+tp50
+(I0
+I160
+I96
+tp51
+(I0
+I168
+I88
+tp52
+(I0
+I176
+I80
+tp53
+(I0
+I184
+I72
+tp54
+(I0
+I192
+I64
+tp55
+(I0
+I200
+I56
+tp56
+(I0
+I208
+I48
+tp57
+(I0
+I216
+I40
+tp58
+(I0
+I224
+I32
+tp59
+(I0
+I232
+I24
+tp60
+(I0
+I240
+I16
+tp61
+(I0
+I248
+I8
+tp62
+(I0
+I255
+I0
+tp63
+(I8
+I255
+I0
+tp64
+(I16
+I255
+I0
+tp65
+(I24
+I255
+I0
+tp66
+(I32
+I255
+I0
+tp67
+(I40
+I255
+I0
+tp68
+(I48
+I255
+I0
+tp69
+(I56
+I255
+I0
+tp70
+(I64
+I255
+I0
+tp71
+(I72
+I255
+I0
+tp72
+(I80
+I255
+I0
+tp73
+(I88
+I255
+I0
+tp74
+(I96
+I255
+I0
+tp75
+(I104
+I255
+I0
+tp76
+(I112
+I255
+I0
+tp77
+(I120
+I255
+I0
+tp78
+(I128
+I255
+I0
+tp79
+(I136
+I255
+I0
+tp80
+(I144
+I255
+I0
+tp81
+(I152
+I255
+I0
+tp82
+(I160
+I255
+I0
+tp83
+(I168
+I255
+I0
+tp84
+(I176
+I255
+I0
+tp85
+(I184
+I255
+I0
+tp86
+(I192
+I255
+I0
+tp87
+(I200
+I255
+I0
+tp88
+(I208
+I255
+I0
+tp89
+(I216
+I255
+I0
+tp90
+(I224
+I255
+I0
+tp91
+(I232
+I255
+I0
+tp92
+(I240
+I255
+I0
+tp93
+(I248
+I255
+I0
+tp94
+(I255
+I255
+I0
+tp95
+(I255
+I248
+I0
+tp96
+(I255
+I240
+I0
+tp97
+(I255
+I232
+I0
+tp98
+(I255
+I224
+I0
+tp99
+(I255
+I216
+I0
+tp100
+(I255
+I208
+I0
+tp101
+(I255
+I200
+I0
+tp102
+(I255
+I192
+I0
+tp103
+(I255
+I184
+I0
+tp104
+(I255
+I176
+I0
+tp105
+(I255
+I168
+I0
+tp106
+(I255
+I160
+I0
+tp107
+(I255
+I152
+I0
+tp108
+(I255
+I144
+I0
+tp109
+(I255
+I136
+I0
+tp110
+(I255
+I128
+I0
+tp111
+(I255
+I120
+I0
+tp112
+(I255
+I112
+I0
+tp113
+(I255
+I104
+I0
+tp114
+(I255
+I96
+I0
+tp115
+(I255
+I88
+I0
+tp116
+(I255
+I80
+I0
+tp117
+(I255
+I72
+I0
+tp118
+(I255
+I64
+I0
+tp119
+(I255
+I56
+I0
+tp120
+(I255
+I48
+I0
+tp121
+(I255
+I40
+I0
+tp122
+(I255
+I32
+I0
+tp123
+(I255
+I24
+I0
+tp124
+(I255
+I16
+I0
+tp125
+(I255
+I8
+I0
+tp126
+(I255
+I0
+I0
+tp127
+tp128
+.
\ No newline at end of file
diff --git a/python/moose/genesis/writeKkit.py b/python/moose/genesis/writeKkit.py
index 7ed8d66e..fbd8058f 100644
--- a/python/moose/genesis/writeKkit.py
+++ b/python/moose/genesis/writeKkit.py
@@ -271,9 +271,9 @@ def writeConcChan(modelpath,f,sceneitems):
else:
error = error + "\n x and y co-ordinates are not specified for `" + cChan.name+ "` zero will be assigned \n "
if color == "" or color == " ":
- color = getRandColor()
+ color = getRandomColor()
if textcolor == "" or textcolor == " ":
- textcolor = getRandColor()
+ textcolor = getRandomColor()
f.write("simundump kchan /kinetics/" + trimPath(cChan)+ " " + str(int(1)) + " " + str(cChan.permeability)+ " " +
str(int(0)) + " " +
str(int(0)) + " " +
@@ -344,9 +344,9 @@ def writeEnz( modelpath,f,sceneitems):
else:
error = error + "\n x and y co-ordinates are not specified for `" + enz.name+ "` zero will be assigned \n "
if color == "" or color == " ":
- color = getRandColor()
+ color = getRandomColor()
if textcolor == "" or textcolor == " ":
- textcolor = getRandColor()
+ textcolor = getRandomColor()
f.write("simundump kenz /kinetics/" + trimPath(enz) + " " + str(int(0))+ " " +
str(concInit) + " " +
@@ -421,9 +421,9 @@ def writeReac(modelpath,f,sceneitems):
y = 0
error = error + "\n x and y co-ordinates are not specified for `" + reac.name+ "` zero will be assigned \n "
if color == "" or color == " ":
- color = getRandColor()
+ color = getRandomColor()
if textcolor == "" or textcolor == " ":
- textcolor = getRandColor()
+ textcolor = getRandomColor()
f.write("simundump kreac /kinetics/" + trimPath(reac) + " " +str(0) +" "+ str(kf) + " " + str(kb) + " \"\" " +
str(color) + " " + str(textcolor) + " " + str(int(x)) + " " + str(int(y)) + " "+ str(0)+"\n")
return reacList,error
@@ -574,9 +574,9 @@ def writeplot( tgraphs,f ):
fg = getColorCheck(fg,GENESIS_COLOR_SEQUENCE)
tabPath = re.sub("\[[0-9]+\]", "", tabPath)
if tabPath.find("conc1") >= 0 or tabPath.find("conc2") >= 0:
- first = first + "simundump xplot " + tabPath + " 3 524288 \\\n" + "\"delete_plot.w <s> <d>; edit_plot.D <w>\" " + fg + " 0 0 1\n"
+ first = first + "simundump xplot " + tabPath + " 3 524288 \\\n" + "\"delete_plot.w <s> <d>; edit_plot.D <w>\" " + str(fg) + " 0 0 1\n"
if tabPath.find("conc3") >= 0 or tabPath.find("conc4") >= 0:
- second = second + "simundump xplot " + tabPath + " 3 524288 \\\n" + "\"delete_plot.w <s> <d>; edit_plot.D <w>\" " + fg + " 0 0 1\n"
+ second = second + "simundump xplot " + tabPath + " 3 524288 \\\n" + "\"delete_plot.w <s> <d>; edit_plot.D <w>\" " + str(fg) + " 0 0 1\n"
return first,second
def writePool(modelpath,f,volIndex,sceneitems):
@@ -628,9 +628,9 @@ def writePool(modelpath,f,volIndex,sceneitems):
geometryName = volIndex[float(poolsCmpt.volume)]
volume = p.volume * NA * 1e-3
if color == "" or color == " ":
- color = getRandColor()
+ color = getRandomColor()
if textcolor == "" or textcolor == " ":
- textcolor = getRandColor()
+ textcolor = getRandomColor()
f.write("simundump kpool /kinetics/" + trimPath(p) + " 0 " +
str(p.diffConst) + " " +
str(0) + " " +
@@ -673,7 +673,7 @@ def getColorCheck(color,GENESIS_COLOR_SEQUENCE):
else:
raise Exception("Invalid Color Value!")
-def getRandColor():
+def getRandomColor():
ignoreColor= ["mistyrose","antiquewhite","aliceblue","azure","bisque","black","blanchedalmond","blue","cornsilk","darkolivegreen","darkslategray","dimgray","floralwhite","gainsboro","ghostwhite","honeydew","ivory","lavender","lavenderblush","lemonchiffon","lightcyan","lightgoldenrodyellow","lightgray","lightyellow","linen","mediumblue","mintcream","navy","oldlace","papayawhip","saddlebrown","seashell","snow","wheat","white","whitesmoke","aquamarine","lightsalmon","moccasin","limegreen","snow","sienna","beige","dimgrey","lightsage"]
matplotcolor = {}
for name,hexno in matplotlib.colors.cnames.items():
@@ -681,7 +681,7 @@ def getRandColor():
k = random.choice(list(matplotcolor.keys()))
if k in ignoreColor:
- return getRandColor()
+ return getRandomColor()
else:
return k
def writeCompartment(modelpath,compts,f):
diff --git a/python/moose/helper.py b/python/moose/helper.py
new file mode 100644
index 00000000..79af6868
--- /dev/null
+++ b/python/moose/helper.py
@@ -0,0 +1,83 @@
+"""helper.py:
+
+Some helper functions which are compatible with both python2 and python3.
+"""
+
+__author__ = "Dilawar Singh"
+__copyright__ = "Copyright 2017-, Dilawar Singh"
+__version__ = "1.0.0"
+__maintainer__ = "Dilawar Singh"
+__email__ = "dilawars@ncbs.res.in"
+__status__ = "Development"
+
+import os
+import re
+import subprocess
+
+def execute(cmd):
+ """execute: Execute a given command.
+
+ :param cmd: string, given command.
+
+ Return:
+ ------
+ Return a iterator over output.
+ """
+ popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
+ for stdout_line in iter(popen.stdout.readline, ""):
+ yield stdout_line
+ popen.stdout.close()
+ return_code = popen.wait()
+ if return_code:
+ raise subprocess.CalledProcessError(return_code, cmd)
+
+
+def find_files( dirname, ext=None, name_contains=None, text_regex_search=None):
+ files = []
+ for d, sd, fs in os.walk(dirname):
+ for f in fs:
+ fpath = os.path.join(d,f)
+ include = True
+ if ext is not None:
+ if f.split('.')[-1] != ext:
+ include = False
+ if name_contains:
+ if name_contains not in os.path.basename(f):
+ include = False
+ if text_regex_search:
+ with open(fpath, 'r' ) as f:
+ txt = f.read()
+ if re.search(text_regex_search, txt) is None:
+ include = False
+ if include:
+ files.append(fpath)
+ return files
+
+# Matplotlib text for running simulation. It make sures at each figure is saved
+# to individual png files.
+matplotlibText = """
+print( '>>>> saving all figues')
+import matplotlib.pyplot as plt
+def multipage(filename, figs=None, dpi=200):
+ pp = PdfPages(filename)
+ if figs is None:
+ figs = [plt.figure(n) for n in plt.get_fignums()]
+ for fig in figs:
+ fig.savefig(pp, format='pdf')
+ pp.close()
+
+def saveall(prefix='results', figs=None):
+ if figs is None:
+ figs = [plt.figure(n) for n in plt.get_fignums()]
+ for i, fig in enumerate(figs):
+ outfile = '%s.%d.png' % (prefix, i)
+ fig.savefig(outfile)
+ print( '>>>> %s saved.' % outfile )
+ plt.close()
+
+try:
+ saveall()
+except Exception as e:
+ print( '>>>> Error in saving: %s' % e )
+ quit(0)
+"""
diff --git a/python/moose/moose.py b/python/moose/moose.py
index c3c6b60d..191d26d4 100644
--- a/python/moose/moose.py
+++ b/python/moose/moose.py
@@ -1,8 +1,9 @@
# -*- coding: utf-8 -*-
+from __future__ import print_function, division, absolute_import
+
# Author: Subhasis Ray
# Maintainer: Dilawar Singh, Harsha Rani, Upi Bhalla
-from __future__ import print_function, division, absolute_import
import warnings
import os
import pydoc
@@ -104,6 +105,7 @@ def loadModel(filename, modelpath, solverclass="gsl"):
else:
mu.error( "Unknown model extenstion '%s'" % extension)
return None
+
# Version
def version( ):
@@ -154,7 +156,21 @@ def mooseReadSBML(filepath, loadpath, solver='ee',validate="on"):
"""
global sbmlImport_
if sbmlImport_:
- return _readSBML.mooseReadSBML(filepath, loadpath, solver, validate)
+ modelpath = _readSBML.mooseReadSBML(filepath, loadpath, solver, validate)
+ sc = solver.lower()
+ if sc in ["gssa","gillespie","stochastic","gsolve"]:
+ method = "gssa"
+ elif sc in ["gsl","runge kutta","deterministic","ksolve","rungekutta","rk5","rkf","rk"]:
+ method = "gsl"
+ elif sc in ["exponential euler","exponentialeuler","neutral"]:
+ method = "ee"
+ else:
+ method = "ee"
+
+ if method != 'ee':
+ chemError = _chemUtil.add_Delete_ChemicalSolver.mooseAddChemSolver(modelpath[0].path, method)
+
+ return modelpath
else:
print( sbmlError_ )
return False
@@ -652,3 +668,7 @@ def doc(arg, inherited=True, paged=True):
pager(text)
else:
print(text)
+
+
+#
+# moose.py ends here
diff --git a/python/moose/server.py b/python/moose/server.py
index 400058a3..9a9820e6 100644
--- a/python/moose/server.py
+++ b/python/moose/server.py
@@ -1,5 +1,11 @@
# -*- coding: utf-8 -*-
-from __future__ import print_function, division
+from __future__ import print_function, division, absolute_import
+
+# This is moose.server.
+# It accepts simulation request on a specified TCP port (default 31417).
+# It simulates the given file (usually a archive file e.g., tar.bz2) and sends
+# back artefacts generated by simulation (mostly images); and streams data from
+# moose.Tables back to client.
__author__ = "Dilawar Singh"
__copyright__ = "Copyright 2019, Dilawar Singh"
@@ -12,18 +18,15 @@ import sys
import re
import os
import time
+import math
import shutil
import socket
import signal
import tarfile
import tempfile
import threading
-import datetime
-import subprocess
import logging
-
-# setup environment variables for the streamer starts working.
-os.environ['MOOSE_SOCKET_STREAMER_ADDRESS'] = 'ghevar.ncbs.res.in:31416'
+import subprocess
# create a logger for this server.
logging.basicConfig(
@@ -34,7 +37,7 @@ logging.basicConfig(
filemode='a'
)
console = logging.StreamHandler()
-console.setLevel(logging.INFO)
+console.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
_logger = logging.getLogger('')
@@ -42,20 +45,54 @@ _logger.addHandler(console)
__all__ = [ 'serve' ]
+# Global variable to stop all running threads.
stop_all_ = False
+sock_ = None
+stop_streamer_ = {}
-def handler(signum, frame):
- global stop_all_
- _logger.info( "User terminated all processes." )
- stop_all_ = True
+# Use prefixL_ bytes to encode the size of stream. One can probably use just one
+# byte to do. Lets go with the inefficient one for now.
+prefixL_ = 9
+
+# Matplotlib text for running simulation. It make sures at each figure is saved
+# to individual png files.
+matplotlibText = """
+print( '>>>> saving all figues')
+import matplotlib.pyplot as plt
+def multipage(filename, figs=None, dpi=200):
+ pp = PdfPages(filename)
+ if figs is None:
+ figs = [plt.figure(n) for n in plt.get_fignums()]
+ for fig in figs:
+ fig.savefig(pp, format='pdf')
+ pp.close()
+
+def saveall(prefix='results', figs=None):
+ if figs is None:
+ figs = [plt.figure(n) for n in plt.get_fignums()]
+ for i, fig in enumerate(figs):
+ outfile = '%s.%d.png' % (prefix, i)
+ fig.savefig(outfile)
+ print( '>>>> %s saved.' % outfile )
+ plt.close()
-signal.signal( signal.SIGINT, handler)
+try:
+ saveall()
+except Exception as e:
+ print( '>>>> Error in saving: %s' % e )
+ quit(0)
+"""
-def split_data( data ):
- prefixLenght = 10
- return data[:prefixLenght].strip(), data[prefixLenght:]
def execute(cmd):
+ """execute: Execute a given command.
+
+ :param cmd: string, given command.
+
+ Return:
+ ------
+ Return a iterator over output.
+ """
popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
for stdout_line in iter(popen.stdout.readline, ""):
yield stdout_line
@@ -64,52 +101,82 @@ def execute(cmd):
if return_code:
raise subprocess.CalledProcessError(return_code, cmd)
-def send_msg(msg, conn):
+
+def find_files( dirname, ext=None, name_contains=None, text_regex_search=None):
+ files = []
+ for d, sd, fs in os.walk(dirname):
+ for f in fs:
+ fpath = os.path.join(d,f)
+ include = True
+ if ext is not None:
+ if f.split('.')[-1] != ext:
+ include = False
+ if name_contains:
+ if name_contains not in os.path.basename(f):
+ include = False
+ if text_regex_search:
+ with open(fpath, 'r' ) as f:
+ txt = f.read()
+ if re.search(text_regex_search, txt) is None:
+ include = False
+ if include:
+ files.append(fpath)
+ return files
+
+def prefix_data_with_size(data):
+ global prefixL_
+ prefix = b'0'*(prefixL_-int(math.log10(len(data)))-1) + b'%d' % len(data)
+ assert len(prefix) == prefixL_
+ return b'%s%s' % (prefix, data)
+
+# Signal handler.
+def signal_handler(signum, frame):
+ global stop_all_
+ global sock_
+ _logger.info( "User terminated all processes." )
+ stop_all_ = True
+ # sock_.shutdown( socket.SHUT_RDWR )
+ sock_.close()
+ time.sleep(1)
+ quit(1)
+
+
+def split_data( data ):
+ global prefixL_
+ return data[:prefixL_].strip(), data[prefixL_:]
+
+def send_msg(msg, conn, prefix='LOG'):
if not msg.strip():
return False
- _logger.debug( msg )
- msg = '%s>>> %s' % (socket.gethostname(), msg)
- conn.sendall( b'%010d%s' % (len(msg), msg))
+ if prefix != 'TAB':
+ _logger.debug(msg)
+ else:
+ _logger.debug( 'Sending msg with size %d' % len(msg))
+ msg = '<%s>%s' % (prefix, msg)
+ conn.sendall(prefix_data_with_size(msg))
def run(cmd, conn, cwd=None):
+ _logger.info( "Executing %s" % cmd )
oldCWD = os.getcwd()
if cwd is not None:
os.chdir(cwd)
- _logger.debug( "Executing in %s" % os.getcwd() )
- for line in execute(cmd.split()):
- send_msg(line, conn)
+ try:
+ for line in execute(cmd.split()):
+ if line:
+ send_msg(line, conn)
+ except Exception as e:
+ send_msg("Simulation failed: %s" % e, conn)
os.chdir(oldCWD)
-def find_files_to_run( files ):
- """Any file name starting with __main is to be run."""
- toRun = []
- for f in files:
- if '__main' in os.path.basename(f):
- toRun.append(f)
- if toRun:
- return toRun
-
- # Then guess.
- if len(files) == 1:
- return files
-
- for f in files:
- with open(f, 'r' ) as fh:
- txt = fh.read()
- if re.search(r'def\s+main\(', txt):
- if re.search('^\s+main\(\S+?\)', txt):
- toRun.append(f)
- return toRun
-
def recv_input(conn, size=1024):
# first 10 bytes always tell how much to read next. Make sure the submit job
# script has it
- d = conn.recv(10, socket.MSG_WAITALL)
- while len(d) < 10:
+ d = conn.recv(prefixL_, socket.MSG_WAITALL)
+ while len(d) < prefixL_:
try:
- d = conn.recv(10, socket.MSG_WAITALL)
+ d = conn.recv(prefixL_, socket.MSG_WAITALL)
except Exception:
- _logger.error( "Error in format. First 6 bytes are size of msg." )
+ _logger.error("MSG FORMAT: %d bytes are size of msg."%prefixL_)
continue
d, data = int(d), b''
while len(data) < d:
@@ -122,7 +189,7 @@ def writeTarfile( data ):
_logger.info( "Writing %d bytes to %s" % (len(data), tfile))
f.write(data)
# Sleep for some time so that file can be written to disk.
- time.sleep(0.2)
+ time.sleep(0.1)
if not tarfile.is_tarfile(tfile):
_logger.warn( 'Not a valid tar file: %s' % tfile)
return None
@@ -133,40 +200,65 @@ def suffixMatplotlibStmt( filename ):
with open(filename, 'r') as f:
txt = f.read()
- matplotlibText = '''
-# from matplotlib.backends.backend_pdf import PdfPages
-import matplotlib.pyplot as plt
-
-def multipage(filename, figs=None, dpi=200):
- pp = PdfPages(filename)
- if figs is None:
- figs = [plt.figure(n) for n in plt.get_fignums()]
- for fig in figs:
- fig.savefig(pp, format='pdf')
- pp.close()
-
-def saveall(prefix='results', figs=None):
- if figs is None:
- figs = [plt.figure(n) for n in plt.get_fignums()]
- for i, fig in enumerate(figs):
- fig.savefig('%s.%d.png' %(prefix,i) )
- plt.close()
-
-try:
- # multipage("results.pdf")
- saveall()
-except Exception as e:
- print( e )
- '''
with open(outfile, 'w' ) as f:
f.write( txt )
+ f.write( '\n' )
f.write( matplotlibText )
return outfile
+def streamer_client(socketPath, conn):
+ # Connect to running socket server.
+ global stop_streamer_
+ stop = False
+ _logger.debug( "Trying to connect to server at : %s" % socketPath )
+ while not os.path.exists( socketPath ):
+ #print( 'socket %s is not available yet.' % socketPath )
+ time.sleep(0.1)
+ stop = stop_streamer_[threading.currentThread().name]
+ if stop:
+ return
+
+ stClient = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ try:
+ stClient.connect(socketPath)
+ except socket.error as e:
+ _logger.warning('Could not connect: %s' % e)
+ return
+
+ # send streaming data back to client. The streamer send fixed size messages
+ # of 1024/2048 bytes each (see the c++ implmenetation).
+ _logger.info( "Socket Streamer is connected with server." )
+ stClient.settimeout(0.05)
+ send_msg( b'Now streaming table data.', conn, 'TAB')
+ while not stop:
+ stop = stop_streamer_[threading.currentThread().name]
+ data = b''
+ try:
+ data = stClient.recv(1024)
+ if len(data.strip()) > 0:
+ send_msg(data, conn, 'TAB')
+ except socket.timeout:
+ continue
+ stClient.close()
+ if os.path.isfile(socketPath):
+ os.unlink(socketPath)
+
def run_file(filename, conn, cwd=None):
+ # set environment variable so that socket streamer can start.
+ global stop_streamer_
+ socketPath = os.path.join(tempfile.mkdtemp(), 'SOCK_TABLE_STREAMER')
+ os.environ['MOOSE_STREAMER_ADDRESS'] = socketPath
+ streamerThread = threading.Thread(target=streamer_client
+ , args=(socketPath, conn,))
+ stop_streamer_[streamerThread.name] = False
+ streamerThread.daemon = True
+ streamerThread.start()
filename = suffixMatplotlibStmt(filename)
run( "%s %s" % (sys.executable, filename), conn, cwd)
- _logger.info( '.... DONE' )
+ stop_streamer_[streamerThread.name] = True
+ streamerThread.join( timeout = 1)
+ if streamerThread.is_alive():
+ _logger.error( "The socket streamer client is still running...")
def extract_files(tfile, to):
userFiles = []
@@ -177,11 +269,9 @@ def extract_files(tfile, to):
except Exception as e:
_logger.warn( e)
# now check if all files have been extracted properly
- success = True
for f in userFiles:
if not os.path.exists(f):
_logger.error( "File %s could not be extracted." % f )
- success = False
return userFiles
def prepareMatplotlib( cwd ):
@@ -189,30 +279,48 @@ def prepareMatplotlib( cwd ):
f.write( 'interactive : True' )
def send_bz2(conn, data):
- data = b'%010d%s' % (len(data), data)
- conn.sendall(data)
+ global prefixL_
+ send_msg(data, conn, 'TAR')
-def sendResults(tdir, conn, fromThisTime):
+def sendResults(tdir, conn, notTheseFiles):
# Only send new files.
resdir = tempfile.mkdtemp()
resfile = os.path.join(resdir, 'results.tar.bz2')
-
with tarfile.open( resfile, 'w|bz2') as tf:
- for d, sd, fs in os.walk(tdir):
- for f in fs:
- fpath = os.path.join(d,f)
- if datetime.datetime.fromtimestamp(os.path.getmtime(fpath)) > fromThisTime:
- _logger.info( "Adding file %s" % f )
- tf.add(os.path.join(d, f), f)
+ for f in find_files(tdir, ext='png'):
+ _logger.info( "Adding file %s" % f )
+ tf.add(f, os.path.basename(f))
time.sleep(0.01)
# now send the tar file back to client
with open(resfile, 'rb' ) as f:
data = f.read()
- _logger.info( 'Total bytes in result: %d' % len(data))
+ _logger.info( 'Total bytes to send to client: %d' % len(data))
send_bz2(conn, data)
shutil.rmtree(resdir)
+def find_files_to_run( files ):
+ """Any file name starting with __main is to be run.
+ Many such files can be recieved by client.
+ """
+ toRun = []
+ for f in files:
+ if '__main' in os.path.basename(f):
+ toRun.append(f)
+ if toRun:
+ return toRun
+ # Else guess.
+ if len(files) == 1:
+ return files
+
+ for f in files:
+ with open(f, 'r' ) as fh:
+ txt = fh.read()
+ if re.search(r'def\s+main\(', txt):
+ if re.search('^\s+main\(\S+?\)', txt):
+ toRun.append(f)
+ return toRun
+
def simulate( tfile, conn ):
"""Simulate a given tar file.
"""
@@ -240,49 +348,56 @@ def savePayload( conn ):
def handle_client(conn, ip, port):
isActive = True
+ _logger.info( "Serving request from %s:%s" % (ip, port) )
while isActive:
tarfileName, nBytes = savePayload(conn)
if tarfileName is None:
_logger.warn( "Could not recieve data." )
isActive = False
- _logger.info( "PAYLOAD RECIEVED: %d" % nBytes )
if not os.path.isfile(tarfileName):
send_msg("[ERROR] %s is not a valid tarfile. Retry"%tarfileName, conn)
break
- startSimTime = datetime.datetime.now()
+
+ # list of files before the simulation.
+ notthesefiles = find_files(os.path.dirname(tarfileName))
res, msg = simulate( tarfileName, conn )
- if not res:
+ if 0 != res:
send_msg( "Failed to run simulation: %s" % msg, conn)
isActive = False
time.sleep(0.1)
- send_msg('>DONE SIMULATION', conn)
+
# Send results after DONE is sent.
- sendResults(os.path.dirname(tarfileName), conn, startSimTime)
+ send_msg('All done', conn, 'EOS')
+ sendResults(os.path.dirname(tarfileName), conn, notthesefiles)
+ break
+
def start_server( host, port, max_requests = 10 ):
global stop_all_
- soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- soc.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ global sock_
+ sock_ = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
- soc.bind( (host, port))
+ sock_.bind( (host, port))
_logger.info( "Server created %s:%s" %(host,port) )
except Exception as e:
_logger.error( "Failed to bind: %s" % e)
quit(1)
- # listen upto 100 of requests
- soc.listen(max_requests)
+ # listen upto 10 of requests
+ sock_.listen(max_requests)
while True:
- conn, (ip, port) = soc.accept()
- _logger.info( "Connected with %s:%s" % (ip, port) )
- try:
- t = threading.Thread(target=handle_client, args=(conn, ip, port))
- t.start()
- except Exception as e:
- _logger.warn(e)
if stop_all_:
break
- soc.close()
+ sock_.settimeout(10)
+ try:
+ conn, (ip, port) = sock_.accept()
+ except socket.timeout as e:
+ continue
+ sock_.settimeout(0.0)
+ t = threading.Thread(target=handle_client, args=(conn, ip, port))
+ t.start()
+ sock_.close()
def serve(host, port):
start_server(host, port)
@@ -290,11 +405,9 @@ def serve(host, port):
def main( args ):
global stop_all_
host, port = args.host, args.port
- try:
- serve(host, port)
- except KeyboardInterrupt:
- stop_all_ = True
- quit(1)
+ # Install a signal handler.
+ signal.signal( signal.SIGINT, signal_handler)
+ serve(host, port)
if __name__ == '__main__':
import argparse
diff --git a/python/moose/streamer_utils.py b/python/moose/streamer_utils.py
new file mode 100644
index 00000000..a89f0502
--- /dev/null
+++ b/python/moose/streamer_utils.py
@@ -0,0 +1,58 @@
+"""streamer_utils.py:
+
+Utility for Table streamer.
+
+"""
+
+__author__ = "Dilawar Singh"
+__copyright__ = "Copyright 2017-, Dilawar Singh"
+__version__ = "1.0.0"
+__maintainer__ = "Dilawar Singh"
+__email__ = "dilawars@ncbs.res.in"
+__status__ = "Development"
+
+import sys
+import os
+import numpy as np
+import math
+import struct
+from collections import defaultdict
+
+def bytes_to_np_arr(data):
+ return np.frombuffer(data, float)
+
+def np_array_to_string(arr):
+ return np.uint8(arr).tostring()
+
+def np_array_to_data(arr):
+ # Make sure that first char of arr is 'H'
+ n = 0
+ res = defaultdict(list)
+ while n < len(arr):
+ assert chr(int(arr[n])) == 'H', 'Expected H, Got %s'%arr[n]
+ hSize = int(arr[n+1])
+ n += 1
+ colName = np_array_to_string(arr[n+1:n+1+hSize])
+ n += hSize + 1
+ assert chr(int(arr[n])) == 'V', 'Expected V'
+ n += 1
+ dataSize = int(arr[n])
+ n += 1
+ res[colName].append(arr[n:n+dataSize])
+ n += dataSize
+ return { k : np.concatenate(v) for k, v in res.items() }
+
+def decode_data(data):
+ arr = bytes_to_np_arr(data)
+ assert int(arr[0]) == ord('H'), "First char must be H"
+ return np_array_to_data(arr)
+
+def test():
+ with open(sys.argv[1], 'rb') as f:
+ data = f.read()
+ print( "[INFO ] Total bytes read %d" % len(data))
+ s = decode_data(data)
+ print(s)
+
+if __name__ == '__main__':
+ test()
diff --git a/python/moose/utils.py b/python/moose/utils.py
index 64fdee8b..17c262d8 100644
--- a/python/moose/utils.py
+++ b/python/moose/utils.py
@@ -338,8 +338,10 @@ def autoposition(root):
def loadModel(filename, target,method='ee'):
moose.loadModel(filename,target)
- moose.mooseaddChemSolver(target,method)
-
+ moose.mooseAddChemSolver(target,method)
+ if moose.exists(target+'/kinetics/info'):
+ moose.element(target+'/kinetics/info').solver = method
+
def readcell_scrambled(filename, target, method='ee'):
"""A special version for handling cases where a .p file has a line
with specified parent yet to be defined.
diff --git a/python/setup.cmake.py b/python/setup.cmake.py
index c7254b40..ca67559d 100644
--- a/python/setup.cmake.py
+++ b/python/setup.cmake.py
@@ -59,5 +59,5 @@ setup(
],
install_requires = [ 'numpy' ],
package_dir = { 'moose' : 'moose', 'rdesigneur' : 'rdesigneur' },
- package_data = { 'moose' : ['_moose' + suffix, 'neuroml2/schema/NeuroMLCoreDimensions.xml'] },
+ package_data = { 'moose' : ['_moose' + suffix, 'neuroml2/schema/NeuroMLCoreDimensions.xml','chemUtil/rainbow2.pkl'] },
)
diff --git a/scheduling/Clock.cpp b/scheduling/Clock.cpp
index dd60aecf..e407bb63 100644
--- a/scheduling/Clock.cpp
+++ b/scheduling/Clock.cpp
@@ -425,8 +425,8 @@ const Cinfo* Clock::initCinfo()
" Ksolve 16 0.1\n"
" Stats 17 0.1\n"
" Table2 18 1\n"
- " SocketStreamer 19 5\n"
- " Streamer 20 10\n"
+ " SocketStreamer 19 1\n"
+ " Streamer 20 5\n"
" HDF5DataWriter 30 1\n"
" HDF5WriterBase 30 1\n"
@@ -1030,8 +1030,8 @@ void Clock::buildDefaultTick()
defaultDt_[16] = 0.1;
defaultDt_[17] = 0.1;
defaultDt_[18] = 1; // For tables for chemical calculations.
- defaultDt_[19] = 5; // For Socket Streamer
- defaultDt_[20] = 10; // For CSV Streamer
+ defaultDt_[19] = 1; // For Socket Streamer
+ defaultDt_[20] = 5; // For CSV Streamer
// 20-29 are not assigned.
defaultDt_[30] = 1; // For the HDF writer
diff --git a/shell/Shell.cpp b/shell/Shell.cpp
index 1043f19d..fe55a708 100644
--- a/shell/Shell.cpp
+++ b/shell/Shell.cpp
@@ -22,6 +22,7 @@
#include "../msg/OneToAllMsg.h"
#include "../msg/SparseMsg.h"
#include "../builtins/SocketStreamer.h"
+#include "../builtins/Streamer.h"
#include "Shell.h"
#include "Wildcard.h"
@@ -63,73 +64,68 @@ const Cinfo* Shell::initCinfo()
&Shell::setCwe,
&Shell::getCwe );
-////////////////////////////////////////////////////////////////
-// Dest Finfos: Functions handled by Shell
-////////////////////////////////////////////////////////////////
+
+ // Dest Finfos: Functions handled by Shell
static DestFinfo handleUseClock( "useClock"
- , "Deals with assignment of path to a given clock."
- " Arguments: path, field, tick number. "
- , new EpFunc4< Shell, string, string, unsigned int, unsigned int >(&Shell::handleUseClock )
- );
+ , "Deals with assignment of path to a given clock."
+ " Arguments: path, field, tick number. "
+ , new EpFunc4< Shell, string, string, unsigned int, unsigned int >(&Shell::handleUseClock )
+ );
static DestFinfo handleCreate( "create"
- , "create( class, parent, newElm, name, numData, isGlobal )"
- , new EpFunc6<Shell, string, ObjId, Id, string, NodeBalance, unsigned int>(&Shell::handleCreate)
- );
+ , "create( class, parent, newElm, name, numData, isGlobal )"
+ , new EpFunc6<Shell, string, ObjId, Id, string, NodeBalance, unsigned int>(&Shell::handleCreate)
+ );
static DestFinfo handleDelete( "delete"
- , "When applied to a regular object, this function operates "
- "on the Id (element) specified by the ObjId argument. "
- "The function deletes the entire object "
- "array on this Id, including all dataEntries on it,"
- "all its messages, and all its children. The DataIndex here "
- "is ignored, and all dataEntries are destroyed. \n"
- "When applied to a message: Destroys only that one specific "
- "message identified by the full ObjId. \n"
- "Args: ObjId\n"
- , new EpFunc1< Shell, ObjId >( & Shell::destroy )
- );
+ , "When applied to a regular object, this function operates "
+ "on the Id (element) specified by the ObjId argument. "
+ "The function deletes the entire object "
+ "array on this Id, including all dataEntries on it,"
+ "all its messages, and all its children. The DataIndex here "
+ "is ignored, and all dataEntries are destroyed. \n"
+ "When applied to a message: Destroys only that one specific "
+ "message identified by the full ObjId. \n"
+ "Args: ObjId\n"
+ , new EpFunc1< Shell, ObjId >( & Shell::destroy )
+ );
static DestFinfo handleAddMsg( "addMsg"
- , "Makes a msg. Arguments are:"
- " msgtype, src object, src field, dest object, dest field"
- , new EpFunc6< Shell, string, ObjId, string, ObjId, string, unsigned int >(&Shell::handleAddMsg)
- );
+ , "Makes a msg. Arguments are:"
+ " msgtype, src object, src field, dest object, dest field"
+ , new EpFunc6< Shell, string, ObjId, string, ObjId, string, unsigned int >(&Shell::handleAddMsg)
+ );
static DestFinfo handleQuit( "quit"
- , "Stops simulation running and quits the simulator"
- , new OpFunc0< Shell >( & Shell::handleQuit )
- );
+ , "Stops simulation running and quits the simulator"
+ , new OpFunc0< Shell >( & Shell::handleQuit )
+ );
+
static DestFinfo handleMove( "move"
- , "handleMove( Id orig, Id newParent ): "
- "moves an Element to a new parent"
- , new EpFunc2<Shell, Id, ObjId>( & Shell::handleMove )
- );
+ , "handleMove( Id orig, Id newParent ): "
+ "moves an Element to a new parent"
+ , new EpFunc2<Shell, Id, ObjId>( & Shell::handleMove )
+ );
static DestFinfo handleCopy( "copy"
- , "handleCopy( vector< Id > args, string newName, unsigned int nCopies, bool toGlobal, bool copyExtMsgs ): "
- " The vector< Id > has Id orig, Id newParent, Id newElm. "
- "This function copies an Element and all its children to a new parent."
- " May also expand out the original into nCopies copies."
- " Normally all messages within the copy tree are also copied. "
- " If the flag copyExtMsgs is true, then all msgs going out are also copied."
- , new EpFunc5< Shell, vector< ObjId >, string, unsigned int, bool, bool >(
- & Shell::handleCopy )
- );
+ , "handleCopy( vector< Id > args, string newName, unsigned int nCopies, bool toGlobal, bool copyExtMsgs ): "
+ " The vector< Id > has Id orig, Id newParent, Id newElm. "
+ "This function copies an Element and all its children to a new parent."
+ " May also expand out the original into nCopies copies."
+ " Normally all messages within the copy tree are also copied. "
+ " If the flag copyExtMsgs is true, then all msgs going out are also copied."
+ , new EpFunc5< Shell, vector< ObjId >, string, unsigned int, bool, bool >(
+ & Shell::handleCopy )
+ );
static DestFinfo setclock( "setclock"
- , "Assigns clock ticks. Args: tick#, dt"
- , new OpFunc2< Shell, unsigned int, double >( & Shell::doSetClock )
- );
+ , "Assigns clock ticks. Args: tick#, dt"
+ , new OpFunc2< Shell, unsigned int, double >( & Shell::doSetClock )
+ );
static Finfo* shellFinfos[] =
{
&setclock,
- ////////////////////////////////////////////////////////////////
- // Shared msg
- ////////////////////////////////////////////////////////////////
- // &master,
- // &worker,
&handleCreate,
&handleDelete,
&handleCopy,
@@ -190,9 +186,8 @@ void Shell::setShellElement( Element* shelle )
*
*/
Id Shell::doCreate( string type, ObjId parent, string name,
- unsigned int numData,
- NodePolicy nodePolicy,
- unsigned int preferredNode )
+ unsigned int numData, NodePolicy nodePolicy, unsigned int preferredNode
+ )
{
const Cinfo* c = Cinfo::find( type );
@@ -650,8 +645,8 @@ bool Shell::isRunning() const
* This gets a bit complicated if the Element is a multidim array.
*/
void Shell::handleCreate( const Eref& e,
- string type, ObjId parent, Id newElm, string name,
- NodeBalance nb, unsigned int parentMsgIndex )
+ string type, ObjId parent, Id newElm, string name,
+ NodeBalance nb, unsigned int parentMsgIndex )
{
innerCreate( type, parent, newElm, name, nb, parentMsgIndex );
}
@@ -702,8 +697,7 @@ bool Shell::adopt( Id parent, Id child, unsigned int msgIndex )
* Assumes we've already done all the argument checking.
*/
void Shell::innerCreate( string type, ObjId parent, Id newElm, string name,
- const NodeBalance& nb, unsigned int msgIndex )
-// unsigned int numData, bool isGlobal
+ const NodeBalance& nb, unsigned int msgIndex )
{
const Cinfo* c = Cinfo::find( type );
if ( c )
@@ -720,7 +714,6 @@ void Shell::innerCreate( string type, ObjId parent, Id newElm, string name,
case MooseSingleNode:
cout << "Error: Shell::innerCreate: Yet to implement SingleNodeDataElement. Making BlockBalance.\n";
ret = new LocalDataElement( newElm, c, name, nb.numData );
- // ret = new SingleNodeDataElement( newElm, c, name, numData, nb.preferredNode );
break;
};
assert( ret );
@@ -728,9 +721,7 @@ void Shell::innerCreate( string type, ObjId parent, Id newElm, string name,
ret->setTick( Clock::lookupDefaultTick( c->name() ) );
}
else
- {
assert( 0 );
- }
}
void Shell::destroy( const Eref& e, ObjId oid)
diff --git a/tests/python/testDisabled_socket_streamer_tcp.py b/tests/python/testDisabled_socket_streamer_tcp.py
new file mode 100644
index 00000000..7a9153f1
--- /dev/null
+++ b/tests/python/testDisabled_socket_streamer_tcp.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, division
+
+__author__ = "Dilawar Singh"
+__copyright__ = "Copyright 2016, Dilawar Singh"
+__credits__ = ["NCBS Bangalore"]
+__license__ = "GNU GPL"
+__version__ = "1.0.0"
+__maintainer__ = "Dilawar Singh"
+__email__ = "dilawars@ncbs.res.in"
+__status__ = "Development"
+
+import os
+import sys
+sys.path.append(os.path.dirname(__file__))
+import time
+import socket
+import numpy as np
+import multiprocessing as mp
+import moose
+import moose.utils as mu
+import models
+from collections import defaultdict
+
+print( '[INFO] Using moose form %s' % moose.__file__ )
+
+port_ = 3114
+
+def socket_client(q, done):
+ global port_
+ # This function waits for socket to be available.
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+ while 1:
+ if 1 == done.value:
+ print( '[INFO] MOOSE is done before I could connect' )
+ q.put({})
+ return
+ s.settimeout(0.5)
+ try:
+ s.connect( ('127.0.0.1', port_) )
+ except socket.timeout as e:
+ print(e)
+ break
+
+ print( 'Py: Connected' )
+
+ # This is client reponsibility to read the data.
+ s.settimeout(0.01)
+ data = b''
+ while True:
+ try:
+ data += s.recv(1024)
+ except socket.timeout:
+ print(end='x')
+ if 1 == done.value:
+ break
+ if data:
+ res = mu.decode_data(data)
+ q.put(res)
+ else:
+ q.put({})
+ s.close()
+ return
+
+def test():
+ global finish_all_
+ os.environ['MOOSE_STREAMER_ADDRESS'] = 'http://127.0.0.1:%d'%port_
+ done = mp.Value('d', 0)
+ q = mp.Queue()
+ client = mp.Process(target=socket_client, args=(q, done))
+ client.start()
+
+ time.sleep(0.1)
+
+ print( '[INFO] Socket client is running now' )
+ ts = models.simple_model_a()
+ moose.reinit()
+ time.sleep(0.1)
+ # If TCP socket is created, some delay is often neccessary before start. Don't
+ # know why. probably some latency in a fresh TCP socket. A TCP guru can
+ # tell.
+ moose.start(50)
+ print( 'MOOSE is done' )
+
+ time.sleep(0.5)
+ done.value = 1
+ res = q.get()
+ client.join()
+
+ if not res:
+ raise RuntimeWarning('Nothing was streamed')
+ for k in res:
+ a = res[k][1::2]
+ b = moose.element(k).vector
+ print(k, len(a), len(b))
+ assert( (a==b).all())
+ print( 'Test 1 passed' )
+
+def main( ):
+ test( )
+ print( '[INFO] All tests passed' )
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/python/test_socket_streamer_multitab.py b/tests/python/test_socket_streamer_multitab.py
new file mode 100644
index 00000000..7f3fe8ab
--- /dev/null
+++ b/tests/python/test_socket_streamer_multitab.py
@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+'''
+Here is some docstring.
+'''
+from __future__ import print_function, division
+
+import os
+import random
+import time
+import multiprocessing as mp
+import socket
+import moose
+import json
+import struct
+import numpy as np
+import moose.utils as mu
+
+
+print('Using from %s. VERSION: %s' % (moose.__file__, moose.__version__))
+
+sockPath = '/tmp/MOOSE'
+os.environ['MOOSE_STREAMER_ADDRESS'] = 'file://%s' % sockPath
+
+
+def streamer_handler(done, q):
+ # Streamer handler
+ global sockPath
+ global all_done_
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ while not os.path.exists(sockPath):
+ time.sleep(0.01)
+ if done.value == 1:
+ break
+ s.connect( sockPath )
+ print( "[INFO ] Connected" )
+ s.settimeout(1e-5)
+ data = b''
+ while True:
+ if done.value == 1:
+ break
+ try:
+ d = s.recv(10240)
+ if d.strip():
+ data += d
+ except socket.timeout:
+ pass
+
+ res = mu.decode_data(data)
+ q.put(res)
+
+def make_network():
+ """
+ This snippet sets up a recurrent network of IntFire objects, using
+ SimpleSynHandlers to deal with spiking events.
+ It isn't very satisfactory as activity runs down after a while.
+ It is a good example for using the IntFire, setting up random
+ connectivity, and using SynHandlers.
+ """
+ global all_done_
+ done = mp.Value('i', 0)
+ q = mp.Queue()
+ th = mp.Process(target=streamer_handler, args=(done, q))
+ th.start()
+
+ size = 1024
+ dt = 0.2
+ runsteps = 10
+ delayMin = 0
+ delayMax = 4
+ weightMax = 1
+ Vmax = 1.0
+ thresh = 0.4
+ refractoryPeriod = 0.4
+ tau = 0.5
+ connectionProbability = 0.01
+ random.seed( 123 )
+ np.random.seed( 456 )
+ t0 = time.time()
+
+ network = moose.IntFire( 'network', size );
+ syns = moose.SimpleSynHandler( '/network/syns', size );
+ moose.connect( syns, 'activationOut', network, 'activation', 'OneToOne' )
+ moose.le( '/network' )
+ syns.vec.numSynapses = [1] * size
+ sv = moose.vec( '/network/syns/synapse' )
+ print('before connect t = %.3f'%(time.time() - t0))
+ mid = moose.connect( network, 'spikeOut', sv, 'addSpike', 'Sparse')
+ print('after connect t = %.3f'%(time.time() - t0))
+ #print mid.destFields
+ m2 = moose.element( mid )
+ m2.setRandomConnectivity( connectionProbability, 5489 )
+ print('after setting connectivity, t=%.3f'%(time.time() - t0))
+ #network.vec.Vm = [(Vmax*random.random()) for r in range(size)]
+ network.vec.Vm = np.random.rand( size ) * Vmax
+ network.vec.thresh = thresh
+ network.vec.refractoryPeriod = refractoryPeriod
+ network.vec.tau = tau
+ numSynVec = syns.vec.numSynapses
+ print('Middle of setup, t = %.3f'%(time.time() - t0))
+ numTotSyn = sum( numSynVec )
+ print((numSynVec.size, ', tot = ', numTotSyn, ', numSynVec = ', numSynVec))
+ for item in syns.vec:
+ h = moose.element(item)
+ h.synapse.delay = delayMin + (delayMax-delayMin) * np.random.rand(len(h.synapse))
+ h.synapse.weight = np.random.rand(len(h.synapse)) * weightMax
+ print('After setup, t = %.3f'%(time.time()-t0))
+
+ numStats = 100
+ stats = moose.SpikeStats( '/stats', numStats )
+ stats.vec.windowLength = 1 # timesteps to put together.
+ plots = moose.Table( '/plot', numStats )
+ convergence = size // numStats
+ for i in range( numStats ):
+ for j in range( size//numStats ):
+ k = i * convergence + j
+ moose.connect( network.vec[k], 'spikeOut', stats.vec[i], 'addSpike' )
+ moose.connect( plots, 'requestOut', stats, 'getMean', 'OneToOne' )
+
+ t1 = time.time()
+ moose.reinit()
+ print('reinit time t = %.3f'%(time.time() - t1))
+ network.vec.Vm = np.random.rand( size ) * Vmax
+ print('setting Vm , t = %.3f'%(time.time() - t1))
+ t1 = time.time()
+ moose.start(runsteps * dt, 1)
+ time.sleep(0.1)
+ done.value = 1
+ print('runtime, t = %.3f'%(time.time() - t1))
+ print(network.vec.Vm[99:103], network.vec.Vm[900:903])
+ res = q.get()
+
+ for tabPath in res:
+ aWithTime = res[tabPath]
+ a = aWithTime[1::2]
+ b = moose.element(tabPath).vector
+ print( tabPath, len(a), len(b) )
+ if len(a) == len(b):
+ assert np.equal(a, b).all()
+ else:
+ print( "Table did not equal size. The last table is allowed to "
+ " have fewer entries." )
+
+
+ th.join()
+ print( 'All done' )
+
+if __name__ == '__main__':
+ make_network()
diff --git a/tests/python/test_socket_streamer_tcp.py b/tests/python/test_socket_streamer_tcp.py
deleted file mode 100644
index 497b66fb..00000000
--- a/tests/python/test_socket_streamer_tcp.py
+++ /dev/null
@@ -1,182 +0,0 @@
-# -*- coding: utf-8 -*-
-"""test_socket_streamer.py:
-
- MOOSE must create a socket server on PORT 31616 (default)
- or setup moose.SocketStreamer port to appropriate port number.
-
- Client can read data from this socket.
-"""
-
-from __future__ import print_function
-
-__author__ = "Dilawar Singh"
-__copyright__ = "Copyright 2016, Dilawar Singh"
-__credits__ = ["NCBS Bangalore"]
-__license__ = "GNU GPL"
-__version__ = "1.0.0"
-__maintainer__ = "Dilawar Singh"
-__email__ = "dilawars@ncbs.res.in"
-__status__ = "Development"
-
-import os
-import sys
-sys.path.append( os.path.dirname(__file__))
-import time
-import socket
-import numpy as np
-import threading
-import moose
-import models
-import json
-from collections import defaultdict
-
-finish_all_ = False
-
-print( '[INFO] Using moose form %s' % moose.__file__ )
-
-def get_msg(s, n=1024):
- d = s.recv( n, socket.MSG_WAITALL)
- while len(d) < n:
- d += s.recv(n-len(d), socket.MSG_WAITALL)
- return d
-
-def socket_client(host='127.0.0.1', port = 31416):
- # This function waits for socket to be available.
- global finish_all_
- s = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
- while 1:
- if finish_all_:
- print( '[INFO] MOOSE is done before I could connect' )
- break
- # print('Py: Trying to connect to %s, %s' % (host, port))
- # print( end = '.' )
- try:
- s.connect( ('', port) )
- break
- except Exception as e:
- continue
- print(e, end = 'x')
- sys.stdout.flush()
- pass
-
- print( 'Connected' )
- if not finish_all_:
- print( 'Py: Connected with socket.' )
-
- # This is client reponsibility to read the data.
- s.settimeout(0.1)
- data = b''
- while not finish_all_:
- try:
- data += get_msg(s)
- except socket.timeout as e:
- print( e, end = ' ' )
-
- s.close()
- assert data, "No data streamed"
- print( 'recieved data:\n', data )
- res = defaultdict(list)
- for x in data.split(b'\n'):
- if not x.strip():
- continue
- d = json.loads(x)
- for k, v in d.items():
- res[k] += v
-
- expected = {u'/compt/tabB/tabC': ([25.,1.07754388], [14.71960144, 0.16830373])
- , u'/compt/a/tab': ([25., 0.42467006], [14.71960144, 0.16766705])
- , u'/compt/tabB': ([25., 2.57797725], [14.71960144, 0.16842971])
- }
- nd = {}
- for k in res:
- v = res[k]
- nd[k] = (np.mean(v, axis=0), np.std(v, axis=0))
- assert np.isclose(expected[k], nd[k]).all(), \
- "Exptected %s, got %s" % (str(expected[k]), str(nd[k]))
-
-def sanity_test( ):
- a = moose.Table( '/t1' )
- b = moose.Table( '/t1/t1' )
- c = moose.Table( '/t1/t1/t1' )
- st = moose.SocketStreamer( '/s' )
- st.addTable( a )
- assert( st.numTables == 1 )
- st.addTable( b )
- assert( st.numTables == 2 )
- st.addTable( c )
- assert( st.numTables == 3 )
- st.addTable( c )
- assert( st.numTables == 3 )
- st.addTable( c )
- assert( st.numTables == 3 )
-
- st.removeTable( c )
- assert( st.numTables == 2 )
- st.removeTable( c )
- assert( st.numTables == 2 )
- st.removeTable( a )
- assert( st.numTables == 1 )
- st.removeTable( b )
- assert( st.numTables == 0 )
- st.removeTable( b )
- assert( st.numTables == 0 )
- print( 'Sanity test passed' )
-
- st.addTables( [a, b, c ])
- assert st.numTables == 3
- st.removeTables( [a, a, c] )
- assert st.numTables == 1
- moose.delete( '/t1' )
- moose.delete( '/s' )
-
-def test():
- global finish_all_
- os.environ['MOOSE_STREAMER_ADDRESS'] = 'http://127.0.0.1:31416'
- client = threading.Thread(target=socket_client, args=())
- # client.daemon = True
- client.start()
- print( '[INFO] Socket client is running now' )
- tables = models.simple_model_a()
- time.sleep(0.1)
- moose.reinit()
- moose.start(50)
- print( 'MOOSE is done' )
-
- # sleep for some time so data can be read.
- time.sleep(1)
- finish_all_ = True
- client.join()
- print( 'Test 1 passed' )
-
-def test_without_env():
- global finish_all_
- os.environ['MOOSE_STREAMER_ADDRESS'] = ''
- client = threading.Thread(target=socket_client, args=())
- #client.daemon = True
- client.start()
- print( '[INFO] Socket client is running now' )
- tables = create_model()
- # Now create a streamer and use it to write to a stream
- st = moose.SocketStreamer( '/compt/streamer' )
- st.address = 'http://127.0.0.1:31416'
- assert st.port == 31416, "Got %s expected %s" % (st.port, expected)
- st.addTable(tables[0])
- st.addTables(tables[1:])
- assert st.numTables == 3
- # Give some time for socket client to make connection.
- moose.reinit()
- moose.start(50)
- time.sleep(1)
- print( 'MOOSE is done' )
- # sleep for some time so data can be read.
- finish_all_ = True
- client.join()
- print( 'Test 2 passed' )
-
-def main( ):
- test( )
- # test_without_env()
- print( '[INFO] All tests passed' )
-
-if __name__ == '__main__':
- main()
diff --git a/tests/python/test_socket_streamer_uds.py b/tests/python/test_socket_streamer_uds.py
index 542c3a46..480af8c6 100644
--- a/tests/python/test_socket_streamer_uds.py
+++ b/tests/python/test_socket_streamer_uds.py
@@ -24,101 +24,81 @@ sys.path.append(os.path.dirname(__file__))
import time
import socket
import numpy as np
-import threading
+import multiprocessing as mp
import moose
+import moose.utils as mu
import json
import models
from collections import defaultdict
-finish_all_ = False
-
print( '[INFO] Using moose form %s' % moose.__file__ )
sockFile_ = '/tmp/MOOSE'
-def get_msg(s, n=1024):
- d = s.recv(n)
- while(len(d) < n):
- d1 = s.recv(n-len(d))
- d += d1
- return d
-
-def socket_client( ):
+def socket_client(done, q):
# This function waits for socket to be available.
- global finish_all_
global sockFile_
- s = socket.socket( socket.AF_UNIX, socket.SOCK_STREAM )
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
address = sockFile_
- while 1:
- if finish_all_:
- print( '[INFO] MOOSE is done before I could connect' )
- break
- # print('Py: Trying to connect to %s, %s' % (host, port))
- # print( end = '.' )
- try:
- s.connect( address )
- break
- except Exception as e:
- pass
-
- if not finish_all_:
- print( 'Py: Connected with socket.' )
+ while not os.path.exists(address):
+ if done.value == 1:
+ return
+ continue
+ s.connect( address )
+ print( 'Py: Connected with socket. %s' % sockFile_ )
# This is client reponsibility to read the data.
+ print( 'Py: Fetching...' )
data = b''
- s.settimeout(1)
- while not finish_all_:
+ s.settimeout(0.01)
+ while True:
try:
- d = get_msg(s, 1024)
- data += d
- except socket.timeout as e:
- pass
- s.close()
+ data += s.recv(64)
+ except socket.timeout:
+ print('x', end = '' )
- assert data, "No data streamed"
- res = defaultdict(list)
- for x in data.split(b'\n'):
- if not x.strip():
- continue
- x = x.decode( 'utf8' )
- try:
- d = json.loads(x)
- except Exception as e:
- print( data )
- raise e
- for k, v in d.items():
- res[k] += v
-
- expected = {u'/compt/tabB/tabC': ([25.,1.07754388], [14.71960144, 0.16830373])
- , u'/compt/a/tab': ([25., 0.42467006], [14.71960144, 0.16766705])
- , u'/compt/tabB': ([25., 2.57797725], [14.71960144, 0.16842971])
- }
- nd = {}
- for k in res:
- v = res[k]
- nd[k] = (np.mean(v, axis=0), np.std(v, axis=0))
- assert np.isclose(expected[k], nd[k]).all(), \
- "Exptected %s, got %s" % (str(expected[k]), str(nd[k]))
+ if done.value == 1:
+ print( 'Simulation is over' )
+ break
+ s.close()
+ if not data:
+ print("No data streamed")
+ done.value = 1
+ q.put({})
+ return
+ res = mu.decode_data(data)
+ q.put(res)
def test():
- global finish_all_
- client = threading.Thread(target=socket_client, args=())
- #client.daemon = True
+ q = mp.Queue()
+ done = mp.Value( 'd', 0.0)
+ client = mp.Process(target=socket_client, args=(done, q))
client.start()
print( '[INFO] Socket client is running now' )
- tables = models.simple_model_a()
+ time.sleep(0.1)
+
# Now create a streamer and use it to write to a stream
- os.environ['MOOSE_STREAMER_ADDRESS'] = 'file:///tmp/moose'
+ os.environ['MOOSE_STREAMER_ADDRESS'] = 'file://%s' % sockFile_
+ models.simple_model_a()
# Give some time for socket client to make connection.
moose.reinit()
moose.start(50)
time.sleep(0.1)
- finish_all_ = True
- print( 'MOOSE is done' )
+ done.value = 1
+
+ res = q.get()
+ if not res:
+ raise RuntimeWarning( 'Nothing was streamed')
+ for k in res:
+ aWithTime = res[k]
+ a = aWithTime[1::2]
+ b = moose.element(k).vector
+ print(k, len(a), len(b))
+ assert (a == b).all()
+
# sleep for some time so data can be read.
client.join()
- print( 'Test 2 passed' )
def main( ):
test()
diff --git a/tests/python/test_streamer.py b/tests/python/test_streamer.py
index 43387df4..3136ec24 100644
--- a/tests/python/test_streamer.py
+++ b/tests/python/test_streamer.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
-"""test_streamer.py:
+from __future__ import print_function
+"""test_streamer.py:
Test script for Streamer class.
-
"""
__author__ = "Dilawar Singh"
@@ -14,13 +14,17 @@ __maintainer__ = "Dilawar Singh"
__email__ = "dilawars@ncbs.res.in"
__status__ = "Development"
-import os
-import sys
-import time
import moose
+import threading
import numpy as np
-print(( '[INFO] Using moose form %s' % moose.__file__ ))
+import time
+import os
+import sys
+print('[INFO] Using moose form %s' % moose.__file__)
+
+all_done_ = False
+# Poll the file to see that we are really writing to it.
def sanity_test( ):
a = moose.Table( '/t1' )
b = moose.Table( '/t1/t1' )
@@ -64,6 +68,7 @@ def sanity_test( ):
def test( ):
compt = moose.CubeMesh( '/compt' )
+ assert compt
r = moose.Reac( '/compt/r' )
a = moose.Pool( '/compt/a' )
a.concInit = 1
@@ -77,6 +82,10 @@ def test( ):
r.Kf = 0.1
r.Kb = 0.01
+ outfile = 'streamer_test.csv'
+ if os.path.exists(outfile):
+ os.remove(outfile)
+
tabA = moose.Table2( '/compt/a/tab' )
tabB = moose.Table2( '/compt/tabB' )
tabC = moose.Table2( '/compt/tabB/tabC' )
@@ -88,21 +97,22 @@ def test( ):
# Now create a streamer and use it to write to a stream
st = moose.Streamer( '/compt/streamer' )
- st.outfile = os.path.join( os.getcwd(), 'temp.npy' )
- print(("outfile set to: %s " % st.outfile ))
- assert st.outfile == os.path.join( os.getcwd(), 'temp.npy' ), st.outfile
+ st.outfile = outfile
+ print("outfile set to: %s " % st.outfile )
st.addTable( tabA )
st.addTables( [ tabB, tabC ] )
-
assert st.numTables == 3
moose.reinit( )
- print( '[INFO] Running for 57 seconds' )
- moose.start( 57 )
+ t = 100
+ print( '[INFO] Running for %d seconds' % t )
+ moose.start(t)
outfile = st.outfile
moose.quit() # Otherwise Streamer won't flush the rest of entries.
+ print('Moose is done. Waiting for monitor to shut down...')
+
# Now read the table and verify that we have written
print( '[INFO] Reading file %s' % outfile )
if 'csv' in outfile:
@@ -110,11 +120,9 @@ def test( ):
else:
data = np.load( outfile )
# Total rows should be 58 (counting zero as well).
- # print(data)
+ # print(data)
# print( data.dtype )
- time = data['time']
- print( time )
- assert data.shape >= (58,), data.shape
+ assert data.shape >= (101,), data.shape
print( '[INFO] Test 2 passed' )
return 0
diff --git a/tests/python/test_table_streaming_support.py b/tests/python/test_table_streaming_support.py
index 0747ef66..42b0b83b 100644
--- a/tests/python/test_table_streaming_support.py
+++ b/tests/python/test_table_streaming_support.py
@@ -59,7 +59,7 @@ def test( ):
moose.reinit( )
[ print_table( x) for x in [tabA, tabB, tabC] ]
runtime = 1000
- print( 'Starting moose for %s' % runtime )
+ print( 'Starting moose for %d secs' % runtime )
moose.start( runtime, 1 )
print( ' MOOSE is done' )
diff --git a/utility/Annotator.cpp b/utility/Annotator.cpp
index 4f1c51bd..f2df4aa1 100644
--- a/utility/Annotator.cpp
+++ b/utility/Annotator.cpp
@@ -32,7 +32,18 @@ const Cinfo* Annotator::initCinfo()
&Annotator::setZ,
&Annotator::getZ
);
-
+ static ValueFinfo< Annotator, double > width(
+ "width",
+ "width field. Typically display width",
+ &Annotator::setwidth,
+ &Annotator::getwidth
+ );
+ static ValueFinfo< Annotator, double > height(
+ "height",
+ "height field. Typically display height",
+ &Annotator::setheight,
+ &Annotator::getheight
+ );
static ValueFinfo< Annotator, string > notes(
"notes",
"A string to hold some text notes about parent object",
@@ -91,6 +102,8 @@ const Cinfo* Annotator::initCinfo()
&x, // Value
&y, // Value
&z, // Value
+ &width,
+ &height,
¬es, // Value
&color, // Value
&textColor, // Value
@@ -116,7 +129,7 @@ const Cinfo* Annotator::initCinfo()
static const Cinfo* annotatorCinfo = Annotator::initCinfo();
Annotator::Annotator()
- : x_( 0.0 ), y_( 0.0 ), z_( 0.0 ),
+ : x_( 0.0 ), y_( 0.0 ), z_( 0.0 ), width_( 0.0 ), height_( 0.0 ),
notes_( "" ), color_( "white" ), textColor_( "black" ),
icon_( "sphere" ),solver_( "ee"),runtime_(100.0),dirpath_(""),modeltype_("")
{
@@ -153,6 +166,26 @@ void Annotator::setZ( double v )
z_ = v;
}
+double Annotator::getheight() const
+{
+ return height_;
+}
+
+void Annotator::setheight( double v )
+{
+ height_ = v;
+}
+
+double Annotator::getwidth() const
+{
+ return width_;
+}
+
+void Annotator::setwidth( double v )
+{
+ width_ = v;
+}
+
string Annotator::getNotes() const
{
return notes_;
diff --git a/utility/Annotator.h b/utility/Annotator.h
index 8dd4e5d5..759a6aaa 100644
--- a/utility/Annotator.h
+++ b/utility/Annotator.h
@@ -23,6 +23,10 @@ class Annotator
double getY() const;
void setY( double v );
double getZ() const;
+ void setwidth( double v );
+ double getwidth() const;
+ void setheight( double v );
+ double getheight() const;
void setZ( double v );
string getNotes() const;
void setNotes( string v );
@@ -45,6 +49,8 @@ class Annotator
double x_;
double y_;
double z_;
+ double height_;
+ double width_;
string notes_;
string color_;
string textColor_;
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index 5fcd5cee..00c014f1 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -16,5 +16,6 @@ add_library(utility
Annotator.cpp
Vec.cpp
cnpy.cpp
+ fileutils.cpp
#matrix_util.cpp
)
diff --git a/utility/fileutils.cpp b/utility/fileutils.cpp
new file mode 100644
index 00000000..221eb165
--- /dev/null
+++ b/utility/fileutils.cpp
@@ -0,0 +1,26 @@
+/*
+ * Description: Filesystem related utilities.
+ *
+ * Author: Dilawar Singh (), dilawars@ncbs.res.in
+ * Organization: NCBS Bangalore
+ */
+
+#include <string>
+#include <map>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+using namespace std;
+
+#include "utility.h"
+
+namespace moose {
+
+bool filepath_exists( const string& path )
+{
+ struct stat buffer;
+ return (stat (path.c_str(), &buffer) == 0);
+}
+
+} //namespace moose.
diff --git a/utility/print_function.hpp b/utility/print_function.hpp
index c2cf0c7f..9d389a06 100644
--- a/utility/print_function.hpp
+++ b/utility/print_function.hpp
@@ -95,7 +95,6 @@ namespace moose {
string mapToString(const map<A, B>& m, bool value=true)
{
unsigned int width = 81;
- unsigned int mapSize = m.size();
unsigned int size = 0;
vector<string> row;
@@ -250,7 +249,9 @@ namespace moose {
#ifdef NDEBUG
#define LOG(t, a ) ((void)0);
#else /* ----- not NDEBUG ----- */
-#define LOG(t, a) { stringstream __ss__; __ss__ << a; moose::__dump__(__ss__.str(), t ); }
+#define LOG(t, a) { stringstream __ss__; \
+ __ss__ << __func__ << ": " << a; moose::__dump__(__ss__.str(), t); \
+}
#endif /* ----- not NDEBUG ----- */
/*-----------------------------------------------------------------------------
diff --git a/utility/setupenv.cpp b/utility/setupenv.cpp
index 422d3471..088f84f4 100644
--- a/utility/setupenv.cpp
+++ b/utility/setupenv.cpp
@@ -33,6 +33,7 @@
#include <string>
#include <sstream>
#include <cstdlib>
+
using namespace std;
extern unsigned getNumCores();
diff --git a/utility/simple_assert.hpp b/utility/simple_assert.hpp
index 78c41d8f..857ad720 100644
--- a/utility/simple_assert.hpp
+++ b/utility/simple_assert.hpp
@@ -62,16 +62,12 @@ namespace moose
// SIMPLE_ASSERT_MSG //
//--------------------------------------------------------------------------------------//
-# undef SIMPLE_ASSERT_MSG
+#undef SIMPLE_ASSERT_MSG
#if defined(SIMPLE_DISABLE_ASSERTS) || defined(NDEBUG)
-
#define SIMPLE_ASSERT_MSG(expr, msg) ((void)0)
-
#elif defined(SIMPLE_ENABLE_ASSERT_HANDLER)
-
#include "current_function.hpp"
-
namespace moose
{
void assertion_failed_msg(char const * expr, char const * msg,
diff --git a/utility/strutil.h b/utility/strutil.h
index 4d95c141..e2480d2b 100644
--- a/utility/strutil.h
+++ b/utility/strutil.h
@@ -9,6 +9,7 @@
#define _STRINGUTIL_H
#include <string>
+#include <sstream>
#include <vector>
namespace moose
diff --git a/utility/testing_macros.hpp b/utility/testing_macros.hpp
index 87189889..30359ef9 100644
--- a/utility/testing_macros.hpp
+++ b/utility/testing_macros.hpp
@@ -68,7 +68,7 @@ static ostringstream assertStream;
}
#define EXPECT_NEQ(a, b, token) \
- if( (a) == (b)) { \
+ if( ! ((a) != (b)) ) { \
assertStream.str(""); \
LOCATION(assertStream); \
assertStream << "Not expected " << a << endl; \
@@ -77,7 +77,7 @@ static ostringstream assertStream;
}
#define EXPECT_GT(a, b, token) \
- if( (a) <= (b)) { \
+ if( !((a) > (b)) ) { \
assertStream.str(""); \
LOCATION(assertStream); \
assertStream << "Expected greater than " << a << ", received " << b << endl; \
@@ -86,7 +86,7 @@ static ostringstream assertStream;
}
#define EXPECT_GTE(a, b, token) \
- if( (a) < (b)) { \
+ if( !((a) >= (b)) ) { \
assertStream.str(""); \
LOCATION(assertStream); \
assertStream << "Expected greater than or equal to " << a \
@@ -96,7 +96,7 @@ static ostringstream assertStream;
}
#define EXPECT_LT(a, b, token) \
- if( (a) >= (b)) { \
+ if( ! ((a) < (b)) ) { \
assertStream.str(""); \
LOCATION(assertStream); \
assertStream << "Expected less than " << a << ", received " << b << endl; \
@@ -105,7 +105,7 @@ static ostringstream assertStream;
}
#define EXPECT_LTE(a, b, token) \
- if( (a) < (b)) { \
+ if( ! ((a) <= (b)) ) { \
assertStream.str(""); \
LOCATION(assertStream); \
assertStream << "Expected less than or equal to " << a \
@@ -129,13 +129,6 @@ static ostringstream assertStream;
throw std::runtime_error(assertStream.str()); \
}
-#define ASSERT_LT( a, b, msg) \
- EXPECT_LT(a, b, msg); \
- assertStream.str(""); \
- assertStream.precision( 9 ); \
- assertStream << msg; \
- throw std::runtime_error( assertStream.str() ); \
-
#define ASSERT_EQ(a, b, token) \
if( ! doubleEq((a), (b)) ) { \
assertStream.str(""); \
@@ -165,5 +158,4 @@ static ostringstream assertStream;
throw std::runtime_error(assertStream.str()); \
}
-
#endif /* ----- #ifndef TESTING_MACROS_INC ----- */
diff --git a/utility/utility.h b/utility/utility.h
index ee0e88d7..33fae676 100644
--- a/utility/utility.h
+++ b/utility/utility.h
@@ -25,6 +25,9 @@ namespace moose
const map<std::string, std::string>& getArgMap();
string getEnv( const string& env);
+ // In fileutils.cpp
+ bool filepath_exists( const string& path );
+
/**
* @brief Givem path of MOOSE element, return its name. It's behaviour is
* like `basename` of unix command e.g. /a/b/c --> c
--
GitLab