From 7b7950cf0f81488c62617db580386ea26f61ec0d Mon Sep 17 00:00:00 2001 From: Thorsten Hater <24411438+thorstenhater@users.noreply.github.com> Date: Tue, 23 Aug 2022 19:04:37 +0200 Subject: [PATCH] Arborio reads from Stream objects now. (#1937) Enable pyArborio input routines to consume Stream objects in addition to files by name. - NML - NeuroLucida ASC - SWC Closes #1459 --- python/cable_cell_io.cpp | 11 ++--- python/morphology.cpp | 46 +++++++----------- python/test/unit/test_io.py | 93 +++++++++++++++++++++++++++++++++++++ python/util.hpp | 25 +++++++++- 4 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 python/test/unit/test_io.py diff --git a/python/cable_cell_io.cpp b/python/cable_cell_io.cpp index 5191c47b..335371df 100644 --- a/python/cable_cell_io.cpp +++ b/python/cable_cell_io.cpp @@ -2,6 +2,7 @@ #include <pybind11/stl.h> #include <fstream> +#include <iostream> #include <iomanip> #include <arbor/cable_cell.hpp> @@ -19,14 +20,10 @@ namespace pyarb { namespace py = pybind11; arborio::cable_cell_component load_component(py::object fn) { - const auto fname = util::to_path(fn); - std::ifstream fid{fname}; - if (!fid.good()) { - throw arb::file_not_found_error(fname); - } - auto component = arborio::parse_component(fid); + auto contents = util::read_file_or_buffer(fn); + auto component = arborio::parse_component(contents); if (!component) { - throw pyarb_error("Error while trying to load component from \"" + fname + "\": " + component.error().what()); + throw pyarb_error(std::string{"Error while trying to load component: "} + component.error().what()); } return component.value(); }; diff --git a/python/morphology.cpp b/python/morphology.cpp index 8f4293ca..ba238754 100644 --- a/python/morphology.cpp +++ b/python/morphology.cpp @@ -252,22 +252,17 @@ void register_morphology(py::module& m) { // Wraps calls to C++ functions arborio::parse_swc() and arborio::load_swc_arbor(). m.def("load_swc_arbor", [](py::object fn) { - const auto fname = util::to_path(fn); - std::ifstream fid{fname}; - if (!fid.good()) { - throw arb::file_not_found_error(fname); - } try { - auto data = arborio::parse_swc(fid); - check_trailing(fid, fname); + auto contents = util::read_file_or_buffer(fn); + auto data = arborio::parse_swc(contents); return arborio::load_swc_arbor(data); } catch (arborio::swc_error& e) { // Try to produce helpful error messages for SWC parsing errors. - throw pyarb_error(util::pprintf("error parsing {}: {}", fname, e.what())); + throw pyarb_error(util::pprintf("Arbor SWC: parse error: {}", e.what())); } }, - "filename"_a, + "filename_or_stream"_a, "Generate a morphology from an SWC file following the rules prescribed by Arbor.\n" "Specifically:\n" "* Single-segment somas are disallowed.\n" @@ -278,23 +273,18 @@ void register_morphology(py::module& m) { m.def("load_swc_neuron", [](py::object fn) { - const auto fname = util::to_path(fn); - std::ifstream fid{fname}; - if (!fid.good()) { - throw arb::file_not_found_error(fname); - } try { - auto data = arborio::parse_swc(fid); - check_trailing(fid, fname); + auto contents = util::read_file_or_buffer(fn); + auto data = arborio::parse_swc(contents); return arborio::load_swc_neuron(data); } catch (arborio::swc_error& e) { // Try to produce helpful error messages for SWC parsing errors. throw pyarb_error( - util::pprintf("NEURON SWC: error parsing {}: {}", fname, e.what())); + util::pprintf("NEURON SWC: parse error: {}", e.what())); } }, - "filename"_a, + "filename_or_stream"_a, "Generate a morphology from an SWC file following the rules prescribed by NEURON.\n" "See the documentation https://docs.arbor-sim.org/en/latest/fileformat/swc.html\n" "for a detailed description of the interpretation."); @@ -343,9 +333,10 @@ void register_morphology(py::module& m) { "The four canonical regions are labeled 'soma', 'axon', 'dend' and 'apic'."); m.def("load_asc", - [](std::string fname) { + [](py::object fn) { try { - return arborio::load_asc(fname); + auto contents = util::read_file_or_buffer(fn); + return arborio::load_asc(contents); } catch (std::exception& e) { // Try to produce helpful error messages for SWC parsing errors. @@ -387,21 +378,16 @@ void register_morphology(py::module& m) { // constructors .def(py::init( [](py::object fn) { - const auto fname = util::to_path(fn); - std::ifstream fid{fname}; - if (!fid.good()) { - throw arb::file_not_found_error(fname); - } try { - std::string string_data((std::istreambuf_iterator<char>(fid)), - std::istreambuf_iterator<char>()); - return arborio::neuroml(string_data); + auto contents = util::read_file_or_buffer(fn); + return arborio::neuroml(contents); } catch (arborio::neuroml_exception& e) { // Try to produce helpful error messages for NeuroML parsing errors. - throw pyarb_error(util::pprintf("NeuroML error processing file {}: {}", fname, e.what())); + throw pyarb_error(util::pprintf("NeuroML error: {}", e.what())); } - })) + }), + "Construct NML morphology from filename or stream.") .def("cell_ids", [](const arborio::neuroml& nml) { try { diff --git a/python/test/unit/test_io.py b/python/test/unit/test_io.py new file mode 100644 index 00000000..b3eda93b --- /dev/null +++ b/python/test/unit/test_io.py @@ -0,0 +1,93 @@ +import unittest + +import arbor as A + +from pathlib import Path +from tempfile import TemporaryDirectory as TD +from io import StringIO + +acc = """(arbor-component + (meta-data + (version "0.1-dev")) + (cable-cell + (morphology + (branch 0 -1 + (segment 0 + (point -3.000000 0.000000 0.000000 3.000000) + (point 3.000000 0.000000 0.000000 3.000000) + 1))) + (label-dict + (region-def "soma" + (tag 1)) + (locset-def "mid" + (location 0 0.5))) + (decor + (default + (membrane-potential -40.000000)) + (default + (ion-internal-concentration "ca" 0.000050)) + (default + (ion-external-concentration "ca" 2.000000)) + (default + (ion-reversal-potential "ca" 132.457934)) + (default + (ion-internal-concentration "k" 54.400000)) + (default + (ion-external-concentration "k" 2.500000)) + (default + (ion-reversal-potential "k" -77.000000)) + (default + (ion-internal-concentration "na" 10.000000)) + (default + (ion-external-concentration "na" 140.000000)) + (default + (ion-reversal-potential "na" 50.000000)) + (paint + (tag 1) + (density + (mechanism "default::hh" + ("gnabar" 0.120000) + ("el" -54.300000) + ("q10" 0.000000) + ("gl" 0.000300) + ("gkbar" 0.036000)))) + (place + (location 0 0.5) + (current-clamp + (envelope + (10.000000 0.800000) + (12.000000 0.000000)) + 0.000000 0.000000) + "I Clamp 0")))) +""" + + +class TestAccIo(unittest.TestCase): + def test_stringio(self): + sio = StringIO(acc) + A.load_component(sio) + + def test_fileio(self): + fn = "test.acc" + with TD() as tmp: + tmp = Path(tmp) + with open(tmp / fn, "w") as fd: + fd.write(acc) + with open(tmp / fn) as fd: + A.load_component(fd) + + def test_nameio(self): + fn = "test.acc" + with TD() as tmp: + tmp = Path(tmp) + with open(tmp / fn, "w") as fd: + fd.write(acc) + A.load_component(str(tmp / fn)) + + def test_pathio(self): + fn = "test.acc" + with TD() as tmp: + tmp = Path(tmp) + with open(tmp / fn, "w") as fd: + fd.write(acc) + A.load_component(tmp / fn) diff --git a/python/util.hpp b/python/util.hpp index fe1b6b07..4a096f6f 100644 --- a/python/util.hpp +++ b/python/util.hpp @@ -1,8 +1,11 @@ #pragma once +#include <fstream> + #include <pybind11/pybind11.h> #include "strprintf.hpp" +#include "error.hpp" namespace pyarb { namespace util { @@ -19,9 +22,29 @@ std::string to_path(py::object fn) { return std::string{py::str(fn)}; } throw std::runtime_error( - util::strprintf("Cannot convert objects of type '{}' to a path-like.", + util::pprintf("Cannot convert objects of type {} to a path-like.", std::string{py::str(fn.get_type())})); } +inline +std::string read_file_or_buffer(py::object fn) { + if (py::hasattr(fn, "read")) { + return py::str(fn.attr("read")(-1)); + } else { + const auto fname = util::to_path(fn); + std::ifstream fid{fname}; + if (!fid.good()) { + throw arb::file_not_found_error(fname); + } + std::string result; + fid.seekg(0, fid.end); + auto sz = fid.tellg(); + fid.seekg(0, fid.beg); + result.resize(sz); + fid.read(result.data(), sz); + return result; + } +} + } } -- GitLab