Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
dedal
Manage
Activity
Members
Labels
Plan
Issues
1
Issue boards
Milestones
Wiki
Code
Merge requests
1
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Harbor Registry
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
EBRAINS RI
Tech Hub
Platform
EBRAINS Software Distribution
dedal
Commits
81cc4cdb
Commit
81cc4cdb
authored
3 months ago
by
Adrian Ciu
Browse files
Options
Downloads
Patches
Plain Diff
esd-spack-installation: fixing tests
parent
76673b42
No related branches found
Branches containing commit
No related tags found
1 merge request
!4
feat(spack_operation): implement setup_spack_env functionality
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
esd/spack_manager/SpackManager.py
+57
-52
57 additions, 52 deletions
esd/spack_manager/SpackManager.py
esd/tests/spack_from_scratch_test.py
+9
-8
9 additions, 8 deletions
esd/tests/spack_from_scratch_test.py
esd/utils/utils.py
+19
-8
19 additions, 8 deletions
esd/utils/utils.py
with
85 additions
and
68 deletions
esd/spack_manager/SpackManager.py
+
57
−
52
View file @
81cc4cdb
import
os
import
os
import
subprocess
from
abc
import
ABC
,
abstractmethod
from
abc
import
ABC
,
abstractmethod
from
pathlib
import
Path
from
pathlib
import
Path
...
@@ -26,11 +25,17 @@ class SpackManager(ABC):
...
@@ -26,11 +25,17 @@ class SpackManager(ABC):
def
__init__
(
self
,
env
:
SpackModel
=
None
,
repos
=
None
,
def
__init__
(
self
,
env
:
SpackModel
=
None
,
repos
=
None
,
upstream_instance
=
None
,
system_name
:
str
=
None
,
logger
=
get_logger
(
__name__
)):
upstream_instance
=
None
,
system_name
:
str
=
None
,
logger
=
get_logger
(
__name__
)):
if
repos
is
None
:
if
repos
is
None
:
self
.
repos
=
list
()
repos
=
[]
self
.
repos
=
repos
self
.
env
=
env
self
.
env
=
env
self
.
upstream_instance
=
upstream_instance
self
.
install_dir
=
Path
(
os
.
environ
.
get
(
"
INSTALLATION_ROOT
"
)
or
os
.
getcwd
()).
resolve
()
self
.
install_dir
=
Path
(
os
.
environ
.
get
(
"
INSTALLATION_ROOT
"
)
or
os
.
getcwd
())
self
.
install_dir
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
self
.
install_dir
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
self
.
env_path
=
None
if
self
.
env
and
self
.
env
.
path
:
self
.
env
.
path
=
self
.
env
.
path
.
resolve
()
self
.
env
.
path
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
self
.
env_path
=
self
.
env
.
path
/
self
.
env
.
env_name
self
.
upstream_instance
=
upstream_instance
self
.
spack_dir
=
self
.
install_dir
/
"
spack
"
self
.
spack_dir
=
self
.
install_dir
/
"
spack
"
self
.
spack_setup_script
=
self
.
spack_dir
/
"
share
"
/
"
spack
"
/
"
setup-env.sh
"
self
.
spack_setup_script
=
self
.
spack_dir
/
"
share
"
/
"
spack
"
/
"
setup-env.sh
"
self
.
logger
=
logger
self
.
logger
=
logger
...
@@ -45,23 +50,17 @@ class SpackManager(ABC):
...
@@ -45,23 +50,17 @@ class SpackManager(ABC):
pass
pass
def
create_fetch_spack_environment
(
self
):
def
create_fetch_spack_environment
(
self
):
env_dir
=
self
.
install_dir
/
self
.
env
.
path
/
self
.
env
.
env_name
if
self
.
env
.
git_path
:
if
self
.
env
.
git_path
:
try
:
git_clone_repo
(
self
.
env
.
env_name
,
self
.
env
.
path
/
self
.
env
.
env_name
,
self
.
env
.
git_path
,
logger
=
self
.
logger
)
git_clone_repo
(
self
.
env
.
env_name
,
env_dir
,
self
.
env
.
git_path
,
logger
=
self
.
logger
)
except
subprocess
.
CalledProcessError
as
e
:
self
.
logger
.
exception
(
f
'
Failed to clone repository:
{
self
.
env
.
env_name
}
:
{
e
}
'
)
raise
BashCommandException
(
f
'
Failed to clone repository:
{
self
.
env
.
env_name
}
:
{
e
}
'
)
else
:
else
:
try
:
os
.
makedirs
(
self
.
env
.
path
/
self
.
env
.
env_name
,
exist_ok
=
True
)
os
.
makedirs
(
self
.
env
.
path
/
self
.
env
.
env_name
,
exist_ok
=
True
)
run_command
(
"
bash
"
,
"
-c
"
,
run_command
(
"
bash
"
,
"
-c
"
,
f
'
source
{
self
.
spack_setup_script
}
&& spack env create -d
{
self
.
env_path
}
'
,
f
'
source
{
self
.
spack_setup_script
}
&& spack env create -d
{
self
.
env
.
path
}
/
{
self
.
env
.
env_name
}
'
,
check
=
True
,
logger
=
self
.
logger
,
check
=
True
,
logger
=
self
.
logger
)
debug_msg
=
f
"
Created
{
self
.
env
.
env_name
}
spack environment
"
,
self
.
logger
.
debug
(
f
"
Created
{
self
.
env
.
env_name
}
spack environment
"
)
exception_msg
=
f
"
Failed to create
{
self
.
env
.
env_name
}
spack environment
"
,
except
subprocess
.
CalledProcessError
as
e
:
exception
=
BashCommandException
)
self
.
logger
.
error
(
f
"
Failed to create
{
self
.
env
.
env_name
}
spack environment
"
)
raise
BashCommandException
(
f
"
Failed to create
{
self
.
env
.
env_name
}
spack environment
"
)
def
setup_spack_env
(
self
):
def
setup_spack_env
(
self
):
"""
"""
...
@@ -94,46 +93,51 @@ class SpackManager(ABC):
...
@@ -94,46 +93,51 @@ class SpackManager(ABC):
def
spack_repo_exists
(
self
,
repo_name
:
str
)
->
bool
:
def
spack_repo_exists
(
self
,
repo_name
:
str
)
->
bool
:
"""
Check if the given Spack repository exists.
"""
"""
Check if the given Spack repository exists.
"""
if
self
.
env
is
None
:
if
self
.
env
is
None
:
try
:
result
=
run_command
(
"
bash
"
,
"
-c
"
,
result
=
run_command
(
"
bash
"
,
"
-c
"
,
f
'
source
{
self
.
spack_setup_script
}
&& spack repo list
'
,
f
'
source
{
self
.
spack_setup_script
}
&& spack repo list
'
,
check
=
True
,
check
=
True
,
capture_output
=
True
,
text
=
True
,
logger
=
self
.
logger
,
capture_output
=
True
,
text
=
True
,
logger
=
self
.
logger
)
debug_msg
=
f
'
Checking if
{
repo_name
}
exists
'
)
except
subprocess
.
CalledProcessError
:
if
result
is
None
:
return
False
return
False
else
:
else
:
try
:
result
=
run_command
(
"
bash
"
,
"
-c
"
,
result
=
run_command
(
"
bash
"
,
"
-c
"
,
f
'
source
{
self
.
spack_setup_script
}
&& spack env activate -p
{
self
.
env_path
}
&& spack repo list
'
,
f
'
source
{
self
.
spack_setup_script
}
&& spack env activate -p
{
self
.
env
.
path
}
/
{
self
.
env
.
env_name
}
&& spack repo list
'
,
check
=
True
,
check
=
True
,
capture_output
=
True
,
text
=
True
,
logger
=
self
.
logger
,
capture_output
=
True
,
text
=
True
,
logger
=
self
.
logger
)
debug_msg
=
f
'
Checking if repository
{
repo_name
}
was added
'
)
except
subprocess
.
CalledProcessError
:
if
result
is
None
:
return
False
return
False
return
any
(
line
.
strip
().
endswith
(
repo_name
)
for
line
in
result
.
stdout
.
splitlines
())
return
any
(
line
.
strip
().
endswith
(
repo_name
)
for
line
in
result
.
stdout
.
splitlines
())
def
spack_env_exists
(
self
):
result
=
run_command
(
"
bash
"
,
"
-c
"
,
f
'
source
{
self
.
spack_setup_script
}
&& spack env activate -p
{
self
.
env_path
}
'
,
check
=
True
,
capture_output
=
True
,
text
=
True
,
logger
=
self
.
logger
,
debug_msg
=
f
'
Checking if environment
{
self
.
env
.
env_name
}
exists
'
)
if
result
is
None
:
return
False
return
True
def
add_spack_repo
(
self
,
repo_path
:
Path
,
repo_name
:
str
):
def
add_spack_repo
(
self
,
repo_path
:
Path
,
repo_name
:
str
):
"""
Add the Spack repository if it does not exist.
"""
"""
Add the Spack repository if it does not exist.
"""
repo_path
=
repo_path
.
resolve
().
as_posix
()
run_command
(
"
bash
"
,
"
-c
"
,
try
:
f
'
source
{
self
.
spack_setup_script
}
&& spack env activate -p
{
self
.
env_path
}
&& spack repo add
{
repo_path
}
/
{
repo_name
}
'
,
run_command
(
"
bash
"
,
"
-c
"
,
check
=
True
,
logger
=
self
.
logger
,
f
'
source
{
self
.
spack_setup_script
}
&& spack env activate -p
{
self
.
env
.
path
}
/
{
self
.
env
.
env_name
}
&& spack repo add
{
repo_path
}
/
{
repo_name
}
'
,
debug_msg
=
f
"
Added
{
repo_name
}
to spack environment
{
self
.
env
.
env_name
}
"
,
check
=
True
,
logger
=
self
.
logger
)
exception_msg
=
f
"
Failed to add
{
repo_name
}
to spack environment
{
self
.
env
.
env_name
}
"
,
self
.
logger
.
debug
(
f
"
Added
{
repo_name
}
to spack environment
{
self
.
env
.
env_name
}
"
)
exception
=
BashCommandException
)
except
subprocess
.
CalledProcessError
as
e
:
self
.
logger
.
error
(
f
"
Failed to add
{
repo_name
}
to spack environment
{
self
.
env
.
env_name
}
"
)
raise
BashCommandException
(
f
"
Failed to add
{
repo_name
}
to spack environment
{
self
.
env
.
env_name
}
:
{
e
}
"
)
def
get_spack_installed_version
(
self
):
def
get_spack_installed_version
(
self
):
try
:
spack_version
=
run_command
(
"
bash
"
,
"
-c
"
,
f
'
source
{
self
.
spack_setup_script
}
&& spack --version
'
,
spack_version
=
run_command
(
"
bash
"
,
"
-c
"
,
f
'
source
{
self
.
spack_setup_script
}
&& spack --version
'
,
capture_output
=
True
,
text
=
True
,
check
=
True
,
capture_output
=
True
,
text
=
True
,
check
=
True
,
logger
=
self
.
logger
,
logger
=
self
.
logger
)
debug_msg
=
f
"
Getting spack version
"
,
spack_version
=
spack_version
.
stdout
.
strip
().
split
()[
0
]
exception_msg
=
f
"
Error retrieving Spack version
"
)
self
.
logger
.
debug
(
f
"
Getting spack version:
{
spack_version
}
"
)
if
spack_version
:
return
spack_version
return
spack_version
.
stdout
.
strip
().
split
()[
0
]
except
subprocess
.
SubprocessError
as
e
:
return
None
self
.
logger
.
error
(
f
"
Error retrieving Spack version:
{
e
}
"
)
return
None
def
install_spack
(
self
,
spack_version
=
"
v0.21.1
"
,
spack_repo
=
'
https://github.com/spack/spack
'
):
def
install_spack
(
self
,
spack_version
=
"
v0.21.1
"
,
spack_repo
=
'
https://github.com/spack/spack
'
):
try
:
try
:
...
@@ -163,8 +167,9 @@ class SpackManager(ABC):
...
@@ -163,8 +167,9 @@ class SpackManager(ABC):
bashrc
.
write
(
f
"
source
{
self
.
spack_setup_script
}
\n
"
)
bashrc
.
write
(
f
"
source
{
self
.
spack_setup_script
}
\n
"
)
self
.
logger
.
info
(
"
Added Spack PATH to .bashrc
"
)
self
.
logger
.
info
(
"
Added Spack PATH to .bashrc
"
)
if
user
:
if
user
:
run_command
(
"
chown
"
,
"
-R
"
,
f
"
{
user
}
:
{
user
}
"
,
self
.
spack_dir
,
check
=
True
,
logger
=
self
.
logger
)
run_command
(
"
chown
"
,
"
-R
"
,
f
"
{
user
}
:
{
user
}
"
,
self
.
spack_dir
,
check
=
True
,
logger
=
self
.
logger
,
run_command
(
"
bash
"
,
"
-c
"
,
f
"
source
{
bashrc_path
}
"
,
check
=
True
,
logger
=
self
.
logger
)
debug_msg
=
'
Adding permissions to the logged in user
'
)
run_command
(
"
bash
"
,
"
-c
"
,
f
"
source
{
bashrc_path
}
"
,
check
=
True
,
logger
=
self
.
logger
,
debug_msg
=
'
Restart bash
'
)
self
.
logger
.
info
(
"
Spack install completed
"
)
self
.
logger
.
info
(
"
Spack install completed
"
)
# Restart Bash after the installation ends
# Restart Bash after the installation ends
os
.
system
(
"
exec bash
"
)
os
.
system
(
"
exec bash
"
)
...
...
This diff is collapsed.
Click to expand it.
esd/tests/spack_from_scratch_test.py
+
9
−
8
View file @
81cc4cdb
...
@@ -23,7 +23,7 @@ def test_spack_from_scratch_setup_1():
...
@@ -23,7 +23,7 @@ def test_spack_from_scratch_setup_1():
install_dir
=
Path
(
'
./install
'
).
resolve
()
install_dir
=
Path
(
'
./install
'
).
resolve
()
env
=
SpackModel
(
'
ebrains-spack-builds
'
,
install_dir
,
env
=
SpackModel
(
'
ebrains-spack-builds
'
,
install_dir
,
'
https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git
'
,
)
'
https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git
'
,
)
spack_manager
=
SpackManagerScratch
(
env
,
[
env
],
system_name
=
'
ebrainslab
'
)
spack_manager
=
SpackManagerScratch
(
env
=
env
,
repos
=
[
env
],
system_name
=
'
ebrainslab
'
)
spack_manager
.
setup_spack_env
()
spack_manager
.
setup_spack_env
()
assert
spack_manager
.
spack_repo_exists
(
env
.
env_name
)
==
True
assert
spack_manager
.
spack_repo_exists
(
env
.
env_name
)
==
True
...
@@ -33,31 +33,32 @@ def test_spack_from_scratch_setup_2():
...
@@ -33,31 +33,32 @@ def test_spack_from_scratch_setup_2():
env
=
SpackModel
(
'
ebrains-spack-builds
'
,
install_dir
,
env
=
SpackModel
(
'
ebrains-spack-builds
'
,
install_dir
,
'
https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git
'
,
)
'
https://gitlab.ebrains.eu/ri/tech-hub/platform/esd/ebrains-spack-builds.git
'
,
)
repo
=
env
repo
=
env
spack_manager
=
SpackManagerScratch
(
env
,
[
repo
,
repo
],
system_name
=
'
ebrainslab
'
)
spack_manager
=
SpackManagerScratch
(
env
=
env
,
repos
=
[
repo
,
repo
],
system_name
=
'
ebrainslab
'
)
spack_manager
.
setup_spack_env
()
spack_manager
.
setup_spack_env
()
assert
spack_manager
.
spack_repo_exists
(
env
.
env_name
)
==
True
assert
spack_manager
.
spack_repo_exists
(
env
.
env_name
)
==
True
def
test_spack_from_scratch_setup_3
():
def
test_spack_from_scratch_setup_3
():
install_dir
=
Path
(
'
./install
'
).
resolve
()
install_dir
=
Path
(
'
./install
'
).
resolve
()
env
=
SpackModel
(
'
new_env
ironment
'
,
install_dir
,
)
env
=
SpackModel
(
'
new_env
1
'
,
install_dir
)
repo
=
env
repo
=
env
spack_manager
=
SpackManagerScratch
(
env
,
[
repo
,
repo
],
system_name
=
'
ebrainslab
'
)
spack_manager
=
SpackManagerScratch
(
env
=
env
,
repos
=
[
repo
,
repo
],
system_name
=
'
ebrainslab
'
)
with
pytest
.
raises
(
BashCommandException
):
with
pytest
.
raises
(
BashCommandException
):
spack_manager
.
setup_spack_env
()
spack_manager
.
setup_spack_env
()
def
test_spack_from_scratch_setup_4
():
def
test_spack_from_scratch_setup_4
():
install_dir
=
Path
(
'
./install
'
).
resolve
()
install_dir
=
Path
(
'
./install
'
).
resolve
()
env
=
SpackModel
(
'
new_env
ironment
'
,
install_dir
,
)
env
=
SpackModel
(
'
new_env
2
'
,
install_dir
)
spack_manager
=
SpackManagerScratch
(
env
,
system_name
=
'
ebrainslab
'
)
spack_manager
=
SpackManagerScratch
(
env
=
env
)
spack_manager
.
setup_spack_env
()
spack_manager
.
setup_spack_env
()
assert
spack_manager
.
spack_
repo
_exists
(
env
.
env_name
)
==
True
assert
spack_manager
.
spack_
env
_exists
()
==
True
def
test_spack_from_scratch_bash_error
():
def
test_spack_from_scratch_bash_error
():
env
=
SpackModel
(
'
ebrains-spack-builds
'
,
Path
(),
None
)
env
=
SpackModel
(
'
ebrains-spack-builds
'
,
Path
(),
None
)
repo
=
env
repo
=
env
spack_manager
=
SpackManagerScratch
(
env
,
[
repo
],
system_name
=
'
ebrainslab
'
)
#
spack_manager
=
SpackManagerScratch
(
env
=
env
,
repos
=
[
repo
],
system_name
=
'
ebrainslab
'
)
with
pytest
.
raises
(
BashCommandException
):
with
pytest
.
raises
(
BashCommandException
):
spack_manager
.
add_spack_repo
(
repo
.
path
,
repo
.
env_name
)
spack_manager
.
add_spack_repo
(
repo
.
path
,
repo
.
env_name
)
This diff is collapsed.
Click to expand it.
esd/utils/utils.py
+
19
−
8
View file @
81cc4cdb
...
@@ -3,6 +3,8 @@ import shutil
...
@@ -3,6 +3,8 @@ import shutil
import
subprocess
import
subprocess
from
pathlib
import
Path
from
pathlib
import
Path
from
esd.error_handling.exceptions
import
BashCommandException
def
clean_up
(
dirs
:
list
[
str
],
logging
,
ignore_errors
=
True
):
def
clean_up
(
dirs
:
list
[
str
],
logging
,
ignore_errors
=
True
):
"""
"""
...
@@ -22,21 +24,30 @@ def clean_up(dirs: list[str], logging, ignore_errors=True):
...
@@ -22,21 +24,30 @@ def clean_up(dirs: list[str], logging, ignore_errors=True):
logging
.
info
(
f
"
{
cleanup_dir
}
does not exist
"
)
logging
.
info
(
f
"
{
cleanup_dir
}
does not exist
"
)
def
run_command
(
*
args
,
logger
:
None
,
**
kwargs
):
def
run_command
(
*
args
,
logger
=
logging
.
getLogger
(
__name__
),
debug_msg
:
str
=
''
,
exception_msg
:
str
=
None
,
if
logger
is
None
:
exception
=
None
,
**
kwargs
):
logger
=
logging
.
getLogger
(
__name__
)
try
:
logger
.
debug
(
f
'
{
args
}
'
)
logger
.
debug
(
f
'
{
debug_msg
}
: args:
{
args
}
'
)
return
subprocess
.
run
(
args
,
**
kwargs
)
return
subprocess
.
run
(
args
,
**
kwargs
)
except
subprocess
.
CalledProcessError
as
e
:
if
exception_msg
is
not
None
:
logger
.
error
(
f
"
{
exception_msg
}
:
{
e
}
"
)
if
exception
is
not
None
:
raise
exception
(
f
'
{
exception_msg
}
:
{
e
}
'
)
else
:
return
None
def
git_clone_repo
(
repo_name
:
str
,
dir
:
Path
,
git_path
:
str
,
logger
:
logging
):
def
git_clone_repo
(
repo_name
:
str
,
dir
:
Path
,
git_path
:
str
,
logger
:
logging
=
logging
.
getLogger
(
__name__
)
):
if
not
dir
.
exists
():
if
not
dir
.
exists
():
run_command
(
run_command
(
"
git
"
,
"
clone
"
,
"
--depth
"
,
"
1
"
,
"
git
"
,
"
clone
"
,
"
--depth
"
,
"
1
"
,
"
-c
"
,
"
advice.detachedHead=false
"
,
"
-c
"
,
"
advice.detachedHead=false
"
,
"
-c
"
,
"
feature.manyFiles=true
"
,
"
-c
"
,
"
feature.manyFiles=true
"
,
git_path
,
dir
git_path
,
dir
,
check
=
True
,
logger
=
logger
)
,
check
=
True
,
logger
=
logger
,
logger
.
debug
(
f
'
Cloned repository
{
repo_name
}
'
)
debug_msg
=
f
'
Cloned repository
{
repo_name
}
'
,
exception_msg
=
f
'
Failed to clone repository:
{
repo_name
}
'
,
exception
=
BashCommandException
)
else
:
else
:
logger
.
debug
(
f
'
Repository
{
repo_name
}
already cloned.
'
)
logger
.
debug
(
f
'
Repository
{
repo_name
}
already cloned.
'
)
This diff is collapsed.
Click to expand it.
Preview
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment