From 82ad8087d56117ee3af0a76a6e8c780cbb683568 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20G=C3=B6bel?= <dgoebel@techfak.uni-bielefeld.de>
Date: Fri, 8 Sep 2023 11:08:32 +0200
Subject: [PATCH] Set workflow parameters as CLI parameters
#46
---
.pre-commit-config.yaml | 2 +-
app/api/utils.py | 3 +-
app/check_slurm_connection.py | 42 ++++++++++++++++++++++++
mako_templates/nextflow_command.template | 4 ++-
requirements-dev.txt | 2 +-
requirements.txt | 2 +-
scripts/prestart.sh | 2 ++
7 files changed, 51 insertions(+), 6 deletions(-)
create mode 100644 app/check_slurm_connection.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 83431d5..8c0e4d5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -21,7 +21,7 @@ repos:
files: app
args: [--check]
- repo: https://github.com/charliermarsh/ruff-pre-commit
- rev: 'v0.0.286'
+ rev: 'v0.0.287'
hooks:
- id: ruff
- repo: https://github.com/PyCQA/isort
diff --git a/app/api/utils.py b/app/api/utils.py
index 93f190f..3a6a362 100644
--- a/app/api/utils.py
+++ b/app/api/utils.py
@@ -160,8 +160,7 @@ async def start_workflow_execution(
nextflow_command = nextflow_command_template.render(
repo=git_repo,
- params_file_name=params_file_name,
- params_mount_path=settings.PARAMS_BUCKET_MOUNT_PATH,
+ parameters=parameters,
execution_id=execution.execution_id,
configuration=settings.NX_CONFIG,
nx_bin=settings.NX_BIN,
diff --git a/app/check_slurm_connection.py b/app/check_slurm_connection.py
new file mode 100644
index 0000000..559b76f
--- /dev/null
+++ b/app/check_slurm_connection.py
@@ -0,0 +1,42 @@
+import logging
+
+import httpx
+from fastapi import status
+from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed
+
+from app.core.config import settings
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+max_tries = 60 * 3 # 3 minutes
+wait_seconds = 2
+
+
+@retry(
+ stop=stop_after_attempt(max_tries),
+ wait=wait_fixed(wait_seconds),
+ before=before_log(logger, logging.INFO),
+ after=after_log(logger, logging.WARN),
+)
+def init() -> None:
+ headers = {"X-SLURM-USER-TOKEN": settings.SLURM_TOKEN, "X-SLURM-USER-NAME": settings.SLURM_USER}
+
+ try:
+ response = httpx.get(
+ f"{settings.SLURM_ENDPOINT}slurm/v0.0.38/ping", timeout=5.0, follow_redirects=False, headers=headers
+ )
+ assert response.status_code == status.HTTP_200_OK
+ except Exception as e:
+ logger.error(e)
+ raise e
+
+
+def main() -> None:
+ logger.info("Check Slurm Cluster connection")
+ init()
+ logger.info("Slurm Cluster connection established")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/mako_templates/nextflow_command.template b/mako_templates/nextflow_command.template
index 7804b17..60eced0 100644
--- a/mako_templates/nextflow_command.template
+++ b/mako_templates/nextflow_command.template
@@ -17,4 +17,6 @@ ${nx_bin} run ${repo.repo_url} \
-c ${configuration} \
% endif
-revision ${repo.git_commit_hash} \
--params-file ${params_mount_path}/${params_file_name}
+% for param_name, param_value in parameters.items():
+--${param_name} "${param_value}" \
+% endfor
diff --git a/requirements-dev.txt b/requirements-dev.txt
index d0d92a7..31f20b7 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -12,4 +12,4 @@ mypy>=1.5.0,<1.6.0
boto3-stubs-lite[s3]>=1.28.0,<1.29.0
types-requests
# Miscellaneous
-pre-commit>=3.3.0,<3.4.0
+pre-commit>=3.4.0,<3.5.0
diff --git a/requirements.txt b/requirements.txt
index 8b72521..305fcb3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@
clowmdb>=2.2.0,<2.3.0
# Webserver packages
-anyio>=3.7.0,<3.8.0
+anyio>=3.7.0,<4.0.0
fastapi>=0.103.0,<0.104.0
pydantic>=2.3.0,<2.4.0
pydantic-settings
diff --git a/scripts/prestart.sh b/scripts/prestart.sh
index d0fbfb3..f7cf08b 100755
--- a/scripts/prestart.sh
+++ b/scripts/prestart.sh
@@ -2,5 +2,7 @@
# Check Connection to Ceph RGW
python app/check_ceph_connection.py
+# Check Connection to Slurm Cluster
+python app/check_slurm_connection.py
# Let the DB start
python app/check_database_connection.py
--
GitLab