Inject arbitrary env variables in nextflow job

#85

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     -   id: check-merge-conflict
     -   id: check-ast
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.4.1'
+    rev: 'v0.4.2'
     hooks:
       - id: ruff
       - id: ruff-format
@@ -25,7 +25,7 @@ repos:
     -   id: isort
         files: app
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
     -   id: mypy
         files: app

README.md

@@ -86,13 +86,12 @@ This is the Workflow service of the CloWM service.
 
 ### Nextflow Variables
 
-All [Nextflow environment variables](https://www.nextflow.io/docs/latest/config.html#environment-variables) with the
-prefix `NXF_` are read from the environment when submitting a job to the slurm cluster, e.g. pinning the Nextflow
-version
+All environment variables with the prefix `WORKFLOW_ENV_` are included in the environment of the nextflow job of the
+cluster with the prefix cut off.
 
-```
-NXF_VER=23.04.0
-```
+Example:
+Pin the nextflow version
+`WORKFLOW_ENV_NXF_VER=23.04.0` -> `NXF_VER=23.04.0`
 
 ## License
 

app/api/background/cluster.py

@@ -34,7 +34,9 @@ s3_file_regex = re.compile(
 tracer = trace.get_tracer_provider().get_tracer(__name__)
 
 dotenv.load_dotenv()
-execution_env: dict[str, str | int | bool] = {key: val for key, val in environ.items() if key.startswith("NXF_")}
+execution_env: dict[str, str | int | bool] = {
+    key[13:]: val for key, val in environ.items() if key.startswith("WORKFLOW_ENV_") and len(key) > 12
+}
 execution_env["SCM_DIR"] = str(Path(settings.cluster.working_directory) / "scm")
 
 

app/templates/scripts/nextflow_command.sh.tmpl

 #!/bin/bash
 <%! from app.scm import SCM, SCMProvider %>
 
+echo $PATH
+env
 if [ -d "$NXF_ASSETS" ]; then
   touch "$NXF_ASSETS"
 fi

requirements-dev.txt

@@ -2,11 +2,11 @@
 pytest>=8.0.0,<8.2.0
 pytest-asyncio>=0.21.0,<0.22.0
 pytest-cov>=5.0.0,<5.1.0
-coverage[toml]>=7.4.0,<7.5.0
+coverage[toml]>=7.4.0,<7.6.0
 # Linters
 ruff>=0.4.0,<0.5.0
 isort>=5.13.0,<5.14.0
-mypy>=1.8.0,<1.10.0
+mypy>=1.8.0,<1.11.0
 # stubs for mypy
 boto3-stubs-lite[s3]>=1.34.0,<1.35.0
 types-requests