Resolve "Support nextflow environment variables"

.pre-commit-config.yaml

@@ -15,13 +15,13 @@ repos:
     -   id: check-merge-conflict
     -   id: check-ast
 -   repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
     -   id: black
         files: app
         args: [--check]
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.257'
+    rev: 'v0.0.261'
     hooks:
     -   id: ruff
 -   repo: https://github.com/pre-commit/mirrors-mypy

README.md

@@ -25,16 +25,23 @@ This is the Workflow service of the CloWM service.
 
 ### Optional Variables
 
-| Variable                          | Default                      | Value                       | Description                                                                                                                      |
-|-----------------------------------|------------------------------|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------|
-| `API_PREFIX`                      | `/api/workflow-service`      | URL path                    | Prefix before every URL path                                                                                                     |
-| `BACKEND_CORS_ORIGINS`            | `[]`                         | json formatted list of urls | List of valid CORS origins                                                                                                       |
-| `SQLALCHEMY_VERBOSE_LOGGER`       | `False`                      | `<"True"&#x7c;"False">`     | Enables verbose SQL output.<br>Should be `false` in production                                                                   |
-| `PARAMS_BUCKET`                   | `nxf-params`                 | Bucket Name                 | Bucket where the nextflow configurations for each execution should be saved                                                      |
-| `WORKFLOW_BUCKET`                 | `clowm-workflows`            | Bucket Name                 | Bucket where to save important workflow files                                                                                    |
-| `ICON_BUCKET`                     | `clowm-icons`                | Bucket name                 | Bucket where to save workflow icons. Should be publicly available.                                                               |
-| `SLURM_USER`                      | `slurm`                      | string                      | User on the slurm cluster who should run the job. Should be the user of the `SLURM_TOKEN`                                        |
-| `SLURM_SHARED_DIR`                | `/vol/spool/clowm-workflows` | Path on slurm cluster       | Shared directory on the cluster where the nextflow jobs will be executed.                                                        |
-| `CONFIG_BUCKET_MOUNT_PATH`        | `/mnt/config-bucket`         | Path on slurm cluster       | Folder where the S3 bucket `NFX_CONFIG_BUCKET` will be mounted on the slurm cluster                                              |
-| `ACTIVE_WORKFLOW_EXECUTION_LIMIT` | `3`                          | Integer                     | Limit of active workflow execution a user is allowed to have. `-1` means infinite.                                               |
-| `DEV_SYSTEM`                      | `False`                      | `<"True"&#x7c;"False">`     | Activates an endpoint that allows execution of an workflow from an arbitrary Git Repository.<br>HAS TO BE `False` in PRODUCTION! |
+| Variable                          | Default                 | Value                       | Description                                                                                                                      |
+|-----------------------------------|-------------------------|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------|
+| `API_PREFIX`                      | `/api/workflow-service` | URL path                    | Prefix before every URL path                                                                                                     |
+| `BACKEND_CORS_ORIGINS`            | `[]`                    | json formatted list of urls | List of valid CORS origins                                                                                                       |
+| `SQLALCHEMY_VERBOSE_LOGGER`       | `False`                 | `<"True"&#x7c;"False">`     | Enables verbose SQL output.<br>Should be `false` in production                                                                   |
+| `PARAMS_BUCKET`                   | `nxf-params`            | Bucket Name                 | Bucket where the nextflow configurations for each execution should be saved                                                      |
+| `WORKFLOW_BUCKET`                 | `clowm-workflows`       | Bucket Name                 | Bucket where to save important workflow files                                                                                    |
+| `ICON_BUCKET`                     | `clowm-icons`           | Bucket name                 | Bucket where to save workflow icons. Should be publicly available.                                                               |
+| `SLURM_USER`                      | `slurm`                 | string                      | User on the slurm cluster who should run the job. Should be the user of the `SLURM_TOKEN`                                        |
+| `PARAMS_BUCKET_MOUNT_PATH`        | `/mnt/params-bucket`    | Path on slurm cluster       | Folder where the S3 bucket `PARAMS_BUCKET` will be mounted on the slurm cluster                                                  |
+| `ACTIVE_WORKFLOW_EXECUTION_LIMIT` | `3`                     | Integer                     | Limit of active workflow execution a user is allowed to have. `-1` means infinite.                                               |
+| `DEV_SYSTEM`                      | `False`                 | `<"True"&#x7c;"False">`     | Activates an endpoint that allows execution of an workflow from an arbitrary Git Repository.<br>HAS TO BE `False` in PRODUCTION! |
+
+### Nextflow Variables
+
+All [Nextflow environment variables](https://www.nextflow.io/docs/latest/config.html#environment-variables) with the
+prefix `NXF_` are also support when submitting a job to the slurm cluster, e.g. pinning the Nextflow version
+```
+NXF_VER=23.04.0
+```

app/api/utils.py

@@ -109,7 +109,7 @@ async def start_workflow_execution(
     nextflow_command = nextflow_command_template.render(
         repo=git_repo,
         params_file_name=params_file_name,
-        config_mount_path=settings.CONFIG_BUCKET_MOUNT_PATH,
+        params_mount_path=settings.PARAMS_BUCKET_MOUNT_PATH,
         execution_id=execution.execution_id,
         report_output_bucket=report_output_bucket[5:]
         if report_output_bucket is not None and report_output_bucket.startswith("s3://")

app/core/config.py

@@ -81,12 +81,8 @@ class Settings(BaseSettings):
     SLURM_ENDPOINT: AnyHttpUrl = Field(..., description="URI of the Slurm Cluster")
     SLURM_TOKEN: str = Field(..., description="JWT for authentication the Slurm REST API")
     SLURM_USER: str = Field("slurm", description="User of communication with slurm")
-    SLURM_SHARED_DIR: str = Field(
-        "/vol/spool/clowm-workflows",
-        description="Shared directory on the slurm cluster where the workflows will be executed",
-    )
-    CONFIG_BUCKET_MOUNT_PATH: str = Field(
-        "/mnt/config-bucket", description="Path on the slurm cluster where the config bucket is mounted."
+    PARAMS_BUCKET_MOUNT_PATH: str = Field(
+        "/mnt/params-bucket", description="Path on the slurm cluster where the params bucket is mounted."
     )
     ACTIVE_WORKFLOW_EXECUTION_LIMIT: int = Field(3, description="The limit of active workflow executions per user.")
     DEV_SYSTEM: bool = Field(False, description="Open a endpoint where to execute arbitrary workflows.")

app/slurm/slurm_rest_client.py

+from os import environ
 from uuid import UUID
 
+import dotenv
 from httpx import AsyncClient
 
 from app.core.config import settings
 
+dotenv.load_dotenv()
+base_env = {key: val for key, val in environ.items() if key.startswith("NXF_")}
+
 
 class SlurmClient:
     def __init__(self, client: AsyncClient, version: str = "v0.0.38"):
@@ -37,16 +42,12 @@ class SlurmClient:
         slurm_job_id : int
             Slurm job ID of submitted job.
         """
-        env = {
-            "NXF_HOME": f"/home/{settings.SLURM_USER}/.nextflow",
-            "NXF_WORK": settings.SLURM_SHARED_DIR,
-            "NXF_ASSETS": settings.SLURM_SHARED_DIR,
-            "TOWER_WORKSPACE_ID": execution_id.hex[:16],
-        }
+        env = base_env.copy()
+        env["TOWER_WORKSPACE_ID"] = execution_id.hex[:16]
+
         body = {
             "script": nextflow_script,
             "job": {
-                "current_working_directory": settings.SLURM_SHARED_DIR,
                 "name": str(execution_id),
                 "requeue": False,
                 "environment": env,

mako_templates/nextflow_command.template

@@ -7,4 +7,4 @@ nextflow run ${repo.repo_url} \
 -with-timeline s3://${report_output_bucket}/timeline-${execution_id.hex}.html \
 % endif
 -revision ${repo.git_commit_hash} \
--params-file ${config_mount_path}/${params_file_name}
+-params-file ${params_mount_path}/${params_file_name}

requirements-dev.txt

 # test packages
 pytest>=7.2.0,<7.3.0
-pytest-asyncio>=0.20.0,<0.21.0
+pytest-asyncio>=0.21.0,<0.22.0
 pytest-cov>=4.0.0,<4.1.0
-coverage[toml]>=7.1.0,<7.2.0
+coverage[toml]>=7.2.0,<7.3.0
 # Linters
 ruff
-black>=23.01.0,<23.02.0
+black>=23.03.0,<23.04.0
 isort>=5.12.0,<5.13.0
-mypy>=1.0.0,<1.1.0
+mypy>=1.1.0,<1.2.0
 # stubs for mypy
 boto3-stubs-lite[s3]>=1.26.0,<1.27.0
 sqlalchemy2-stubs
 types-requests
 # Miscellaneous
-pre-commit>=3.1.0,<3.2.0
-python-dotenv
+pre-commit>=3.2.0,<3.3.0

requirements.txt

@@ -21,3 +21,4 @@ httpx>=0.23.0,<0.24.0
 itsdangerous
 jsonschema>=4.0.0,<5.0.0
 mako
+python-dotenv