Resolve "Nextflow config for whole cluster"

README.md

@@ -35,6 +35,8 @@ This is the Workflow service of the CloWM service.
 | `ICON_BUCKET`                     | `clowm-icons`           | Bucket name                 | Bucket where to save workflow icons. Should be publicly available.                                                               |
 | `SLURM_USER`                      | `slurm`                 | string                      | User on the slurm cluster who should run the job. Should be the user of the `SLURM_TOKEN`                                        |
 | `PARAMS_BUCKET_MOUNT_PATH`        | `/mnt/params-bucket`    | Path on slurm cluster       | Folder where the S3 bucket `PARAMS_BUCKET` will be mounted on the slurm cluster                                                  |
+| `NX_CONFIG`                       | unset                   | Path on slurm cluster       | Configuration file on the slurm cluster that is the same for every nextflow run                                                  |
+| `NX_BIN`                          | `nextflow`              | Path on slurm cluster       | Path to the nextflow executable. Default it is in the `PATH`                                                                     |
 | `ACTIVE_WORKFLOW_EXECUTION_LIMIT` | `3`                     | Integer                     | Limit of active workflow execution a user is allowed to have. `-1` means infinite.                                               |
 | `DEV_SYSTEM`                      | `False`                 | `<"True"&#x7c;"False">`     | Activates an endpoint that allows execution of an workflow from an arbitrary Git Repository.<br>HAS TO BE `False` in PRODUCTION! |
 

app/api/utils.py

@@ -111,10 +111,13 @@ async def start_workflow_execution(
         params_file_name=params_file_name,
         params_mount_path=settings.PARAMS_BUCKET_MOUNT_PATH,
         execution_id=execution.execution_id,
+        configuration=settings.NX_CONFIG,
+        nx_bin=settings.NX_BIN,
         report_output_bucket=report_output_bucket[5:]
         if report_output_bucket is not None and report_output_bucket.startswith("s3://")
         else report_output_bucket,
     )
+
     try:
         slurm_job_id = await slurm_client.submit_job(
             nextflow_script=nextflow_command, execution_id=execution.execution_id

app/core/config.py

@@ -84,6 +84,8 @@ class Settings(BaseSettings):
     PARAMS_BUCKET_MOUNT_PATH: str = Field(
         "/mnt/params-bucket", description="Path on the slurm cluster where the params bucket is mounted."
     )
+    NX_CONFIG: str | None = Field(None, description="Path to a nextflow configuration for every run")
+    NX_BIN: str = Field("nextflow", description="Path to the nextflow executable")
     ACTIVE_WORKFLOW_EXECUTION_LIMIT: int = Field(3, description="The limit of active workflow executions per user.")
     DEV_SYSTEM: bool = Field(False, description="Open a endpoint where to execute arbitrary workflows.")
 

mako_templates/nextflow_command.template

 #!/bin/bash
 
-nextflow run ${repo.repo_url} \
+${nx_bin} run ${repo.repo_url} \
 -hub ${repo.provider} \
 % if report_output_bucket is not None:
 -with-report s3://${report_output_bucket}/report-${execution_id.hex}.html \
 -with-timeline s3://${report_output_bucket}/timeline-${execution_id.hex}.html \
 % endif
+% if configuration is not None:
+-c ${configuration} \
+% endif
 -revision ${repo.git_commit_hash} \
 -params-file ${params_mount_path}/${params_file_name}