Merge branch 'feature/33-write-workflow-parameters-in-parameter-file' into 'development'

Resolve "Write workflow parameters in parameter file"

Closes #33

See merge request cmg/clowm/clowm-workflow-service!31

app/api/utils.py

@@ -21,7 +21,7 @@ if TYPE_CHECKING:
 else:
     S3ServiceResource = object
 
-config_template = Template(filename="mako_templates/config_template.config.template")
+parameter_file_template = Template(filename="mako_templates/workflow-params.yaml.template")
 nextflow_command_template = Template(filename="mako_templates/nextflow_command.template")
 # regex to find S3 files in parameters of workflow execution
 s3_file_regex = re.compile(
@@ -129,22 +129,24 @@ async def start_workflow_execution(
     slurm_client : app.slurm.slurm_rest_client.SlurmClient
         Slurm Rest Client to communicate with Slurm cluster.
     """
-    config_name = f"{execution.execution_id.hex}.config"
+    params_file_name = f"params-{execution.execution_id.hex}.yaml"
     with SpooledTemporaryFile(max_size=512000) as f:
         ctx = BinaryWriterContext(
             f,
-            execution_id=execution.execution_id,
-            output_bucket=report_output_bucket[5:]
-            if report_output_bucket is not None and report_output_bucket.startswith("s3://")
-            else report_output_bucket,
             parameters=parameters,
         )
-        config_template.render_context(ctx)
+        parameter_file_template.render_context(ctx)
         f.seek(0)
-        s3.Bucket(name=settings.NXF_CONFIG_BUCKET).Object(key=config_name).upload_fileobj(f)
+        s3.Bucket(name=settings.NXF_CONFIG_BUCKET).Object(key=params_file_name).upload_fileobj(f)
 
     nextflow_command = nextflow_command_template.render(
-        repo=git_repo, config_name=config_name, config_mount_path=settings.CONFIG_BUCKET_MOUNT_PATH
+        repo=git_repo,
+        params_file_name=params_file_name,
+        config_mount_path=settings.CONFIG_BUCKET_MOUNT_PATH,
+        execution_id=execution.execution_id,
+        report_output_bucket=report_output_bucket[5:]
+        if report_output_bucket is not None and report_output_bucket.startswith("s3://")
+        else report_output_bucket,
     )
     try:
         slurm_job_id = await slurm_client.submit_job(

app/slurm/slurm_rest_client.py

@@ -43,7 +43,10 @@ class SlurmClient:
                 "current_working_directory": settings.SLURM_SHARED_DIR,
                 "name": str(execution_id),
                 "requeue": False,
-                "environment": {"NXF_HOME": f"/home/{settings.SLURM_USER}/.nextflow"},
+                "environment": {
+                    "NXF_HOME": f"/home/{settings.SLURM_USER}/.nextflow",
+                    "TOWER_WORKSPACE_ID": execution_id.hex[:16],
+                },
             },
         }
         response = await self._client.post(

app/tests/api/test_workflow_execution.py

@@ -52,7 +52,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
         assert execution_response["workflow_version_id"] == random_workflow_version.git_commit_hash
 
         assert (
-            UUID(hex=execution_response["execution_id"]).hex + ".config"
+            f"params-{UUID(hex=execution_response['execution_id']).hex }.yaml"
             in mock_s3_service.Bucket(settings.NXF_CONFIG_BUCKET).objects.all_keys()
         )
 
@@ -97,7 +97,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
         assert execution_response["workflow_version_id"] == random_workflow_version.git_commit_hash
 
         assert (
-            UUID(hex=execution_response["execution_id"]).hex + ".config"
+            f"params-{UUID(hex=execution_response['execution_id']).hex }.yaml"
             in mock_s3_service.Bucket(settings.NXF_CONFIG_BUCKET).objects.all_keys()
         )
 
@@ -400,7 +400,7 @@ class TestDevWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
         assert execution_response["status"] == WorkflowExecution.WorkflowExecutionStatus.PENDING
 
         assert (
-            UUID(hex=execution_response["execution_id"]).hex + ".config"
+            f"params-{UUID(hex=execution_response['execution_id']).hex }.yaml"
             in mock_s3_service.Bucket(settings.NXF_CONFIG_BUCKET).objects.all_keys()
         )
 

mako_templates/config_template.config.template

-tower {
-    workspaceId = "${execution_id.hex[:16]}"
-}
-
-report {
-    enabled = ${ str(output_bucket is not None).lower()}
-% if output_bucket is not None:
-    file = "s3://${ output_bucket }/report-${ execution_id.hex }.html"
-% endif
-}
-
-timeline {
-    enabled = ${ str(output_bucket is not None).lower()}
-% if output_bucket is not None:
-    file = "s3://${ output_bucket }/timeline-${ execution_id.hex }.html"
-% endif
-}
-
-params {
-% for param, val in parameters.items():
-    % if isinstance(val, str):
-    ${param} = "${val}"
-    % elif isinstance(val, bool):
-    ${param} = ${str(val).lower()}
-    % else:
-    ${param} = ${val}
-    %endif
-% endfor
-}

mako_templates/nextflow_command.template

 #!/bin/bash
 
-nextflow run ${repo.repo_url} -c ${config_mount_path}/${config_name} -hub ${repo.provider} -revision ${repo.git_commit_hash}
+nextflow run ${repo.repo_url} \
+-hub ${repo.provider} \
+% if report_output_bucket is not None:
+-with-report s3://${report_output_bucket}/report-${execution_id.hex}.html \
+-with-timeline s3://${report_output_bucket}/timeline-${execution_id.hex}.html \
+% endif
+-revision ${repo.git_commit_hash} \
+-params-file ${config_mount_path}/${params_file_name}

mako_templates/workflow-params.yaml.template

+% for param, val in parameters.items():
+% if isinstance(val, str):
+${param}: "${val}"
+% elif isinstance(val, bool):
+${param}: ${str(val).lower()}
+% else:
+${param}: ${val}
+%endif
+% endfor