diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index be43dbf448aeb3d979b507209404ac4a231e9310..92cff7208f3fc64d14861ca9cb3dfd2bac5527cf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,7 +13,7 @@ variables:
SLURM_ENDPOINT: "http://127.0.0.1:8002"
ACTIVE_WORKFLOW_EXECUTION_LIMIT: 3
DEV_SYSTEM: "True"
- SLURM_JOB_STATUS_CHECK_INTERVAL: 1
+ SLURM_JOB_STATUS_CHECK_INTERVAL: 0
cache:
paths:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 29dce2edcc2d2a38a3834d107976a1ca7c3e76a6..399192f0085161a6a07561d48994cdf79b74ab67 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,13 +15,13 @@ repos:
- id: check-merge-conflict
- id: check-ast
- repo: https://github.com/psf/black
- rev: 23.9.1
+ rev: 23.10.0
hooks:
- id: black
files: app
args: [--check]
- repo: https://github.com/charliermarsh/ruff-pre-commit
- rev: 'v0.0.292'
+ rev: 'v0.1.1'
hooks:
- id: ruff
- repo: https://github.com/PyCQA/isort
@@ -31,7 +31,7 @@ repos:
files: app
args: [-c]
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.5.1
+ rev: v1.6.1
hooks:
- id: mypy
files: app
diff --git a/README.md b/README.md
index 2e5b3206ea718c9a64e04115da0b4738f2fe926a..1d5e494c63eb3075e421d62e800669d1640e8334 100644
--- a/README.md
+++ b/README.md
@@ -25,24 +25,24 @@ This is the Workflow service of the CloWM service.
### Optional Variables
-| Variable | Default | Value | Description |
-|-----------------------------------|-------------------------|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------|
-| `API_PREFIX` | `/api/workflow-service` | URL path | Prefix before every URL path |
-| `BACKEND_CORS_ORIGINS` | `[]` | json formatted list of urls | List of valid CORS origins |
-| `SQLALCHEMY_VERBOSE_LOGGER` | `False` | `<"True"|"False">` | Enables verbose SQL output.<br>Should be `false` in production |
-| `PARAMS_BUCKET` | `nxf-params` | Bucket Name | Bucket where the nextflow configurations for each execution should be saved |
-| `WORKFLOW_BUCKET` | `clowm-workflows` | Bucket Name | Bucket where to save important workflow files |
-| `ICON_BUCKET` | `clowm-icons` | Bucket name | Bucket where to save workflow icons. Should be publicly available. |
-| `SLURM_USER` | `slurm` | string | User on the slurm cluster who should run the job. Should be the user of the `SLURM_TOKEN` |
-| `PARAMS_BUCKET_MOUNT_PATH` | `/mnt/params-bucket` | Path on slurm cluster | Folder where the S3 bucket `PARAMS_BUCKET` will be mounted on the slurm cluster |
-| `NX_CONFIG` | unset | Path on slurm cluster | Configuration file on the slurm cluster that is the same for every nextflow run |
-| `NX_BIN` | `nextflow` | Path on slurm cluster | Path to the nextflow executable. Default it is in the `PATH` |
-| `SLURM_WORKING_DIRECTORY` | `/tmp` | Path on slurm cluster | Working directory for the slurm job with the nextflow command |
-| `ACTIVE_WORKFLOW_EXECUTION_LIMIT` | `3` | Integer | Limit of active workflow execution a user is allowed to have. `-1` means infinite. |
-| `DEV_SYSTEM` | `False` | `<"True"|"False">` | Activates an endpoint that allows execution of an workflow from an arbitrary Git Repository.<br>HAS TO BE `False` in PRODUCTION! |
-| `OPA_POLICY_PATH` | `/clowm/authz/allow` | URL path | Path to the OPA Policy for Authorization |
-| `SLURM_JOB_STATUS_CHECK_INTERVAL` | 30 | integer (seconds) | Interval for checking the slurm jobs status after starting a workflow execution |
-| `OTLP_GRPC_ENDPOINT` | unset | <hostname / IP> | OTLP compatible endpoint to send traces via gRPC, e.g. Jaeger |
+| Variable | Default | Value | Description |
+|-----------------------------------|-------------------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|
+| `API_PREFIX` | `/api/workflow-service` | URL path | Prefix before every URL path |
+| `BACKEND_CORS_ORIGINS` | `[]` | json formatted list of urls | List of valid CORS origins |
+| `SQLALCHEMY_VERBOSE_LOGGER` | `False` | `<"True"|"False">` | Enables verbose SQL output.<br>Should be `false` in production |
+| `PARAMS_BUCKET` | `nxf-params` | Bucket Name | Bucket where the nextflow configurations for each execution should be saved |
+| `WORKFLOW_BUCKET` | `clowm-workflows` | Bucket Name | Bucket where to save important workflow files |
+| `ICON_BUCKET` | `clowm-icons` | Bucket name | Bucket where to save workflow icons. Should be publicly available. |
+| `SLURM_USER` | `slurm` | string | User on the slurm cluster who should run the job. Should be the user of the `SLURM_TOKEN` |
+| `PARAMS_BUCKET_MOUNT_PATH` | `/mnt/params-bucket` | Path on slurm cluster | Folder where the S3 bucket `PARAMS_BUCKET` will be mounted on the slurm cluster |
+| `NX_CONFIG` | unset | Path on slurm cluster | Configuration file on the slurm cluster that is the same for every nextflow run |
+| `NX_BIN` | `nextflow` | Path on slurm cluster | Path to the nextflow executable. Default it is in the `PATH` |
+| `SLURM_WORKING_DIRECTORY` | `/tmp` | Path on slurm cluster | Working directory for the slurm job with the nextflow command |
+| `ACTIVE_WORKFLOW_EXECUTION_LIMIT` | `3` | Integer | Limit of active workflow execution a user is allowed to have. `-1` means infinite. |
+| `DEV_SYSTEM` | `False` | `<"True"|"False">` | Activates an endpoint that allows execution of an workflow from an arbitrary Git Repository.<br>HAS TO BE `False` in PRODUCTION! |
+| `OPA_POLICY_PATH` | `/clowm/authz/allow` | URL path | Path to the OPA Policy for Authorization |
+| `SLURM_JOB_STATUS_CHECK_INTERVAL` | 30 | integer (seconds) | Interval for checking the slurm jobs status after starting a workflow execution in seconds. If 0, then workflow execution is not monitored |
+| `OTLP_GRPC_ENDPOINT` | unset | <hostname / IP> | OTLP compatible endpoint to send traces via gRPC, e.g. Jaeger |
### Nextflow Variables
diff --git a/app/api/dependencies.py b/app/api/dependencies.py
index 26db81478d0d8c6e9a11750c56aebe44fcca504d..a7f2456b2ec7366f04411daba47c64f4b0e8081d 100644
--- a/app/api/dependencies.py
+++ b/app/api/dependencies.py
@@ -163,10 +163,8 @@ class AuthorizationDependency:
"""
async def authorization_wrapper(operation: str) -> AuthzResponse:
- with tracer.start_as_current_span("authorization") as span:
- span.set_attributes({"resource": self.resource, "operation": operation})
- params = AuthzRequest(operation=operation, resource=self.resource, uid=token.sub)
- return await request_authorization(request_params=params, client=client)
+ params = AuthzRequest(operation=operation, resource=self.resource, uid=token.sub)
+ return await request_authorization(request_params=params, client=client)
return authorization_wrapper
diff --git a/app/api/endpoints/workflow.py b/app/api/endpoints/workflow.py
index 99067f84d8a8bd294807ab3fa680dbea11ff4592..1890bc07aab851b1f772c4015dd50f7d5b050985 100644
--- a/app/api/endpoints/workflow.py
+++ b/app/api/endpoints/workflow.py
@@ -73,7 +73,7 @@ async def list_workflows(
current_span = trace.get_current_span()
if developer_id is not None:
current_span.set_attribute("developer_id", developer_id)
- if name_substring is not None:
+ if name_substring is not None: # pragma: no cover
current_span.set_attribute("name_substring", name_substring)
if version_status is not None and len(version_status) > 0:
current_span.set_attribute("version_status", [stat.name for stat in version_status])
diff --git a/app/api/endpoints/workflow_execution.py b/app/api/endpoints/workflow_execution.py
index 6abc40d23075ef6363a3226a0cb9edfb6bdd4269..50228d8e84e7a504bced5470317dafac88171fbe 100644
--- a/app/api/endpoints/workflow_execution.py
+++ b/app/api/endpoints/workflow_execution.py
@@ -351,11 +351,11 @@ async def list_workflow_executions(
List of filtered workflow executions.
"""
current_span = trace.get_current_span()
- if user_id is not None:
+ if user_id is not None: # pragma: no cover
current_span.set_attribute("user_id", user_id)
- if execution_status is not None and len(execution_status) > 0:
+ if execution_status is not None and len(execution_status) > 0: # pragma: no cover
current_span.set_attribute("execution_status", [stat.name for stat in execution_status])
- if workflow_version_id is not None:
+ if workflow_version_id is not None: # pragma: no cover
current_span.set_attribute("git_commit_hash", workflow_version_id)
rbac_operation = "list" if user_id is not None and user_id == current_user.uid else "list_all"
diff --git a/app/api/utils.py b/app/api/utils.py
index 4b0061a1a5a04f6eb845bdac6df65c9a3107eda4..b581948587bad1801f6bbf616b159335c9904e48 100644
--- a/app/api/utils.py
+++ b/app/api/utils.py
@@ -222,10 +222,11 @@ async def start_workflow_execution(
await CRUDWorkflowExecution.update_slurm_job_id(
db, slurm_job_id=slurm_job_id, execution_id=execution.execution_id
)
- await _monitor_proper_job_execution(
- db=db, slurm_client=slurm_client, execution_id=execution.execution_id, slurm_job_id=slurm_job_id
- )
- except (ConnectError, ConnectTimeout): # pragma: no cover
+ if settings.SLURM_JOB_STATUS_CHECK_INTERVAL > 0: # pragma: no cover
+ await _monitor_proper_job_execution(
+ db=db, slurm_client=slurm_client, execution_id=execution.execution_id, slurm_job_id=slurm_job_id
+ )
+ except (ConnectError, ConnectTimeout, KeyError):
# Mark job as aborted when there is an error
await CRUDWorkflowExecution.cancel(
db, execution_id=execution.execution_id, status=WorkflowExecution.WorkflowExecutionStatus.ERROR
diff --git a/app/core/config.py b/app/core/config.py
index af3dd9a67806708eab25f8fdb5989abdad0f64e9..af7eab115a36cb594221cdd206ede3922eb565fe 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -107,9 +107,9 @@ class Settings(BaseSettings):
ACTIVE_WORKFLOW_EXECUTION_LIMIT: int = Field(3, description="The limit of active workflow executions per user.")
SLURM_JOB_STATUS_CHECK_INTERVAL: int = Field(
30,
- ge=1,
+ ge=0,
le=600,
- description="Interval for checking the slurm jobs status after starting a workflow execution in seconds",
+ description="Interval for checking the slurm jobs status after starting a workflow execution in seconds. If 0, then workflow execution is not monitored",
)
DEV_SYSTEM: bool = Field(False, description="Open a endpoint where to execute arbitrary workflows.")
OTLP_GRPC_ENDPOINT: Optional[str] = Field(
diff --git a/app/core/security.py b/app/core/security.py
index c09999a308fb7fc31cad83f4c59a94dc82b29dc4..d30faf1c89694dddd3cb1dba1e9e437fecd47984 100644
--- a/app/core/security.py
+++ b/app/core/security.py
@@ -3,6 +3,7 @@ from typing import Dict
from authlib.jose import JsonWebToken
from fastapi import HTTPException, status
from httpx import AsyncClient
+from opentelemetry import trace
from app.core.config import settings
from app.schemas.security import AuthzRequest, AuthzResponse
@@ -11,6 +12,8 @@ ISSUER = "clowm"
ALGORITHM = "RS256"
jwt = JsonWebToken([ALGORITHM])
+tracer = trace.get_tracer_provider().get_tracer(__name__)
+
def decode_token(token: str) -> Dict[str, str]: # pragma: no cover
"""
@@ -55,13 +58,17 @@ async def request_authorization(request_params: AuthzRequest, client: AsyncClien
response : app.schemas.security.AuthzResponse
Response by the Auth service about the authorization request
"""
- response = await client.post(
- f"{settings.OPA_URI}v1/data{settings.OPA_POLICY_PATH}", json={"input": request_params.model_dump()}
- )
- parsed_response = AuthzResponse(**response.json())
+ with tracer.start_as_current_span("authorization") as span:
+ span.set_attributes({"resource": request_params.resource, "operation": request_params.operation})
+ response = await client.post(
+ f"{settings.OPA_URI}v1/data{settings.OPA_POLICY_PATH}", json={"input": request_params.model_dump()}
+ )
+ parsed_response = AuthzResponse(**response.json())
+ span.set_attribute("decision_id", str(parsed_response.decision_id))
if not parsed_response.result: # pragma: no cover
raise HTTPException(
- status_code=status.HTTP_403_FORBIDDEN, detail=f"Action forbidden. Decision ID {parsed_response.decision_id}"
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail=f"Action forbidden. Decision ID {parsed_response.decision_id}",
)
return parsed_response
diff --git a/app/main.py b/app/main.py
index 313629df80f18f520be2c5837f51805a75ce21a2..6144739671506b61e81e68299e8fe2f8184048e5 100644
--- a/app/main.py
+++ b/app/main.py
@@ -63,7 +63,9 @@ if settings.OTLP_GRPC_ENDPOINT is not None and len(settings.OTLP_GRPC_ENDPOINT)
return await request_validation_exception_handler(request, exc)
-FastAPIInstrumentor.instrument_app(app, excluded_urls="health", tracer_provider=trace.get_tracer_provider())
+FastAPIInstrumentor.instrument_app(
+ app, excluded_urls="health,docs,openapi.json", tracer_provider=trace.get_tracer_provider()
+)
# CORS Settings for the API
app.add_middleware(
diff --git a/app/slurm/slurm_rest_client.py b/app/slurm/slurm_rest_client.py
index 4325a04e4961c2e20ad14374b687402b5c510d64..84ed6b6a44a8ac9b6dea6f33a7a27be6267ad5c8 100644
--- a/app/slurm/slurm_rest_client.py
+++ b/app/slurm/slurm_rest_client.py
@@ -69,6 +69,8 @@ class SlurmClient:
response = await self._client.post(
f"{settings.SLURM_ENDPOINT}slurm/{self.version}/job/submit", headers=self._headers, json=body
)
+ if response.status_code != status.HTTP_200_OK:
+ raise KeyError("Error at slurm")
return int(response.json()["job_id"])
async def cancel_job(self, job_id: int) -> None:
@@ -85,7 +87,7 @@ class SlurmClient:
f"{settings.SLURM_ENDPOINT}slurm/{self.version}/job/{job_id}", headers=self._headers
)
- async def is_job_finished(self, job_id: int) -> bool:
+ async def is_job_finished(self, job_id: int) -> bool: # pragma: no cover
"""
Check if the job with the given is completed
@@ -106,7 +108,7 @@ class SlurmClient:
span.set_attribute("slurm.job-status.request.code", response.status_code)
if response.status_code != status.HTTP_200_OK:
return True
- try: # pragma: no cover
+ try:
job_state = response.json()["jobs"][0]["job_state"]
span.set_attribute("slurm.job-status.state", job_state)
return job_state == "COMPLETED" or job_state == "FAILED" or job_state == "CANCELLED"
diff --git a/app/tests/api/test_security.py b/app/tests/api/test_security.py
index 40d0c665718af6cb3b5ba913e43814ee7ce8934e..3f6df23dc5f4bce2c76b44e6321e5b247987cc87 100644
--- a/app/tests/api/test_security.py
+++ b/app/tests/api/test_security.py
@@ -84,3 +84,24 @@ class TestJWTProtectedRoutes:
self.protected_route, params={"user": random_user.user.uid}, headers=random_user.auth_headers
)
assert response.status_code == status.HTTP_404_NOT_FOUND
+
+ @pytest.mark.asyncio
+ async def test_routed_with_insufficient_permissions(
+ self, client: AsyncClient, random_user: UserWithAuthHeader
+ ) -> None:
+ """
+ Test with correct authorization header but with insufficient permissions.
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing.
+ """
+ response = await client.get(
+ self.protected_route,
+ params={"raise_opa_error": True},
+ headers=random_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_403_FORBIDDEN
diff --git a/app/tests/api/test_workflow_execution.py b/app/tests/api/test_workflow_execution.py
index 73cdc60ef254df58593c3fbf31eb8f1da439bab6..e33c0e91751fed99503d70547a07cf22dc16cd23 100644
--- a/app/tests/api/test_workflow_execution.py
+++ b/app/tests/api/test_workflow_execution.py
@@ -5,7 +5,7 @@ import pytest
from clowmdb.models import Bucket, Workflow, WorkflowExecution, WorkflowMode, WorkflowVersion
from fastapi import status
from httpx import AsyncClient
-from sqlalchemy import update
+from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
@@ -14,7 +14,8 @@ from app.schemas.workflow import WorkflowOut
from app.schemas.workflow_execution import DevWorkflowExecutionIn, WorkflowExecutionIn
from app.schemas.workflow_mode import WorkflowModeIn
from app.scm import SCM, Provider
-from app.tests.mocks import MockS3ServiceResource, MockSlurmCluster
+from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
+from app.tests.mocks.mock_slurm_cluster import MockSlurmCluster
from app.tests.utils.bucket import add_permission_for_bucket
from app.tests.utils.user import UserWithAuthHeader
from app.tests.utils.utils import random_hex_string, random_lower_string
@@ -607,6 +608,46 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in)
assert response.status_code == status.HTTP_400_BAD_REQUEST
+ @pytest.mark.asyncio
+ async def test_start_execution_with_slurm_error(
+ self,
+ client: AsyncClient,
+ db: AsyncSession,
+ random_user: UserWithAuthHeader,
+ random_workflow_version: WorkflowVersion,
+ random_workflow: WorkflowOut,
+ ) -> None:
+ """
+ Test for starting a workflow execution from a public GitHub repository.
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing.
+ random_workflow : app.schemas.workflow.WorkflowOut
+ Random workflow for testing.
+ random_workflow_version : clowmdb.models.WorkflowVersion
+ Random workflow version for testing.
+ """
+ execution_in = WorkflowExecutionIn(workflow_version_id=random_workflow_version.git_commit_hash, parameters={})
+ response = await client.post(
+ self.base_path,
+ headers=random_user.auth_headers,
+ json=execution_in.model_dump(),
+ params={"raise_slurm_error": True},
+ )
+ assert response.status_code == status.HTTP_201_CREATED
+ execution_id = UUID(response.json()["execution_id"])
+
+ stmt = select(WorkflowExecution).where(WorkflowExecution._execution_id == execution_id.bytes)
+ execution_db = await db.scalar(stmt)
+ assert execution_db
+ assert execution_db.status == WorkflowExecution.WorkflowExecutionStatus.ERROR
+
class TestDevWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
@pytest.mark.asyncio
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index 4fa2b806b68adc77efd39ab1b50da1e08138cbba..7e7a2210d3eb0e9152ae976d47d6d4d327819ce7 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -2,7 +2,7 @@ import asyncio
from functools import partial
from io import BytesIO
from secrets import token_urlsafe
-from typing import AsyncIterator, Iterator
+from typing import AsyncIterator, Dict, Iterator
import httpx
import pytest
@@ -16,7 +16,7 @@ from clowmdb.models import (
WorkflowVersion,
workflow_mode_association_table,
)
-from fastapi import status
+from pytrie import SortedStringTrie as Trie
from sqlalchemy import insert, select, update
from sqlalchemy.ext.asyncio import AsyncSession
@@ -26,7 +26,10 @@ from app.git_repository import build_repository
from app.main import app
from app.schemas.workflow import WorkflowOut
from app.scm import SCM, Provider
-from app.tests.mocks import MockOpaService, MockS3ServiceResource, MockSlurmCluster
+from app.tests.mocks import DefaultMockHTTPService, MockHTTPService
+from app.tests.mocks.mock_opa_service import MockOpaService
+from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
+from app.tests.mocks.mock_slurm_cluster import MockSlurmCluster
from app.tests.utils.bucket import create_random_bucket
from app.tests.utils.user import UserWithAuthHeader, create_random_user, decode_mock_token, get_authorization_headers
from app.tests.utils.utils import random_hex_string, random_lower_string
@@ -82,26 +85,40 @@ async def client(
) -> AsyncIterator[httpx.AsyncClient]:
"""
Fixture for creating a TestClient and perform HTTP Request on it.
- Overrides serveral dependencies.
+ Overrides several dependencies.
"""
+ endpoints: Dict[str, MockHTTPService] = {
+ str(settings.SLURM_ENDPOINT): mock_slurm_cluster,
+ str(settings.OPA_URI): mock_opa_service,
+ }
+ # data structure to easily find the appropriate mock request based on the URL
+ t = Trie(**endpoints)
+
+ async def get_mock_httpx_client(
+ raise_opa_error: bool = False, raise_slurm_error: bool = False, raise_error: bool = False
+ ) -> AsyncIterator[httpx.AsyncClient]:
+ """
+ FastAPI Dependency to get an async httpx client with mock transport.
+
+ Parameters
+ ----------
+ raise_opa_error : bool
+ Flag to raise an error when querying the OPA service. Query parameter.
+ raise_slurm_error : bool
+ Flag to raise an error when querying the Slurm service. Query parameter.
+ raise_error : bool
+ Flag to raise an error. Query parameter.
+
+ Returns
+ -------
+ client : AsyncIterator[httpx.AsyncClient]
+ Http client with mock transports.
+ """
+ errors = locals() # catch all error flags in a dict
- async def get_mock_httpx_client(raise_error: bool = False) -> AsyncIterator[httpx.AsyncClient]:
- # raises an 404 error if the query parameter 'raise_error' is true
def mock_request_handler(request: httpx.Request) -> httpx.Response:
url = str(request.url)
- if url.startswith(str(settings.OPA_URI)):
- return mock_opa_service.handle_request(request)
- elif url.startswith(str(settings.SLURM_ENDPOINT)):
- return mock_slurm_cluster.handle_request(request)
- elif raise_error:
- return httpx.Response(status_code=status.HTTP_404_NOT_FOUND, json={})
- return httpx.Response(
- status_code=status.HTTP_200_OK,
- json={
- # When checking if a file exists in a git repository, the GitHub API expects this in a response
- "download_url": "https://example.com"
- },
- )
+ return t.longest_prefix_value(url, DefaultMockHTTPService()).handle_request(request, **errors)
async with httpx.AsyncClient(transport=httpx.MockTransport(mock_request_handler)) as http_client:
yield http_client
diff --git a/app/tests/mocks/__init__.py b/app/tests/mocks/__init__.py
index 9fb9a28c031d86669c1518f79dec5a0c2dafc3e7..56f370d8a08ba9a3bb5696e22b43c5f1caf4d182 100644
--- a/app/tests/mocks/__init__.py
+++ b/app/tests/mocks/__init__.py
@@ -1,3 +1,22 @@
-from .mock_opa_service import MockOpaService # noqa: F401
-from .mock_s3_resource import MockS3ServiceResource # noqa: F401
-from .mock_slurm_cluster import MockSlurmCluster # noqa: F401
+from abc import ABC, abstractmethod
+
+from fastapi import status
+from httpx import Request, Response
+
+
+class MockHTTPService(ABC):
+ @abstractmethod
+ def handle_request(self, request: Request, **kwargs: bool) -> Response:
+ ...
+
+
+class DefaultMockHTTPService(MockHTTPService):
+ def handle_request(self, request: Request, **kwargs: bool) -> Response:
+ raise_error = kwargs.get("raise_error", False)
+ return Response(
+ status_code=status.HTTP_404_NOT_FOUND if raise_error else status.HTTP_200_OK,
+ json={
+ # When checking if a file exists in a git repository, the GitHub API expects this in a response
+ "download_url": "https://example.com"
+ },
+ )
diff --git a/app/tests/mocks/mock_opa_service.py b/app/tests/mocks/mock_opa_service.py
index 639e0a0431ebf05b5f4a7f311298b4ebfebb7c3c..1c1c105913dc9596aae3a33dd66db6210101930b 100644
--- a/app/tests/mocks/mock_opa_service.py
+++ b/app/tests/mocks/mock_opa_service.py
@@ -6,9 +6,10 @@ from fastapi import status
from httpx import Request, Response
from app.schemas.security import AuthzRequest, AuthzResponse
+from app.tests.mocks import MockHTTPService
-class MockOpaService:
+class MockOpaService(MockHTTPService):
"""
Class to mock the Open Policy Agent service.
Has a simplified role management. A user can be either "Admin" or "Normal User".
@@ -48,7 +49,7 @@ class MockOpaService:
"""
self._users = {}
- def handle_request(self, request: Request) -> Response:
+ def handle_request(self, request: Request, **kwargs: bool) -> Response:
"""
Handle the raw request that is sent to the mock service.
@@ -62,8 +63,9 @@ class MockOpaService:
response : httpx.Response
Appropriate response to the received request.
"""
+ raise_error = kwargs.get("raise_opa_error", False)
authz_request = AuthzRequest(**json.loads(request.read().decode("utf-8"))["input"])
- if authz_request.uid not in self._users:
+ if raise_error or authz_request.uid not in self._users:
result = False
else:
result = not MockOpaService.request_admin_permission(authz_request) or self._users[authz_request.uid]
diff --git a/app/tests/mocks/mock_slurm_cluster.py b/app/tests/mocks/mock_slurm_cluster.py
index 15d56dc28f77b6af8e8ee7d3aeb28b254ab3b98a..65ac0fa2ae49695e361756021cad181790aaf786 100644
--- a/app/tests/mocks/mock_slurm_cluster.py
+++ b/app/tests/mocks/mock_slurm_cluster.py
@@ -5,10 +5,12 @@ from typing import Any, Dict, List, Optional
from fastapi import status
from httpx import Headers, Request, Response
+from app.tests.mocks import MockHTTPService
+
SlurmRequestBody = Dict[str, Any]
-class MockSlurmCluster:
+class MockSlurmCluster(MockHTTPService):
"""
Class to mock the Rest API of a Slurm cluster
"""
@@ -23,7 +25,7 @@ class MockSlurmCluster:
self.base_path = f"slurm/{version}"
self._job_path_regex = re.compile(f"^/slurm/{re.escape(version)}/job/[\d]*$")
- def handle_request(self, request: Request) -> Response:
+ def handle_request(self, request: Request, **kwargs: bool) -> Response:
"""
Handle the raw request that is sent to the API.
@@ -37,6 +39,8 @@ class MockSlurmCluster:
response : httpx.Response
Appropriate response to the request
"""
+ if kwargs.get("raise_slurm_error", False):
+ return Response(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Authorize request
error_response = MockSlurmCluster.authorize_request(request.headers)
if error_response is not None:
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 810ec9548bdb7ead51ed359cc8ff11a8c26bee07..f63739f710d32f34ab3dd2dc88c361bb5369421a 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -4,12 +4,13 @@ pytest-asyncio>=0.21.0,<0.22.0
pytest-cov>=4.1.0,<4.2.0
coverage[toml]>=7.3.0,<7.4.0
# Linters
-ruff
-black>=23.09.0,<23.10.0
+ruff>=0.1.1,<0.1.2
+black>=23.10.0,<23.11.0
isort>=5.12.0,<5.13.0
-mypy>=1.5.0,<1.6.0
+mypy>=1.6.0,<1.7.0
# stubs for mypy
boto3-stubs-lite[s3]>=1.28.0,<1.29.0
types-requests
# Miscellaneous
-pre-commit>=3.4.0,<3.5.0
+pre-commit>=3.5.0,<3.6.0
+PyTrie>=0.4.0,<0.5.0
diff --git a/requirements.txt b/requirements.txt
index a4d2e77cfec71690bd3f53c4b3395ef32836b8aa..abb97a4507fe5732b5a7a651fcbab0678f7d9708 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ clowmdb>=2.2.0,<2.3.0
# Webserver packages
anyio>=3.7.0,<4.0.0
-fastapi>=0.103.0,<0.104.0
+fastapi>=0.104.0,<0.105.0
pydantic>=2.4.0,<2.5.0
pydantic-settings
uvicorn>=0.23.0,<0.24.0
@@ -23,7 +23,7 @@ itsdangerous
jsonschema>=4.0.0,<5.0.0
mako
python-dotenv
-Pillow>=10.0.0,<10.1.0
+Pillow>=10.1.0,<10.2.0
# Monitoring
opentelemetry-instrumentation-fastapi