diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cd3a549c1217eabb2ff186be5d067735fdd51d1c..8aa44886588ae5feeacad5da6e55c8a8724a4d54 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,9 +4,8 @@ variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PYTHONPATH: "$CI_PROJECT_DIR"
OBJECT_GATEWAY_URI: "http://127.0.0.1:8001"
- CEPH_ACCESS_KEY: ""
- CEPH_SECRET_KEY: ""
- CEPH_USERNAME: ""
+ BUCKET_CEPH_ACCESS_KEY: ""
+ BUCKET_CEPH_SECRET_KEY: ""
FF_NETWORK_PER_BUILD: 1
PUBLIC_KEY_VALUE: "empty"
OPA_URI: "http://127.0.0.1:8181"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b0c5251a83107b080366ebfd6da4ce551e98283f..f547a4fe938f063d7c07547e799a4fb7f3491c07 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,17 +15,17 @@ repos:
- id: check-merge-conflict
- id: check-ast
- repo: https://github.com/psf/black
- rev: 23.3.0
+ rev: 23.7.0
hooks:
- id: black
files: app
args: [--check]
- repo: https://github.com/charliermarsh/ruff-pre-commit
- rev: 'v0.0.262'
+ rev: 'v0.0.281'
hooks:
- id: ruff
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.2.0
+ rev: v1.4.1
hooks:
- id: mypy
files: app
diff --git a/README.md b/README.md
index 534f2693216da226fc1ce74c980a0215006b758e..6ce47f4a1ec8eb9b83809df3c14d45158c0c48f3 100644
--- a/README.md
+++ b/README.md
@@ -10,14 +10,14 @@ This is the Workflow service of the CloWM service.
| Variable | Default | Value | Description |
|----------------------------------------|--------------------|---------------------------------|----------------------------------------------------------------------------------------------|
| `PUBLIC_KEY_VALUE` / `PUBLIC_KEY_FILE` | randomly generated | Public Key / Path to Public Key | Public part of RSA Key in PEM format to verify JWTs |
-| `DB_HOST` | unset | <db hostname / IP> | IP or Hostname Adress of DB |
+| `DB_HOST` | unset | <db hostname / IP> | IP or Hostname Address of DB |
| `DB_PORT` | 3306 | Number | Port of the database |
| `DB_USER` | unset | \<db username> | Username of the database user |
| `DB_PASSWORD` | unset | \<db password> | Password of the database user |
| `DB_DATABASE` | unset | \<db name> | Name of the database |
| `OBJECT_GATEWAY_URI` | unset | HTTP URL | HTTP URL of the Ceph Object Gateway |
-| `CEPH_ACCESS_KEY` | unset | \<access key> | Ceph access key with admin privileges |
-| `CEPH_SECRET_KEY` | unset | \<secret key> | Ceph secret key with admin privileges |
+| `BUCKET_CEPH_ACCESS_KEY` | unset | \<access key> | Ceph access key with admin privileges |
+| `BUCKET_CEPH_SECRET_KEY` | unset | \<secret key> | Ceph secret key with admin privileges |
| `OPA_URI` | unset | HTTP URL | HTTP URL of the OPA service |
| `SLURM_ENDPOINT` | unset | HTTP URL | HTTP URL to communicate with the Slurm cluster |
| `SLURM_TOKEN` | unset | \<JWT> | JWT for communication with the Slurm REST API. Should belong to the user of the `SLURM_USER` |
diff --git a/app/api/dependencies.py b/app/api/dependencies.py
index eeba6f3cdb672e54ff8b78f263d33ebf291ad431..3dad80d4ef50a6e15e6c096ab2cdad0e643f7832 100644
--- a/app/api/dependencies.py
+++ b/app/api/dependencies.py
@@ -41,7 +41,7 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]:
Async session object with the database
"""
async with get_async_session(
- settings.SQLALCHEMY_DATABASE_ASYNC_URI, verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
+ str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
)() as db:
yield db
@@ -179,7 +179,7 @@ async def get_current_user(token: JWT = Depends(decode_bearer_token), db: AsyncS
async def get_current_workflow(
- wid: UUID = Path(..., description="ID of a workflow", example="0cc78936-381b-4bdd-999d-736c40591078"),
+ wid: UUID = Path(..., description="ID of a workflow", examples=["0cc78936-381b-4bdd-999d-736c40591078"]),
db: AsyncSession = Depends(get_db),
) -> Workflow:
"""
@@ -207,7 +207,7 @@ async def get_current_workflow(
async def get_current_workflow_execution(
- eid: UUID = Path(..., description="ID of a workflow execution.", example="0cc78936-381b-4bdd-999d-736c40591078"),
+ eid: UUID = Path(..., description="ID of a workflow execution.", examples=["0cc78936-381b-4bdd-999d-736c40591078"]),
db: AsyncSession = Depends(get_db),
) -> WorkflowExecution:
"""
diff --git a/app/api/endpoints/workflow.py b/app/api/endpoints/workflow.py
index 966548006c8583d33c9c4f6e26eb86fd0446ec93..236bbc9c3ef7a4d8dc534caee506ac94294e7251 100644
--- a/app/api/endpoints/workflow.py
+++ b/app/api/endpoints/workflow.py
@@ -1,7 +1,8 @@
from typing import Annotated, Any, Awaitable, Callable
from clowmdb.models import Workflow, WorkflowVersion
-from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Query, Response, UploadFile, status
+from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status
+from pydantic import AnyHttpUrl
from app.api.dependencies import AuthorizationDependency, CurrentUser, CurrentWorkflow, DBSession, HTTPClient, S3Service
from app.api.utils import check_repo, upload_icon
@@ -38,7 +39,7 @@ async def list_workflows(
| None = Query(
None,
description="Filter for workflow by developer. If current user is the same as developer ID, permission 'workflow:list' required, otherwise 'workflow:list_filter'.", # noqa: E501
- example="28c5353b8bb34984a8bd4169ba94c606",
+ examples=["28c5353b8bb34984a8bd4169ba94c606"],
),
) -> list[WorkflowOut]:
"""
@@ -91,7 +92,40 @@ async def create_workflow(
authorization: Authorization,
client: HTTPClient,
s3: S3Service,
- workflow: WorkflowIn = Depends(WorkflowIn.as_form), # type: ignore[attr-defined]
+ name: str = Form(
+ ...,
+ description="Short descriptive name of the workflow",
+ examples=["RNA ReadMapper"],
+ max_length=64,
+ min_length=3,
+ ),
+ short_description: str = Form(
+ ...,
+ description="Short description of the workflow",
+ examples=["This should be a very good example of a short and descriptive description"],
+ max_length=256,
+ min_length=64,
+ ),
+ repository_url: AnyHttpUrl = Form(
+ ...,
+ description="URL to the Git repository belonging to this workflow",
+ examples=["https://github.com/example-user/example"],
+ ),
+ git_commit_hash: str = Form(
+ ...,
+ description="Hash of the git commit",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
+ pattern=r"^[0-9a-f]{40}$",
+ min_length=40,
+ max_length=40,
+ ),
+ initial_version: str = Form(
+ default="v1.0.0",
+ description="Initial version of the Workflow. Should follow semantic versioning",
+ examples=["v1.0.0"],
+ min_length=5,
+ max_length=10,
+ ),
icon: UploadFile | None = File(None, description="Optional Icon for the Workflow."),
) -> WorkflowOut:
"""
@@ -102,10 +136,18 @@ async def create_workflow(
----------
background_tasks : fastapi.BackgroundTasks
Entrypoint for new BackgroundTasks. Provided by FastAPI.
- workflow : app.schemas.workflow.WorkflowIn
- Parameters to create a new Workflow. HTML Form.
icon : fastapi.UploadFile | None, default None
Optional Icon for the first workflow version. HTML Form
+ name : fastapi.Form
+ Required Name for the workflow. HTML Form.
+ short_description : fastapi.Form
+ Required short description for the workflow. HTML Form.
+ repository_url : fastapi.Form
+ Required repository URL of the workflow. HTML Form.
+ git_commit_hash : fastapi.Form
+ Required Git commit hash of the workflow version. HTML Form.
+ initial_version : fastapi.Form
+ Initials version of the workflow. HTML Form.
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
current_user : clowmdb.models.User
@@ -123,6 +165,13 @@ async def create_workflow(
The newly created workflow
"""
await authorization("create")
+ workflow = WorkflowIn(
+ name=name,
+ short_description=short_description,
+ repository_url=repository_url,
+ git_commit_hash=git_commit_hash,
+ initial_version=initial_version,
+ )
# Check if name is workflow name is already taken
if await CRUDWorkflow.get_by_name(db, workflow.name) is not None:
raise HTTPException(
@@ -156,8 +205,8 @@ async def create_workflow(
icon_slug = upload_icon(s3=s3, background_tasks=background_tasks, icon=icon)
workflow_db = await CRUDWorkflow.create(db, workflow, current_user.uid, icon_slug=icon_slug)
- initial_version = await CRUDWorkflowVersion.get(db, workflow.git_commit_hash)
- return WorkflowOut.from_db_workflow(workflow_db, [initial_version])
+ initial_version_workflow = await CRUDWorkflowVersion.get(db, workflow.git_commit_hash)
+ return WorkflowOut.from_db_workflow(workflow_db, [initial_version_workflow])
@router.get("/{wid}", status_code=status.HTTP_200_OK, summary="Get a workflow")
@@ -285,11 +334,25 @@ async def update_workflow(
current_user: CurrentUser,
s3: S3Service,
authorization: Authorization,
- version_update: WorkflowVersionUpdate = Depends(WorkflowVersionUpdate.as_form), # type: ignore[attr-defined]
icon: UploadFile
| None = File(
None, description="Optional Icon for the workflow version. If None, then the previous one will be reused."
),
+ version: str = Form(
+ ...,
+ description="Version of the Workflow. Should follow semantic versioning",
+ examples=["v1.1.0"],
+ min_length=5,
+ max_length=10,
+ ),
+ git_commit_hash: str = Form(
+ ...,
+ description="Hash of the git commit",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
+ pattern=r"^[0-9a-f]{40}$",
+ min_length=40,
+ max_length=40,
+ ),
) -> WorkflowVersionFull:
"""
Create a new workflow version.\n
@@ -301,10 +364,12 @@ async def update_workflow(
Entrypoint for new BackgroundTasks. Provided by FastAPI
workflow : clowmdb.models.Workflow
Workflow with given ID. Dependency Injection.
- version_update : app.schemas.workflow_version.WorkflowVersionUpdate
- Parameters to create a new Workflow version. HTML Form.
icon : fastapi.UploadFile | None, default None
Optional Icon for the workflow version. If None, then the previous one will be reused. HTML Form.
+ version : Fastapi.Form
+ Required version for the updated workflow. HTML Form.
+ git_commit_hash : fastapi.Form
+ Required git commit hash for the updated workflow. HTML Form
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
current_user : clowmdb.models.User
@@ -322,6 +387,7 @@ async def update_workflow(
The new workflow version
"""
await authorization("update")
+ version_update = WorkflowVersionUpdate(version=version, git_commit_hash=git_commit_hash)
if current_user.uid != workflow.developer_id:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Only the developer can update his workflow")
diff --git a/app/api/endpoints/workflow_execution.py b/app/api/endpoints/workflow_execution.py
index 677b08dd440d6b2360f2097faf8ed28c5fa831c2..c49b396ed12cce962919268a2018d5edd0a7f6d6 100644
--- a/app/api/endpoints/workflow_execution.py
+++ b/app/api/endpoints/workflow_execution.py
@@ -237,7 +237,7 @@ async def list_workflow_executions(
| None = Query(
None,
description="Filter for workflow executions by a user. If none, Permission 'workflow_execution:read_any' required.", # noqa: E501
- example="28c5353b8bb34984a8bd4169ba94c606",
+ examples=["28c5353b8bb34984a8bd4169ba94c606"],
),
execution_status: list[WorkflowExecution.WorkflowExecutionStatus]
| None = Query(None, description="Filter for status of workflow execution"),
@@ -245,7 +245,7 @@ async def list_workflow_executions(
| None = Query(
None,
description="Filter for workflow version",
- example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
regex=r"^[0-9a-f]{40}$",
),
) -> list[WorkflowExecutionOut]:
diff --git a/app/api/endpoints/workflow_version.py b/app/api/endpoints/workflow_version.py
index 50e4ad73875aa53971d7d01ec57606a794ccfec5..ae1c356caf3e358a6f8b5884c95504eafc5a907b 100644
--- a/app/api/endpoints/workflow_version.py
+++ b/app/api/endpoints/workflow_version.py
@@ -18,7 +18,7 @@ GitCommitHash = Annotated[
...,
description="Git commit git_commit_hash of specific version.",
regex=r"^([0-9a-f]{40}|latest)$",
- example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
),
]
@@ -89,18 +89,7 @@ async def get_workflow_version(
...,
description="Git commit git_commit_hash of specific version or 'latest'.",
regex=r"^([0-9a-f]{40}|latest)$",
- examples={
- "normal": {
- "summary": "Latest version",
- "description": "Get the latest version of the workflow",
- "value": "latest",
- },
- "pseudo-folder": {
- "summary": "Specific version",
- "description": "Get a specific version of the workflow",
- "value": "ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
- },
- },
+ examples=["latest", "ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
),
) -> WorkflowVersionFull:
"""
diff --git a/app/ceph/rgw.py b/app/ceph/rgw.py
index 5c8cc331ff6c6585e161e8c9a10f21ae16a530a1..ca195d8afbae065f2a1fb20923ee45645afa43d3 100644
--- a/app/ceph/rgw.py
+++ b/app/ceph/rgw.py
@@ -11,8 +11,8 @@ else:
s3_resource: S3ServiceResource = resource(
service_name="s3",
- endpoint_url=settings.OBJECT_GATEWAY_URI,
- aws_access_key_id=settings.CEPH_ACCESS_KEY,
- aws_secret_access_key=settings.CEPH_SECRET_KEY,
- verify=settings.OBJECT_GATEWAY_URI.startswith("https"),
+ endpoint_url=str(settings.OBJECT_GATEWAY_URI)[:-1],
+ aws_access_key_id=settings.BUCKET_CEPH_ACCESS_KEY,
+ aws_secret_access_key=settings.BUCKET_CEPH_SECRET_KEY,
+ verify=str(settings.OBJECT_GATEWAY_URI).startswith("https"),
)
diff --git a/app/check_ceph_connection.py b/app/check_ceph_connection.py
index 23a95bd156effde36c1145291e0e7764403c9ea6..0e73fd3076b7877a306cbe83fe3b2fc2843bd7cc 100644
--- a/app/check_ceph_connection.py
+++ b/app/check_ceph_connection.py
@@ -21,7 +21,7 @@ wait_seconds = 2
)
def init() -> None:
try:
- response = httpx.get(settings.OBJECT_GATEWAY_URI, timeout=5.0, follow_redirects=False)
+ response = httpx.get(str(settings.OBJECT_GATEWAY_URI), timeout=5.0, follow_redirects=False)
assert response.status_code == status.HTTP_200_OK
except Exception as e:
logger.error(e)
diff --git a/app/check_database_connection.py b/app/check_database_connection.py
index 62db2c5a1b46b37bfcd13abe267efeec942e75e2..ae54e93d69f91628dd1b90245652923206877406 100644
--- a/app/check_database_connection.py
+++ b/app/check_database_connection.py
@@ -22,7 +22,7 @@ wait_seconds = 2
)
def init() -> None:
try:
- with get_session(url=settings.SQLALCHEMY_DATABASE_NORMAL_URI)() as db:
+ with get_session(url=str(settings.SQLALCHEMY_DATABASE_NORMAL_URI))() as db:
# Try to create session to check if DB is awake
db_revision = db.execute(text("SELECT version_num FROM alembic_version LIMIT 1")).scalar_one_or_none()
if db_revision != latest_revision:
diff --git a/app/core/config.py b/app/core/config.py
index ddd9d4fb1bed6fe9e49dd72d427fd4499327403a..b389531b465ec4674fc2028f243cfc595ff273c0 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -1,79 +1,93 @@
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
-from pydantic import AnyHttpUrl, AnyUrl, BaseSettings, Field, validator
+from pydantic import AnyHttpUrl, AnyUrl, Field, computed_field
+from pydantic_settings import BaseSettings, SettingsConfigDict
def _assemble_db_uri(values: Dict[str, Any], async_flag: bool = True) -> Any:
return AnyUrl.build(
scheme=f"mysql+{'aiomysql' if async_flag else 'pymysql'}",
password=values.get("DB_PASSWORD"),
- user=values.get("DB_USER"),
- port=str(values.get("DB_PORT")),
- host=values.get("DB_HOST"),
- path=f"/{values.get('DB_DATABASE') or ''}",
+ username=values.get("DB_USER"),
+ port=values.get("DB_PORT"),
+ host=values.get("DB_HOST"), # type: ignore[arg-type]
+ path=f"{values.get('DB_DATABASE') or ''}",
)
+def _load_public_key(pub_key_val: Optional[str], pub_key_file: Optional[Path]) -> str:
+ pub_key = ""
+ if pub_key_val is not None:
+ pub_key = pub_key_val
+ if pub_key_file is not None:
+ with open(pub_key_file) as f:
+ pub_key = f.read()
+ if len(pub_key) == 0:
+ raise ValueError("PUBLIC_KEY_VALUE or PUBLIC_KEY_FILE must be set")
+ return pub_key
+
+
class Settings(BaseSettings):
API_PREFIX: str = Field("/api/workflow-service", description="Path Prefix for all API endpoints.")
public_key_value: str | None = Field(
- None, description="Public RSA Key in PEM format to sign the JWTs.", env="PUBLIC_KEY_VALUE"
+ None, description="Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_VALUE"
)
public_key_file: Path | None = Field(
- None, description="Path to Public RSA Key in PEM format to sign the JWTs.", env="PUBLIC_KEY_FILE"
+ None, description="Path to Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_FILE"
)
- PUBLIC_KEY: str = ""
-
- @validator("PUBLIC_KEY", pre=True)
- def load_public_key(cls, v: str, values: Dict[str, Any]) -> str:
- pub_key = ""
- if values["public_key_value"] is not None:
- pub_key = values["public_key_value"]
- if values["public_key_file"] is not None:
- with open(values["public_key_file"]) as f:
- pub_key = f.read()
- if len(pub_key) == 0:
- raise ValueError("PUBLIC_KEY_VALUE or PUBLIC_KEY_FILE must be set")
- return pub_key
+
+ @computed_field # type: ignore[misc]
+ @property
+ def PUBLIC_KEY(self) -> str:
+ return _load_public_key(self.public_key_value, self.public_key_file)
# BACKEND_CORS_ORIGINS is a JSON-formatted list of origins
BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = Field([], description="List of all valid CORS origins")
- @validator("BACKEND_CORS_ORIGINS", pre=True)
- def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]:
- if isinstance(v, str) and not v.startswith("["):
- return [i.strip() for i in v.split(",")]
- elif isinstance(v, (list, str)):
- return v
- raise ValueError(v)
-
DB_HOST: str = Field(..., description="Host of the database.")
DB_USER: str = Field(..., description="Username in the database.")
DB_PASSWORD: str = Field(..., description="Password for the database user.")
DB_DATABASE: str = Field(..., description="Name of the database.")
DB_PORT: int = Field(3306, description="Port of the database.")
SQLALCHEMY_VERBOSE_LOGGER: bool = Field(False, description="Flag whether to print the SQL Queries in the logs.")
- SQLALCHEMY_DATABASE_ASYNC_URI: AnyUrl | None = None
-
- @validator("SQLALCHEMY_DATABASE_ASYNC_URI", pre=True)
- def assemble_async_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
- if isinstance(v, str):
- return v
- return _assemble_db_uri(values, async_flag=True)
- SQLALCHEMY_DATABASE_NORMAL_URI: AnyUrl | None = None
-
- @validator("SQLALCHEMY_DATABASE_NORMAL_URI", pre=True)
- def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
- if isinstance(v, str):
- return v
- return _assemble_db_uri(values, async_flag=False)
+ @computed_field # type: ignore[misc]
+ @property
+ def SQLALCHEMY_DATABASE_ASYNC_URI(self) -> AnyUrl:
+ return _assemble_db_uri(
+ {
+ "DB_HOST": self.DB_HOST,
+ "DB_USER": self.DB_USER,
+ "DB_PASSWORD": self.DB_PASSWORD,
+ "DB_DATABASE": self.DB_DATABASE,
+ "DB_PORT": self.DB_PORT,
+ },
+ async_flag=True,
+ )
+
+ @computed_field # type: ignore[misc]
+ @property
+ def SQLALCHEMY_DATABASE_NORMAL_URI(self) -> AnyUrl:
+ return _assemble_db_uri(
+ {
+ "DB_HOST": self.DB_HOST,
+ "DB_USER": self.DB_USER,
+ "DB_PASSWORD": self.DB_PASSWORD,
+ "DB_DATABASE": self.DB_DATABASE,
+ "DB_PORT": self.DB_PORT,
+ },
+ async_flag=False,
+ )
OBJECT_GATEWAY_URI: AnyHttpUrl = Field(..., description="URI of the Ceph Object Gateway.")
- CEPH_ACCESS_KEY: str = Field(..., description="Access key for the Ceph Object Gateway with admin privileges.")
- CEPH_SECRET_KEY: str = Field(..., description="Secret key for the Ceph Object Gateway with admin privileges.")
+ BUCKET_CEPH_ACCESS_KEY: str = Field(
+ ..., description="Access key for the Ceph Object Gateway with admin privileges."
+ )
+ BUCKET_CEPH_SECRET_KEY: str = Field(
+ ..., description="Secret key for the Ceph Object Gateway with admin privileges."
+ )
OPA_URI: AnyHttpUrl = Field(..., description="URI of the OPA Service")
OPA_POLICY_PATH: str = Field("/clowm/authz/allow", description="Path to the OPA Policy for Authorization")
PARAMS_BUCKET: str = Field("nxf-params", description="Bucket where the nextflow configurations should be saved")
@@ -92,11 +106,7 @@ class Settings(BaseSettings):
)
ACTIVE_WORKFLOW_EXECUTION_LIMIT: int = Field(3, description="The limit of active workflow executions per user.")
DEV_SYSTEM: bool = Field(False, description="Open a endpoint where to execute arbitrary workflows.")
-
- class Config:
- case_sensitive = True
- env_file = ".env"
- secrets_dir = "/run/secrets"
+ model_config = SettingsConfigDict(case_sensitive=True, env_file=".env", secrets_dir="/run/secrets", extra="ignore")
settings = Settings()
diff --git a/app/core/security.py b/app/core/security.py
index f34d133aefcc09f58909bdd4204b001bdb7cc61a..60f9fa2f6074b78fdcca0a4c73910d58189f8016 100644
--- a/app/core/security.py
+++ b/app/core/security.py
@@ -54,7 +54,7 @@ async def request_authorization(request_params: AuthzRequest, client: AsyncClien
Response by the Auth service about the authorization request
"""
response = await client.post(
- f"{settings.OPA_URI}/v1/data{settings.OPA_POLICY_PATH}", json={"input": request_params.dict()}
+ f"{settings.OPA_URI}v1/data{settings.OPA_POLICY_PATH}", json={"input": request_params.model_dump()}
)
parsed_response = AuthzResponse(**response.json())
diff --git a/app/git_repository/__init__.py b/app/git_repository/__init__.py
index be15a9752b1c11297e7e1c913ba6b4530c619718..2474334881f98f2a85600c7b5d1737f9f068f6f0 100644
--- a/app/git_repository/__init__.py
+++ b/app/git_repository/__init__.py
@@ -1,11 +1,13 @@
from urllib.parse import urlparse
+from pydantic import AnyHttpUrl
+
from app.git_repository.abstract_repository import GitRepository
from app.git_repository.github import GithubRepository
from app.git_repository.gitlab import GitlabRepository
-def build_repository(url: str, git_commit_hash: str) -> GitRepository:
+def build_repository(url: AnyHttpUrl, git_commit_hash: str) -> GitRepository:
"""
Build the right git repository object based on the url
@@ -14,16 +16,16 @@ def build_repository(url: str, git_commit_hash: str) -> GitRepository:
url : str
URL of the git repository
git_commit_hash : str
- Pin dowm git commit hash
+ Pin down git commit hash
Returns
-------
repo : GitRepository
Specialized Git repository object
"""
- domain = urlparse(url).netloc
+ domain = urlparse(str(url)).netloc
if "github" in domain:
- return GithubRepository(url=url, git_commit_hash=git_commit_hash)
+ return GithubRepository(url=str(url), git_commit_hash=git_commit_hash)
elif "gitlab" in domain:
- return GitlabRepository(url=url, git_commit_hash=git_commit_hash)
+ return GitlabRepository(url=str(url), git_commit_hash=git_commit_hash)
raise NotImplementedError("Unknown Git repository Provider")
diff --git a/app/git_repository/abstract_repository.py b/app/git_repository/abstract_repository.py
index 054f5cd213c7cc5a8aee938f97078c389bcd4b7e..180b64518705e0e18ab4a63584278852c5f9e2f2 100644
--- a/app/git_repository/abstract_repository.py
+++ b/app/git_repository/abstract_repository.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
from fastapi import HTTPException, status
from httpx import AsyncClient
+from pydantic import AnyHttpUrl
if TYPE_CHECKING:
from mypy_boto3_s3.service_resource import Object
@@ -47,7 +48,7 @@ class GitRepository(ABC):
...
@abstractmethod
- def downloadFileURL(self, filepath: str) -> str:
+ def downloadFileURL(self, filepath: str) -> AnyHttpUrl:
"""
Construct an URL where to download a file from
@@ -86,7 +87,7 @@ class GitRepository(ABC):
exist : bool
Flag if the file exists.
"""
- response = await client.head(self.downloadFileURL(filepath), follow_redirects=True)
+ response = await client.head(str(self.downloadFileURL(filepath)), follow_redirects=True)
return response.status_code == status.HTTP_200_OK
async def check_files_exist(self, files: list[str], client: AsyncClient, raise_error: bool = True) -> list[bool]:
@@ -148,6 +149,6 @@ class GitRepository(ABC):
file_handle : BytesIO
Write the file into this stream in binary mode.
"""
- async with client.stream("GET", self.downloadFileURL(filepath)) as response:
+ async with client.stream("GET", str(self.downloadFileURL(filepath))) as response:
async for chunk in response.aiter_bytes():
file_handle.write(chunk)
diff --git a/app/git_repository/github.py b/app/git_repository/github.py
index 73bbda133e2c131bb759f56dafa19488ee1a4804..c8a634bb54aa36a2a69705473a130f430513f8fb 100644
--- a/app/git_repository/github.py
+++ b/app/git_repository/github.py
@@ -30,7 +30,7 @@ class GithubRepository(GitRepository):
self.repository = bla[1]
self.commit = git_commit_hash
- def downloadFileURL(self, filepath: str) -> str:
+ def downloadFileURL(self, filepath: str) -> AnyHttpUrl:
return AnyHttpUrl.build(
scheme="https",
host="raw.githubusercontent.com/",
diff --git a/app/git_repository/gitlab.py b/app/git_repository/gitlab.py
index 548f9e1ccf5d8e250d482336062e1580567a1072..55a9016f6f627fbcc8393b51edfec5883e4d6c13 100644
--- a/app/git_repository/gitlab.py
+++ b/app/git_repository/gitlab.py
@@ -31,7 +31,7 @@ class GitlabRepository(GitRepository):
self.repository = path_parts[-1]
self.commit = git_commit_hash
- def downloadFileURL(self, filepath: str) -> str:
+ def downloadFileURL(self, filepath: str) -> AnyHttpUrl:
return AnyHttpUrl.build(
scheme="https",
host=self.domain,
diff --git a/app/schemas/security.py b/app/schemas/security.py
index 5bd3799182272053553920c247ddf47010b7e177..3c0f596a45e25b0eaa0ea6b86f961733c0d7e3c7 100644
--- a/app/schemas/security.py
+++ b/app/schemas/security.py
@@ -9,7 +9,7 @@ class AuthzResponse(BaseModel):
decision_id: str = Field(
...,
description="Decision ID for for the specific decision",
- example="8851dce0-7546-4e81-a89d-111cbec376c1",
+ examples=["8851dce0-7546-4e81-a89d-111cbec376c1"],
)
result: bool = Field(..., description="Result of the Authz request")
@@ -17,9 +17,9 @@ class AuthzResponse(BaseModel):
class AuthzRequest(BaseModel):
"""Schema for a Request to OPA"""
- uid: str = Field(..., description="UID of user", example="28c5353b8bb34984a8bd4169ba94c606")
- operation: str = Field(..., description="Operation the user wants to perform", example="read")
- resource: str = Field(..., description="Resource the operation should be performed on", example="bucket")
+ uid: str = Field(..., description="UID of user", examples=["28c5353b8bb34984a8bd4169ba94c606"])
+ operation: str = Field(..., description="Operation the user wants to perform", examples=["read"])
+ resource: str = Field(..., description="Resource the operation should be performed on", examples=["bucket"])
class JWT(BaseModel):
diff --git a/app/schemas/utils.py b/app/schemas/utils.py
deleted file mode 100644
index b8f42b8152d3527ff14325674fea8f660c839b45..0000000000000000000000000000000000000000
--- a/app/schemas/utils.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import inspect
-from typing import Any, Type
-
-from fastapi import Form
-from pydantic import BaseModel
-from pydantic.fields import ModelField
-
-
-# https://stackoverflow.com/a/60670614
-def as_form(cls: Type[BaseModel]) -> Type[BaseModel]:
- new_parameters = []
-
- for field_name, model_field in cls.__fields__.items():
- model_field: ModelField # type: ignore
-
- new_parameters.append(
- inspect.Parameter(
- model_field.alias,
- inspect.Parameter.POSITIONAL_ONLY,
- default=Form(...) if model_field.required else Form(model_field.default),
- annotation=model_field.outer_type_,
- )
- )
-
- async def as_form_func(**data: dict[str, Any]) -> BaseModel:
- return cls(**data)
-
- sig = inspect.signature(as_form_func)
- sig = sig.replace(parameters=new_parameters)
- as_form_func.__signature__ = sig # type: ignore
- setattr(cls, "as_form", as_form_func)
- return cls
diff --git a/app/schemas/workflow.py b/app/schemas/workflow.py
index 9b5e6d6242b394144183b8c6c64e7f0e1bbc3958..00722197d3c88f6e03a60669136f6daae3e5fc2e 100644
--- a/app/schemas/workflow.py
+++ b/app/schemas/workflow.py
@@ -3,54 +3,60 @@ from uuid import UUID
from clowmdb.models import Workflow as WorkflowDB
from clowmdb.models import WorkflowVersion as WorkflowVersionDB
-from pydantic import AnyHttpUrl, BaseModel, Field
+from pydantic import AnyHttpUrl, BaseModel, Field, FieldSerializationInfo, field_serializer
-from app.schemas.utils import as_form
from app.schemas.workflow_version import WorkflowVersionReduced
class _BaseWorkflow(BaseModel):
name: str = Field(
- ..., description="Short descriptive name of the workflow", example="RNA ReadMapper", max_length=64, min_length=3
+ ...,
+ description="Short descriptive name of the workflow",
+ examples=["RNA ReadMapper"],
+ max_length=64,
+ min_length=3,
)
short_description: str = Field(
...,
description="Short description of the workflow",
- example="This should be a very good example of a short and descriptive description",
+ examples=["This should be a very good example of a short and descriptive description"],
max_length=256,
min_length=64,
)
repository_url: AnyHttpUrl = Field(
...,
description="URL to the Git repository belonging to this workflow",
- example="https://github.com/example-user/example",
+ examples=["https://github.com/example-user/example"],
)
+ @field_serializer("repository_url")
+ def serialize_dt(self, url: AnyHttpUrl, _info: FieldSerializationInfo) -> str:
+ return str(url)
+
-@as_form
class WorkflowIn(_BaseWorkflow):
git_commit_hash: str = Field(
...,
description="Hash of the git commit",
- example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
- regex=r"^[0-9a-f]{40}$",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
+ pattern=r"^[0-9a-f]{40}$",
min_length=40,
max_length=40,
)
initial_version: str = Field(
"v1.0.0",
description="Initial version of the Workflow. Should follow semantic versioning",
- example="v1.0.0",
- minlength=5,
+ examples=["v1.0.0"],
+ min_length=5,
max_length=10,
)
class WorkflowOut(_BaseWorkflow):
- workflow_id: UUID = Field(..., description="Id of the workflow", example="0cc78936-381b-4bdd-999d-736c40591078")
+ workflow_id: UUID = Field(..., description="Id of the workflow", examples=["0cc78936-381b-4bdd-999d-736c40591078"])
versions: list[WorkflowVersionReduced] = Field(..., description="Versions of the workflow")
developer_id: str = Field(
- ..., description="Id of developer of the workflow", example="28c5353b8bb34984a8bd4169ba94c606"
+ ..., description="Id of developer of the workflow", examples=["28c5353b8bb34984a8bd4169ba94c606"]
)
@staticmethod
@@ -71,5 +77,5 @@ class WorkflowOut(_BaseWorkflow):
class WorkflowStatistic(BaseModel):
- day: date = Field(..., description="Day of the datapoint", example=date(day=1, month=1, year=2023))
- count: int = Field(..., description="Number of started workflows on that day", example=1)
+ day: date = Field(..., description="Day of the datapoint", examples=[date(day=1, month=1, year=2023)])
+ count: int = Field(..., description="Number of started workflows on that day", examples=[1])
diff --git a/app/schemas/workflow_execution.py b/app/schemas/workflow_execution.py
index d59ab60762204e689e7272863715a96ee9d8c35c..19df2e5ceb05a2d480cab72b44ec1d4b3bded920 100644
--- a/app/schemas/workflow_execution.py
+++ b/app/schemas/workflow_execution.py
@@ -3,15 +3,15 @@ from typing import Any
from uuid import UUID
from clowmdb.models import WorkflowExecution
-from pydantic import AnyHttpUrl, BaseModel, Field
+from pydantic import AnyHttpUrl, BaseModel, Field, FieldSerializationInfo, field_serializer
class _BaseWorkflowExecution(BaseModel):
workflow_version_id: str = Field(
...,
description="Workflow version git commit hash",
- example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
- regex=r"^[0-9a-f]{40}$",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
+ pattern=r"^[0-9a-f]{40}$",
min_length=40,
max_length=40,
)
@@ -19,7 +19,7 @@ class _BaseWorkflowExecution(BaseModel):
None,
description="Optional notes for this workflow execution",
max_length=2**16,
- example="Some workflow execution specific notes",
+ examples=["Some workflow execution specific notes"],
)
@@ -30,33 +30,33 @@ class WorkflowExecutionIn(_BaseWorkflowExecution):
description="Bucket where to save the Nextflow report. If None, no report will be generated. With our without prefix 's3://'", # noqa: E501
min_length=3,
max_length=63,
- example="some-bucket",
+ examples=["some-bucket"],
)
class WorkflowExecutionOut(_BaseWorkflowExecution):
execution_id: UUID = Field(
- ..., description="ID of the workflow execution", example="0cc78936-381b-4bdd-999d-736c40591078"
+ ..., description="ID of the workflow execution", examples=["0cc78936-381b-4bdd-999d-736c40591078"]
)
user_id: str = Field(
- ..., description="UID of user who started the workflow", example="28c5353b8bb34984a8bd4169ba94c606"
+ ..., description="UID of user who started the workflow", examples=["28c5353b8bb34984a8bd4169ba94c606"]
)
start_time: datetime = Field(
- ..., description="Start time of the workflow execution", example=datetime(year=2023, month=1, day=1)
+ ..., description="Start time of the workflow execution", examples=[datetime(year=2023, month=1, day=1)]
)
end_time: datetime | None = Field(
- None, description="End time of the workflow execution", example=datetime(year=2023, month=1, day=1)
+ None, description="End time of the workflow execution", examples=[datetime(year=2023, month=1, day=1)]
)
status: WorkflowExecution.WorkflowExecutionStatus = Field(
...,
description="Status of the workflow execution",
- example=WorkflowExecution.WorkflowExecutionStatus.RUNNING,
+ examples=[WorkflowExecution.WorkflowExecutionStatus.RUNNING],
)
workflow_version_id: str | None = Field( # type: ignore[assignment]
- ..., description="Workflow version git commit hash", example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f"
+ None, description="Workflow version git commit hash", examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"]
)
workflow_id: UUID | None = Field(
- ..., description="Id of the workflow", example="0cc78936-381b-4bdd-999d-736c40591078"
+ None, description="Id of the workflow", examples=["0cc78936-381b-4bdd-999d-736c40591078"]
)
@staticmethod
@@ -80,18 +80,22 @@ class DevWorkflowExecutionIn(BaseModel):
description="Bucket where to save the Nextflow report. If None, no report will be generated",
min_length=3,
max_length=63,
- example="some-bucket",
+ examples=["some-bucket"],
)
git_commit_hash: str = Field(
...,
description="Hash of the git commit",
- example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
- regex=r"^[0-9a-f]{40}$",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
+ pattern=r"^[0-9a-f]{40}$",
min_length=40,
max_length=40,
)
repository_url: AnyHttpUrl = Field(
...,
description="URL to the Git repository belonging to this workflow",
- example="https://github.com/example-user/example",
+ examples=["https://github.com/example-user/example"],
)
+
+ @field_serializer("repository_url")
+ def serialize_dt(self, url: AnyHttpUrl, _info: FieldSerializationInfo) -> str:
+ return str(url)
diff --git a/app/schemas/workflow_version.py b/app/schemas/workflow_version.py
index d1e4890a08bc9640d77a83f621bd5da99b98ab13..e8a75615c8c92d757801f6152f7cba5a7b22a90f 100644
--- a/app/schemas/workflow_version.py
+++ b/app/schemas/workflow_version.py
@@ -6,12 +6,11 @@ from pydantic import AnyHttpUrl, BaseModel, Field
from app.core.config import settings
from app.git_repository.abstract_repository import GitRepository
-from app.schemas.utils import as_form
class WorkflowVersionStatus(BaseModel):
status: WorkflowVersionDB.Status = Field(
- ..., description="Status of the workflow version", example=WorkflowVersionDB.Status.PUBLISHED
+ ..., description="Status of the workflow version", examples=[WorkflowVersionDB.Status.PUBLISHED]
)
@@ -19,32 +18,34 @@ class WorkflowVersionReduced(WorkflowVersionStatus):
version: str = Field(
...,
description="Version of the Workflow. Should follow semantic versioning",
- example="v1.0.0",
+ examples=["v1.0.0"],
minlength=5,
max_length=10,
)
git_commit_hash: str = Field(
...,
description="Hash of the git commit",
- example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
- regex=r"^[0-9a-f]{40}$",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
+ pattern=r"^[0-9a-f]{40}$",
min_length=40,
max_length=40,
)
icon_url: AnyHttpUrl | None = Field(
None,
description="URL of the icon for this workflow version",
- example=f"{settings.OBJECT_GATEWAY_URI}/{settings.ICON_BUCKET}/{uuid4().hex}.png",
+ examples=[f"{settings.OBJECT_GATEWAY_URI}{settings.ICON_BUCKET}/{uuid4().hex}.png"],
)
created_at: datetime = Field(
- ..., description="Timestamp when the version was created", example=datetime(year=2023, month=1, day=1)
+ ...,
+ description="Timestamp when the version was created",
+ examples=[datetime(year=2023, month=1, day=1)],
)
@staticmethod
def from_db_version(db_version: WorkflowVersionDB) -> "WorkflowVersionReduced":
icon_url = None
if db_version.icon_slug is not None:
- icon_url = "/".join([settings.OBJECT_GATEWAY_URI, settings.ICON_BUCKET, db_version.icon_slug])
+ icon_url = str(settings.OBJECT_GATEWAY_URI) + "/".join([settings.ICON_BUCKET, db_version.icon_slug])
return WorkflowVersionReduced(
version=db_version.version,
git_commit_hash=db_version.git_commit_hash,
@@ -55,31 +56,31 @@ class WorkflowVersionReduced(WorkflowVersionStatus):
class WorkflowVersionFull(WorkflowVersionReduced):
- workflow_id: UUID = Field(..., description="ID of the corresponding workflow", example=uuid4())
+ workflow_id: UUID = Field(..., description="ID of the corresponding workflow", examples=[uuid4()])
readme_url: AnyHttpUrl = Field(
...,
description="URL to download README.md from",
- example="https://raw.githubusercontent.com/example/example/README.md",
+ examples=["https://raw.githubusercontent.com/example/example/README.md"],
)
changelog_url: AnyHttpUrl = Field(
...,
description="URL to download CHANGELOG.md from",
- example="https://raw.githubusercontent.com/example/example/CHANGELOG.md",
+ examples=["https://raw.githubusercontent.com/example/example/CHANGELOG.md"],
)
usage_url: AnyHttpUrl = Field(
...,
description="URL to download usage.md from",
- example="https://raw.githubusercontent.com/example/example/docs/usage.md",
+ examples=["https://raw.githubusercontent.com/example/example/docs/usage.md"],
)
output_url: AnyHttpUrl = Field(
...,
description="URL to download output.md from",
- example="https://raw.githubusercontent.com/example/example/docs/output.md",
+ examples=["https://raw.githubusercontent.com/example/example/docs/output.md"],
)
parameter_schema_url: AnyHttpUrl = Field(
...,
description="URL to download nextflow_schema.json from",
- example="https://raw.githubusercontent.com/example/example/nextflow_schema.json",
+ examples=["https://raw.githubusercontent.com/example/example/nextflow_schema.json"],
)
@staticmethod
@@ -91,24 +92,23 @@ class WorkflowVersionFull(WorkflowVersionReduced):
usage_url=repo.downloadFileURL("docs/usage.md"),
output_url=repo.downloadFileURL("docs/output.md"),
parameter_schema_url=repo.downloadFileURL("nextflow_schema.json"),
- **WorkflowVersionReduced.from_db_version(db_version).dict(),
+ **WorkflowVersionReduced.from_db_version(db_version).model_dump(),
)
-@as_form
class WorkflowVersionUpdate(BaseModel):
version: str = Field(
...,
description="Version of the Workflow. Should follow semantic versioning",
- example="v1.1.0",
+ examples=["v1.1.0"],
minlength=5,
max_length=10,
)
git_commit_hash: str = Field(
...,
description="Hash of the git commit",
- example="ba8bcd9294c2c96aedefa1763a84a18077c50c0f",
- regex=r"^[0-9a-f]{40}$",
+ examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
+ pattern=r"^[0-9a-f]{40}$",
min_length=40,
max_length=40,
)
diff --git a/app/slurm/slurm_rest_client.py b/app/slurm/slurm_rest_client.py
index 2ad1ef1d551b788a274ade94f588391a287929b1..fdbe486b208983754c96845659baba2bbb8e7e01 100644
--- a/app/slurm/slurm_rest_client.py
+++ b/app/slurm/slurm_rest_client.py
@@ -55,7 +55,7 @@ class SlurmClient:
},
}
response = await self._client.post(
- f"{settings.SLURM_ENDPOINT}/slurm/{self.version}/job/submit", headers=self._headers, json=body
+ f"{settings.SLURM_ENDPOINT}slurm/{self.version}/job/submit", headers=self._headers, json=body
)
return int(response.json()["job_id"])
diff --git a/app/tests/api/test_workflow.py b/app/tests/api/test_workflow.py
index 549b2873a20394dae1256cab4b18052e3700ce48..a488e06fadfd6c5a443ba068b6d218a04b13249e 100644
--- a/app/tests/api/test_workflow.py
+++ b/app/tests/api/test_workflow.py
@@ -47,7 +47,7 @@ class TestWorkflowRoutesCreate(_TestWorkflowRoutes):
name=random_lower_string(10),
short_description=random_lower_string(65),
repository_url="https://github.de/example-user/example",
- ).dict()
+ ).model_dump()
response = await client.post(self.base_path, data=workflow, headers=random_user.auth_headers)
assert response.status_code == status.HTTP_201_CREATED
created_workflow = response.json()
@@ -89,7 +89,7 @@ class TestWorkflowRoutesCreate(_TestWorkflowRoutes):
name=random_lower_string(10),
short_description=random_lower_string(65),
repository_url="https://gitlab.de/example-user/example",
- ).dict()
+ ).model_dump()
response = await client.post(self.base_path, data=workflow, headers=random_user.auth_headers)
assert response.status_code == status.HTTP_201_CREATED
created_workflow = response.json()
@@ -126,7 +126,7 @@ class TestWorkflowRoutesCreate(_TestWorkflowRoutes):
name=random_lower_string(10),
short_description=random_lower_string(65),
repository_url="https://github.de/example-user/example",
- ).dict()
+ ).model_dump()
response = await client.post(
self.base_path, params={"raise_error": True}, data=workflow, headers=random_user.auth_headers
)
@@ -162,7 +162,7 @@ class TestWorkflowRoutesCreate(_TestWorkflowRoutes):
name=random_lower_string(10),
short_description=random_lower_string(65),
repository_url="https://github.de/example-user/example",
- ).dict()
+ ).model_dump()
files = {"icon": ("RickRoll.txt", BytesIO(b"Never gonna give you up"), "plain/text")}
response = await client.post(self.base_path, data=workflow, headers=random_user.auth_headers, files=files)
assert response.status_code == status.HTTP_201_CREATED
@@ -197,7 +197,7 @@ class TestWorkflowRoutesCreate(_TestWorkflowRoutes):
name=random_workflow.name,
short_description=random_lower_string(65),
repository_url="https://github.de/example-user/example",
- ).dict()
+ ).model_dump()
response = await client.post(self.base_path, data=workflow, headers=random_user.auth_headers)
assert response.status_code == status.HTTP_400_BAD_REQUEST
@@ -222,7 +222,7 @@ class TestWorkflowRoutesCreate(_TestWorkflowRoutes):
name=random_lower_string(10),
short_description=random_lower_string(65),
repository_url="https://github.de/example-user/example",
- ).dict()
+ ).model_dump()
response = await client.post(self.base_path, data=workflow, headers=random_user.auth_headers)
assert response.status_code == status.HTTP_400_BAD_REQUEST
@@ -247,7 +247,7 @@ class TestWorkflowRoutesCreate(_TestWorkflowRoutes):
name=random_lower_string(10),
short_description=random_lower_string(65),
repository_url="https://example.org",
- ).dict()
+ ).model_dump()
response = await client.post(self.base_path, data=workflow, headers=random_user.auth_headers)
assert response.status_code == status.HTTP_400_BAD_REQUEST
@@ -465,7 +465,7 @@ class TestWorkflowRoutesUpdate(_TestWorkflowRoutes):
version_update = WorkflowVersionUpdate(
git_commit_hash=git_commit_hash,
version=random_lower_string(8),
- ).dict()
+ ).model_dump()
response = await client.post(
"/".join([self.base_path, str(random_workflow.workflow_id), "update"]),
data=version_update,
@@ -475,7 +475,7 @@ class TestWorkflowRoutesUpdate(_TestWorkflowRoutes):
created_version = response.json()
assert created_version["git_commit_hash"] == git_commit_hash
assert created_version["status"] == WorkflowVersion.Status.CREATED
- assert created_version["icon_url"] == random_workflow.versions[0].icon_url
+ assert created_version["icon_url"] == str(random_workflow.versions[0].icon_url)
stmt = select(WorkflowVersion).where(WorkflowVersion.git_commit_hash == git_commit_hash)
db_version = (await db.execute(stmt)).scalar()
@@ -501,7 +501,7 @@ class TestWorkflowRoutesUpdate(_TestWorkflowRoutes):
version_update = WorkflowVersionUpdate(
git_commit_hash=random_hex_string(),
version=random_lower_string(8),
- ).dict()
+ ).model_dump()
response = await client.post(
"/".join([self.base_path, str(random_workflow.workflow_id), "update"]),
params={"raise_error": True},
@@ -540,7 +540,7 @@ class TestWorkflowRoutesUpdate(_TestWorkflowRoutes):
version_update = WorkflowVersionUpdate(
git_commit_hash=random_hex_string(),
version=random_lower_string(8),
- ).dict()
+ ).model_dump()
files = {"icon": ("RickRoll.txt", BytesIO(b"Never gonna give you up"), "plain/text")}
response = await client.post(
@@ -575,7 +575,7 @@ class TestWorkflowRoutesUpdate(_TestWorkflowRoutes):
version_update = WorkflowVersionUpdate(
git_commit_hash=random_workflow.versions[0].git_commit_hash,
version=random_lower_string(8),
- ).dict()
+ ).model_dump()
response = await client.post(
"/".join([self.base_path, str(random_workflow.workflow_id), "update"]),
data=version_update,
@@ -600,7 +600,7 @@ class TestWorkflowRoutesUpdate(_TestWorkflowRoutes):
version_update = WorkflowVersionUpdate(
git_commit_hash=random_hex_string(),
version=random_lower_string(8),
- ).dict()
+ ).model_dump()
response = await client.post(
"/".join([self.base_path, str(random_workflow.workflow_id), "update"]),
data=version_update,
diff --git a/app/tests/api/test_workflow_execution.py b/app/tests/api/test_workflow_execution.py
index a186dec235c80b4df357f74d54f8a3a829bccaf5..a374d868d002e16785d96f2fcdb1f90f9f3dc4e3 100644
--- a/app/tests/api/test_workflow_execution.py
+++ b/app/tests/api/test_workflow_execution.py
@@ -43,7 +43,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
Mock S3 Service to manipulate objects. pytest fixture.
"""
execution_in = WorkflowExecutionIn(workflow_version_id=random_workflow_version.git_commit_hash, parameters={})
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_201_CREATED
execution_response = response.json()
assert execution_response["workflow_version_id"] == execution_in.workflow_version_id
@@ -88,7 +88,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
await db.execute(stmt)
await db.commit()
execution_in = WorkflowExecutionIn(workflow_version_id=random_workflow_version.git_commit_hash, parameters={})
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_201_CREATED
execution_response = response.json()
assert execution_response["workflow_version_id"] == execution_in.workflow_version_id
@@ -134,7 +134,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
active_execution_counter += 1
await db.commit()
execution_in = WorkflowExecutionIn(workflow_version_id=random_workflow_version.git_commit_hash, parameters={})
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_403_FORBIDDEN
@pytest.mark.asyncio
@@ -155,7 +155,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
workflow_version_id=random_hex_string(),
parameters={},
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_404_NOT_FOUND
@pytest.mark.asyncio
@@ -190,7 +190,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
workflow_version_id=random_workflow_version.git_commit_hash,
parameters={},
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_403_FORBIDDEN
@pytest.mark.asyncio
@@ -216,7 +216,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
workflow_version_id=random_workflow_version.git_commit_hash,
parameters={"dir": "s3://" + random_lower_string()},
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_400_BAD_REQUEST
@pytest.mark.asyncio
@@ -245,7 +245,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
workflow_version_id=random_workflow_version.git_commit_hash,
parameters={"dir": f"s3://{random_bucket.name}"},
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_201_CREATED
@pytest.mark.asyncio
@@ -278,7 +278,9 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
workflow_version_id=random_workflow_version.git_commit_hash,
parameters={"dir": f"s3://{random_bucket.name}"},
)
- response = await client.post(self.base_path, headers=random_second_user.auth_headers, json=execution_in.dict())
+ response = await client.post(
+ self.base_path, headers=random_second_user.auth_headers, json=execution_in.model_dump()
+ )
assert response.status_code == status.HTTP_201_CREATED
@pytest.mark.asyncio
@@ -307,7 +309,9 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
workflow_version_id=random_workflow_version.git_commit_hash,
parameters={"dir": f"s3://{random_bucket.name}"},
)
- response = await client.post(self.base_path, headers=random_second_user.auth_headers, json=execution_in.dict())
+ response = await client.post(
+ self.base_path, headers=random_second_user.auth_headers, json=execution_in.model_dump()
+ )
assert response.status_code == status.HTTP_400_BAD_REQUEST
@pytest.mark.asyncio
@@ -337,7 +341,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
parameters={},
report_output_bucket=random_bucket.name,
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_201_CREATED
@pytest.mark.asyncio
@@ -364,7 +368,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
parameters={},
report_output_bucket=random_lower_string(),
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.dict())
+ response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
assert response.status_code == status.HTTP_400_BAD_REQUEST
@@ -392,7 +396,7 @@ class TestDevWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
git_commit_hash=random_hex_string(), repository_url="https://github.com/example/example", parameters={}
)
response = await client.post(
- f"{self.base_path}/arbitrary", headers=random_user.auth_headers, json=execution_in.dict()
+ f"{self.base_path}/arbitrary", headers=random_user.auth_headers, json=execution_in.model_dump()
)
assert response.status_code == status.HTTP_201_CREATED
execution_response = response.json()
@@ -427,8 +431,10 @@ class TestDevWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
git_commit_hash=random_hex_string(), repository_url="https://bitbucket.com/example/example", parameters={}
)
response = await client.post(
- f"{self.base_path}/arbitrary", headers=random_user.auth_headers, json=execution_in.dict()
+ f"{self.base_path}/arbitrary", headers=random_user.auth_headers, json=execution_in.model_dump()
)
+ print(execution_in.model_dump())
+ print(response.json())
assert response.status_code == status.HTTP_400_BAD_REQUEST
diff --git a/app/tests/api/test_workflow_version.py b/app/tests/api/test_workflow_version.py
index e92257bc3585c8645da587941bb48ff7d1b02ac9..c87dc024d10420bdd0a41042c16105c3c12f2818 100644
--- a/app/tests/api/test_workflow_version.py
+++ b/app/tests/api/test_workflow_version.py
@@ -127,7 +127,7 @@ class TestWorkflowVersionRoutesUpdate(_TestWorkflowVersionRoutes):
]
),
headers=random_user.auth_headers,
- json=WorkflowVersionStatus(status=WorkflowVersion.Status.PUBLISHED).dict(),
+ json=WorkflowVersionStatus(status=WorkflowVersion.Status.PUBLISHED).model_dump(),
)
assert response.status_code == status.HTTP_200_OK
@@ -188,7 +188,7 @@ class TestWorkflowVersionRoutesUpdate(_TestWorkflowVersionRoutes):
response = await client.patch(
"/".join([self.base_path, str(random_workflow.workflow_id), "versions", random_hex_string(), "status"]),
headers=random_user.auth_headers,
- json=WorkflowVersionStatus(status=WorkflowVersion.Status.PUBLISHED).dict(),
+ json=WorkflowVersionStatus(status=WorkflowVersion.Status.PUBLISHED).model_dump(),
)
assert response.status_code == status.HTTP_404_NOT_FOUND
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index 55e72fc8e9a6dfa1bb7b3833d4e933800fbc262c..5224b747ebd7544b823edaf91939c41e726aa4ad 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -77,7 +77,7 @@ async def db() -> AsyncGenerator[AsyncSession, None]:
Fixture for creating a database session to connect to.
"""
async with get_async_session(
- url=settings.SQLALCHEMY_DATABASE_ASYNC_URI, verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
+ url=str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
)() as dbSession:
yield dbSession
diff --git a/app/tests/mocks/authorization_service.py b/app/tests/mocks/authorization_service.py
index 8680e3ac368320d8f279eecdb93fc7ae3eb37be2..f2a02e8e1a60a6f8bc10b81a13bb960a5dcc7789 100644
--- a/app/tests/mocks/authorization_service.py
+++ b/app/tests/mocks/authorization_service.py
@@ -20,7 +20,7 @@ def handle_request(body: dict[str, str]) -> Response:
response : httpx.Response
Mock response.
"""
- response_body = AuthzResponse(result=not request_admin_permission(body), decision_id=str(uuid4())).dict()
+ response_body = AuthzResponse(result=not request_admin_permission(body), decision_id=str(uuid4())).model_dump()
return Response(status_code=status.HTTP_200_OK, json=response_body)
diff --git a/app/tests/utils/utils.py b/app/tests/utils/utils.py
index 584503d63799d69eaf5e9f4403e665a3a890d286..1fb5e6265861ff3676fbf3c2904bc9772973590d 100644
--- a/app/tests/utils/utils.py
+++ b/app/tests/utils/utils.py
@@ -94,10 +94,10 @@ def handle_http_request(request: httpx.Request, raise_error: bool = False) -> ht
Generated mock request.
"""
url = str(request.url)
- if url.startswith(settings.OPA_URI):
+ if url.startswith(str(settings.OPA_URI)):
request_body: dict[str, str] = eval(request.content.decode("utf-8"))["input"]
return auth_handle_request(body=request_body)
- elif url.startswith(settings.SLURM_ENDPOINT):
+ elif url.startswith(str(settings.SLURM_ENDPOINT)):
return slurm_handle_request(request.method)
elif raise_error:
return httpx.Response(status_code=status.HTTP_404_NOT_FOUND, json={})
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 822a0542b20fe265f8dbf49556080ef11d56e4e9..c9635291c502c7b89302fb150639b53d396eff23 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,16 +1,16 @@
# test packages
-pytest>=7.3.0,<7.4.0
+pytest>=7.4.0,<7.5.0
pytest-asyncio>=0.21.0,<0.22.0
-pytest-cov>=4.0.0,<4.1.0
+pytest-cov>=4.1.0,<4.2.0
coverage[toml]>=7.2.0,<7.3.0
# Linters
ruff
-black>=23.03.0,<23.04.0
+black>=23.07.0,<23.08.0
isort>=5.12.0,<5.13.0
-mypy>=1.2.0,<1.3.0
+mypy>=1.4.0,<1.5.0
# stubs for mypy
-boto3-stubs-lite[s3]>=1.26.0,<1.27.0
+boto3-stubs-lite[s3]>=1.28.0,<1.29.0
sqlalchemy2-stubs
types-requests
# Miscellaneous
-pre-commit>=3.2.0,<3.3.0
+pre-commit>=3.3.0,<3.4.0
diff --git a/requirements.txt b/requirements.txt
index 47d8fb8e74646b8af7b7a737451617daa5b30dd4..ccedf9f65648c017d52970f35ed6234713e27784 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,10 +2,11 @@
clowmdb>=1.3.0,<1.4.0
# Webserver packages
-anyio>=3.6.0,<3.7.0
-fastapi>=0.95.0,<0.96.0
-pydantic>=1.10.0,<2.0.0
-uvicorn>=0.21.0,<0.22.0
+anyio>=3.7.0,<3.8.0
+fastapi>=0.100.0,<0.101.0
+pydantic>=2.1.0,<2.2.0
+pydantic-settings
+uvicorn>=0.23.0,<0.24.0
python-multipart
# Database packages
PyMySQL>=1.0.2,<1.1.0
@@ -14,7 +15,7 @@ aiomysql>=0.1.0,<0.2.0
# Security packages
authlib>=1.2.0,<1.3.0
# Ceph and S3 packages
-boto3>=1.26.0,<1.27.0
+boto3>=1.28.0,<1.29.0
# Miscellaneous
tenacity>=8.1.0,<8.2.0
httpx>=0.24.0,<0.25.0