diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8aa44886588ae5feeacad5da6e55c8a8724a4d54..f3d7002763b64080264731efebc0cb5f7b979019 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: python:3.10-slim +image: python:3.11-slim variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -50,7 +50,7 @@ integration-test-job: # Runs integration tests with the database MYSQL_DATABASE: "$DB_DATABASE" MYSQL_USER: "$DB_USER" MYSQL_PASSWORD: "$DB_PASSWORD" - - name: $CI_REGISTRY/cmg/clowm/clowm-database:v1.3 + - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0 alias: upgrade-db script: - python app/check_database_connection.py @@ -78,7 +78,7 @@ e2e-test-job: # Runs e2e tests on the API endpoints MYSQL_DATABASE: "$DB_DATABASE" MYSQL_USER: "$DB_USER" MYSQL_PASSWORD: "$DB_PASSWORD" - - name: $CI_REGISTRY/cmg/clowm/clowm-database:v1.3 + - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0 alias: upgrade-db script: - python app/check_database_connection.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f547a4fe938f063d7c07547e799a4fb7f3491c07..b4e13c9cff2566cfbaed0f7b898ae5922597d084 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: files: app args: [--check] - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.281' + rev: 'v0.0.283' hooks: - id: ruff - repo: https://github.com/pre-commit/mirrors-mypy @@ -31,9 +31,8 @@ repos: files: app args: [--config=pyproject.toml] additional_dependencies: - - sqlalchemy2-stubs - boto3-stubs-lite[s3] - - sqlalchemy<2.0.0 + - sqlalchemy>=2.0.0.<2.1.0 - pydantic - types-requests - repo: https://github.com/PyCQA/isort diff --git a/Dockerfile b/Dockerfile index c80d4d76b8766555e52d1a5c04ed8b3e1ae9147d..d364a9b2efbb0abeb5800317b3544c8c059f56f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM python:3.10-slim +FROM python:3.11-slim EXPOSE 8000 # dumb-init forwards the kill signal to the python process RUN apt-get update && apt-get -y install dumb-init curl ENTRYPOINT ["/usr/bin/dumb-init", "--"] -HEALTHCHECK --interval=35s --timeout=4s CMD curl -f http://localhost:8000/health || exit 1 +HEALTHCHECK --interval=30s --timeout=4s CMD curl -f http://localhost:8000/health || exit 1 RUN useradd -m worker USER worker diff --git a/Dockerfile-Gunicorn b/Dockerfile-Gunicorn index 19aae7fe449c5500aa264e3b665c1ecc02f4c18b..9c460ba123efe26687258bc06f8f76c5e36fa43d 100644 --- a/Dockerfile-Gunicorn +++ b/Dockerfile-Gunicorn @@ -1,10 +1,10 @@ -FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10-slim +FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim EXPOSE 8000 ENV PORT=8000 RUN pip install --no-cache-dir httpx[cli] -HEALTHCHECK --interval=35s --timeout=4s CMD httpx http://localhost:$PORT/health || exit 1 +HEALTHCHECK --interval=30s --timeout=4s CMD httpx http://localhost:$PORT/health || exit 1 COPY ./scripts/prestart.sh /app/prestart.sh COPY ./requirements.txt /app/requirements.txt diff --git a/app/api/api.py b/app/api/api.py index b8b5baa47cfbab4b8c2411439b7bbce3afb9a2d0..5b224b4adaa71bdc0ac04bf0ef917ec0ca41a5cd 100644 --- a/app/api/api.py +++ b/app/api/api.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Dict, Union from fastapi import APIRouter, Depends, status @@ -8,7 +8,7 @@ from app.api.endpoints.workflow_execution import router as execution_router from app.api.endpoints.workflow_version import router as version_router from app.schemas.security import ErrorDetail -alternative_responses: dict[int | str, dict[str, Any]] = { +alternative_responses: Dict[Union[int, str], Dict[str, Any]] = { status.HTTP_400_BAD_REQUEST: { "model": ErrorDetail, "description": "Error decoding JWT Token", diff --git a/app/api/dependencies.py b/app/api/dependencies.py index 3dad80d4ef50a6e15e6c096ab2cdad0e643f7832..724bef058e26a845e61ec3ac98895880fe0da0a1 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Annotated, AsyncGenerator, Awaitable, Callable +from typing import TYPE_CHECKING, Annotated, AsyncGenerator, Awaitable, Callable, Dict from uuid import UUID from authlib.jose.errors import BadSignatureError, DecodeError, ExpiredTokenError @@ -42,7 +42,7 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]: """ async with get_async_session( str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER - )() as db: + ) as db: yield db @@ -63,7 +63,7 @@ def get_slurm_client(client: AsyncClient = Depends(get_httpx_client)) -> SlurmCl return SlurmClient(client=client) -def get_decode_jwt_function() -> Callable[[str], dict[str, str]]: # pragma: no cover +def get_decode_jwt_function() -> Callable[[str], Dict[str, str]]: # pragma: no cover """ Get function to decode and verify the JWT. @@ -72,7 +72,7 @@ def get_decode_jwt_function() -> Callable[[str], dict[str, str]]: # pragma: no Returns ------- - decode : Callable[[str], dict[str, str]] + decode : Callable[[str], Dict[str, str]] Function to decode & verify the token. raw_token -> claims. Dependency Injection """ return decode_token @@ -80,7 +80,7 @@ def get_decode_jwt_function() -> Callable[[str], dict[str, str]]: # pragma: no async def decode_bearer_token( token: HTTPAuthorizationCredentials = Depends(bearer_token), - decode: Callable[[str], dict[str, str]] = Depends(get_decode_jwt_function), + decode: Callable[[str], Dict[str, str]] = Depends(get_decode_jwt_function), db: AsyncSession = Depends(get_db), ) -> JWT: """ @@ -92,7 +92,7 @@ async def decode_bearer_token( ---------- token : fastapi.security.http.HTTPAuthorizationCredentials Bearer token sent with the HTTP request. Dependency Injection. - decode : Callable[[str], dict[str, str]] + decode : Callable[[str], Dict[str, str]] Function to decode & verify the token. raw_token -> claims. Dependency Injection db : sqlalchemy.ext.asyncio.AsyncSession. Async database session to perform query on. Dependency Injection. diff --git a/app/api/endpoints/workflow.py b/app/api/endpoints/workflow.py index 236bbc9c3ef7a4d8dc534caee506ac94294e7251..7ef3e17eff660597bbc8039b5031eb518c61db4d 100644 --- a/app/api/endpoints/workflow.py +++ b/app/api/endpoints/workflow.py @@ -1,4 +1,4 @@ -from typing import Annotated, Any, Awaitable, Callable +from typing import Annotated, Any, Awaitable, Callable, List, Optional from clowmdb.models import Workflow, WorkflowVersion from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, Query, Response, UploadFile, status @@ -23,25 +23,22 @@ async def list_workflows( db: DBSession, authorization: Authorization, current_user: CurrentUser, - name_substring: str - | None = Query( + name_substring: Optional[str] = Query( None, min_length=3, max_length=30, description="Filter workflows by a substring in their name.", ), - version_status: list[WorkflowVersion.Status] - | None = Query( + version_status: Optional[List[WorkflowVersion.Status]] = Query( None, description=f"Which versions of the workflow to include in the response. Permission 'workflow:list_filter required', unless 'developer_id' is provided and current user is developer, then only permission 'workflow:list' required. Default {WorkflowVersion.Status.PUBLISHED.name} and {WorkflowVersion.Status.DEPRECATED.name}.", # noqa: E501 ), - developer_id: str - | None = Query( + developer_id: Optional[str] = Query( None, description="Filter for workflow by developer. If current user is the same as developer ID, permission 'workflow:list' required, otherwise 'workflow:list_filter'.", # noqa: E501 examples=["28c5353b8bb34984a8bd4169ba94c606"], ), -) -> list[WorkflowOut]: +) -> List[WorkflowOut]: """ List all workflows.\n Permission "workflow:list" required. @@ -54,7 +51,7 @@ async def list_workflows( Filter workflows by a developer. Query Parameter. name_substring : string | None, default None Filter workflows by a substring in their name. Query Parameter. - version_status : list[clowmdb.models.WorkflowVersionReduced.Status] | None, default None + version_status : List[clowmdb.models.WorkflowVersionReduced.Status] | None, default None Status of Workflow versions to filter for to fetch. Query Parameter. authorization : Callable[[str], Awaitable[Any]] Async function to ask the auth service for authorization. Dependency Injection. @@ -63,7 +60,7 @@ async def list_workflows( Returns ------- - workflows : list[app.schemas.workflow.WorkflowOut] + workflows : List[app.schemas.workflow.WorkflowOut] Workflows in the system """ rbac_operation = "list" @@ -73,7 +70,7 @@ async def list_workflows( rbac_operation = "list_filter" await authorization(rbac_operation) - workflows: list[Workflow] = await CRUDWorkflow.list_workflows( + workflows: List[Workflow] = await CRUDWorkflow.list_workflows( db, name_substring=name_substring, developer_id=developer_id, @@ -126,7 +123,7 @@ async def create_workflow( min_length=5, max_length=10, ), - icon: UploadFile | None = File(None, description="Optional Icon for the Workflow."), + icon: Optional[UploadFile] = File(None, description="Optional Icon for the Workflow."), ) -> WorkflowOut: """ Create a new workflow.\nR @@ -200,7 +197,7 @@ async def create_workflow( client=client, ) - icon_slug: str | None = None + icon_slug: Optional[str] = None if icon is not None: icon_slug = upload_icon(s3=s3, background_tasks=background_tasks, icon=icon) @@ -215,8 +212,7 @@ async def get_workflow( db: DBSession, current_user: CurrentUser, authorization: Authorization, - version_status: list[WorkflowVersion.Status] - | None = Query( + version_status: Optional[List[WorkflowVersion.Status]] = Query( None, description=f"Which versions of the workflow to include in the response. Permission 'workflow:read_any' required if you are not the developer of this workflow. Default {WorkflowVersion.Status.PUBLISHED.name} and {WorkflowVersion.Status.DEPRECATED.name}", # noqa: E501 ), @@ -229,7 +225,7 @@ async def get_workflow( ---------- workflow : clowmdb.models.Workflow Workflow with given ID. Dependency Injection. - version_status : list[clowmdb.models.WorkflowVersionReduced.Status] | None, default None + version_status : List[clowmdb.models.WorkflowVersionReduced.Status] | None, default None Status of Workflow versions to filter for to fetch. Query Parameter db : sqlalchemy.ext.asyncio.AsyncSession. Async database session to perform query on. Dependency Injection. @@ -257,7 +253,7 @@ async def get_workflow( @router.get("/{wid}/statistics", status_code=status.HTTP_200_OK, summary="Get statistics for a workflow") async def get_workflow_statistics( workflow: CurrentWorkflow, db: DBSession, authorization: Authorization, response: Response -) -> list[WorkflowStatistic]: +) -> List[WorkflowStatistic]: """ Get the number of started workflow per day. \f @@ -274,7 +270,7 @@ async def get_workflow_statistics( Returns ------- - statistics : list[app.schema.Workflow.WorkflowStatistic] + statistics : List[app.schema.Workflow.WorkflowStatistic] """ await authorization("read") # Instruct client to cache response for 1 hour @@ -334,8 +330,7 @@ async def update_workflow( current_user: CurrentUser, s3: S3Service, authorization: Authorization, - icon: UploadFile - | None = File( + icon: Optional[UploadFile] = File( None, description="Optional Icon for the workflow version. If None, then the previous one will be reused." ), version: str = Form( diff --git a/app/api/endpoints/workflow_execution.py b/app/api/endpoints/workflow_execution.py index c49b396ed12cce962919268a2018d5edd0a7f6d6..59ce14a8d7eea62ba2124f011d970430dcca1a7f 100644 --- a/app/api/endpoints/workflow_execution.py +++ b/app/api/endpoints/workflow_execution.py @@ -1,6 +1,6 @@ import json from tempfile import SpooledTemporaryFile -from typing import Annotated, Any, Awaitable, Callable +from typing import Annotated, Any, Awaitable, Callable, Dict, List, Optional import jsonschema from clowmdb.models import WorkflowExecution, WorkflowVersion @@ -233,22 +233,21 @@ async def list_workflow_executions( db: DBSession, current_user: CurrentUser, authorization: Authorization, - user_id: str - | None = Query( + user_id: Optional[str] = Query( None, description="Filter for workflow executions by a user. If none, Permission 'workflow_execution:read_any' required.", # noqa: E501 examples=["28c5353b8bb34984a8bd4169ba94c606"], ), - execution_status: list[WorkflowExecution.WorkflowExecutionStatus] - | None = Query(None, description="Filter for status of workflow execution"), - workflow_version_id: str - | None = Query( + execution_status: Optional[List[WorkflowExecution.WorkflowExecutionStatus]] = Query( + None, description="Filter for status of workflow execution" + ), + workflow_version_id: Optional[str] = Query( None, description="Filter for workflow version", examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"], regex=r"^[0-9a-f]{40}$", ), -) -> list[WorkflowExecutionOut]: +) -> List[WorkflowExecutionOut]: """ Get all workflow executions.\n Permission "workflow_execution:list" required, if 'user_id' is the same as the current user, @@ -258,7 +257,7 @@ async def list_workflow_executions( ---------- user_id : str | None, default None Filter for workflow executions by a user. Query Parameter. - execution_status : list[clowmdb.models.WorkflowExecution.WorkflowExecutionStatus] | None, default None + execution_status : List[clowmdb.models.WorkflowExecution.WorkflowExecutionStatus] | None, default None Filter for status of workflow execution. Query Parameter. workflow_version_id : str | None, default None Filter for workflow version, Query Parameter. @@ -271,7 +270,7 @@ async def list_workflow_executions( Returns ------- - executions : list[clowmdb.models.WorkflowExecution] + executions : List[clowmdb.models.WorkflowExecution] List of filtered workflow executions. """ rbac_operation = "list" if user_id is not None and user_id == current_user.uid else "list_all" @@ -326,7 +325,7 @@ async def get_workflow_execution_params( current_user: CurrentUser, authorization: Authorization, s3: S3Service, -) -> dict[str, Any]: +) -> Dict[str, Any]: """ Get the parameters of a specific workflow execution.\n Permission "workflow_execution:read" required if the current user started the workflow execution, diff --git a/app/api/endpoints/workflow_version.py b/app/api/endpoints/workflow_version.py index ae1c356caf3e358a6f8b5884c95504eafc5a907b..4f5b531528908a72fa72d621f2256789ec5c9c82 100644 --- a/app/api/endpoints/workflow_version.py +++ b/app/api/endpoints/workflow_version.py @@ -1,4 +1,4 @@ -from typing import Annotated, Any, Awaitable, Callable +from typing import Annotated, Any, Awaitable, Callable, List, Optional from clowmdb.models import WorkflowVersion from fastapi import APIRouter, Depends, HTTPException, Path, Query, status @@ -29,12 +29,11 @@ async def list_workflow_version( workflow: CurrentWorkflow, db: DBSession, authorization: Authorization, - version_status: list[WorkflowVersion.Status] - | None = Query( + version_status: Optional[List[WorkflowVersion.Status]] = Query( None, description=f"Which versions of the workflow to include in the response. Permission 'workflow:list_filter' required if you are not the developer of this workflow. Default {WorkflowVersion.Status.PUBLISHED.name} and {WorkflowVersion.Status.DEPRECATED.name}", # noqa: E501 ), -) -> list[WorkflowVersionFull]: +) -> List[WorkflowVersionFull]: """ List all versions of a Workflow.\n Permission "workflow:list" required. @@ -43,7 +42,7 @@ async def list_workflow_version( ---------- workflow : clowmdb.models.Workflow Workflow with given ID. Dependency Injection. - version_status : list[clowmdb.models.WorkflowVersionReduced.Status] | None, default None + version_status : List[clowmdb.models.WorkflowVersionReduced.Status] | None, default None Status of Workflow versions to filter for to fetch. Query Parameter db : sqlalchemy.ext.asyncio.AsyncSession. Async database session to perform query on. Dependency Injection. diff --git a/app/api/miscellaneous_endpoints.py b/app/api/miscellaneous_endpoints.py index 8736dbbe0ec593f9a3a9e862630390b9a22346e5..0f4f786b06506ed0843fceb5f84a7a791441ab2e 100644 --- a/app/api/miscellaneous_endpoints.py +++ b/app/api/miscellaneous_endpoints.py @@ -1,3 +1,5 @@ +from typing import Dict + from fastapi import APIRouter, status miscellaneous_router = APIRouter(include_in_schema=False) @@ -13,14 +15,14 @@ miscellaneous_router = APIRouter(include_in_schema=False) }, }, ) -def health_check() -> dict[str, str]: +def health_check() -> Dict[str, str]: """ Check if the service is reachable. \f Returns ------- - response : dict[str, str] + response : Dict[str, str] status ok """ return {"status": "OK"} diff --git a/app/api/utils.py b/app/api/utils.py index 7d5a6e980539e8d40ca34099b45c4001a7faab00..1cd09661707389093d6d458b2e4fae5a70f2a6cc 100644 --- a/app/api/utils.py +++ b/app/api/utils.py @@ -1,7 +1,7 @@ import json import re from tempfile import SpooledTemporaryFile -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Dict, Optional from uuid import uuid4 from clowmdb.models import WorkflowExecution @@ -75,8 +75,8 @@ async def start_workflow_execution( s3: S3ServiceResource, db: AsyncSession, execution: WorkflowExecution, - report_output_bucket: str | None, - parameters: dict[str, Any], + report_output_bucket: Optional[str], + parameters: Dict[str, Any], git_repo: GitRepository, slurm_client: SlurmClient, ) -> None: @@ -93,7 +93,7 @@ async def start_workflow_execution( Workflow execution to execute. report_output_bucket : str | None Bucket where to save the Nextflow report. - parameters : dict[str, Any] + parameters : Dict[str, Any] Parameters for the workflow. git_repo : app.git_repository.abstract_repository.GitRepository Git repository of the workflow version. @@ -156,7 +156,7 @@ async def check_active_workflow_execution_limit(db: AsyncSession, uid: str) -> N async def check_buckets_access( - db: AsyncSession, parameters: dict[str, Any], uid: str, report_bucket: str | None = None + db: AsyncSession, parameters: Dict[str, Any], uid: str, report_bucket: Optional[str] = None ) -> None: """ Check if the user has access to the buckets referenced in the workflow execution parameters. @@ -166,7 +166,7 @@ async def check_buckets_access( ---------- db : sqlalchemy.ext.asyncio.AsyncSession. Async database session to perform query on. - parameters : dict[str, Any] + parameters : Dict[str, Any] Parameters of the workflow. uid : str UID of a user. @@ -193,7 +193,7 @@ async def check_buckets_access( ) -async def _check_bucket_access(db: AsyncSession, uid: str, bucket_path: str) -> str | None: +async def _check_bucket_access(db: AsyncSession, uid: str, bucket_path: str) -> Optional[str]: """ Check if the bucket exists and the user has READWRITE access to it. diff --git a/app/check_database_connection.py b/app/check_database_connection.py index ae54e93d69f91628dd1b90245652923206877406..bcfc53e6e7529901ee7c15d480c204ee771f4ba2 100644 --- a/app/check_database_connection.py +++ b/app/check_database_connection.py @@ -22,7 +22,7 @@ wait_seconds = 2 ) def init() -> None: try: - with get_session(url=str(settings.SQLALCHEMY_DATABASE_NORMAL_URI))() as db: + with get_session(url=str(settings.SQLALCHEMY_DATABASE_NORMAL_URI)) as db: # Try to create session to check if DB is awake db_revision = db.execute(text("SELECT version_num FROM alembic_version LIMIT 1")).scalar_one_or_none() if db_revision != latest_revision: diff --git a/app/core/config.py b/app/core/config.py index b389531b465ec4674fc2028f243cfc595ff273c0..7c703197371662081c6d5d2e5e3fd0a45db56a52 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -31,10 +31,10 @@ def _load_public_key(pub_key_val: Optional[str], pub_key_file: Optional[Path]) - class Settings(BaseSettings): API_PREFIX: str = Field("/api/workflow-service", description="Path Prefix for all API endpoints.") - public_key_value: str | None = Field( + public_key_value: Optional[str] = Field( None, description="Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_VALUE" ) - public_key_file: Path | None = Field( + public_key_file: Optional[Path] = Field( None, description="Path to Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_FILE" ) @@ -99,7 +99,7 @@ class Settings(BaseSettings): PARAMS_BUCKET_MOUNT_PATH: str = Field( "/mnt/params-bucket", description="Path on the slurm cluster where the params bucket is mounted." ) - NX_CONFIG: str | None = Field(None, description="Path to a nextflow configuration for every run") + NX_CONFIG: Optional[str] = Field(None, description="Path to a nextflow configuration for every run") NX_BIN: str = Field("nextflow", description="Path to the nextflow executable") SLURM_WORKING_DIRECTORY: str = Field( "/tmp", description="Working directory for the slurm job with the nextflow command" diff --git a/app/core/security.py b/app/core/security.py index 60f9fa2f6074b78fdcca0a4c73910d58189f8016..c09999a308fb7fc31cad83f4c59a94dc82b29dc4 100644 --- a/app/core/security.py +++ b/app/core/security.py @@ -1,3 +1,5 @@ +from typing import Dict + from authlib.jose import JsonWebToken from fastapi import HTTPException, status from httpx import AsyncClient @@ -10,7 +12,7 @@ ALGORITHM = "RS256" jwt = JsonWebToken([ALGORITHM]) -def decode_token(token: str) -> dict[str, str]: # pragma: no cover +def decode_token(token: str) -> Dict[str, str]: # pragma: no cover """ Decode and verify a JWT token. @@ -21,7 +23,7 @@ def decode_token(token: str) -> dict[str, str]: # pragma: no cover Returns ------- - decoded_token : dict[str, str] + decoded_token : Dict[str, str] Payload of the decoded token. """ claims = jwt.decode( diff --git a/app/crud/crud_bucket.py b/app/crud/crud_bucket.py index a01f8107a9ffbffca66337d1272751c2c5b62094..266041bae8b6961b8a6a0ea3f35fc94d935ef424 100644 --- a/app/crud/crud_bucket.py +++ b/app/crud/crud_bucket.py @@ -1,3 +1,5 @@ +from typing import Optional + from clowmdb.models import Bucket, BucketPermission from sqlalchemy import func, or_, select from sqlalchemy.ext.asyncio import AsyncSession @@ -26,7 +28,7 @@ class CRUDBucket: return bucket is not None @staticmethod - async def check_access(db: AsyncSession, bucket_name: str, uid: str, key: str | None = None) -> bool: + async def check_access(db: AsyncSession, bucket_name: str, uid: str, key: Optional[str] = None) -> bool: """ Check if the given user has access to the bucket. @@ -71,7 +73,7 @@ class CRUDBucket: ) ) - permission: BucketPermission | None = (await db.execute(stmt)).scalar() + permission: Optional[BucketPermission] = (await db.execute(stmt)).scalar() # If the user has no active READWRITE Permission for the bucket -> user has no access if permission is None: return False diff --git a/app/crud/crud_user.py b/app/crud/crud_user.py index 8477ed1203730ed3661e8f0f21600b1cac208699..254a14c2ae0f43dd84ddc9d0eb2875cdd209e81e 100644 --- a/app/crud/crud_user.py +++ b/app/crud/crud_user.py @@ -1,3 +1,5 @@ +from typing import Optional + from clowmdb.models import User from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -5,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession class CRUDUser: @staticmethod - async def get(db: AsyncSession, uid: str) -> User | None: + async def get(db: AsyncSession, uid: str) -> Optional[User]: """ Get a user by its UID. diff --git a/app/crud/crud_workflow.py b/app/crud/crud_workflow.py index 106e4a67850d4f72a6646e2f1df6b4a70806167d..9124ebaf693a793b401d600b92ffd22fe77257e7 100644 --- a/app/crud/crud_workflow.py +++ b/app/crud/crud_workflow.py @@ -1,3 +1,4 @@ +from typing import List, Optional, Union from uuid import UUID from clowmdb.models import Workflow, WorkflowExecution, WorkflowVersion @@ -13,10 +14,10 @@ class CRUDWorkflow: @staticmethod async def list_workflows( db: AsyncSession, - name_substring: str | None = None, - developer_id: str | None = None, - version_status: list[WorkflowVersion.Status] | None = None, - ) -> list[Workflow]: + name_substring: Optional[str] = None, + developer_id: Optional[str] = None, + version_status: Optional[List[WorkflowVersion.Status]] = None, + ) -> List[Workflow]: """ List all workflows. Populates the version attribute of the workflows. @@ -28,12 +29,12 @@ class CRUDWorkflow: Substring to filter for in the name of a workflow. developer_id : str | None, default None Filter workflows by developer. - version_status : list[clowmdb.models.WorkflowVersion.Status] | None, default None + version_status : List[clowmdb.models.WorkflowVersion.Status] | None, default None Filter versions of a workflow based on the status. Removes workflows that have no version after this filter. Returns ------- - workflows : list[app.models.user.User] + workflows : List[app.models.user.User] List of workflows. """ stmt = select(Workflow).options(joinedload(Workflow.versions)) @@ -50,7 +51,7 @@ class CRUDWorkflow: return [w for w in (await db.execute(stmt)).scalars().unique().all() if len(w.versions) > 0] @staticmethod - async def delete(db: AsyncSession, workflow_id: UUID | bytes) -> None: + async def delete(db: AsyncSession, workflow_id: Union[UUID, bytes]) -> None: """ Delete a workflow. @@ -67,7 +68,7 @@ class CRUDWorkflow: await db.commit() @staticmethod - async def statistics(db: AsyncSession, workflow_id: bytes | UUID) -> list[WorkflowStatistic]: + async def statistics(db: AsyncSession, workflow_id: Union[bytes, UUID]) -> List[WorkflowStatistic]: """ Calculate the number of workflows started per day for a specific workflow @@ -80,7 +81,7 @@ class CRUDWorkflow: Returns ------- - stat : list[app.schemas.Workflow.WorkflowStatistic] + stat : List[app.schemas.Workflow.WorkflowStatistic] List of datapoints """ wid = workflow_id.bytes if isinstance(workflow_id, UUID) else workflow_id @@ -95,7 +96,7 @@ class CRUDWorkflow: return [WorkflowStatistic(day=row.day, count=row.count) for row in await db.execute(stmt)] @staticmethod - async def get(db: AsyncSession, workflow_id: UUID | bytes) -> Workflow | None: + async def get(db: AsyncSession, workflow_id: Union[UUID, bytes]) -> Optional[Workflow]: """ Get a workflow by its ID. @@ -116,7 +117,7 @@ class CRUDWorkflow: return (await db.execute(stmt)).scalar() @staticmethod - async def get_by_name(db: AsyncSession, workflow_name: str) -> Workflow | None: + async def get_by_name(db: AsyncSession, workflow_name: str) -> Optional[Workflow]: """ Get a workflow by its name. @@ -136,7 +137,9 @@ class CRUDWorkflow: return (await db.execute(stmt)).scalar() @staticmethod - async def create(db: AsyncSession, workflow: WorkflowIn, developer: str, icon_slug: str | None = None) -> Workflow: + async def create( + db: AsyncSession, workflow: WorkflowIn, developer: str, icon_slug: Optional[str] = None + ) -> Workflow: """ Create a workflow and the initial version in the database diff --git a/app/crud/crud_workflow_execution.py b/app/crud/crud_workflow_execution.py index 99d14a2caf8924c9234ab9c05bc81e3978ca6710..725b4a76d0c9427aad6fc59f1b959795a22f3a87 100644 --- a/app/crud/crud_workflow_execution.py +++ b/app/crud/crud_workflow_execution.py @@ -1,3 +1,4 @@ +from typing import List, Optional, Sequence, Union from uuid import UUID from clowmdb.models import WorkflowExecution @@ -12,9 +13,9 @@ class CRUDWorkflowExecution: @staticmethod async def create( db: AsyncSession, - execution: WorkflowExecutionIn | DevWorkflowExecutionIn, + execution: Union[WorkflowExecutionIn, DevWorkflowExecutionIn], owner_id: str, - notes: str | None = None, + notes: Optional[str] = None, ) -> WorkflowExecution: """ Create a workflow execution in the database. @@ -52,7 +53,7 @@ class CRUDWorkflowExecution: return workflow_execution @staticmethod - async def get(db: AsyncSession, execution_id: bytes | UUID) -> WorkflowExecution | None: + async def get(db: AsyncSession, execution_id: Union[bytes, UUID]) -> Optional[WorkflowExecution]: """ Get a workflow execution by its execution id from the database. @@ -80,10 +81,10 @@ class CRUDWorkflowExecution: @staticmethod async def list( db: AsyncSession, - uid: str | None = None, - workflow_version_id: str | None = None, - status_list: list[WorkflowExecution.WorkflowExecutionStatus] | None = None, - ) -> list[WorkflowExecution]: + uid: Optional[str] = None, + workflow_version_id: Optional[str] = None, + status_list: Optional[List[WorkflowExecution.WorkflowExecutionStatus]] = None, + ) -> Sequence[WorkflowExecution]: """ List all workflow executions and apply filter. @@ -95,12 +96,12 @@ class CRUDWorkflowExecution: Filter for the user who started the workflow execution. workflow_version_id : str | None, default None Filter for the workflow version - status_list : list[clowmdb.models.WorkflowExecution.WorkflowExecutionStatus] | None, default None + status_list : List[clowmdb.models.WorkflowExecution.WorkflowExecutionStatus] | None, default None Filter for the status of the workflow executions. Returns ------- - workflow_executions : list[clowmdb.models.WorkflowExecution] + workflow_executions : List[clowmdb.models.WorkflowExecution] List of all workflow executions with applied filters. """ stmt = select(WorkflowExecution).options(joinedload(WorkflowExecution.workflow_version)) @@ -114,7 +115,7 @@ class CRUDWorkflowExecution: return executions @staticmethod - async def delete(db: AsyncSession, execution_id: bytes | UUID) -> None: + async def delete(db: AsyncSession, execution_id: Union[bytes, UUID]) -> None: """ Delete a workflow execution from the database. @@ -133,7 +134,7 @@ class CRUDWorkflowExecution: @staticmethod async def cancel( db: AsyncSession, - execution_id: bytes | UUID, + execution_id: Union[bytes, UUID], status: WorkflowExecution.WorkflowExecutionStatus = WorkflowExecution.WorkflowExecutionStatus.CANCELED, ) -> None: """ @@ -158,7 +159,7 @@ class CRUDWorkflowExecution: await db.commit() @staticmethod - async def update_slurm_job_id(db: AsyncSession, execution_id: bytes | UUID, slurm_job_id: int) -> None: + async def update_slurm_job_id(db: AsyncSession, execution_id: Union[bytes, UUID], slurm_job_id: int) -> None: """ Update the status of a workflow execution to CANCELED in the database. diff --git a/app/crud/crud_workflow_version.py b/app/crud/crud_workflow_version.py index 9e1f5cdc8ec23c834c19b4612365dd0d92541642..0269725005b8ab8755b706c30506ff61f121d456 100644 --- a/app/crud/crud_workflow_version.py +++ b/app/crud/crud_workflow_version.py @@ -1,3 +1,4 @@ +from typing import List, Optional, Sequence, Union from uuid import UUID from clowmdb.models import WorkflowVersion @@ -8,7 +9,7 @@ from sqlalchemy.orm import joinedload class CRUDWorkflowVersion: @staticmethod - async def get(db: AsyncSession, git_commit_hash: str, populate_workflow: bool = False) -> WorkflowVersion | None: + async def get(db: AsyncSession, git_commit_hash: str, populate_workflow: bool = False) -> Optional[WorkflowVersion]: """ Get a workflow version by its commit git_commit_hash. @@ -32,7 +33,7 @@ class CRUDWorkflowVersion: return (await db.execute(stmt)).scalar() @staticmethod - async def get_latest(db: AsyncSession, wid: bytes | UUID, published: bool = True) -> WorkflowVersion | None: + async def get_latest(db: AsyncSession, wid: bytes | UUID, published: bool = True) -> Optional[WorkflowVersion]: """ Get the latest version of a workflow. @@ -52,7 +53,9 @@ class CRUDWorkflowVersion: """ stmt = ( select(WorkflowVersion) - .where(WorkflowVersion._workflow_id == wid.bytes if isinstance(wid, UUID) else wid) + .where( + WorkflowVersion._workflow_id == wid.bytes if isinstance(wid, UUID) else wid # type: ignore[arg-type] + ) .order_by(WorkflowVersion.created_at) .limit(1) ) @@ -69,8 +72,8 @@ class CRUDWorkflowVersion: @staticmethod async def list( - db: AsyncSession, wid: bytes | UUID, version_status: list[WorkflowVersion.Status] | None = None - ) -> list[WorkflowVersion]: + db: AsyncSession, wid: Union[bytes, UUID], version_status: Optional[List[WorkflowVersion.Status]] = None + ) -> Sequence[WorkflowVersion]: """ List all versions of a workflow. @@ -80,16 +83,16 @@ class CRUDWorkflowVersion: Async database session to perform query on. wid : bytes | uuid.UUID Git commit git_commit_hash of the version. - version_status : list[clowmdb.models.WorkflowVersion.Status] | None, default None + version_status : List[clowmdb.models.WorkflowVersion.Status] | None, default None Filter versions based on the status Returns ------- - user : list[clowmdb.models.WorkflowVersion] + user : List[clowmdb.models.WorkflowVersion] All workflow version of the given workflow """ stmt = select(WorkflowVersion).where( - WorkflowVersion._workflow_id == wid.bytes if isinstance(wid, UUID) else wid + WorkflowVersion._workflow_id == wid.bytes if isinstance(wid, UUID) else wid # type: ignore[arg-type] ) if version_status is not None: stmt = stmt.where(or_(*[WorkflowVersion.status == status for status in version_status])) @@ -101,9 +104,9 @@ class CRUDWorkflowVersion: db: AsyncSession, git_commit_hash: str, version: str, - wid: bytes | UUID, - icon_slug: str | None = None, - previous_version: str | None = None, + wid: Union[bytes, UUID], + icon_slug: Optional[str] = None, + previous_version: Optional[str] = None, ) -> WorkflowVersion: """ Create a new workflow version. diff --git a/app/git_repository/abstract_repository.py b/app/git_repository/abstract_repository.py index 180b64518705e0e18ab4a63584278852c5f9e2f2..90fe38ec4639facff352d101549cee03726f2c9f 100644 --- a/app/git_repository/abstract_repository.py +++ b/app/git_repository/abstract_repository.py @@ -1,7 +1,7 @@ import asyncio from abc import ABC, abstractmethod from tempfile import SpooledTemporaryFile -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List from fastapi import HTTPException, status from httpx import AsyncClient @@ -90,13 +90,13 @@ class GitRepository(ABC): response = await client.head(str(self.downloadFileURL(filepath)), follow_redirects=True) return response.status_code == status.HTTP_200_OK - async def check_files_exist(self, files: list[str], client: AsyncClient, raise_error: bool = True) -> list[bool]: + async def check_files_exist(self, files: List[str], client: AsyncClient, raise_error: bool = True) -> List[bool]: """ Check if multiple files exists in the Git Repository Parameters ---------- - files : list[str] + files : List[str] Paths to the file to check client : httpx.AsyncClient Async HTTP Client with an open connection @@ -104,7 +104,7 @@ class GitRepository(ABC): Raise an HTTPException if any of the files doesn't exist. Returns ------- - exist : list[bool] + exist : List[bool] Flags if the files exist. """ tasks = [asyncio.ensure_future(self.check_file_exists(file, client=client)) for file in files] diff --git a/app/git_repository/github.py b/app/git_repository/github.py index c8a634bb54aa36a2a69705473a130f430513f8fb..883198f0213191cb2e9b3a70d89b3c37d9b830d5 100644 --- a/app/git_repository/github.py +++ b/app/git_repository/github.py @@ -24,21 +24,21 @@ class GithubRepository(GitRepository): def __init__(self, url: str, git_commit_hash: str): parse_result = urlparse(url) - bla = parse_result.path[1:].split("/") + path_parts = parse_result.path[1:].split("/") self.url = url - self.account = bla[0] - self.repository = bla[1] + self.account = path_parts[0] + self.repository = path_parts[1] self.commit = git_commit_hash def downloadFileURL(self, filepath: str) -> AnyHttpUrl: return AnyHttpUrl.build( scheme="https", - host="raw.githubusercontent.com/", + host="raw.githubusercontent.com", path="/".join([self.account, self.repository, self.commit, filepath]), ) def __repr__(self) -> str: url = AnyHttpUrl.build( - scheme="https", host="raw.githubusercontent.com/", path="/".join([self.account, self.repository]) + scheme="https", host="raw.githubusercontent.com", path="/".join([self.account, self.repository]) ) return f"Github(repo={url} commit={self.commit}))" diff --git a/app/git_repository/gitlab.py b/app/git_repository/gitlab.py index 55a9016f6f627fbcc8393b51edfec5883e4d6c13..665ad8bdd87f73b26691f8b34e8a6dac0a50c957 100644 --- a/app/git_repository/gitlab.py +++ b/app/git_repository/gitlab.py @@ -35,9 +35,9 @@ class GitlabRepository(GitRepository): return AnyHttpUrl.build( scheme="https", host=self.domain, - path="/".join(["", self.account, self.repository, "-", "raw", self.commit, filepath]), + path="/".join([self.account, self.repository, "-", "raw", self.commit, filepath]), ) def __repr__(self) -> str: - url = AnyHttpUrl.build(scheme="https", host=self.domain, path="/".join(["", self.account, self.repository])) + url = AnyHttpUrl.build(scheme="https", host=self.domain, path="/".join([self.account, self.repository])) return f"Gitlab(repo={url} commit={self.commit}))" diff --git a/app/schemas/workflow.py b/app/schemas/workflow.py index 00722197d3c88f6e03a60669136f6daae3e5fc2e..1e04f08fbfb34148be22678406150a14f8cf24fd 100644 --- a/app/schemas/workflow.py +++ b/app/schemas/workflow.py @@ -1,4 +1,5 @@ from datetime import date +from typing import List, Sequence, Union from uuid import UUID from clowmdb.models import Workflow as WorkflowDB @@ -54,14 +55,14 @@ class WorkflowIn(_BaseWorkflow): class WorkflowOut(_BaseWorkflow): workflow_id: UUID = Field(..., description="Id of the workflow", examples=["0cc78936-381b-4bdd-999d-736c40591078"]) - versions: list[WorkflowVersionReduced] = Field(..., description="Versions of the workflow") + versions: List[WorkflowVersionReduced] = Field(..., description="Versions of the workflow") developer_id: str = Field( ..., description="Id of developer of the workflow", examples=["28c5353b8bb34984a8bd4169ba94c606"] ) @staticmethod def from_db_workflow( - db_workflow: WorkflowDB, versions: list[WorkflowVersionReduced | WorkflowVersionDB] + db_workflow: WorkflowDB, versions: Sequence[Union[WorkflowVersionReduced, WorkflowVersionDB]] ) -> "WorkflowOut": temp_versions = versions if len(versions) > 0 and isinstance(versions[0], WorkflowVersionDB): diff --git a/app/schemas/workflow_execution.py b/app/schemas/workflow_execution.py index 19df2e5ceb05a2d480cab72b44ec1d4b3bded920..eda50d880808180ab9fbf2d349723d08c055b777 100644 --- a/app/schemas/workflow_execution.py +++ b/app/schemas/workflow_execution.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any +from typing import Any, Dict, Optional from uuid import UUID from clowmdb.models import WorkflowExecution @@ -15,7 +15,7 @@ class _BaseWorkflowExecution(BaseModel): min_length=40, max_length=40, ) - notes: str | None = Field( + notes: Optional[str] = Field( None, description="Optional notes for this workflow execution", max_length=2**16, @@ -24,8 +24,8 @@ class _BaseWorkflowExecution(BaseModel): class WorkflowExecutionIn(_BaseWorkflowExecution): - parameters: dict[str, Any] = Field(..., description="Parameters for this workflow") - report_output_bucket: str | None = Field( + parameters: Dict[str, Any] = Field(..., description="Parameters for this workflow") + report_output_bucket: Optional[str] = Field( None, description="Bucket where to save the Nextflow report. If None, no report will be generated. With our without prefix 's3://'", # noqa: E501 min_length=3, @@ -44,7 +44,7 @@ class WorkflowExecutionOut(_BaseWorkflowExecution): start_time: datetime = Field( ..., description="Start time of the workflow execution", examples=[datetime(year=2023, month=1, day=1)] ) - end_time: datetime | None = Field( + end_time: Optional[datetime] = Field( None, description="End time of the workflow execution", examples=[datetime(year=2023, month=1, day=1)] ) status: WorkflowExecution.WorkflowExecutionStatus = Field( @@ -52,15 +52,17 @@ class WorkflowExecutionOut(_BaseWorkflowExecution): description="Status of the workflow execution", examples=[WorkflowExecution.WorkflowExecutionStatus.RUNNING], ) - workflow_version_id: str | None = Field( # type: ignore[assignment] + workflow_version_id: Optional[str] = Field( # type: ignore[assignment] None, description="Workflow version git commit hash", examples=["ba8bcd9294c2c96aedefa1763a84a18077c50c0f"] ) - workflow_id: UUID | None = Field( + workflow_id: Optional[UUID] = Field( None, description="Id of the workflow", examples=["0cc78936-381b-4bdd-999d-736c40591078"] ) @staticmethod - def from_db_model(workflow_execution: WorkflowExecution, workflow_id: UUID | None = None) -> "WorkflowExecutionOut": + def from_db_model( + workflow_execution: WorkflowExecution, workflow_id: Optional[UUID] = None + ) -> "WorkflowExecutionOut": return WorkflowExecutionOut( execution_id=workflow_execution.execution_id, user_id=workflow_execution.user_id, @@ -74,8 +76,8 @@ class WorkflowExecutionOut(_BaseWorkflowExecution): class DevWorkflowExecutionIn(BaseModel): - parameters: dict[str, Any] = Field(..., description="Parameters for this workflow") - report_output_bucket: str | None = Field( + parameters: Dict[str, Any] = Field(..., description="Parameters for this workflow") + report_output_bucket: Optional[str] = Field( None, description="Bucket where to save the Nextflow report. If None, no report will be generated", min_length=3, diff --git a/app/schemas/workflow_version.py b/app/schemas/workflow_version.py index e8a75615c8c92d757801f6152f7cba5a7b22a90f..a51900eebdbe028a1a45d79150b58dd1ac1cbb71 100644 --- a/app/schemas/workflow_version.py +++ b/app/schemas/workflow_version.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from uuid import UUID, uuid4 from clowmdb.models import WorkflowVersion as WorkflowVersionDB @@ -30,7 +31,7 @@ class WorkflowVersionReduced(WorkflowVersionStatus): min_length=40, max_length=40, ) - icon_url: AnyHttpUrl | None = Field( + icon_url: Optional[AnyHttpUrl] = Field( None, description="URL of the icon for this workflow version", examples=[f"{settings.OBJECT_GATEWAY_URI}{settings.ICON_BUCKET}/{uuid4().hex}.png"], diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 5224b747ebd7544b823edaf91939c41e726aa4ad..3b92150de94f182e70b3172c7715c5a312b2e251 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -1,7 +1,7 @@ import asyncio from functools import partial from secrets import token_urlsafe -from typing import AsyncGenerator, Callable, Generator +from typing import AsyncGenerator, Callable, Dict, Generator import httpx import pytest @@ -51,7 +51,7 @@ async def client(mock_s3_service: MockS3ServiceResource) -> AsyncGenerator: def get_mock_s3() -> MockS3ServiceResource: return mock_s3_service - def get_decode_token_function() -> Callable[[str], dict[str, str]]: + def get_decode_token_function() -> Callable[[str], Dict[str, str]]: # Override the decode_jwt function with mock function for tests and inject random shared secret return partial(decode_mock_token, secret=jwt_secret) @@ -78,7 +78,7 @@ async def db() -> AsyncGenerator[AsyncSession, None]: """ async with get_async_session( url=str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER - )() as dbSession: + ) as dbSession: yield dbSession diff --git a/app/tests/mocks/authorization_service.py b/app/tests/mocks/authorization_service.py index f2a02e8e1a60a6f8bc10b81a13bb960a5dcc7789..3793001f15975715966479367ba8f8becfeab0a2 100644 --- a/app/tests/mocks/authorization_service.py +++ b/app/tests/mocks/authorization_service.py @@ -1,3 +1,4 @@ +from typing import Dict from uuid import uuid4 from fastapi import status @@ -6,13 +7,13 @@ from httpx import Response from app.schemas.security import AuthzResponse -def handle_request(body: dict[str, str]) -> Response: +def handle_request(body: Dict[str, str]) -> Response: """ Handle a request to the authorization service during testing. Parameters ---------- - body : dict[str, str] + body : Dict[str, str] Body of the request. Returns @@ -24,13 +25,13 @@ def handle_request(body: dict[str, str]) -> Response: return Response(status_code=status.HTTP_200_OK, json=response_body) -def request_admin_permission(body: dict[str, str]) -> bool: +def request_admin_permission(body: Dict[str, str]) -> bool: """ Helper function to determine if the authorization request needs the 'administrator' role. Parameters ---------- - body : dict[str, str] + body : Dict[str, str] Body of the request. Returns diff --git a/app/tests/mocks/mock_s3_resource.py b/app/tests/mocks/mock_s3_resource.py index 8b742a4eb56e8699eb1b37d432aab4cd66addb9f..d78555d3b0bfba1ea52fb7226dcc4a52106cc0c4 100644 --- a/app/tests/mocks/mock_s3_resource.py +++ b/app/tests/mocks/mock_s3_resource.py @@ -1,6 +1,6 @@ from datetime import datetime from io import BytesIO -from typing import Any +from typing import Any, Dict, List, Optional from botocore.exceptions import ClientError @@ -38,15 +38,15 @@ class MockS3Object: def __repr__(self) -> str: return f"MockS3Object(key={self.key}, bucket={self.bucket_name})" - def upload_fileobj(self, Fileobj: BytesIO, ExtraArgs: dict[str, Any] | None = None) -> None: + def upload_fileobj(self, Fileobj: BytesIO, ExtraArgs: Optional[Dict[str, Any]] = None) -> None: """ - Mcok function for uploading a file from a ByteStream + Mock function for uploading a file from a ByteStream Parameters ---------- Fileobj : io.BytesIO Stream to read from. Ignored. - ExtraArgs : dict[str, Any] | None, default None + ExtraArgs : Dict[str, Any] | None, default None Extra arguments for file upload. Ignored. """ Fileobj.close() @@ -75,9 +75,9 @@ class MockS3Bucket: Create the bucket in the mock service. delete() -> None Delete the bucket in the mock service - delete_objects(Delete: dict[str, list[dict[str, str]]]) -> None + delete_objects(Delete: Dict[str, List[Dict[str, str]]]) -> None Delete multiple objects in the bucket. - get_objects() -> list[app.tests.mocks.mock_s3_resource.MockS3Object] + get_objects() -> List[app.tests.mocks.mock_s3_resource.MockS3Object] List of MockS3Object in the bucket. add_object(obj: app.tests.mocks.mock_s3_resource.MockS3Object) -> None Add a MockS3Object to the bucket. @@ -101,7 +101,7 @@ class MockS3Bucket: Functions --------- - all() -> list[app.tests.mocks.mock_s3_resource.MockS3Object] + all() -> List[app.tests.mocks.mock_s3_resource.MockS3Object] Get the saved list. filter(Prefix: str) -> app.tests.mocks.mock_s3_resource.MockS3Bucket.MockS3ObjectList Filter the object in the list by the prefix all their keys should have. @@ -111,27 +111,27 @@ class MockS3Bucket: Delete a MockS3Object from the list """ - def __init__(self, obj_list: list[MockS3Object] | None = None) -> None: - self._objs: list[MockS3Object] = [] if obj_list is None else obj_list + def __init__(self, obj_list: Optional[List[MockS3Object]] = None) -> None: + self._objs: List[MockS3Object] = [] if obj_list is None else obj_list - def all(self) -> list[MockS3Object]: + def all(self) -> List[MockS3Object]: """ Get the saved list. Returns ------- - objects : list[app.tests.mocks.mock_s3_resource.MockS3Object] + objects : List[app.tests.mocks.mock_s3_resource.MockS3Object] List of MockS3Object """ return self._objs - def all_keys(self) -> list[str]: + def all_keys(self) -> List[str]: """ Get the keys of all objects Returns ------- - keys : list[str] + keys : List[str] List of all keys """ return [o.key for o in self._objs] @@ -202,13 +202,13 @@ class MockS3Bucket: """ self._parent_service.delete_bucket(self.name) - def delete_objects(self, Delete: dict[str, list[dict[str, str]]]) -> None: + def delete_objects(self, Delete: Dict[str, List[Dict[str, str]]]) -> None: """ Delete multiple objects in the bucket. Parameters ---------- - Delete : dict[str, list[dict[str, str]]] + Delete : Dict[str, List[Dict[str, str]]] The keys of the objects to delete. Notes @@ -223,14 +223,14 @@ class MockS3Bucket: for key_object in Delete["Objects"]: self.objects.delete(key=key_object["Key"]) - def get_objects(self) -> list[MockS3Object]: + def get_objects(self) -> List[MockS3Object]: """ Get the MockS3Object in the bucket. Convenience function for testing. Returns ------- - objects : list[app.tests.mocks.mock_s3_resource.MockS3Object] + objects : List[app.tests.mocks.mock_s3_resource.MockS3Object] List of MockS3Object in the bucket. """ return self.objects.all() @@ -284,7 +284,7 @@ class MockS3ServiceResource: """ def __init__(self) -> None: - self._buckets: dict[str, MockS3Bucket] = {} + self._buckets: Dict[str, MockS3Bucket] = {} def Bucket(self, name: str) -> MockS3Bucket: """ diff --git a/app/tests/utils/bucket.py b/app/tests/utils/bucket.py index f8a11eb67609a0d3f7e96f1e5908117567350fa0..0ff7b82b782caaf99cf3d4443253fa7dceba5097 100644 --- a/app/tests/utils/bucket.py +++ b/app/tests/utils/bucket.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional import pytest from clowmdb.models import Bucket, BucketPermission, User @@ -39,10 +40,10 @@ async def add_permission_for_bucket( db: AsyncSession, bucket_name: str, uid: str, - from_: datetime | None = None, - to: datetime | None = None, + from_: Optional[datetime] = None, + to: Optional[datetime] = None, permission: BucketPermission.Permission = BucketPermission.Permission.READWRITE, - file_prefix: str | None = None, + file_prefix: Optional[str] = None, ) -> None: """ Creates Permission to a bucket for a user in the database. diff --git a/app/tests/utils/user.py b/app/tests/utils/user.py index b83070aa07dcb6a678be34ead07e2013c99acb8d..109cc0a9e56bf546a3d23c060026f70ceb31f200 100644 --- a/app/tests/utils/user.py +++ b/app/tests/utils/user.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from datetime import datetime, timedelta +from typing import Dict import pytest from authlib.jose import JsonWebToken @@ -13,11 +14,11 @@ _jwt = JsonWebToken(["HS256"]) @dataclass class UserWithAuthHeader: - auth_headers: dict[str, str] + auth_headers: Dict[str, str] user: User -def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> dict[str, str]: +def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> Dict[str, str]: """ Create a valid JWT and return the correct headers for subsequent requests. @@ -29,7 +30,7 @@ def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> dict[str Secret to sign the JWT with Returns ------- - headers : dict[str,str] + headers : Dict[str,str] HTTP Headers to authorize each request. """ to_encode = {"sub": uid, "exp": datetime.utcnow() + timedelta(hours=1)} @@ -39,7 +40,7 @@ def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> dict[str return headers -def decode_mock_token(token: str, secret: str = "SuperSecret") -> dict[str, str]: +def decode_mock_token(token: str, secret: str = "SuperSecret") -> Dict[str, str]: """ Decode and verify a test JWT token. @@ -52,7 +53,7 @@ def decode_mock_token(token: str, secret: str = "SuperSecret") -> dict[str, str] Returns ------- - decoded_token : dict[str, str] + decoded_token : Dict[str, str] Payload of the decoded token. """ claims = _jwt.decode( diff --git a/app/tests/utils/utils.py b/app/tests/utils/utils.py index 1fb5e6265861ff3676fbf3c2904bc9772973590d..3a8d698cccc199ba9bff2320a1c615f2813805d3 100644 --- a/app/tests/utils/utils.py +++ b/app/tests/utils/utils.py @@ -1,7 +1,7 @@ import random import string from datetime import datetime -from typing import Any +from typing import Any, Dict, Optional import httpx from fastapi import status @@ -57,7 +57,7 @@ def random_ipv4_string() -> str: return ".".join(str(random.randint(0, 255)) for _ in range(4)) -def json_datetime_converter(obj: Any) -> str | None: +def json_datetime_converter(obj: Any) -> Optional[str]: """ Helper function for the json converter to covert the object into a string format if it is a datetime object.\n Parse a datetime object into the format YYYY-MM-DDTHH:MM:SS, e.g. 2022-01-01T00:00:00 @@ -95,7 +95,7 @@ def handle_http_request(request: httpx.Request, raise_error: bool = False) -> ht """ url = str(request.url) if url.startswith(str(settings.OPA_URI)): - request_body: dict[str, str] = eval(request.content.decode("utf-8"))["input"] + request_body: Dict[str, str] = eval(request.content.decode("utf-8"))["input"] return auth_handle_request(body=request_body) elif url.startswith(str(settings.SLURM_ENDPOINT)): return slurm_handle_request(request.method) diff --git a/requirements-dev.txt b/requirements-dev.txt index c9635291c502c7b89302fb150639b53d396eff23..89c110972a69971461215284bd048e7124bc42c1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,6 @@ isort>=5.12.0,<5.13.0 mypy>=1.4.0,<1.5.0 # stubs for mypy boto3-stubs-lite[s3]>=1.28.0,<1.29.0 -sqlalchemy2-stubs types-requests # Miscellaneous pre-commit>=3.3.0,<3.4.0 diff --git a/requirements.txt b/requirements.txt index ccedf9f65648c017d52970f35ed6234713e27784..ad9dd86139bc74883a7be7cc9b45d823cf80d8e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,23 @@ --extra-index-url https://gitlab.ub.uni-bielefeld.de/api/v4/projects/5493/packages/pypi/simple -clowmdb>=1.3.0,<1.4.0 +clowmdb>=2.0.0,<2.1.0 # Webserver packages anyio>=3.7.0,<3.8.0 -fastapi>=0.100.0,<0.101.0 +fastapi>=0.101.0,<0.102.0 pydantic>=2.1.0,<2.2.0 pydantic-settings uvicorn>=0.23.0,<0.24.0 python-multipart # Database packages -PyMySQL>=1.0.2,<1.1.0 -SQLAlchemy>=1.4.0,<2.0.0 -aiomysql>=0.1.0,<0.2.0 +PyMySQL>=1.1.0,<1.2.0 +SQLAlchemy>=2.0.0,<2.1.0 +aiomysql>=0.2.0,<0.3.0 # Security packages authlib>=1.2.0,<1.3.0 # Ceph and S3 packages boto3>=1.28.0,<1.29.0 # Miscellaneous -tenacity>=8.1.0,<8.2.0 +tenacity>=8.2.0,<8.3.0 httpx>=0.24.0,<0.25.0 itsdangerous jsonschema>=4.0.0,<5.0.0