From 23f829876681db7a5152b5705a2607391ad1baa7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20G=C3=B6bel?= <dgoebel@techfak.uni-bielefeld.de>
Date: Fri, 18 Aug 2023 14:27:00 +0200
Subject: [PATCH] Add endpoints to update/delete the credentials for a workflow
#42
---
app/api/api.py | 4 +
app/api/endpoints/workflow.py | 15 +-
app/api/endpoints/workflow_credentials.py | 122 +++++++++
app/api/endpoints/workflow_version.py | 35 +--
app/crud/crud_workflow.py | 29 +-
app/crud/crud_workflow_version.py | 2 +-
app/schemas/workflow.py | 23 +-
app/schemas/workflow_version.py | 50 +---
app/tests/api/test_workflow_credentials.py | 292 +++++++++++++++++++++
app/tests/api/test_workflow_execution.py | 13 +-
app/tests/conftest.py | 45 +++-
app/tests/crud/test_workflow.py | 48 +++-
12 files changed, 587 insertions(+), 91 deletions(-)
create mode 100644 app/api/endpoints/workflow_credentials.py
create mode 100644 app/tests/api/test_workflow_credentials.py
diff --git a/app/api/api.py b/app/api/api.py
index 5b224b4..261a792 100644
--- a/app/api/api.py
+++ b/app/api/api.py
@@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends, status
from app.api.dependencies import decode_bearer_token
from app.api.endpoints.workflow import router as workflow_router
+from app.api.endpoints.workflow_credentials import router as credentials_router
from app.api.endpoints.workflow_execution import router as execution_router
from app.api.endpoints.workflow_version import router as version_router
from app.schemas.security import ErrorDetail
@@ -29,6 +30,9 @@ alternative_responses: Dict[Union[int, str], Dict[str, Any]] = {
api_router = APIRouter()
workflow_router.include_router(version_router)
api_router.include_router(workflow_router, responses=alternative_responses, dependencies=[Depends(decode_bearer_token)])
+api_router.include_router(
+ credentials_router, responses=alternative_responses, dependencies=[Depends(decode_bearer_token)]
+)
api_router.include_router(
execution_router, responses=alternative_responses, dependencies=[Depends(decode_bearer_token)]
)
diff --git a/app/api/endpoints/workflow.py b/app/api/endpoints/workflow.py
index 5d6c265..e5dfc35 100644
--- a/app/api/endpoints/workflow.py
+++ b/app/api/endpoints/workflow.py
@@ -10,7 +10,8 @@ from app.core.config import settings
from app.crud import CRUDWorkflow, CRUDWorkflowVersion
from app.git_repository import GitHubRepository, build_repository
from app.schemas.workflow import WorkflowIn, WorkflowOut, WorkflowStatistic
-from app.schemas.workflow_version import WorkflowVersionFull, WorkflowVersionUpdate
+from app.schemas.workflow_version import WorkflowVersion as WorkflowVersionSchema
+from app.schemas.workflow_version import WorkflowVersionUpdate
from app.scm import SCM, Provider
router = APIRouter(prefix="/workflows", tags=["Workflow"])
@@ -52,7 +53,7 @@ async def list_workflows(
Filter workflows by a developer. Query Parameter.
name_substring : string | None, default None
Filter workflows by a substring in their name. Query Parameter.
- version_status : List[clowmdb.models.WorkflowVersionReduced.Status] | None, default None
+ version_status : List[clowmdb.models.WorkflowVersion.Status] | None, default None
Status of Workflow versions to filter for to fetch. Query Parameter.
authorization : Callable[[str], Awaitable[Any]]
Async function to ask the auth service for authorization. Dependency Injection.
@@ -257,7 +258,7 @@ async def get_workflow(
----------
workflow : clowmdb.models.Workflow
Workflow with given ID. Dependency Injection.
- version_status : List[clowmdb.models.WorkflowVersionReduced.Status] | None, default None
+ version_status : List[clowmdb.models.WorkflowVersion.Status] | None, default None
Status of Workflow versions to filter for to fetch. Query Parameter
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
@@ -386,7 +387,7 @@ async def update_workflow(
min_length=40,
max_length=40,
),
-) -> WorkflowVersionFull:
+) -> WorkflowVersionSchema:
"""
Create a new workflow version.\n
Permission "workflow:update" required.
@@ -416,14 +417,14 @@ async def update_workflow(
Returns
-------
- version : app.schemas.workflow_version.WorkflowVersionFull
+ version : app.schemas.workflow_version.WorkflowVersion
The new workflow version
"""
await authorization("update")
- version_update = WorkflowVersionUpdate(version=version, git_commit_hash=git_commit_hash)
if current_user.uid != workflow.developer_id:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Only the developer can update his workflow")
+ version_update = WorkflowVersionUpdate(version=version, git_commit_hash=git_commit_hash)
# Check if git commit is already used
if await CRUDWorkflowVersion.get(db, version_update.git_commit_hash) is not None:
raise HTTPException(
@@ -457,4 +458,4 @@ async def update_workflow(
icon_slug=icon_slug,
previous_version=previous_version.git_commit_hash if previous_version else None,
)
- return WorkflowVersionFull.from_db_version_with_repo(version, repo)
+ return WorkflowVersionSchema.from_db_version(version)
diff --git a/app/api/endpoints/workflow_credentials.py b/app/api/endpoints/workflow_credentials.py
new file mode 100644
index 0000000..c4c62cb
--- /dev/null
+++ b/app/api/endpoints/workflow_credentials.py
@@ -0,0 +1,122 @@
+from typing import Annotated, Any, Awaitable, Callable
+
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status
+
+from app.api.dependencies import AuthorizationDependency, CurrentUser, CurrentWorkflow, DBSession, HTTPClient, S3Service
+from app.api.utils import check_repo, upload_scm_file
+from app.core.config import settings
+from app.crud.crud_workflow import CRUDWorkflow
+from app.crud.crud_workflow_version import CRUDWorkflowVersion
+from app.git_repository import GitHubRepository, build_repository
+from app.schemas.workflow import WorkflowCredentialsIn
+from app.scm import SCM, Provider
+
+router = APIRouter(prefix="/workflows/{wid}/credentials", tags=["Workflow Credentials"])
+workflow_authorization = AuthorizationDependency(resource="workflow")
+
+Authorization = Annotated[Callable[[str], Awaitable[Any]], Depends(workflow_authorization)]
+
+
+@router.delete("", status_code=status.HTTP_204_NO_CONTENT, summary="Delete the credentials of a workflow")
+async def delete_workflow_credentials(
+ workflow: CurrentWorkflow,
+ db: DBSession,
+ current_user: CurrentUser,
+ authorization: Authorization,
+ s3: S3Service,
+) -> None:
+ """
+ Delete the credentials for the repository of a workflow.\n
+ Permission "workflow:delete" required.
+ \f
+ Parameters
+ ----------
+ workflow : clowmdb.models.Workflow
+ Workflow with given ID. Dependency Injection.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. Dependency Injection.
+ current_user : clowmdb.models.User
+ Current user. Dependency Injection.
+ authorization : Callable[[str], Awaitable[Any]]
+ Async function to ask the auth service for authorization. Dependency Injection.
+ s3 : boto3_type_annotations.s3.ServiceResource
+ S3 Service to perform operations on buckets in Ceph. Dependency Injection.
+
+ Returns
+ -------
+ workflow : app.schemas.workflow.WorkflowOut
+ Workflow with existing ID
+ """
+ rbac_operation = "delete" if workflow.developer_id == current_user.uid else "delete_any"
+ await authorization(rbac_operation)
+ s3.Bucket(settings.PARAMS_BUCKET).Object(f"{workflow.workflow_id.hex}.scm").delete()
+ await CRUDWorkflow.update_credentials(db, workflow.workflow_id, token=None, username=None)
+
+
+@router.put("", status_code=status.HTTP_200_OK, summary="Delete the credentials of a workflow")
+async def update_workflow_credentials(
+ credentials: WorkflowCredentialsIn,
+ workflow: CurrentWorkflow,
+ db: DBSession,
+ current_user: CurrentUser,
+ authorization: Authorization,
+ s3: S3Service,
+ background_tasks: BackgroundTasks,
+ client: HTTPClient,
+) -> None:
+ """
+ Update the credentials for the repository of a workflow.\n
+ Permission "workflow:update" required.
+ \f
+ Parameters
+ ----------
+ credentials : app.schemas.workflow.WorkflowCredentialsIn
+ Updated credentials for the workflow git repository
+ workflow : clowmdb.models.Workflow
+ Workflow with given ID. Dependency Injection.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. Dependency Injection.
+ current_user : clowmdb.models.User
+ Current user. Dependency Injection.
+ authorization : Callable[[str], Awaitable[Any]]
+ Async function to ask the auth service for authorization. Dependency Injection.
+ s3 : boto3_type_annotations.s3.ServiceResource
+ S3 Service to perform operations on buckets in Ceph. Dependency Injection.
+ client : httpx.AsyncClient
+ Http client with an open connection. Dependency Injection.
+ background_tasks : fastapi.BackgroundTasks
+ Entrypoint for new BackgroundTasks. Provided by FastAPI.
+
+ Returns
+ -------
+ workflow : app.schemas.workflow.WorkflowOut
+ Workflow with existing ID
+ """
+ await authorization("update")
+ if current_user.uid != workflow.developer_id:
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN, detail="Only the developer can update the repository credentials"
+ )
+ latest_version = await CRUDWorkflowVersion.get_latest(db, workflow.workflow_id, published=False)
+ # Build a git repository object based on the repository url
+ repo = build_repository(
+ workflow.repository_url,
+ latest_version.git_commit_hash, # type: ignore[union-attr]
+ token=credentials.token,
+ username=credentials.username,
+ )
+ # if the workflow is in a private GitHub repo and the username is None, then take the username from the url
+ if credentials.username is None and isinstance(repo, GitHubRepository):
+ credentials.username = repo.user
+
+ await check_repo(repo=repo, client=client)
+ scm_provider = Provider.from_repo(repo=repo, name=f"repo{workflow.workflow_id.hex}")
+ background_tasks.add_task(
+ upload_scm_file,
+ s3=s3,
+ scm=SCM(providers=[scm_provider]), # type: ignore[list-item]
+ scm_file_id=workflow.workflow_id.hex,
+ )
+ await CRUDWorkflow.update_credentials(
+ db, workflow.workflow_id, token=credentials.token, username=credentials.username
+ )
diff --git a/app/api/endpoints/workflow_version.py b/app/api/endpoints/workflow_version.py
index 4f5b531..cd2b188 100644
--- a/app/api/endpoints/workflow_version.py
+++ b/app/api/endpoints/workflow_version.py
@@ -5,8 +5,8 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query, status
from app.api.dependencies import AuthorizationDependency, CurrentUser, CurrentWorkflow, DBSession
from app.crud import CRUDWorkflowVersion
-from app.git_repository import build_repository
-from app.schemas.workflow_version import WorkflowVersionFull, WorkflowVersionStatus
+from app.schemas.workflow_version import WorkflowVersion as WorkflowVersionSchema
+from app.schemas.workflow_version import WorkflowVersionStatus
router = APIRouter(prefix="/{wid}/versions", tags=["WorkflowVersion"])
workflow_authorization = AuthorizationDependency(resource="workflow")
@@ -33,7 +33,7 @@ async def list_workflow_version(
None,
description=f"Which versions of the workflow to include in the response. Permission 'workflow:list_filter' required if you are not the developer of this workflow. Default {WorkflowVersion.Status.PUBLISHED.name} and {WorkflowVersion.Status.DEPRECATED.name}", # noqa: E501
),
-) -> List[WorkflowVersionFull]:
+) -> List[WorkflowVersionSchema]:
"""
List all versions of a Workflow.\n
Permission "workflow:list" required.
@@ -42,7 +42,7 @@ async def list_workflow_version(
----------
workflow : clowmdb.models.Workflow
Workflow with given ID. Dependency Injection.
- version_status : List[clowmdb.models.WorkflowVersionReduced.Status] | None, default None
+ version_status : List[clowmdb.models.WorkflowVersion.Status] | None, default None
Status of Workflow versions to filter for to fetch. Query Parameter
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
@@ -53,7 +53,7 @@ async def list_workflow_version(
Returns
-------
- versions : [app.schemas.workflow_version.WorkflowVersionFull]
+ versions : [app.schemas.workflow_version.WorkflowVersion]
All versions of the workflow
"""
rbac_operation = (
@@ -68,10 +68,7 @@ async def list_workflow_version(
if version_status is not None
else [WorkflowVersion.Status.PUBLISHED, WorkflowVersion.Status.DEPRECATED],
)
- return [
- WorkflowVersionFull.from_db_version_with_repo(v, build_repository(workflow.repository_url, v.git_commit_hash))
- for v in versions
- ]
+ return [WorkflowVersionSchema.from_db_version(v) for v in versions]
@router.get(
@@ -90,7 +87,7 @@ async def get_workflow_version(
regex=r"^([0-9a-f]{40}|latest)$",
examples=["latest", "ba8bcd9294c2c96aedefa1763a84a18077c50c0f"],
),
-) -> WorkflowVersionFull:
+) -> WorkflowVersionSchema:
"""
Get a specific version of a workflow.\n
Permission "workflow:read" required if the version is public or you are the developer of the workflow,
@@ -111,7 +108,7 @@ async def get_workflow_version(
Returns
-------
- version : app.schemas.workflow_version.WorkflowVersionFull
+ version : app.schemas.workflow_version.WorkflowVersion
The specified WorkflowVersion
"""
rbac_operation = "read"
@@ -133,9 +130,7 @@ async def get_workflow_version(
):
rbac_operation = "read_any"
await authorization(rbac_operation)
- return WorkflowVersionFull.from_db_version_with_repo(
- version, repo=build_repository(workflow.repository_url, version.git_commit_hash)
- )
+ return WorkflowVersionSchema.from_db_version(version)
@router.patch("/{git_commit_hash}/status", status_code=status.HTTP_200_OK, summary="Update status of workflow version")
@@ -145,7 +140,7 @@ async def update_workflow_version_status(
git_commit_hash: GitCommitHash,
db: DBSession,
authorization: Authorization,
-) -> WorkflowVersionFull:
+) -> WorkflowVersionSchema:
"""
Update the status of a workflow version.\n
Permission "workflow:update_status"
@@ -177,9 +172,7 @@ async def update_workflow_version_status(
)
await CRUDWorkflowVersion.update_status(db, git_commit_hash, version_status.status)
version.status = version_status.status
- return WorkflowVersionFull.from_db_version_with_repo(
- version, repo=build_repository(workflow.repository_url, version.git_commit_hash)
- )
+ return WorkflowVersionSchema.from_db_version(version)
@router.post("/{git_commit_hash}/deprecate", status_code=status.HTTP_200_OK, summary="Deprecate a workflow version")
@@ -189,7 +182,7 @@ async def deprecate_workflow_version(
db: DBSession,
authorization: Authorization,
current_user: CurrentUser,
-) -> WorkflowVersionFull:
+) -> WorkflowVersionSchema:
"""
Deprecate a workflow version.\n
Permission "workflow:update" required if you are the developer of the workflow,
@@ -222,6 +215,4 @@ async def deprecate_workflow_version(
)
await CRUDWorkflowVersion.update_status(db, git_commit_hash, WorkflowVersion.Status.DEPRECATED)
version.status = WorkflowVersion.Status.DEPRECATED
- return WorkflowVersionFull.from_db_version_with_repo(
- version, repo=build_repository(workflow.repository_url, version.git_commit_hash)
- )
+ return WorkflowVersionSchema.from_db_version(version)
diff --git a/app/crud/crud_workflow.py b/app/crud/crud_workflow.py
index 25b8011..af8fa55 100644
--- a/app/crud/crud_workflow.py
+++ b/app/crud/crud_workflow.py
@@ -2,7 +2,7 @@ from typing import List, Optional, Union
from uuid import UUID
from clowmdb.models import Workflow, WorkflowExecution, WorkflowVersion
-from sqlalchemy import Date, cast, delete, func, or_, select
+from sqlalchemy import Date, cast, delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import joinedload
@@ -67,6 +67,33 @@ class CRUDWorkflow:
await db.execute(stmt)
await db.commit()
+ @staticmethod
+ async def update_credentials(
+ db: AsyncSession, workflow_id: Union[UUID, bytes], token: Optional[str] = None, username: Optional[str] = None
+ ) -> None:
+ """
+ Delete a workflow.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ workflow_id : bytes | uuid.UUID
+ UID of a workflow
+ token : str | None
+ Token to save in the database. If None, the token in the database gets deleted
+ username : str | None
+ Username to save in the database. If None, the token in the database gets deleted
+ """
+ wid = workflow_id.bytes if isinstance(workflow_id, UUID) else workflow_id
+ stmt = (
+ update(Workflow)
+ .where(Workflow._workflow_id == wid)
+ .values(credentials_token=token, credentials_username=username)
+ )
+ await db.execute(stmt)
+ await db.commit()
+
@staticmethod
async def statistics(db: AsyncSession, workflow_id: Union[bytes, UUID]) -> List[WorkflowStatistic]:
"""
diff --git a/app/crud/crud_workflow_version.py b/app/crud/crud_workflow_version.py
index 0269725..b29797b 100644
--- a/app/crud/crud_workflow_version.py
+++ b/app/crud/crud_workflow_version.py
@@ -129,7 +129,7 @@ class CRUDWorkflowVersion:
Returns
-------
workflow_version : clowmdb.models.WorkflowVersion
- Newly create WorkflowVersionReduced
+ Newly create WorkflowVersion
"""
workflow_version = WorkflowVersion(
git_commit_hash=git_commit_hash,
diff --git a/app/schemas/workflow.py b/app/schemas/workflow.py
index 9b57760..40d488b 100644
--- a/app/schemas/workflow.py
+++ b/app/schemas/workflow.py
@@ -6,7 +6,7 @@ from clowmdb.models import Workflow as WorkflowDB
from clowmdb.models import WorkflowVersion as WorkflowVersionDB
from pydantic import AnyHttpUrl, BaseModel, Field, FieldSerializationInfo, field_serializer
-from app.schemas.workflow_version import WorkflowVersionReduced
+from app.schemas.workflow_version import WorkflowVersion
class _BaseWorkflow(BaseModel):
@@ -67,7 +67,7 @@ class WorkflowIn(_BaseWorkflow):
class WorkflowOut(_BaseWorkflow):
workflow_id: UUID = Field(..., description="Id of the workflow", examples=["0cc78936-381b-4bdd-999d-736c40591078"])
- versions: List[WorkflowVersionReduced] = Field(..., description="Versions of the workflow")
+ versions: List[WorkflowVersion] = Field(..., description="Versions of the workflow")
developer_id: str = Field(
..., description="Id of developer of the workflow", examples=["28c5353b8bb34984a8bd4169ba94c606"]
)
@@ -75,11 +75,11 @@ class WorkflowOut(_BaseWorkflow):
@staticmethod
def from_db_workflow(
- db_workflow: WorkflowDB, versions: Sequence[Union[WorkflowVersionReduced, WorkflowVersionDB]]
+ db_workflow: WorkflowDB, versions: Sequence[Union[WorkflowVersion, WorkflowVersionDB]]
) -> "WorkflowOut":
temp_versions = versions
if len(versions) > 0 and isinstance(versions[0], WorkflowVersionDB):
- temp_versions = [WorkflowVersionReduced.from_db_version(v) for v in versions]
+ temp_versions = [WorkflowVersion.from_db_version(v) for v in versions]
return WorkflowOut(
workflow_id=db_workflow.workflow_id,
name=db_workflow.name,
@@ -94,3 +94,18 @@ class WorkflowOut(_BaseWorkflow):
class WorkflowStatistic(BaseModel):
day: date = Field(..., description="Day of the datapoint", examples=[date(day=1, month=1, year=2023)])
count: int = Field(..., description="Number of started workflows on that day", examples=[1])
+
+
+class WorkflowCredentialsIn(BaseModel):
+ token: str = Field(
+ ...,
+ description="Token to access the content git repository",
+ examples=["vnpau89avpa48iunga984gh9h89pvhj"],
+ max_length=128,
+ )
+ username: Optional[str] = Field(
+ default=None,
+ description="Username belonging to the token. If not provided, it will be parsed from the git repository URL",
+ examples=["bilbobaggins"],
+ max_length=128,
+ )
diff --git a/app/schemas/workflow_version.py b/app/schemas/workflow_version.py
index ce0dda7..b86b86d 100644
--- a/app/schemas/workflow_version.py
+++ b/app/schemas/workflow_version.py
@@ -6,7 +6,6 @@ from clowmdb.models import WorkflowVersion as WorkflowVersionDB
from pydantic import AnyHttpUrl, BaseModel, Field
from app.core.config import settings
-from app.git_repository.abstract_repository import GitRepository
class WorkflowVersionStatus(BaseModel):
@@ -15,7 +14,8 @@ class WorkflowVersionStatus(BaseModel):
)
-class WorkflowVersionReduced(WorkflowVersionStatus):
+class WorkflowVersion(WorkflowVersionStatus):
+ workflow_id: UUID = Field(..., description="ID of the corresponding workflow", examples=[uuid4()])
version: str = Field(
...,
description="Version of the Workflow. Should follow semantic versioning",
@@ -43,11 +43,12 @@ class WorkflowVersionReduced(WorkflowVersionStatus):
)
@staticmethod
- def from_db_version(db_version: WorkflowVersionDB) -> "WorkflowVersionReduced":
+ def from_db_version(db_version: WorkflowVersionDB) -> "WorkflowVersion":
icon_url = None
if db_version.icon_slug is not None:
icon_url = str(settings.OBJECT_GATEWAY_URI) + "/".join([settings.ICON_BUCKET, db_version.icon_slug])
- return WorkflowVersionReduced(
+ return WorkflowVersion(
+ workflow_id=db_version.workflow_id,
version=db_version.version,
git_commit_hash=db_version.git_commit_hash,
icon_url=icon_url,
@@ -56,47 +57,6 @@ class WorkflowVersionReduced(WorkflowVersionStatus):
)
-class WorkflowVersionFull(WorkflowVersionReduced):
- workflow_id: UUID = Field(..., description="ID of the corresponding workflow", examples=[uuid4()])
- readme_url: AnyHttpUrl = Field(
- ...,
- description="URL to download README.md from",
- examples=["https://raw.githubusercontent.com/example/example/README.md"],
- )
- changelog_url: AnyHttpUrl = Field(
- ...,
- description="URL to download CHANGELOG.md from",
- examples=["https://raw.githubusercontent.com/example/example/CHANGELOG.md"],
- )
- usage_url: AnyHttpUrl = Field(
- ...,
- description="URL to download usage.md from",
- examples=["https://raw.githubusercontent.com/example/example/docs/usage.md"],
- )
- output_url: AnyHttpUrl = Field(
- ...,
- description="URL to download output.md from",
- examples=["https://raw.githubusercontent.com/example/example/docs/output.md"],
- )
- parameter_schema_url: AnyHttpUrl = Field(
- ...,
- description="URL to download nextflow_schema.json from",
- examples=["https://raw.githubusercontent.com/example/example/nextflow_schema.json"],
- )
-
- @staticmethod
- def from_db_version_with_repo(db_version: WorkflowVersionDB, repo: GitRepository) -> "WorkflowVersionFull":
- return WorkflowVersionFull(
- workflow_id=db_version.workflow_id,
- readme_url=AnyHttpUrl("https://example.org/README.md"),
- changelog_url=AnyHttpUrl("https://example.org/CHANGELOG.md"),
- usage_url=AnyHttpUrl("https://example.org/docs/usage.md"),
- output_url=AnyHttpUrl("https://example.org/docs/output.md"),
- parameter_schema_url=AnyHttpUrl("https://example.org/nextflow_schema.json"),
- **WorkflowVersionReduced.from_db_version(db_version).model_dump(),
- )
-
-
class WorkflowVersionUpdate(BaseModel):
version: str = Field(
...,
diff --git a/app/tests/api/test_workflow_credentials.py b/app/tests/api/test_workflow_credentials.py
new file mode 100644
index 0000000..e9e0013
--- /dev/null
+++ b/app/tests/api/test_workflow_credentials.py
@@ -0,0 +1,292 @@
+from io import BytesIO
+
+import pytest
+from botocore.client import ClientError
+from clowmdb.models import Workflow
+from fastapi import status
+from httpx import AsyncClient
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import settings
+from app.schemas.workflow import WorkflowCredentialsIn, WorkflowOut
+from app.scm import SCM
+from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
+from app.tests.utils.user import UserWithAuthHeader
+from app.tests.utils.utils import random_lower_string
+
+
+class _TestWorkflowCredentialRoutes:
+ base_path: str = "/workflows"
+
+
+class TestWorkflowCredentialsRoutesUpdate(_TestWorkflowCredentialRoutes):
+ @pytest.mark.asyncio
+ async def test_update_workflow_credentials_on_public_workflow(
+ self,
+ db: AsyncSession,
+ client: AsyncClient,
+ random_user: UserWithAuthHeader,
+ random_workflow: WorkflowOut,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for updating the credentials on a workflow formerly hosted in a public git repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on. pytest fixture.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing. pytest fixture.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects. pytest fixture.
+ random_workflow : app.schemas.workflow.WorkflowOut
+ Random workflow for testing. pytest fixture.
+ """
+ credentials = WorkflowCredentialsIn(token=random_lower_string(15), username=random_lower_string(15))
+ response = await client.put(
+ "/".join([self.base_path, str(random_workflow.workflow_id), "credentials"]),
+ json=credentials.model_dump(),
+ headers=random_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_200_OK
+
+ stmt = select(Workflow).where(Workflow._workflow_id == random_workflow.workflow_id.bytes)
+ db_workflow = (await db.execute(stmt)).scalar()
+ assert db_workflow is not None
+
+ assert db_workflow.credentials_token == credentials.token
+ assert db_workflow.credentials_username == credentials.username
+
+ scm_file = mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(f"{db_workflow.workflow_id.hex}.scm")
+ with BytesIO() as f:
+ scm_file.download_fileobj(f)
+ f.seek(0)
+ scm = SCM.deserialize(f)
+
+ assert len(scm.providers) == 1
+ assert scm.providers[0].password == credentials.token
+
+ # Clean up after test
+ scm_file.delete()
+
+ @pytest.mark.asyncio
+ async def test_update_workflow_credentials_on_private_workflow(
+ self,
+ db: AsyncSession,
+ client: AsyncClient,
+ random_user: UserWithAuthHeader,
+ random_private_workflow: WorkflowOut,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for updating the credentials on a workflow hosted in a private git repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on. pytest fixture.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random second user for testing. pytest fixture.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects. pytest fixture.
+ random_private_workflow : app.schemas.workflow.WorkflowOut
+ Random private workflow for testing. pytest fixture.
+ """
+ credentials = WorkflowCredentialsIn(token=random_lower_string(15), username=random_lower_string(15))
+ response = await client.put(
+ "/".join([self.base_path, str(random_private_workflow.workflow_id), "credentials"]),
+ json=credentials.model_dump(),
+ headers=random_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_200_OK
+
+ stmt = select(Workflow).where(Workflow._workflow_id == random_private_workflow.workflow_id.bytes)
+ db_workflow = (await db.execute(stmt)).scalar()
+ assert db_workflow is not None
+
+ assert db_workflow.credentials_token == credentials.token
+ assert db_workflow.credentials_username == credentials.username
+
+ scm_file = mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(f"{db_workflow.workflow_id.hex}.scm")
+ with BytesIO() as f:
+ scm_file.download_fileobj(f)
+ f.seek(0)
+ scm = SCM.deserialize(f)
+
+ assert len(scm.providers) == 1
+ assert scm.providers[0].password == credentials.token
+
+ @pytest.mark.asyncio
+ async def test_update_workflow_credentials_without_username(
+ self,
+ db: AsyncSession,
+ client: AsyncClient,
+ random_user: UserWithAuthHeader,
+ random_private_workflow: WorkflowOut,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for updating the credentials on a workflow hosted in a private git repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on. pytest fixture.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing. pytest fixture.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects. pytest fixture.
+ random_private_workflow : app.schemas.workflow.WorkflowOut
+ Random private workflow for testing. pytest fixture.
+ """
+ credentials = WorkflowCredentialsIn(token=random_lower_string(15))
+ response = await client.put(
+ "/".join([self.base_path, str(random_private_workflow.workflow_id), "credentials"]),
+ json=credentials.model_dump(),
+ headers=random_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_200_OK
+
+ stmt = select(Workflow).where(Workflow._workflow_id == random_private_workflow.workflow_id.bytes)
+ db_workflow = (await db.execute(stmt)).scalar()
+ assert db_workflow is not None
+
+ assert db_workflow.credentials_token == credentials.token
+
+ scm_file = mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(f"{db_workflow.workflow_id.hex}.scm")
+ with BytesIO() as f:
+ scm_file.download_fileobj(f)
+ f.seek(0)
+ scm = SCM.deserialize(f)
+
+ assert len(scm.providers) == 1
+ assert scm.providers[0].password == credentials.token
+
+ @pytest.mark.asyncio
+ async def test_update_workflow_credentials_on_foreign_workflow(
+ self,
+ db: AsyncSession,
+ client: AsyncClient,
+ random_second_user: UserWithAuthHeader,
+ random_private_workflow: WorkflowOut,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for updating the credentials on a workflow hosted in a private git repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on. pytest fixture.
+ random_second_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing. pytest fixture.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects. pytest fixture.
+ random_private_workflow : app.schemas.workflow.WorkflowOut
+ Random private workflow for testing. pytest fixture.
+ """
+ credentials = WorkflowCredentialsIn(token=random_lower_string(15), username=random_lower_string(15))
+ response = await client.put(
+ "/".join([self.base_path, str(random_private_workflow.workflow_id), "credentials"]),
+ json=credentials.model_dump(),
+ headers=random_second_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_403_FORBIDDEN
+
+
+class TestWorkflowCredentialsRoutesDelete(_TestWorkflowCredentialRoutes):
+ @pytest.mark.asyncio
+ async def test_delete_workflow_credentials_on_public_workflow(
+ self,
+ db: AsyncSession,
+ client: AsyncClient,
+ random_user: UserWithAuthHeader,
+ random_workflow: WorkflowOut,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for updating the credentials on a workflow formerly hosted in a public git repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on. pytest fixture.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing. pytest fixture.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects. pytest fixture.
+ random_workflow : app.schemas.workflow.WorkflowOut
+ Random workflow for testing. pytest fixture.
+ """
+ response = await client.delete(
+ "/".join([self.base_path, str(random_workflow.workflow_id), "credentials"]),
+ headers=random_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_204_NO_CONTENT
+
+ stmt = select(Workflow).where(Workflow._workflow_id == random_workflow.workflow_id.bytes)
+ db_workflow = (await db.execute(stmt)).scalar()
+ assert db_workflow is not None
+
+ assert db_workflow.credentials_token is None
+ assert db_workflow.credentials_username is None
+
+ scm_file = mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(f"{db_workflow.workflow_id.hex}.scm")
+ with pytest.raises(ClientError):
+ with BytesIO() as f:
+ scm_file.download_fileobj(f)
+
+ @pytest.mark.asyncio
+ async def test_delete_workflow_credentials_on_private_workflow(
+ self,
+ db: AsyncSession,
+ client: AsyncClient,
+ random_user: UserWithAuthHeader,
+ random_private_workflow: WorkflowOut,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for updating the credentials on a workflow formerly hosted in a public git repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on. pytest fixture.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing. pytest fixture.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects. pytest fixture.
+ random_private_workflow : app.schemas.workflow.WorkflowOut
+ Random private workflow for testing. pytest fixture.
+ """
+ response = await client.delete(
+ "/".join([self.base_path, str(random_private_workflow.workflow_id), "credentials"]),
+ headers=random_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_204_NO_CONTENT
+
+ stmt = select(Workflow).where(Workflow._workflow_id == random_private_workflow.workflow_id.bytes)
+ db_workflow = (await db.execute(stmt)).scalar()
+ assert db_workflow is not None
+
+ assert db_workflow.credentials_token is None
+ assert db_workflow.credentials_username is None
+
+ scm_file = mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(f"{db_workflow.workflow_id.hex}.scm")
+ with pytest.raises(ClientError):
+ with BytesIO() as f:
+ scm_file.download_fileobj(f)
diff --git a/app/tests/api/test_workflow_execution.py b/app/tests/api/test_workflow_execution.py
index 2b5bf01..aea05f1 100644
--- a/app/tests/api/test_workflow_execution.py
+++ b/app/tests/api/test_workflow_execution.py
@@ -9,6 +9,7 @@ from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
+from app.schemas.workflow import WorkflowOut
from app.schemas.workflow_execution import DevWorkflowExecutionIn, WorkflowExecutionIn
from app.scm import SCM
from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
@@ -64,6 +65,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
client: AsyncClient,
random_user: UserWithAuthHeader,
random_workflow_version: WorkflowVersion,
+ random_private_workflow: WorkflowOut,
mock_s3_service: MockS3ServiceResource,
) -> None:
"""
@@ -77,14 +79,11 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
Random user for testing. pytest fixture.
random_workflow_version : clowmdb.models.WorkflowVersion
Random workflow version for testing. pytest fixture.
+ random_private_workflow : app.schemas.workflow.WorkflowOut
+ Random private workflow for testing. pytest fixture.
mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
Mock S3 Service to manipulate objects. pytest fixture.
"""
- # Create an SCM file in S3 to mark this workflow as a private repository
- scm_file = mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(
- f"{random_workflow_version.workflow_id.hex}.scm"
- )
- scm_file.upload_fileobj(BytesIO(b""))
execution_in = WorkflowExecutionIn(workflow_version_id=random_workflow_version.git_commit_hash, parameters={})
response = await client.post(self.base_path, headers=random_user.auth_headers, json=execution_in.model_dump())
@@ -99,8 +98,6 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
f"params-{UUID(hex=execution_response['execution_id']).hex }.json"
in mock_s3_service.Bucket(settings.PARAMS_BUCKET).objects.all_keys()
)
- # Clean up after tests
- scm_file.delete()
@pytest.mark.asyncio
async def test_start_gitlab_workflow_execution(
@@ -205,7 +202,7 @@ class TestWorkflowExecutionRoutesCreate(_TestWorkflowExecutionRoutes):
assert response.status_code == status.HTTP_404_NOT_FOUND
@pytest.mark.asyncio
- async def test_start_workflow_execution_with_unaccessable_workflow_version(
+ async def test_start_workflow_execution_with_deprecated_workflow_version(
self,
client: AsyncClient,
db: AsyncSession,
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index d88b3ea..9a5f51c 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -9,13 +9,15 @@ import pytest
import pytest_asyncio
from clowmdb.db.session import get_async_session
from clowmdb.models import Bucket, Workflow, WorkflowExecution, WorkflowVersion
-from sqlalchemy import select
+from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.dependencies import get_decode_jwt_function, get_httpx_client, get_s3_resource
from app.core.config import settings
+from app.git_repository import build_repository
from app.main import app
from app.schemas.workflow import WorkflowOut
+from app.scm import SCM, Provider
from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
from app.tests.utils.bucket import create_random_bucket
from app.tests.utils.user import UserWithAuthHeader, create_random_user, decode_mock_token, get_authorization_headers
@@ -147,7 +149,7 @@ async def random_workflow(
"""
workflow_db = Workflow(
name=random_lower_string(10),
- repository_url="https://github.de/example/example",
+ repository_url="https://github.de/example-user/example",
short_description=random_lower_string(65),
developer_id=random_user.user.uid,
)
@@ -171,6 +173,45 @@ async def random_workflow(
await db.commit()
+@pytest_asyncio.fixture(scope="function")
+async def random_private_workflow(
+ db: AsyncSession,
+ random_workflow: WorkflowOut,
+ random_workflow_version: WorkflowVersion,
+ mock_s3_service: MockS3ServiceResource,
+) -> AsyncIterator[WorkflowOut]:
+ """
+ Transform the random workflow into a private workflow.
+ """
+ # Update credentials in Database
+ token = random_lower_string(15)
+ username = random_lower_string(15)
+ await db.execute(
+ update(Workflow)
+ .where(Workflow._workflow_id == random_workflow.workflow_id.bytes)
+ .values(credentials_token=token, credentials_username=username)
+ )
+ await db.commit()
+
+ # Upload SCM file to PARAMS_BUCKET
+ scm_provider = Provider.from_repo(
+ build_repository(
+ random_workflow.repository_url, random_workflow_version.git_commit_hash, token=token, username=username
+ ),
+ name=f"repo{random_workflow.workflow_id.hex}",
+ )
+ assert scm_provider is not None
+ with BytesIO() as f:
+ SCM([scm_provider]).serialize(f)
+ f.seek(0)
+ mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(f"{random_workflow.workflow_id.hex}.scm").upload_fileobj(
+ f
+ )
+ yield random_workflow
+ # Delete SCM in PARAMS_BUCKET after tests
+ mock_s3_service.Bucket(settings.PARAMS_BUCKET).Object(f"{random_workflow.workflow_id.hex}.scm").delete()
+
+
@pytest_asyncio.fixture(scope="function")
async def random_workflow_version(db: AsyncSession, random_workflow: WorkflowOut) -> WorkflowVersion:
"""
diff --git a/app/tests/crud/test_workflow.py b/app/tests/crud/test_workflow.py
index aa7680f..95d1655 100644
--- a/app/tests/crud/test_workflow.py
+++ b/app/tests/crud/test_workflow.py
@@ -185,6 +185,30 @@ class TestWorkflowCRUDCreate:
await db.execute(delete(Workflow).where(Workflow._workflow_id == workflow.workflow_id.bytes))
await db.commit()
+ @pytest.mark.asyncio
+ async def test_create_workflow_credentials(self, db: AsyncSession, random_workflow: WorkflowOut) -> None:
+ """
+ Test for updating the workflow credentials in CRUD Repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ random_workflow : app.schemas.workflow.WorkflowOut
+ Random workflow for testing. pytest fixture.
+ """
+ token = random_lower_string(15)
+ username = random_lower_string(15)
+ await CRUDWorkflow.update_credentials(
+ db, workflow_id=random_workflow.workflow_id, token=token, username=username
+ )
+
+ stmt = select(Workflow).where(Workflow._workflow_id == random_workflow.workflow_id.bytes)
+ workflow = (await db.execute(stmt)).scalar()
+ assert workflow is not None
+ assert workflow.credentials_token == token
+ assert workflow.credentials_username == username
+
class TestWorkflowCRUDDelete:
@pytest.mark.asyncio
@@ -197,7 +221,7 @@ class TestWorkflowCRUDDelete:
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. pytest fixture.
random_workflow : app.schemas.workflow.WorkflowOut
- Random bucket for testing. pytest fixture.
+ Random workflow for testing. pytest fixture.
"""
await CRUDWorkflow.delete(db, workflow_id=random_workflow.workflow_id)
@@ -210,3 +234,25 @@ class TestWorkflowCRUDDelete:
versions = (await db.execute(stmt)).scalars().all()
assert len(versions) == 0
+
+ @pytest.mark.asyncio
+ async def test_delete_workflow_credentials(self, db: AsyncSession, random_private_workflow: WorkflowOut) -> None:
+ """
+ Test for deleting the workflow credentials in CRUD Repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. pytest fixture.
+ random_private_workflow : app.schemas.workflow.WorkflowOut
+ Random private workflow for testing. pytest fixture.
+ """
+ await CRUDWorkflow.update_credentials(
+ db, workflow_id=random_private_workflow.workflow_id, token=None, username=None
+ )
+
+ stmt = select(Workflow).where(Workflow._workflow_id == random_private_workflow.workflow_id.bytes)
+ workflow = (await db.execute(stmt)).scalar()
+ assert workflow is not None
+ assert workflow.credentials_token is None
+ assert workflow.credentials_username is None
--
GitLab