From 45de67c85579b37462ca0936a3429add7a742c96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20G=C3=B6bel?= <dgoebel@techfak.uni-bielefeld.de>
Date: Wed, 13 Sep 2023 11:22:27 +0200
Subject: [PATCH] Return URL when uploading an icon
* Resize icon before uploading it
#47
---
.pre-commit-config.yaml | 4 +-
app/api/endpoints/workflow_credentials.py | 2 +-
app/api/endpoints/workflow_version.py | 10 ++++-
app/api/utils.py | 37 +++++++++++++---
app/tests/api/test_workflow_version.py | 51 ++++++++++++++++++++++-
requirements-dev.txt | 2 +-
requirements.txt | 3 +-
7 files changed, 95 insertions(+), 14 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 8c0e4d5..c187c08 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,13 +15,13 @@ repos:
- id: check-merge-conflict
- id: check-ast
- repo: https://github.com/psf/black
- rev: 23.7.0
+ rev: 23.9.1
hooks:
- id: black
files: app
args: [--check]
- repo: https://github.com/charliermarsh/ruff-pre-commit
- rev: 'v0.0.287'
+ rev: 'v0.0.288'
hooks:
- id: ruff
- repo: https://github.com/PyCQA/isort
diff --git a/app/api/endpoints/workflow_credentials.py b/app/api/endpoints/workflow_credentials.py
index c4c62cb..a8d1090 100644
--- a/app/api/endpoints/workflow_credentials.py
+++ b/app/api/endpoints/workflow_credentials.py
@@ -53,7 +53,7 @@ async def delete_workflow_credentials(
await CRUDWorkflow.update_credentials(db, workflow.workflow_id, token=None, username=None)
-@router.put("", status_code=status.HTTP_200_OK, summary="Delete the credentials of a workflow")
+@router.put("", status_code=status.HTTP_200_OK, summary="Update the credentials of a workflow")
async def update_workflow_credentials(
credentials: WorkflowCredentialsIn,
workflow: CurrentWorkflow,
diff --git a/app/api/endpoints/workflow_version.py b/app/api/endpoints/workflow_version.py
index 6d75cab..4c667ae 100644
--- a/app/api/endpoints/workflow_version.py
+++ b/app/api/endpoints/workflow_version.py
@@ -16,6 +16,7 @@ from app.api.dependencies import (
S3Service,
)
from app.api.utils import delete_remote_icon, upload_icon
+from app.core.config import settings
from app.crud import CRUDWorkflowVersion
from app.git_repository import build_repository
from app.schemas.workflow_version import WorkflowVersion as WorkflowVersionSchema
@@ -307,7 +308,7 @@ async def upload_workflow_version_icon(
icon: UploadFile = File(..., description="Optional Icon for the Workflow."),
) -> str:
"""
- Upload an icon for the workflow version.\n
+ Upload an icon for the workflow version and returns the new icon URL.\n
Permission "workflow:update" required.
\f
Parameters
@@ -328,6 +329,11 @@ async def upload_workflow_version_icon(
Async database session to perform query on. Dependency Injection.
icon : fastapi.UploadFile
New Icon for the workflow version. HTML Form.
+
+ Returns
+ -------
+ icon_url : str
+ URL where the icon can be downloaded
"""
await authorization("update")
if current_user.uid != workflow.developer_id:
@@ -338,7 +344,7 @@ async def upload_workflow_version_icon(
# Delete old icon if possible
if old_slug is not None:
background_tasks.add_task(delete_remote_icon, s3=s3, db=db, icon_slug=old_slug)
- return icon_slug
+ return str(settings.OBJECT_GATEWAY_URI) + "/".join([settings.ICON_BUCKET, icon_slug])
@router.delete(
diff --git a/app/api/utils.py b/app/api/utils.py
index 3a6a362..101c0e7 100644
--- a/app/api/utils.py
+++ b/app/api/utils.py
@@ -2,7 +2,7 @@ import json
import re
from io import BytesIO
from tempfile import SpooledTemporaryFile
-from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Union
+from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Sequence, Union
from uuid import uuid4
import botocore.client
@@ -10,6 +10,7 @@ from clowmdb.models import WorkflowExecution, WorkflowMode
from fastapi import BackgroundTasks, HTTPException, UploadFile, status
from httpx import AsyncClient, ConnectError, ConnectTimeout
from mako.template import Template
+from PIL import Image, UnidentifiedImageError
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
@@ -49,14 +50,40 @@ def upload_icon(s3: S3ServiceResource, background_tasks: BackgroundTasks, icon:
icon_slug : str
Slug of the icon in the bucket
"""
- file_ending = icon.filename.split(".")[-1] # type: ignore[union-attr]
- icon_slug = f"{uuid4().hex}.{file_ending}"
- obj = s3.Bucket(name=settings.ICON_BUCKET).Object(key=icon_slug)
+ try:
+ Image.open(icon.file)
+ except UnidentifiedImageError:
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="icon needs to be an image")
+ icon_slug = f"{uuid4().hex}.png"
# Write the icon to the icon bucket in a background task
- background_tasks.add_task(obj.upload_fileobj, Fileobj=icon.file, ExtraArgs={"ContentType": icon.content_type})
+ background_tasks.add_task(process_and_upload_icon, s3=s3, icon_slug=icon_slug, icon_buffer=icon.file)
return icon_slug
+def process_and_upload_icon(s3: S3ServiceResource, icon_slug: str, icon_buffer: BinaryIO) -> None:
+ """
+ Process the icon and upload it to the S3 Icon Bucket
+
+ Parameters
+ ----------
+ s3 : boto3_type_annotations.s3.ServiceResource
+ S3 Service to perform operations on buckets in Ceph.
+ icon_slug : str
+ Slug of the icon
+ icon_buffer : typing.BinaryIO
+ Binary stream containing the image
+ """
+ im = Image.open(icon_buffer)
+ im.thumbnail((64, 64)) # Crop to 64x64 image
+ thumbnail_buffer = BytesIO()
+ im.save(thumbnail_buffer, "PNG") # save in buffer as PNG image
+ thumbnail_buffer.seek(0)
+ # Upload to bucket
+ s3.Bucket(name=settings.ICON_BUCKET).Object(key=icon_slug).upload_fileobj(
+ Fileobj=thumbnail_buffer, ExtraArgs={"ContentType": "image/png"}
+ )
+
+
async def delete_remote_icon(s3: S3ServiceResource, db: AsyncSession, icon_slug: str) -> None:
"""
Delete icon in S3 Bucket if there are no other workflow versions that depend on it
diff --git a/app/tests/api/test_workflow_version.py b/app/tests/api/test_workflow_version.py
index c601d70..fb1dbb6 100644
--- a/app/tests/api/test_workflow_version.py
+++ b/app/tests/api/test_workflow_version.py
@@ -5,6 +5,7 @@ import pytest
from clowmdb.models import WorkflowMode, WorkflowVersion
from fastapi import status
from httpx import AsyncClient
+from PIL import Image
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -361,7 +362,10 @@ class TestWorkflowVersionIconRoutes(_TestWorkflowVersionRoutes):
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on.
"""
- files = {"icon": ("RickRoll.txt", BytesIO(b"Never gonna give you up"), "plain/text")}
+ img_buffer = BytesIO()
+ Image.linear_gradient(mode="L").save(img_buffer, "PNG")
+ img_buffer.seek(0)
+ files = {"icon": ("RickRoll.png", img_buffer, "image/png")}
response = await client.post(
"/".join(
[
@@ -376,7 +380,8 @@ class TestWorkflowVersionIconRoutes(_TestWorkflowVersionRoutes):
files=files,
)
assert response.status_code == status.HTTP_201_CREATED
- icon_slug = response.json()
+ icon_url = response.json()
+ icon_slug = icon_url.split("/")[-1]
assert icon_slug in mock_s3_service.Bucket(settings.ICON_BUCKET).objects.all_keys()
db_version = await db.scalar(
select(WorkflowVersion).where(WorkflowVersion.git_commit_hash == random_workflow_version.git_commit_hash)
@@ -387,6 +392,48 @@ class TestWorkflowVersionIconRoutes(_TestWorkflowVersionRoutes):
# Clean up
mock_s3_service.Bucket(settings.ICON_BUCKET).Object(icon_slug).delete()
+ @pytest.mark.asyncio
+ async def test_upload_new_icon_as_text(
+ self,
+ client: AsyncClient,
+ random_user: UserWithAuthHeader,
+ random_workflow_version: WorkflowVersion,
+ mock_s3_service: MockS3ServiceResource,
+ db: AsyncSession,
+ ) -> None:
+ """
+ Test for uploading a textfile as a new icon for a workflow version
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing.
+ random_workflow_version : clowmdb.models.WorkflowVersion
+ Random workflow version for testing.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ """
+
+ files = {"icon": ("RickRoll.png", BytesIO(b"Never gonna give you up"), "text/plain")}
+ response = await client.post(
+ "/".join(
+ [
+ self.base_path,
+ str(random_workflow_version.workflow_id),
+ "versions",
+ random_workflow_version.git_commit_hash,
+ "icon",
+ ]
+ ),
+ headers=random_user.auth_headers,
+ files=files,
+ )
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+
@pytest.mark.asyncio
async def test_upload_new_icon_as_non_developer(
self,
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 31f20b7..810ec95 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -5,7 +5,7 @@ pytest-cov>=4.1.0,<4.2.0
coverage[toml]>=7.3.0,<7.4.0
# Linters
ruff
-black>=23.07.0,<23.08.0
+black>=23.09.0,<23.10.0
isort>=5.12.0,<5.13.0
mypy>=1.5.0,<1.6.0
# stubs for mypy
diff --git a/requirements.txt b/requirements.txt
index 305fcb3..2dfa1b7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,8 +18,9 @@ authlib>=1.2.0,<1.3.0
boto3>=1.28.0,<1.29.0
# Miscellaneous
tenacity>=8.2.0,<8.3.0
-httpx>=0.24.0,<0.25.0
+httpx>=0.25.0,<0.26.0
itsdangerous
jsonschema>=4.0.0,<5.0.0
mako
python-dotenv
+Pillow>=10.0.0,<10.1.0
--
GitLab