Update clowmdb to 3

#8

app/tests/api/test_security.py

@@ -54,8 +54,9 @@ class TestJWTProtectedRoutes:
             Random user for testing.
         """
         response = await client.get(
-            self.protected_route, params={"user": random_user.user.uid}, headers=random_user.auth_headers
+            self.protected_route, params={"maintainer_id": str(random_user.user.uid)}, headers=random_user.auth_headers
         )
+        print(response.json())
         assert response.status_code == status.HTTP_200_OK
 
     @pytest.mark.asyncio

app/tests/conftest.py

@@ -84,6 +84,13 @@ async def mock_client(
         yield http_client
 
 
+@pytest.fixture(autouse=True)
+def monkeypatch_environment(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("OTLP_GRPC_ENDPOINT", "")
+
+
 @pytest.fixture(autouse=True)
 def monkeypatch_background_tasks(
     monkeypatch: pytest.MonkeyPatch,
@@ -107,7 +114,6 @@ def monkeypatch_background_tasks(
     monkeypatch.setattr(dependencies, "get_db", get_patch_db)
     monkeypatch.setattr(resource_s3_utils, "get_s3_resource", get_s3)
     monkeypatch.setattr(resource_cluster_utils, "get_async_http_client", get_http_client)
-    monkeypatch.setenv("OTLP_GRPC_ENDPOINT", "")
 
 
 @pytest_asyncio.fixture(scope="module")
@@ -184,9 +190,9 @@ async def random_user(db: AsyncSession, mock_opa_service: MockOpaService) -> Asy
     Create a random user and deletes him afterward.
     """
     user = await create_random_user(db)
-    mock_opa_service.add_user(user.uid, privileged=True)
+    mock_opa_service.add_user(user.lifescience_id, privileged=True)
     yield UserWithAuthHeader(user=user, auth_headers=get_authorization_headers(uid=user.uid, secret=jwt_secret))
-    mock_opa_service.delete_user(user.uid)
+    mock_opa_service.delete_user(user.lifescience_id)
     await db.delete(user)
     await db.commit()
 
@@ -197,9 +203,9 @@ async def random_second_user(db: AsyncSession, mock_opa_service: MockOpaService)
     Create a random second user and deletes him afterward.
     """
     user = await create_random_user(db)
-    mock_opa_service.add_user(user.uid)
+    mock_opa_service.add_user(user.lifescience_id)
     yield UserWithAuthHeader(user=user, auth_headers=get_authorization_headers(uid=user.uid, secret=jwt_secret))
-    mock_opa_service.delete_user(user.uid)
+    mock_opa_service.delete_user(user.lifescience_id)
     await db.delete(user)
     await db.commit()
 
@@ -210,9 +216,9 @@ async def random_third_user(db: AsyncSession, mock_opa_service: MockOpaService)
     Create a random third user and deletes him afterward.
     """
     user = await create_random_user(db)
-    mock_opa_service.add_user(user.uid)
+    mock_opa_service.add_user(user.lifescience_id)
     yield UserWithAuthHeader(user=user, auth_headers=get_authorization_headers(uid=user.uid, secret=jwt_secret))
-    mock_opa_service.delete_user(user.uid)
+    mock_opa_service.delete_user(user.lifescience_id)
     await db.delete(user)
     await db.commit()
 
@@ -231,7 +237,7 @@ async def random_resource(
         name=random_lower_string(8),
         short_description=random_lower_string(32),
         source=random_lower_string(32),
-        maintainer_id=random_user.user.uid,
+        _maintainer_id=random_user.user.uid.bytes,
     )
     db.add(resource_db)
     await db.commit()

app/tests/crud/test_user.py

+from uuid import uuid4
+
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.crud import CRUDUser
 from app.tests.utils.user import UserWithAuthHeader
-from app.tests.utils.utils import random_hex_string
 
 
 class TestUserCRUD:
@@ -33,5 +34,5 @@ class TestUserCRUD:
         db : sqlalchemy.ext.asyncio.AsyncSession.
             Async database session to perform query on.
         """
-        user = await CRUDUser.get(db, random_hex_string())
+        user = await CRUDUser.get(db, uuid4())
         assert user is None

app/tests/mocks/__init__.py

@@ -6,8 +6,7 @@ from httpx import Request, Response
 
 class MockHTTPService(ABC):
     @abstractmethod
-    def handle_request(self, request: Request, **kwargs: bool) -> Response:
-        ...
+    def handle_request(self, request: Request, **kwargs: bool) -> Response: ...
 
 
 class DefaultMockHTTPService(MockHTTPService):

app/tests/utils/bucket.py

 from datetime import datetime
 from typing import Optional
+from uuid import UUID
 
-import pytest
 from clowmdb.models import Bucket, BucketPermission, User
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from .utils import random_lower_string
 
 
-@pytest.mark.asyncio
 async def create_random_bucket(db: AsyncSession, user: User) -> Bucket:
     """
     Creates a random bucket in the database.
@@ -28,18 +27,17 @@ async def create_random_bucket(db: AsyncSession, user: User) -> Bucket:
     bucket = Bucket(
         name=random_lower_string(),
         description=random_lower_string(length=127),
-        owner_id=user.uid,
+        _owner_id=user.uid.bytes,
     )
     db.add(bucket)
     await db.commit()
     return bucket
 
 
-@pytest.mark.asyncio
 async def add_permission_for_bucket(
     db: AsyncSession,
     bucket_name: str,
-    uid: str,
+    uid: UUID,
     from_: Optional[datetime] = None,
     to: Optional[datetime] = None,
     permission: BucketPermission.Permission = BucketPermission.Permission.READWRITE,
@@ -66,7 +64,7 @@ async def add_permission_for_bucket(
         The file prefix for the permission.
     """
     perm = BucketPermission(
-        user_id=uid,
+        _uid=uid.bytes,
         bucket_name=bucket_name,
         from_=round(from_.timestamp()) if from_ is not None else None,
         to=round(to.timestamp()) if to is not None else None,

app/tests/utils/user.py

 from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import UTC, datetime, timedelta
 from typing import Dict
+from uuid import UUID
 
-import pytest
 from authlib.jose import JsonWebToken
 from clowmdb.models import User
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -18,7 +18,7 @@ class UserWithAuthHeader:
     user: User
 
 
-def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> Dict[str, str]:
+def get_authorization_headers(uid: UUID, secret: str = "SuperSecret") -> Dict[str, str]:
     """
     Create a valid JWT and return the correct headers for subsequent requests.
 
@@ -33,7 +33,7 @@ def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> Dict[str
     headers : Dict[str,str]
         HTTP Headers to authorize each request.
     """
-    to_encode = {"sub": uid, "exp": datetime.utcnow() + timedelta(hours=1)}
+    to_encode = {"sub": str(uid), "exp": datetime.now(UTC) + timedelta(hours=1)}
     encoded_jwt = _jwt.encode(header={"alg": "HS256"}, payload=to_encode, key=secret)
 
     headers = {"Authorization": f"Bearer {encoded_jwt.decode('utf-8')}"}
@@ -68,7 +68,6 @@ def decode_mock_token(token: str, secret: str = "SuperSecret") -> Dict[str, str]
     return claims
 
 
-@pytest.mark.asyncio
 async def create_random_user(db: AsyncSession) -> User:
     """
     Creates a random user in the database.
@@ -84,7 +83,7 @@ async def create_random_user(db: AsyncSession) -> User:
         Newly created user.
     """
     user = User(
-        uid=random_hex_string(),
+        lifescience_id=random_hex_string(),
         display_name=random_lower_string(),
     )
     db.add(user)

gunicorn_conf.py

+import json
+import multiprocessing
+import os
+
+workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1")
+max_workers_str = os.getenv("MAX_WORKERS")
+use_max_workers = None
+if max_workers_str:
+    use_max_workers = int(max_workers_str)
+web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
+
+host = os.getenv("HOST", "0.0.0.0")
+port = os.getenv("PORT", "80")
+bind_env = os.getenv("BIND", None)
+use_loglevel = os.getenv("LOG_LEVEL", "info")
+if bind_env:
+    use_bind = bind_env
+else:
+    use_bind = f"{host}:{port}"
+
+cores = multiprocessing.cpu_count()
+workers_per_core = float(workers_per_core_str)
+default_web_concurrency = workers_per_core * cores
+if web_concurrency_str:
+    web_concurrency = int(web_concurrency_str)
+    assert web_concurrency > 0
+else:
+    web_concurrency = max(int(default_web_concurrency), 2)
+    if use_max_workers:
+        web_concurrency = min(web_concurrency, use_max_workers)
+accesslog_var = os.getenv("ACCESS_LOG", "-")
+use_accesslog = accesslog_var or None
+errorlog_var = os.getenv("ERROR_LOG", "-")
+use_errorlog = errorlog_var or None
+graceful_timeout_str = os.getenv("GRACEFUL_TIMEOUT", "120")
+timeout_str = os.getenv("TIMEOUT", "120")
+keepalive_str = os.getenv("KEEP_ALIVE", "5")
+
+# Gunicorn config variables
+loglevel = use_loglevel
+workers = web_concurrency
+bind = use_bind
+errorlog = use_errorlog
+worker_tmp_dir = "/dev/shm"
+accesslog = use_accesslog
+graceful_timeout = int(graceful_timeout_str)
+timeout = int(timeout_str)
+keepalive = int(keepalive_str)
+
+
+# For debugging and testing
+log_data = {
+    "loglevel": loglevel,
+    "workers": workers,
+    "bind": bind,
+    "graceful_timeout": graceful_timeout,
+    "timeout": timeout,
+    "keepalive": keepalive,
+    "errorlog": errorlog,
+    "accesslog": accesslog,
+    # Additional, non-gunicorn variables
+    "workers_per_core": workers_per_core,
+    "use_max_workers": use_max_workers,
+    "host": host,
+    "port": port,
+}
+print(json.dumps(log_data))

pyproject.toml

@@ -8,7 +8,7 @@ line-length = 120
 
 [tool.ruff]
 line-length = 120
-target-version = "py312"
+target-version = "py311"
 
 [tool.mypy]
 plugins = ["pydantic.mypy", "sqlalchemy.ext.mypy.plugin"]

requirements-dev.txt

 # test packages
-pytest>=7.4.0,<7.5.0
+pytest>=8.0.0,<8.1.0
 pytest-asyncio>=0.21.0,<0.22.0
 pytest-cov>=4.1.0,<4.2.0
 coverage[toml]>=7.4.0,<7.5.0
 # Linters
 ruff>=0.1.0,<0.2.0
-black>=23.12.0,<24.0.0
+black>=24.1.0,<24.2.0
 isort>=5.13.0,<5.14.0
 mypy>=1.8.0,<1.9.0
 # stubs for mypy
@@ -13,3 +13,4 @@ types-aiobotocore-lite[s3]>=2.9.0,<2.10.0
 types-requests
 # Miscellaneous
 pre-commit>=3.6.0,<3.7.0
+uvicorn>=0.27.0,<0.28.0

requirements.txt

 --extra-index-url https://gitlab.ub.uni-bielefeld.de/api/v4/projects/5493/packages/pypi/simple
-clowmdb>=2.3.0,<2.4.0
+clowmdb>=3.0.0,<3.1.0
 
 # Webserver packages
 fastapi>=0.109.0,<0.110.0
-pydantic>=2.5.0,<2.6.0
+pydantic>=2.6.0,<2.7.0
 pydantic-settings>=2.1.0,<2.2.0
-uvicorn>=0.25.0,<0.26.0
 # Database packages
 PyMySQL>=1.1.0,<1.2.0
 SQLAlchemy>=2.0.0,<2.1.0

start_service_gunicorn.sh

+#! /usr/bin/env sh
+set -e
+
+./prestart.sh
+
+# Start Gunicorn
+exec gunicorn -k uvicorn.workers.UvicornWorker -c /app/gunicorn_conf.py app.main:app

start_service.sh → start_service_uvicorn.sh

 #! /usr/bin/env bash
+set -e
 
-./scripts/prestart.sh
+./prestart.sh
 
 # Start webserver
 uvicorn app.main:app --host 0.0.0.0 --port 8000 --no-server-header