diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1e95cae38c963b08757fed01b201e737d46cae04..2fbd1cea20033ecdee30bace8ee94f1b80d5eeb6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: python:3.10-slim
+image: python:3.11-slim
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -52,7 +52,7 @@ integration-test-job: # Runs integration tests with the database
MYSQL_DATABASE: "$DB_DATABASE"
MYSQL_USER: "$DB_USER"
MYSQL_PASSWORD: "$DB_PASSWORD"
- - name: $CI_REGISTRY/cmg/clowm/clowm-database:v1.3
+ - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0
alias: upgrade-db
script:
- python app/check_database_connection.py
@@ -80,7 +80,7 @@ e2e-test-job: # Runs e2e tests on the API endpoints
MYSQL_DATABASE: "$DB_DATABASE"
MYSQL_USER: "$DB_USER"
MYSQL_PASSWORD: "$DB_PASSWORD"
- - name: $CI_REGISTRY/cmg/clowm/clowm-database:v1.3
+ - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0
alias: upgrade-db
script:
- python app/check_database_connection.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c9c2bec414a7395906f242a2f1b1cf6104fc39e1..18698343b729f362c7000d45a7f24683cbba680e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -33,9 +33,8 @@ repos:
files: app
args: [--config=pyproject.toml]
additional_dependencies:
- - sqlalchemy2-stubs
- boto3-stubs-lite[s3]
- - sqlalchemy<2.0.0
+ - sqlalchemy>=2.0.0,<2.1.0
- pydantic
- types-requests
- repo: https://github.com/PyCQA/isort
diff --git a/Dockerfile b/Dockerfile
index c5154b0e0f4dcdbaa2251f6dcb9c9db3c4161790..b791fde634f0c5dcc5dbbec9286cf43a47feb1f4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
EXPOSE 8000
# dumb-init forwards the kill signal to the python process
diff --git a/Dockerfile-Gunicorn b/Dockerfile-Gunicorn
index 7201aafe5df03369ca30def9b3e2ab9e189370ef..64c2d08e3a2ab2e2f746e0562b1c252008f2cfaa 100644
--- a/Dockerfile-Gunicorn
+++ b/Dockerfile-Gunicorn
@@ -1,4 +1,4 @@
-FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10-slim
+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim
EXPOSE 8000
ENV PORT=8000
diff --git a/README.md b/README.md
index 80e906327478d9b229abd30c9ea56987d51cf66d..cfbff3d3dc826cbaff2fef99d21a20ad8548bfe3 100644
--- a/README.md
+++ b/README.md
@@ -45,7 +45,6 @@ user-friendly manner. ðŸ‘
| `BACKEND_CORS_ORIGINS` | `[]` | json formatted list of urls | List of valid CORS origins |
| `SQLALCHEMY_VERBOSE_LOGGER` | `false` | `<"true"|"false">` | Enables verbose SQL output.<br>Should be `false` in production |
| `OPA_POLICY_PATH` | `/clowm/authz/allow` | URL path | Path to the OPA Policy for Authorization |
-| `CEPH_TENANT` | unset | string | Tenant in the Ceph RGW where the users are buckets are created |
## Getting started
This service depends on multiple other services. See [DEVELOPING.md](DEVELOPING.md) how to set these up for developing
diff --git a/app/api/api.py b/app/api/api.py
index c092480f721152454a195a8ab512be595b151bd6..3ba0060f85d9dc0c78a3ea8ff2d2afded3f9e462 100644
--- a/app/api/api.py
+++ b/app/api/api.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Dict, Union
from fastapi import APIRouter, Depends, status
@@ -6,7 +6,7 @@ from app.api.dependencies import decode_bearer_token
from app.api.endpoints import bucket_permissions, buckets, s3key
from app.schemas.security import ErrorDetail
-alternative_responses: dict[int | str, dict[str, Any]] = {
+alternative_responses: Dict[Union[int, str], Dict[str, Any]] = {
status.HTTP_400_BAD_REQUEST: {
"model": ErrorDetail,
"description": "Error decoding JWT Token",
diff --git a/app/api/dependencies.py b/app/api/dependencies.py
index 7463e439e5de730c5915963bc67fda10cdb52573..b34e92d7ebac52da32941a62fd209c759f5fdef5 100644
--- a/app/api/dependencies.py
+++ b/app/api/dependencies.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Annotated, AsyncGenerator, Awaitable, Callable
+from typing import TYPE_CHECKING, Annotated, AsyncGenerator, Awaitable, Callable, Dict
from authlib.jose.errors import BadSignatureError, DecodeError, ExpiredTokenError
from clowmdb.db.session import get_async_session
@@ -52,7 +52,7 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]:
"""
async with get_async_session(
str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
- )() as db:
+ ) as db:
yield db
@@ -75,7 +75,7 @@ async def get_httpx_client() -> AsyncGenerator[AsyncClient, None]: # pragma: no
HTTPXClient = Annotated[AsyncClient, Depends(get_httpx_client)]
-def get_decode_jwt_function() -> Callable[[str], dict[str, str]]: # pragma: no cover
+def get_decode_jwt_function() -> Callable[[str], Dict[str, str]]: # pragma: no cover
"""
Get function to decode and verify the JWT.
@@ -84,7 +84,7 @@ def get_decode_jwt_function() -> Callable[[str], dict[str, str]]: # pragma: no
Returns
-------
- decode : Callable[[str], dict[str, str]]
+ decode : Callable[[str], Dict[str, str]]
Function to decode & verify the token. raw_token -> claims. Dependency Injection
"""
return decode_token
@@ -92,7 +92,7 @@ def get_decode_jwt_function() -> Callable[[str], dict[str, str]]: # pragma: no
def decode_bearer_token(
token: HTTPAuthorizationCredentials = Depends(bearer_token),
- decode: Callable[[str], dict[str, str]] = Depends(get_decode_jwt_function),
+ decode: Callable[[str], Dict[str, str]] = Depends(get_decode_jwt_function),
) -> JWT:
"""
Get the decoded JWT or reject request if it is not valid.
@@ -103,7 +103,7 @@ def decode_bearer_token(
----------
token : fastapi.security.http.HTTPAuthorizationCredentials
Bearer token sent with the HTTP request. Dependency Injection.
- decode : Callable[[str], dict[str, str]]
+ decode : Callable[[str], Dict[str, str]]
Function to decode & verify the token. raw_token -> claims. Dependency Injection
Returns
-------
diff --git a/app/api/endpoints/bucket_permissions.py b/app/api/endpoints/bucket_permissions.py
index f6c5a57f50bd8b2ff500df8303516f4272f87b35..ccb337b9b908c4a8653e8c1eea855f2ed932bcf5 100644
--- a/app/api/endpoints/bucket_permissions.py
+++ b/app/api/endpoints/bucket_permissions.py
@@ -1,5 +1,5 @@
import json
-from typing import Annotated, Any, Awaitable, Callable
+from typing import Annotated, Any, Awaitable, Callable, List, Optional
from clowmdb.models import BucketPermission
from clowmdb.models import User as UserDB
@@ -133,7 +133,7 @@ async def delete_permission_for_bucket(
@router.get(
"/bucket/{bucket_name}",
- response_model=list[PermissionSchemaOut],
+ response_model=List[PermissionSchemaOut],
summary="Get all permissions for a bucket.",
response_model_exclude_none=True,
)
@@ -142,11 +142,13 @@ async def list_permissions_per_bucket(
db: DBSession,
current_user: CurrentUser,
authorization: Authorization,
- permission_type: list[BucketPermission.Permission]
- | None = Query(None, description="Type of Bucket Permissions to fetch"),
- permission_status: CRUDBucketPermission.PermissionStatus
- | None = Query(None, description="Status of Bucket Permissions to fetch"),
-) -> list[PermissionSchemaOut]:
+ permission_type: Optional[List[BucketPermission.Permission]] = Query(
+ None, description="Type of Bucket Permissions to fetch"
+ ),
+ permission_status: Optional[CRUDBucketPermission.PermissionStatus] = Query(
+ None, description="Status of Bucket Permissions to fetch"
+ ),
+) -> List[PermissionSchemaOut]:
"""
List all the bucket permissions for the given bucket.\n
Permission "bucket_permission:read" required if current user is owner of the bucket,
@@ -154,7 +156,7 @@ async def list_permissions_per_bucket(
\f
Parameters
----------
- permission_type : list[clowmdb.models.BucketPermission.Permission] | None, default None
+ permission_type : List[clowmdb.models.BucketPermission.Permission] | None, default None
Type of Bucket Permissions to fetch. Query Parameter
permission_status : app.crud.crud_bucket_permission.CRUDBucketPermission.PermissionStatus | None, default None
Status of Bucket Permissions to fetch. Query Parameter.
@@ -169,7 +171,7 @@ async def list_permissions_per_bucket(
Returns
-------
- permissions : list[app.schemas.bucket_permission.BucketPermissionOut]
+ permissions : List[app.schemas.bucket_permission.BucketPermissionOut]
List of all permissions for this bucket.
"""
rbac_operation = "list_bucket" if bucket.owner_id == current_user.uid else "list_all"
@@ -182,7 +184,7 @@ async def list_permissions_per_bucket(
@router.get(
"/user/{uid}",
- response_model=list[PermissionSchemaOut],
+ response_model=List[PermissionSchemaOut],
summary="Get all permissions for a user.",
response_model_exclude_none=True,
)
@@ -190,12 +192,14 @@ async def list_permissions_per_user(
db: DBSession,
current_user: CurrentUser,
authorization: Authorization,
- permission_type: list[BucketPermission.Permission]
- | None = Query(None, description="Type of Bucket Permissions to fetch"),
- permission_status: CRUDBucketPermission.PermissionStatus
- | None = Query(None, description="Status of Bucket Permissions to fetch"),
+ permission_type: Optional[List[BucketPermission.Permission]] = Query(
+ None, description="Type of Bucket Permissions to fetch"
+ ),
+ permission_status: Optional[CRUDBucketPermission.PermissionStatus] = Query(
+ None, description="Status of Bucket Permissions to fetch"
+ ),
user: UserDB = Depends(get_user_by_path_uid),
-) -> list[PermissionSchemaOut]:
+) -> List[PermissionSchemaOut]:
"""
List all the bucket permissions for the given user.\n
Permission "bucket_permission:read" required if current user is the target the bucket permission,
@@ -203,7 +207,7 @@ async def list_permissions_per_user(
\f
Parameters
----------
- permission_type : list[clowmdb.models.BucketPermission.Permission] | None, default None
+ permission_type : List[clowmdb.models.BucketPermission.Permission] | None, default None
Type of Bucket Permissions to fetch. Query Parameter
permission_status : app.crud.crud_bucket_permission.CRUDBucketPermission.PermissionStatus | None, default None
Status of Bucket Permissions to fetch. Query Parameter.
@@ -218,7 +222,7 @@ async def list_permissions_per_user(
Returns
-------
- permissions : list[app.schemas.bucket_permission.BucketPermissionOut]
+ permissions : List[app.schemas.bucket_permission.BucketPermissionOut]
List of all permissions for this user.
"""
rbac_operation = "list_user" if user == current_user else "list_all"
diff --git a/app/api/endpoints/buckets.py b/app/api/endpoints/buckets.py
index fa09f6ebbdf71912cd897b6cecc3ad17cda9d291..8888ad51786d01aedffe848a55f9f8ee099bf222 100644
--- a/app/api/endpoints/buckets.py
+++ b/app/api/endpoints/buckets.py
@@ -1,6 +1,6 @@
import json
from functools import reduce
-from typing import TYPE_CHECKING, Annotated, Any, Awaitable, Callable
+from typing import TYPE_CHECKING, Annotated, Any, Awaitable, Callable, List, Optional
from botocore.exceptions import ClientError
from fastapi import APIRouter, Depends, HTTPException, Path, Query, status
@@ -23,21 +23,20 @@ bucket_authorization = AuthorizationDependency(resource="bucket")
Authorization = Annotated[Callable[[str], Awaitable[Any]], Depends(bucket_authorization)]
-@router.get("", response_model=list[BucketOutSchema], summary="List buckets of user")
+@router.get("", response_model=List[BucketOutSchema], summary="List buckets of user")
async def list_buckets(
db: DBSession,
s3: S3Resource,
current_user: CurrentUser,
authorization: Authorization,
- user: str
- | None = Query(
+ user: Optional[str] = Query(
None,
description="UID of the user for whom to fetch the buckets for. Permission 'bucket:read_any' required if current user is not the target.", # noqa:E501
),
bucket_type: CRUDBucket.BucketType = Query(
CRUDBucket.BucketType.ALL, description="Type of the bucket to get. Ignored when `user` parameter not set"
),
-) -> list[BucketOutSchema]:
+) -> List[BucketOutSchema]:
"""
List all the buckets in the system or of the desired user where the user has READ permissions for.\n
Permission "bucket:read" required.
@@ -58,7 +57,7 @@ async def list_buckets(
Async function to ask the auth service for authorization. Dependency Injection.
Returns
-------
- buckets : list[app.schemas.bucket.BucketOut]
+ buckets : List[app.schemas.bucket.BucketOut]
All the buckets for which the user has READ permissions.
"""
await authorization("list_all" if user is None or current_user.uid != user else "list")
@@ -202,7 +201,7 @@ async def get_bucket(
)
await authorization(rbac_operation)
s3bucket = s3.Bucket(name=bucket.name)
- objects: list[ObjectSummary] = list(s3bucket.objects.all())
+ objects: List[ObjectSummary] = list(s3bucket.objects.all())
return BucketOutSchema(
**{
"description": bucket.description,
@@ -260,7 +259,7 @@ async def delete_bucket(
@router.get(
"/{bucket_name}/objects",
- response_model=list[S3ObjectMetaInformation],
+ response_model=List[S3ObjectMetaInformation],
tags=["Object"],
summary="Get the metadata of the objects in the bucket",
deprecated=True,
@@ -271,8 +270,8 @@ async def get_bucket_objects(
current_user: CurrentUser,
db: DBSession,
authorization: Authorization,
- file_prefix: str | None = Query(None, description="Get only objects with the specified prefix."),
-) -> list[S3ObjectMetaInformation]:
+ file_prefix: Optional[str] = Query(None, description="Get only objects with the specified prefix."),
+) -> List[S3ObjectMetaInformation]:
"""
Get the metadata of the objects in the bucket.
@@ -298,7 +297,7 @@ async def get_bucket_objects(
Returns
-------
- objs : list[app.schemas.bucket.S3ObjectMetaInformation]
+ objs : List[app.schemas.bucket.S3ObjectMetaInformation]
Meta information about all objects in the bucket.
"""
rbac_operation = (
diff --git a/app/api/endpoints/s3key.py b/app/api/endpoints/s3key.py
index 562b05d20a5fdb045c20338ab050935793a0137b..9e15727f0ffbc82448b3aadaaf5667d67121b8e4 100644
--- a/app/api/endpoints/s3key.py
+++ b/app/api/endpoints/s3key.py
@@ -1,4 +1,4 @@
-from typing import Annotated, Any, Awaitable, Callable
+from typing import Annotated, Any, Awaitable, Callable, List
from clowmdb.models import User as UserDB
from fastapi import APIRouter, Depends, HTTPException, Path, status
@@ -23,7 +23,7 @@ AccessID = Annotated[
@router.get(
"",
- response_model=list[S3Key],
+ response_model=List[S3Key],
summary="Get the S3 Access keys from a user",
)
async def get_user_keys(
@@ -31,7 +31,7 @@ async def get_user_keys(
current_user: CurrentUser,
authorization: Authorization,
user: UserDB = Depends(get_user_by_path_uid),
-) -> list[S3Key]:
+) -> List[S3Key]:
"""
Get all the S3 Access keys for a specific user.\n
Permission "s3_key:list" required.
@@ -49,7 +49,7 @@ async def get_user_keys(
Returns
-------
- keys : list(app.schemas.user.S3Key)
+ keys : List(app.schemas.user.S3Key)
All S3 keys from the user.
"""
if current_user.uid != user.uid:
diff --git a/app/api/miscellaneous_endpoints.py b/app/api/miscellaneous_endpoints.py
index 8736dbbe0ec593f9a3a9e862630390b9a22346e5..0f4f786b06506ed0843fceb5f84a7a791441ab2e 100644
--- a/app/api/miscellaneous_endpoints.py
+++ b/app/api/miscellaneous_endpoints.py
@@ -1,3 +1,5 @@
+from typing import Dict
+
from fastapi import APIRouter, status
miscellaneous_router = APIRouter(include_in_schema=False)
@@ -13,14 +15,14 @@ miscellaneous_router = APIRouter(include_in_schema=False)
},
},
)
-def health_check() -> dict[str, str]:
+def health_check() -> Dict[str, str]:
"""
Check if the service is reachable.
\f
Returns
-------
- response : dict[str, str]
+ response : Dict[str, str]
status ok
"""
return {"status": "OK"}
diff --git a/app/check_ceph_connection.py b/app/check_ceph_connection.py
index 71a7cb7d2d2dda71d13efc8deb6560e83814e363..4a3716ea39af355a8183d2441fdf8560aa1409ea 100644
--- a/app/check_ceph_connection.py
+++ b/app/check_ceph_connection.py
@@ -8,7 +8,7 @@ from app.core.config import settings
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-max_tries = 60 * 3 # 3 minutes
+max_tries = 30 # 2*30 seconds
wait_seconds = 2
diff --git a/app/check_database_connection.py b/app/check_database_connection.py
index ae54e93d69f91628dd1b90245652923206877406..30102f8561f941951fee1bef510f4ab8b2376b81 100644
--- a/app/check_database_connection.py
+++ b/app/check_database_connection.py
@@ -10,8 +10,8 @@ from app.core.config import settings
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-max_tries = 60 * 3 # 3 minutes
-wait_seconds = 2
+max_tries = 30 # 3*30 seconds
+wait_seconds = 3
@retry(
@@ -22,7 +22,7 @@ wait_seconds = 2
)
def init() -> None:
try:
- with get_session(url=str(settings.SQLALCHEMY_DATABASE_NORMAL_URI))() as db:
+ with get_session(url=str(settings.SQLALCHEMY_DATABASE_NORMAL_URI)) as db:
# Try to create session to check if DB is awake
db_revision = db.execute(text("SELECT version_num FROM alembic_version LIMIT 1")).scalar_one_or_none()
if db_revision != latest_revision:
diff --git a/app/core/config.py b/app/core/config.py
index 15eda68dd79ff1ec76e5464ed5218896afbb19d7..1492a586239abdac5f4112e18790ba006c8d4f0f 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -31,10 +31,10 @@ def _load_public_key(pub_key_val: Optional[str], pub_key_file: Optional[Path]) -
class Settings(BaseSettings):
API_PREFIX: str = Field("/api", description="Path Prefix for all API endpoints.")
- public_key_value: str | None = Field(
+ public_key_value: Optional[str] = Field(
None, description="Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_VALUE"
)
- public_key_file: Path | None = Field(
+ public_key_file: Optional[Path] = Field(
None, description="Path to Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_FILE"
)
@@ -106,7 +106,6 @@ class Settings(BaseSettings):
BUCKET_CEPH_USERNAME: str = Field(
..., description="ID of the user in ceph who owns all the buckets. Owner of 'BUCKET_CEPH_ACCESS_KEY'"
)
- CEPH_TENANT: str = Field("", description="Tenant in the Ceph RGW where the users are buckets are created")
OPA_URI: AnyHttpUrl = Field(..., description="URI of the OPA Service")
OPA_POLICY_PATH: str = Field("/clowm/authz/allow", description="Path to the OPA Policy for Authorization")
model_config = SettingsConfigDict(case_sensitive=True, env_file=".env", secrets_dir="/run/secrets")
diff --git a/app/core/security.py b/app/core/security.py
index 60f9fa2f6074b78fdcca0a4c73910d58189f8016..c09999a308fb7fc31cad83f4c59a94dc82b29dc4 100644
--- a/app/core/security.py
+++ b/app/core/security.py
@@ -1,3 +1,5 @@
+from typing import Dict
+
from authlib.jose import JsonWebToken
from fastapi import HTTPException, status
from httpx import AsyncClient
@@ -10,7 +12,7 @@ ALGORITHM = "RS256"
jwt = JsonWebToken([ALGORITHM])
-def decode_token(token: str) -> dict[str, str]: # pragma: no cover
+def decode_token(token: str) -> Dict[str, str]: # pragma: no cover
"""
Decode and verify a JWT token.
@@ -21,7 +23,7 @@ def decode_token(token: str) -> dict[str, str]: # pragma: no cover
Returns
-------
- decoded_token : dict[str, str]
+ decoded_token : Dict[str, str]
Payload of the decoded token.
"""
claims = jwt.decode(
diff --git a/app/crud/crud_bucket.py b/app/crud/crud_bucket.py
index d069cb9905a90da3ee5f2eb10608d46ff925a4bd..7d7460acf247531d424196e0c62d940059dbf279 100644
--- a/app/crud/crud_bucket.py
+++ b/app/crud/crud_bucket.py
@@ -1,4 +1,5 @@
from enum import Enum, unique
+from typing import Optional, Sequence
from clowmdb.models import Bucket
from clowmdb.models import BucketPermission as BucketPermissionDB
@@ -24,7 +25,7 @@ class CRUDBucket:
PERMISSION: str = "PERMISSION"
@staticmethod
- async def get(db: AsyncSession, bucket_name: str) -> Bucket | None:
+ async def get(db: AsyncSession, bucket_name: str) -> Optional[Bucket]:
"""
Get a bucket by its name.
@@ -45,13 +46,13 @@ class CRUDBucket:
return row.scalar()
@staticmethod
- async def get_all(db: AsyncSession) -> list[Bucket]:
+ async def get_all(db: AsyncSession) -> Sequence[Bucket]:
stmt = select(Bucket)
buckets = (await db.execute(stmt)).scalars().all()
return buckets
@staticmethod
- async def get_for_user(db: AsyncSession, uid: str, bucket_type: BucketType = BucketType.ALL) -> list[Bucket]:
+ async def get_for_user(db: AsyncSession, uid: str, bucket_type: BucketType = BucketType.ALL) -> Sequence[Bucket]:
"""
Get all buckets for a user. Depending on the `bucket_type`, the user is either owner of the bucket or has
permission for the bucket
@@ -67,7 +68,7 @@ class CRUDBucket:
Returns
-------
- buckets : list[clowmdb.models.Bucket]
+ buckets : List[clowmdb.models.Bucket]
A list of all buckets where the given user has READ permissions for.
Notes
@@ -143,7 +144,7 @@ class CRUDBucket:
return buckets
@staticmethod
- async def create(db: AsyncSession, bucket_in: BucketInSchema, uid: str) -> Bucket | None:
+ async def create(db: AsyncSession, bucket_in: BucketInSchema, uid: str) -> Optional[Bucket]:
"""
Create a bucket for a given user.
diff --git a/app/crud/crud_bucket_permission.py b/app/crud/crud_bucket_permission.py
index 0fe5ca5f51c50291cbe4953d6a9f9b2b41ecc935..1295fe522aefed5206637946a7170d9d1b52e246 100644
--- a/app/crud/crud_bucket_permission.py
+++ b/app/crud/crud_bucket_permission.py
@@ -1,4 +1,5 @@
from enum import Enum, unique
+from typing import List, Optional, Sequence
from clowmdb.models import BucketPermission as BucketPermissionDB
from sqlalchemy import and_, func, or_, select
@@ -24,7 +25,7 @@ class CRUDBucketPermission:
INACTIVE: str = "INACTIVE"
@staticmethod
- async def get(db: AsyncSession, bucket_name: str, user_id: str) -> BucketPermissionDB | None:
+ async def get(db: AsyncSession, bucket_name: str, user_id: str) -> Optional[BucketPermissionDB]:
stmt = select(BucketPermissionDB).where(
and_(BucketPermissionDB.user_id == user_id, BucketPermissionDB.bucket_name == bucket_name)
)
@@ -35,9 +36,9 @@ class CRUDBucketPermission:
async def get_permissions_for_bucket(
db: AsyncSession,
bucket_name: str,
- permission_types: list[BucketPermissionDB.Permission] | None = None,
- permission_status: PermissionStatus | None = None,
- ) -> list[BucketPermissionDB]:
+ permission_types: Optional[List[BucketPermissionDB.Permission]] = None,
+ permission_status: Optional[PermissionStatus] = None,
+ ) -> Sequence[BucketPermissionDB]:
"""
Get the permissions for the given bucket.
@@ -47,14 +48,14 @@ class CRUDBucketPermission:
Async database session to perform query on.
bucket_name : str
Name of the bucket which to query.
- permission_types : list[clowmdb.models.BucketPermission.Permission] | None, default None
+ permission_types : List[clowmdb.models.BucketPermission.Permission] | None, default None
Type of Bucket Permissions to fetch.
permission_status : app.crud.crud_bucket_permission.CRUDBucketPermission.PermissionStatus | None, default None
Status of Bucket Permissions to fetch.
Returns
-------
- buckets : list[BucketPermission]
+ buckets : List[BucketPermission]
Returns the permissions for the given bucket.
"""
stmt = (
@@ -73,9 +74,9 @@ class CRUDBucketPermission:
async def get_permissions_for_user(
db: AsyncSession,
user_id: str,
- permission_types: list[BucketPermissionDB.Permission] | None = None,
- permission_status: PermissionStatus | None = None,
- ) -> list[BucketPermissionDB]:
+ permission_types: Optional[List[BucketPermissionDB.Permission]] = None,
+ permission_status: Optional[PermissionStatus] = None,
+ ) -> Sequence[BucketPermissionDB]:
"""
Get the permissions for the given user.
@@ -85,14 +86,14 @@ class CRUDBucketPermission:
Async database session to perform query on.
user_id : str
UID of the user which to query.
- permission_types : list[clowmdb.models.BucketPermission.Permission] | None, default None
+ permission_types : List[clowmdb.models.BucketPermission.Permission] | None, default None
Type of Bucket Permissions to fetch.
permission_status : app.crud.crud_bucket_permission.CRUDBucketPermission.PermissionStatus | None, default None
Status of Bucket Permissions to fetch.
Returns
-------
- buckets : list[BucketPermission]
+ buckets : List[BucketPermission]
Returns the permissions for the given user.
"""
stmt = select(BucketPermissionDB).where(BucketPermissionDB.user_id == user_id)
@@ -223,7 +224,7 @@ class CRUDBucketPermission:
return permission
@staticmethod
- def _filter_permission_types(stmt: SQLSelect, permission_types: list[BucketPermissionDB.Permission]) -> SQLSelect:
+ def _filter_permission_types(stmt: SQLSelect, permission_types: List[BucketPermissionDB.Permission]) -> SQLSelect:
"""
Add a where clauses to the SQL Statement where the type of permission is filtered.
@@ -231,7 +232,7 @@ class CRUDBucketPermission:
----------
stmt : sqlalchemy.sql.Select
Declarative Select statement from SQLAlchemy.
- permission_types : list[clowmdb.models.BucketPermission.Permission]
+ permission_types : List[clowmdb.models.BucketPermission.Permission]
Type of Bucket Permissions to filter for.
Returns
diff --git a/app/crud/crud_user.py b/app/crud/crud_user.py
index 8477ed1203730ed3661e8f0f21600b1cac208699..254a14c2ae0f43dd84ddc9d0eb2875cdd209e81e 100644
--- a/app/crud/crud_user.py
+++ b/app/crud/crud_user.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
from clowmdb.models import User
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -5,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
class CRUDUser:
@staticmethod
- async def get(db: AsyncSession, uid: str) -> User | None:
+ async def get(db: AsyncSession, uid: str) -> Optional[User]:
"""
Get a user by its UID.
diff --git a/app/schemas/bucket.py b/app/schemas/bucket.py
index fc755dd365f42308f072549d800626def6396aae..c860ef12e13fc66de3bee575685ce4e5c77f7316 100644
--- a/app/schemas/bucket.py
+++ b/app/schemas/bucket.py
@@ -1,11 +1,9 @@
import re
from datetime import datetime
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
from clowmdb.models import Bucket
-from pydantic import BaseModel, ConfigDict, Field, FieldSerializationInfo, field_serializer, field_validator
-
-from app.core.config import settings
+from pydantic import BaseModel, ConfigDict, Field, field_validator
if TYPE_CHECKING:
from mypy_boto3_s3.service_resource import ObjectSummary
@@ -60,12 +58,6 @@ class BucketOut(_BaseBucket):
Schema for answering a request with a bucket.
"""
- name: str = Field(
- ...,
- examples=["tenant:test-bucket"],
- description="Name of the bucket. Can have the tenant of Ceph as prefix.",
- min_length=3,
- )
created_at: datetime = Field(
...,
examples=[datetime(2022, 1, 1, 0, 0)],
@@ -74,17 +66,13 @@ class BucketOut(_BaseBucket):
owner: str = Field(..., description="UID of the owner", examples=["28c5353b8bb34984a8bd4169ba94c606"])
num_objects: int = Field(..., description="Number of Objects in this bucket", examples=[6])
size: int = Field(..., description="Total size of objects in this bucket in bytes", examples=[3256216])
- owner_constraint: Bucket.Constraint | None = Field(None, description="Constraint for the owner of the bucket")
+ owner_constraint: Optional[Bucket.Constraint] = Field(None, description="Constraint for the owner of the bucket")
description: str = Field(
...,
description="Description of the bucket",
)
model_config = ConfigDict(from_attributes=True)
- @field_serializer("name")
- def serialize_dt(self, name: str, _info: FieldSerializationInfo) -> str:
- return f"{settings.CEPH_TENANT}:{name}" if settings.CEPH_TENANT else name
-
class S3ObjectMetaInformation(BaseModel):
"""
diff --git a/app/schemas/bucket_permission.py b/app/schemas/bucket_permission.py
index b9fcd5fc6ac18be891acf338dff5866a9c06bba6..b384f9f0bfe00a5dcbf549216c452ac7cb3a4989 100644
--- a/app/schemas/bucket_permission.py
+++ b/app/schemas/bucket_permission.py
@@ -1,6 +1,6 @@
import hashlib
from datetime import datetime
-from typing import Any
+from typing import Any, Dict, List, Optional, Union
from clowmdb.models import BucketPermission as BucketPermissionDB
from pydantic import BaseModel, Field
@@ -11,14 +11,14 @@ class BucketPermissionParameters(BaseModel):
Schema for the parameters of a bucket permission.
"""
- from_timestamp: datetime | None = Field(
+ from_timestamp: Optional[datetime] = Field(
None, description="Start date of permission", examples=[datetime(2022, 1, 1, 0, 0)]
)
- to_timestamp: datetime | None = Field(
+ to_timestamp: Optional[datetime] = Field(
None, description="End date of permission", examples=[datetime(2023, 1, 1, 0, 0)]
)
- file_prefix: str | None = Field(None, description="Prefix of subfolder", examples=["pseudo/sub/folder/"])
- permission: BucketPermissionDB.Permission | str = Field(
+ file_prefix: Optional[str] = Field(None, description="Prefix of subfolder", examples=["pseudo/sub/folder/"])
+ permission: Union[BucketPermissionDB.Permission, str] = Field(
BucketPermissionDB.Permission.READ, description="Permission", examples=[BucketPermissionDB.Permission.READ]
)
@@ -44,7 +44,7 @@ class BucketPermissionIn(BucketPermissionParameters):
str_for_id_hash = self.bucket_name + user_id
return hashlib.md5(str_for_id_hash.encode("utf-8")).hexdigest()
- def map_to_bucket_policy_statement(self, user_id: str) -> list[dict[str, Any]]:
+ def map_to_bucket_policy_statement(self, user_id: str) -> List[Dict[str, Any]]:
"""
Create a bucket policy statement from the schema and the user_id.\n
The Sid is unique for every bucket and user combination.
@@ -56,10 +56,10 @@ class BucketPermissionIn(BucketPermissionParameters):
Returns
-------
- statements : list[dict[str, Any]]
+ statements : List[Dict[str, Any]]
Bucket and object permission statements.
"""
- obj_policy: dict[str, Any] = {
+ obj_policy: Dict[str, Any] = {
"Sid": self.to_hash(user_id),
"Effect": "Allow",
"Principal": {"AWS": f"arn:aws:iam:::user/{self.uid}"},
@@ -67,7 +67,7 @@ class BucketPermissionIn(BucketPermissionParameters):
"Action": [],
"Condition": {},
}
- bucket_policy: dict[str, Any] = {
+ bucket_policy: Dict[str, Any] = {
"Sid": self.to_hash(user_id),
"Effect": "Allow",
"Principal": {"AWS": f"arn:aws:iam:::user/{self.uid}"},
@@ -115,7 +115,7 @@ class BucketPermissionOut(BucketPermissionIn):
@staticmethod
def from_db_model(
- permission: BucketPermissionDB, uid: str | None = None, grantee_display_name: str | None = None
+ permission: BucketPermissionDB, uid: Optional[str] = None, grantee_display_name: Optional[str] = None
) -> "BucketPermissionOut":
"""
Create a bucket permission schema from the database model.
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index 8f1e4883fa9cbe9dbe8d3177897f2d4ce755bee7..c3e84d9d9613a1459a092bba9d24de0d9406e532 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -2,7 +2,7 @@ import asyncio
import json
from functools import partial
from secrets import token_urlsafe
-from typing import AsyncGenerator, Callable, Generator
+from typing import AsyncGenerator, Callable, Dict, Generator
from uuid import uuid4
import httpx
@@ -66,7 +66,7 @@ async def client(mock_rgw_admin: MockRGWAdmin, mock_s3_service: MockS3ServiceRes
def get_mock_s3() -> MockS3ServiceResource:
return mock_s3_service
- def get_decode_token_function() -> Callable[[str], dict[str, str]]:
+ def get_decode_token_function() -> Callable[[str], Dict[str, str]]:
# Override the decode_jwt function with mock function for tests and inject random shared secret
return partial(decode_mock_token, secret=jwt_secret)
@@ -98,7 +98,7 @@ async def db() -> AsyncGenerator[AsyncSession, None]:
"""
async with get_async_session(
url=str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
- )() as dbSession:
+ ) as dbSession:
yield dbSession
diff --git a/app/tests/mocks/mock_rgw_admin.py b/app/tests/mocks/mock_rgw_admin.py
index fc2ba9460b253f223fc02e7a7fe9b358c8611a15..6100a232c7e4fb22c594b945322be26900f00464 100644
--- a/app/tests/mocks/mock_rgw_admin.py
+++ b/app/tests/mocks/mock_rgw_admin.py
@@ -1,3 +1,5 @@
+from typing import Dict, List
+
from fastapi import status
from rgwadmin.exceptions import RGWAdminException
@@ -10,9 +12,9 @@ class MockRGWAdmin:
Functions
---------
- get_user(uid: str, stats: bool = False) -> dict[str, list[dict[str, str]]]
+ get_user(uid: str, stats: bool = False) -> Dict[str, List[Dict[str, str]]]
Returns a dict with only one key 'keys'.
- create_key(uid: str, key_type: str = "s3", generate_key: bool = True) -> dict[str, list[dict[str, str]]]
+ create_key(uid: str, key_type: str = "s3", generate_key: bool = True) -> Dict[str, List[Dict[str, str]]]
Creates a new key for a user.
remove_key(access_key: str, uid: str) -> None
Remove a key for a user.
@@ -20,7 +22,7 @@ class MockRGWAdmin:
Deletes all keys for a user.
"""
- _keys: dict[str, list[dict[str, str]]]
+ _keys: Dict[str, List[Dict[str, str]]]
def __init__(self) -> None:
self._keys = {}
@@ -28,7 +30,7 @@ class MockRGWAdmin:
def create_user(self, uid: str, max_buckets: int, display_name: str) -> None:
self.create_key(uid)
- def get_user(self, uid: str, stats: bool = False) -> dict[str, list[dict[str, str]]]: # noqa
+ def get_user(self, uid: str, stats: bool = False) -> Dict[str, List[Dict[str, str]]]: # noqa
"""
Get the keys from a user.
@@ -41,7 +43,7 @@ class MockRGWAdmin:
Returns
-------
- user_keys : dict[str, list[dict[str, str]]]
+ user_keys : Dict[str, List[Dict[str, str]]]
The user object with the associated keys. See Notes.
Notes
@@ -62,7 +64,7 @@ class MockRGWAdmin:
return {"keys": self._keys[uid]}
return {"keys": []}
- def create_key(self, uid: str, key_type: str = "s3", generate_key: bool = True) -> list[dict[str, str]]: # noqa
+ def create_key(self, uid: str, key_type: str = "s3", generate_key: bool = True) -> List[Dict[str, str]]: # noqa
"""
Create a S3 key for a user.
@@ -77,7 +79,7 @@ class MockRGWAdmin:
Returns
-------
- keys : list[dict[str, str]]
+ keys : List[Dict[str, str]]
All keys for the user including the new one.
"""
new_key = {"user": uid, "access_key": random_lower_string(20).upper(), "secret_key": random_lower_string(40)}
diff --git a/app/tests/mocks/mock_s3_resource.py b/app/tests/mocks/mock_s3_resource.py
index 3561daaabdd68f2d5994882208f499fe7d894816..e67ad2caff404c2e55b1aa94c3db6a011f3b3ec5 100644
--- a/app/tests/mocks/mock_s3_resource.py
+++ b/app/tests/mocks/mock_s3_resource.py
@@ -1,4 +1,5 @@
from datetime import datetime
+from typing import Dict, List, Optional
from botocore.exceptions import ClientError
@@ -131,9 +132,9 @@ class MockS3Bucket:
Create the bucket in the mock service.
delete() -> None
Delete the bucket in the mock service
- delete_objects(Delete: dict[str, list[dict[str, str]]]) -> None
+ delete_objects(Delete: Dict[str, List[Dict[str, str]]]) -> None
Delete multiple objects in the bucket.
- get_objects() -> list[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
+ get_objects() -> List[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
List of MockS3ObjectSummary in the bucket.
add_object(obj: app.tests.mocks.mock_s3_resource.MockS3ObjectSummary) -> None
Add a MockS3ObjectSummary to the bucket.
@@ -157,7 +158,7 @@ class MockS3Bucket:
Functions
---------
- all() -> list[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
+ all() -> List[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
Get the saved list.
filter(Prefix: str) -> app.tests.mocks.mock_s3_resource.MockS3Bucket.MockS3ObjectList
Filter the object in the list by the prefix all their keys should have.
@@ -167,16 +168,16 @@ class MockS3Bucket:
Delete a MockS3ObjectSummary from the list
"""
- def __init__(self, obj_list: list[MockS3ObjectSummary] | None = None) -> None:
- self._objs: list[MockS3ObjectSummary] = [] if obj_list is None else obj_list
+ def __init__(self, obj_list: Optional[List[MockS3ObjectSummary]] = None) -> None:
+ self._objs: List[MockS3ObjectSummary] = [] if obj_list is None else obj_list
- def all(self) -> list[MockS3ObjectSummary]:
+ def all(self) -> List[MockS3ObjectSummary]:
"""
Get the saved list.
Returns
-------
- objects : list[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
+ objects : List[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
List of MockS3ObjectSummary
"""
return self._objs
@@ -259,13 +260,13 @@ class MockS3Bucket:
"""
self._parent_service.delete_bucket(self.name)
- def delete_objects(self, Delete: dict[str, list[dict[str, str]]]) -> None:
+ def delete_objects(self, Delete: Dict[str, List[Dict[str, str]]]) -> None:
"""
Delete multiple objects in the bucket.
Parameters
----------
- Delete : dict[str, list[dict[str, str]]]
+ Delete : Dict[str, List[Dict[str, str]]]
The keys of the objects to delete.
Notes
@@ -280,14 +281,14 @@ class MockS3Bucket:
for key_object in Delete["Objects"]:
self.objects.delete(key=key_object["Key"])
- def get_objects(self) -> list[MockS3ObjectSummary]:
+ def get_objects(self) -> List[MockS3ObjectSummary]:
"""
Get the MockS3ObjectSummary in the bucket.
Convenience function for testing.
Returns
-------
- objects : list[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
+ objects : List[app.tests.mocks.mock_s3_resource.MockS3ObjectSummary]
List of MockS3ObjectSummary in the bucket.
"""
return self.objects.all()
@@ -331,7 +332,7 @@ class MockS3ServiceResource:
"""
def __init__(self) -> None:
- self._buckets: dict[str, MockS3Bucket] = {}
+ self._buckets: Dict[str, MockS3Bucket] = {}
def Bucket(self, name: str) -> MockS3Bucket:
"""
diff --git a/app/tests/utils/bucket.py b/app/tests/utils/bucket.py
index dc8b370f53af8b71f95eb16179a8c58032c6fafd..a5369a2c238b84d2024d4943104c3f6582575abc 100644
--- a/app/tests/utils/bucket.py
+++ b/app/tests/utils/bucket.py
@@ -1,4 +1,5 @@
from datetime import datetime
+from typing import Optional
import pytest
from clowmdb.models import Bucket, BucketPermission, User
@@ -39,8 +40,8 @@ async def add_permission_for_bucket(
db: AsyncSession,
bucket_name: str,
uid: str,
- from_: datetime | None = None,
- to: datetime | None = None,
+ from_: Optional[datetime] = None,
+ to: Optional[datetime] = None,
permission: BucketPermission.Permission = BucketPermission.Permission.READ,
) -> None:
"""
diff --git a/app/tests/utils/user.py b/app/tests/utils/user.py
index b83070aa07dcb6a678be34ead07e2013c99acb8d..109cc0a9e56bf546a3d23c060026f70ceb31f200 100644
--- a/app/tests/utils/user.py
+++ b/app/tests/utils/user.py
@@ -1,5 +1,6 @@
from dataclasses import dataclass
from datetime import datetime, timedelta
+from typing import Dict
import pytest
from authlib.jose import JsonWebToken
@@ -13,11 +14,11 @@ _jwt = JsonWebToken(["HS256"])
@dataclass
class UserWithAuthHeader:
- auth_headers: dict[str, str]
+ auth_headers: Dict[str, str]
user: User
-def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> dict[str, str]:
+def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> Dict[str, str]:
"""
Create a valid JWT and return the correct headers for subsequent requests.
@@ -29,7 +30,7 @@ def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> dict[str
Secret to sign the JWT with
Returns
-------
- headers : dict[str,str]
+ headers : Dict[str,str]
HTTP Headers to authorize each request.
"""
to_encode = {"sub": uid, "exp": datetime.utcnow() + timedelta(hours=1)}
@@ -39,7 +40,7 @@ def get_authorization_headers(uid: str, secret: str = "SuperSecret") -> dict[str
return headers
-def decode_mock_token(token: str, secret: str = "SuperSecret") -> dict[str, str]:
+def decode_mock_token(token: str, secret: str = "SuperSecret") -> Dict[str, str]:
"""
Decode and verify a test JWT token.
@@ -52,7 +53,7 @@ def decode_mock_token(token: str, secret: str = "SuperSecret") -> dict[str, str]
Returns
-------
- decoded_token : dict[str, str]
+ decoded_token : Dict[str, str]
Payload of the decoded token.
"""
claims = _jwt.decode(
diff --git a/app/tests/utils/utils.py b/app/tests/utils/utils.py
index 81183197f2f42c42c10aeec73247b3b400292a36..e20970d9ce67751af07a4fac165f3527fa906569 100644
--- a/app/tests/utils/utils.py
+++ b/app/tests/utils/utils.py
@@ -1,7 +1,7 @@
import random
import string
from datetime import datetime
-from typing import Any
+from typing import Any, Dict, Optional
import httpx
@@ -35,7 +35,7 @@ def random_ipv4_string() -> str:
return ".".join(str(random.randint(0, 255)) for _ in range(4))
-def json_datetime_converter(obj: Any) -> str | None:
+def json_datetime_converter(obj: Any) -> Optional[str]:
"""
Helper function for the json converter to covert the object into a string format if it is a datetime object.\n
Parse a datetime object into the format YYYY-MM-DDTHH:MM:SS, e.g. 2022-01-01T00:00:00
@@ -69,7 +69,7 @@ def request_admin_permission(request: httpx.Request) -> bool:
decision : bool
Flag if the request needs the 'administrator' role
"""
- request_body: dict[str, str] = eval(request.content.decode("utf-8"))["input"]
+ request_body: Dict[str, str] = eval(request.content.decode("utf-8"))["input"]
operation = request_body["operation"]
checks = "any" in operation
if "bucket_permission" in request_body["resource"]:
diff --git a/requirements-dev.txt b/requirements-dev.txt
index e750f64bada01ec1e85e00175fa4efe2de0b9771..d7cfa3bfe8150f30e795be49704788c9d6389faf 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -11,7 +11,6 @@ isort>=5.12.0,<5.13.0
mypy>=1.4.0,<1.5.0
# stubs for mypy
boto3-stubs-lite[s3]>=1.28.0,<1.29.0
-sqlalchemy2-stubs
types-requests
# Miscellaneous
pre-commit>=3.3.0,<3.4.0
diff --git a/requirements.txt b/requirements.txt
index 0eefe6d9249b16065ed6088068841621d319de90..afa2dba60c0c25992954e8d45154e85ed47bcb90 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,22 +1,22 @@
--extra-index-url https://gitlab.ub.uni-bielefeld.de/api/v4/projects/5493/packages/pypi/simple
-clowmdb>=1.3.0,<1.4.0
+clowmdb>=2.0.0,<2.1.0
# Webserver packages
anyio>=3.7.0,<3.8.0
-fastapi>=0.100.0,<0.101.0
+fastapi>=0.101.0,<0.102.0
pydantic>=2.1.0,<2.2.0
pydantic-settings>=2.0.0
uvicorn>=0.23.0,<0.24.0
# Database packages
-PyMySQL>=1.0.2,<1.1.0
-SQLAlchemy>=1.4.0,<2.0.0
-aiomysql>=0.1.0,<0.2.0
+PyMySQL>=1.1.0,<1.2.0
+SQLAlchemy>=2.0.0,<2.1.0
+aiomysql>=0.2.0,<0.3.0
# Security packages
authlib>=1.2.0,<1.3.0
# Ceph and S3 packages
boto3>=1.28.0,<1.29.0
rgwadmin>=2.4.0,<2.5.0
# Miscellaneous
-tenacity>=8.1.0,<8.2.0
+tenacity>=8.2.0,<8.3.0
httpx>=0.24.0,<0.25.0
itsdangerous