diff --git a/.gitignore b/.gitignore
index 1a91d71400bc0ab62c6332878eb62cbd63213d48..e00963ccc4a37100a59a7cd02cec34ccf14d2c67 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,5 +6,9 @@ env/
venv/
ENV/
.coverage
-oidc_dev/
-traefik
+config.yaml
+config.json
+config.toml
+test-config.yaml
+test-config.json
+test-config.toml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9ae62486846705eb350d515dd492200b20bc2133..8fa9c3b423e8bd526327603778b98d847c52cc3b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,21 +1,20 @@
image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.12-slim
variables:
+ FF_NETWORK_PER_BUILD: 1
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PYTHONPATH: "$CI_PROJECT_DIR"
- OBJECT_GATEWAY_URI: "http://127.0.0.1:8000"
- BUCKET_CEPH_ACCESS_KEY: ""
- BUCKET_CEPH_SECRET_KEY: ""
- USER_CEPH_ACCESS_KEY: ""
- USER_CEPH_SECRET_KEY: ""
- BUCKET_CEPH_USERNAME: ""
- FF_NETWORK_PER_BUILD: 1
- PUBLIC_KEY_VALUE: "empty"
- OPA_URI: "http://127.0.0.1:8181"
- DB_PASSWORD: "random"
- DB_USER: "random"
- DB_DATABASE: "random"
- DB_HOST: "random"
+ CLOWM_S3__ADMIN_ACCESS_KEY: "nonempty"
+ CLOWM_S3__ADMIN_SECRET_KEY: "nonempty"
+ CLOWM_S3__USERNAME: "clowm-bucket-manager"
+ CLOWM_S3__URI: "http://127.0.0.1:8001"
+ CLOWM_S3__ACCESS_KEY: "nonempty"
+ CLOWM_S3__SECRET_KEY: "nonempty"
+ CLOWM_PUBLIC_KEY: "nonempty"
+ CLOWM_OPA__URI: "http://127.0.0.1:8181"
+ CLOWM_CLUSTER__SLURM__TOKEN: "empty"
+ CLOWM_CLUSTER__SLURM__URI: "http://127.0.0.1:8002"
+ CLOWM_UI_URI: "http://localhost"
cache:
paths:
@@ -33,26 +32,43 @@ default:
- python -m pip install --disable-pip-version-check --upgrade -r requirements.txt -r requirements-dev.txt
stages: # List of stages for jobs, and their order of execution
+ - lint
- test
- deploy
+lint-test-job: # Runs linters checks on code
+ stage: lint
+ before_script:
+ - python --version # For debugging
+ - pip install --disable-pip-version-check virtualenv
+ - virtualenv venv
+ - source venv/bin/activate
+ - python -m pip install --disable-pip-version-check --upgrade -r requirements.txt -r requirements-dev.txt --upgrade-strategy=eager
+ script:
+ - ./scripts/lint.sh
+
integration-test-job: # Runs integration tests with the database
stage: test
variables:
- DB_PASSWORD: "$TEST_DB_PASSWORD"
- DB_USER: "test_api_user"
- DB_DATABASE: "integration-test-db"
- DB_HOST: "integration-test-db"
+ CLOWM_DB__PASSWORD: "$TEST_DB_PASSWORD"
+ CLOWM_DB__USER: "test_api_user"
+ CLOWM_DB__NAME: "integration-test-db"
+ CLOWM_DB__HOST: "integration-test-db"
services:
- name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/mysql:8
alias: integration-test-db
variables:
MYSQL_RANDOM_ROOT_PASSWORD: "yes"
- MYSQL_DATABASE: "$DB_DATABASE"
- MYSQL_USER: "$DB_USER"
- MYSQL_PASSWORD: "$DB_PASSWORD"
+ MYSQL_DATABASE: "$CLOWM_DB__NAME"
+ MYSQL_USER: "$CLOWM_DB__USER"
+ MYSQL_PASSWORD: "$CLOWM_DB__PASSWORD"
- name: $CI_REGISTRY/cmg/clowm/clowm-database:v3.1
alias: upgrade-db
+ variables:
+ DB_HOST: "$CLOWM_DB__HOST"
+ DB_DATABASE: "$CLOWM_DB__NAME"
+ DB_USER: "$CLOWM_DB__USER"
+ DB_PASSWORD: "$CLOWM_DB__PASSWORD"
script:
- python app/check_database_connection.py
- pytest --junitxml=integration-report.xml --cov=app --cov-report=term-missing app/tests/crud
@@ -67,20 +83,25 @@ integration-test-job: # Runs integration tests with the database
e2e-test-job: # Runs e2e tests on the API endpoints
stage: test
variables:
- DB_PASSWORD: "$TEST_DB_PASSWORD"
- DB_USER: "test_api_user"
- DB_DATABASE: "e2e-test-db"
- DB_HOST: "e2e-test-db"
+ CLOWM_DB__PASSWORD: "$TEST_DB_PASSWORD"
+ CLOWM_DB__USER: "test_api_user"
+ CLOWM_DB__NAME: "e2e-test-db"
+ CLOWM_DB__HOST: "e2e-test-db"
services:
- name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/mysql:8
alias: e2e-test-db
variables:
MYSQL_RANDOM_ROOT_PASSWORD: "yes"
- MYSQL_DATABASE: "$DB_DATABASE"
- MYSQL_USER: "$DB_USER"
- MYSQL_PASSWORD: "$DB_PASSWORD"
+ MYSQL_DATABASE: "$CLOWM_DB__NAME"
+ MYSQL_USER: "$CLOWM_DB__USER"
+ MYSQL_PASSWORD: "$CLOWM_DB__PASSWORD"
- name: $CI_REGISTRY/cmg/clowm/clowm-database:v3.1
alias: upgrade-db
+ variables:
+ DB_HOST: "$CLOWM_DB__HOST"
+ DB_DATABASE: "$CLOWM_DB__NAME"
+ DB_USER: "$CLOWM_DB__USER"
+ DB_PASSWORD: "$CLOWM_DB__PASSWORD"
script:
- python app/check_database_connection.py
- pytest --junitxml=e2e-report.xml --cov=app --cov-report=term-missing app/tests/api
@@ -94,8 +115,13 @@ e2e-test-job: # Runs e2e tests on the API endpoints
unit-test-job: # Runs unit tests
stage: test
+ variables:
+ CLOWM_DB__PASSWORD: "nonempty"
+ CLOWM_DB__USER: "nonempty"
+ CLOWM_DB__NAME: "nonempty"
+ CLOWM_DB__HOST: "nonempty"
script:
- - pytest --junitxml=unit-report.xml --noconftest --cov=app --cov-report=term-missing app/tests/unit
+ - pytest --junitxml=unit-report.xml --cov=app --cov-report=term-missing app/tests/unit
- mkdir coverage-unit
- mv .coverage coverage-unit
artifacts:
@@ -124,45 +150,48 @@ combine-test-coverage-job: # Combine coverage reports from different test jobs
coverage_format: cobertura
path: $CI_PROJECT_DIR/coverage.xml
-lint-test-job: # Runs linters checks on code
- stage: test
- script:
- - ./scripts/lint.sh
-
-publish-main-docker-container-job:
+.publish-docker-container:
stage: deploy
image:
- name: gcr.io/kaniko-project/executor:v1.20.1-debug
- entrypoint: [""]
- dependencies: []
+ name: gcr.io/kaniko-project/executor:v1.22.0-debug
+ entrypoint: [ "" ]
+ dependencies: [ ]
+ cache: [ ]
+ before_script:
+ - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
+
+.publish-main-docker-container:
+ extends: .publish-docker-container
only:
refs:
- main
- before_script:
- - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
+
+publish-main-docker-container-job:
+ extends: .publish-main-docker-container
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
--destination "${CI_REGISTRY_IMAGE}:main-${CI_COMMIT_SHA}"
--destination "${CI_REGISTRY_IMAGE}:main-latest"
- --cleanup
+
+publish-main-gunicorn-docker-container-job:
+ extends: .publish-main-docker-container
+ script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile-Gunicorn"
- --destination "${CI_REGISTRY_IMAGE}:main-${CI_COMMIT_SHA}-gunicorn"
- --destination "${CI_REGISTRY_IMAGE}:main-latest-gunicorn"
+ --destination "${CI_REGISTRY_IMAGE}:main-gunicorn-${CI_COMMIT_SHA}"
+ --destination "${CI_REGISTRY_IMAGE}:main-gunicorn-latest"
-publish-docker-container-job:
- stage: deploy
- image:
- name: gcr.io/kaniko-project/executor:v1.20.1-debug
- entrypoint: [""]
- dependencies: []
+
+.publish-version-docker-container:
+ extends: .publish-docker-container
only:
- tags
- before_script:
- - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
+
+publish-docker-container-job:
+ extends: .publish-version-docker-container
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
@@ -171,11 +200,15 @@ publish-docker-container-job:
--destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1-2)"
--destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1)"
--destination "${CI_REGISTRY_IMAGE}:latest"
- --cleanup
+
+
+publish-gunicorn-docker-container-job:
+ extends: .publish-version-docker-container
+ script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile-Gunicorn"
- --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}-gunicorn"
- --destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1-2)-gunicorn"
- --destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1)-gunicorn"
- --destination "${CI_REGISTRY_IMAGE}:latest-gunicorn"
+ --destination "${CI_REGISTRY_IMAGE}:gunicorn-${CI_COMMIT_TAG}"
+ --destination "${CI_REGISTRY_IMAGE}:gunicorn-$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1-2)"
+ --destination "${CI_REGISTRY_IMAGE}:gunicorn-$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1)"
+ --destination "${CI_REGISTRY_IMAGE}:gunicorn-latest"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index dfe22a4e1c0381be7e5f6756580484f5cfb761d4..ff91b0d74cffb33ef2ced1ef0de704b7ffe3171f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.5.0
+ rev: v4.6.0
hooks:
- id: end-of-file-fixer
- id: check-added-large-files
@@ -15,13 +15,13 @@ repos:
- id: check-merge-conflict
- id: check-ast
- repo: https://github.com/psf/black
- rev: 24.2.0
+ rev: 24.4.0
hooks:
- id: black
files: app
args: [--check]
- repo: https://github.com/charliermarsh/ruff-pre-commit
- rev: 'v0.2.2'
+ rev: 'v0.3.7'
hooks:
- id: ruff
- repo: https://github.com/PyCQA/isort
@@ -31,7 +31,7 @@ repos:
files: app
args: [-c]
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.8.0
+ rev: v1.9.0
hooks:
- id: mypy
files: app
@@ -39,5 +39,5 @@ repos:
additional_dependencies:
- boto3-stubs-lite[s3]<1.35.0
- sqlalchemy>=2.0.0,<2.1.0
- - pydantic<2.7.0
+ - pydantic<2.8.0
- types-requests
diff --git a/Dockerfile b/Dockerfile
index 5650e0243160cc2f3003094c0fcbc06e3cc9ed74..60a1b2ad47d72ec322409f29fff467bddb7ab16e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,13 +3,12 @@ ENV PORT=8000
EXPOSE $PORT
# dumb-init forwards the kill signal to the python process
-RUN apt-get update && apt-get -y install dumb-init
-RUN apt-get clean
+RUN apt-get update && apt-get -y install dumb-init && apt-get clean
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
STOPSIGNAL SIGINT
-RUN pip install --disable-pip-version-check --no-cache-dir httpx[cli] "uvicorn<0.28.0"
+RUN pip install --disable-pip-version-check --no-cache-dir httpx[cli] "uvicorn<0.30.0"
-HEALTHCHECK --interval=30s --timeout=2s CMD httpx http://localhost:$PORT/health || exit 1
+HEALTHCHECK --interval=5s --timeout=2s CMD httpx http://localhost:$PORT/health || exit 1
RUN useradd -m worker
USER worker
diff --git a/Dockerfile-Gunicorn b/Dockerfile-Gunicorn
index 916c9c2afcd7dcbda7b4f2b9dd9cb7ed83750d21..f5a25296798f3ba88366ac6f9ea9f3c551a1432f 100644
--- a/Dockerfile-Gunicorn
+++ b/Dockerfile-Gunicorn
@@ -4,11 +4,11 @@ EXPOSE $PORT
WORKDIR /app/
ENV PYTHONPATH=/app
-RUN pip install --disable-pip-version-check --no-cache-dir httpx[cli] "gunicorn<21.3.0" "uvicorn<0.28.0"
+RUN pip install --disable-pip-version-check --no-cache-dir httpx[cli] "gunicorn<21.3.0" "uvicorn<0.30.0"
COPY ./gunicorn_conf.py /app/gunicorn_conf.py
COPY ./start_service_gunicorn.sh /app/entrypoint.sh
-HEALTHCHECK --interval=30s --timeout=2s CMD httpx http://localhost:$PORT/health || exit 1
+HEALTHCHECK --interval=5s --timeout=2s CMD httpx http://localhost:$PORT/health || exit 1
COPY ./scripts/prestart.sh /app/prestart.sh
COPY ./requirements.txt /app/requirements.txt
diff --git a/README.md b/README.md
index 7f4f57c555dfbf235a13bf4a87c0006c3ffd22a1..78b3c31e9850546d555f364268aece47cbb4bfbd 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,9 @@
# CloWM S3Proxy Service
## Description
-Openstack is shipping with an integrated UI to access the Object Store provided by Ceph. Unfortunately, this UI does not allow
+
+Openstack is shipping with an integrated UI to access the Object Store provided by Ceph. Unfortunately, this UI does not
+allow
fine-grained control who can access a bucket or object. You can either make it accessible for everyone or nobody, but
Ceph can do this and much more. 👎
This is the backend for a new UI which can leverage the additional powerful functionality provided by Ceph in a
@@ -15,38 +17,59 @@ user-friendly manner. ðŸ‘
| Fine-grained Access Control | ⌠| ✅ |
### Concept
+
-## Environment Variables
-
-### Mandatory / Recommended Variables
-
-| Variable | Default | Value | Description |
-|----------------------------------------|-------------------------|---------------------------------|------------------------------------------------------------------------------------|
-| `DB_HOST` | unset | <db hostname / IP> | IP or Hostname Address of DB |
-| `DB_PORT` | 3306 | Number | Port of the database |
-| `DB_USER` | unset | \<db username> | Username of the database user |
-| `DB_PASSWORD` | unset | \<db password> | Password of the database user |
-| `DB_DATABASE` | unset | \<db name> | Name of the database |
-| `OBJECT_GATEWAY_URI` | unset | HTTP URL | HTTP URL of the Ceph Object Gateway |
-| `BUCKET_CEPH_ACCESS_KEY` | unset | \<access key> | Access key for the Ceph Object Gateway user with unlimited buckets. |
-| `BUCKET_CEPH_SECRET_KEY` | unset | \<secret key> | Secret key for the Ceph Object Gateway user with unlimited buckets. |
-| `BUCKET_CEPH_USERNAME` | unset | \<ceph username> | ID of the user in ceph who owns all the buckets. Owner of `BUCKET_CEPH_ACCESS_KEY` |
-| `USER_CEPH_ACCESS_KEY` | unset | \<access key> | Access key for the Ceph Object Gateway user with `user:*` privileges |
-| `USER_CEPH_SECRET_KEY` | unset | \<secret key> | Secret key for the Ceph Object Gateway user with `user:*` privileges. |
-| `PUBLIC_KEY_VALUE` / `PUBLIC_KEY_FILE` | randomly generated | Public Key / Path to Public Key | Public part of RSA Key in PEM format to verify JWTs |
-| `OPA_URI` | unset | HTTP URL | HTTP URL of the OPA service |
-| `CLOWM_URL` | `http://localhost:8080` | HTTP URL | HTTP URL of the CloWM website |
-
-### Optional Variables
-
-| Variable | Default | Value | Description |
-|-----------------------------|----------------------|-----------------------------|----------------------------------------------------------------|
-| `API_PREFIX` | `/api` | URL path | Prefix before every URL path |
-| `SQLALCHEMY_VERBOSE_LOGGER` | `false` | `<"true"|"false">` | Enables verbose SQL output.<br>Should be `false` in production |
-| `OPA_POLICY_PATH` | `/clowm/authz/allow` | URL path | Path to the OPA Policy for Authorization |
-| `OTLP_GRPC_ENDPOINT` | unset | <hostname / IP> | OTLP compatible endpoint to send traces via gRPC, e.g. Jaeger |
+## Configuration
+
+### General
+
+| Env variable | Config file key | Default | Value | Example | Description |
+|--------------------------|-----------------|---------------|----------|------------------------|-----------------------------------------------------------------------------------------------------------------------|
+| `CLOWM_CONFIG_FILE_YAML` | - | `config.yaml` | Filepath | `/path/to/config.yaml` | Path to a YAML file to read the config. See [example-config/example-config.yaml](example-config/example-config.yaml). |
+| `CLOWM_CONFIG_FILE_TOML` | - | `config.toml` | Filepath | `/path/to/config.toml` | Path to a TOML file to read the config. See [example-config/example-config.toml](example-config/example-config.toml). |
+| `CLOWM_CONFIG_FILE_JSON` | - | `config.json` | Filepath | `/path/to/config.json` | Path to a JSON file to read the config. See [example-config/example-config.json](example-config/example-config.json). |
+| `CLOWM_API_PREFIX` | `api_prefix` | unset | URI path | `/api` | Prefix before every URL path |
+| * `CLOWM_UI_URI` | `ui_uri` | unset | HTTP URL | `https://localhost` | HTTP URL of the CloWM website |
+
+### Database
+
+| Env variable | Config file key | Default | Value | Example | Description |
+|------------------------|-----------------|-------------|--------------------|---------------|----------------------------------------------------------------|
+| `CLOWM_DB__HOST` | `db.host` | `localhost` | <db hostname / IP> | `localhost` | IP or Hostname Address of DB |
+| `CLOWM_DB__PORT` | `db.port` | 3306 | Integer | 3306 | Port of the database |
+| * `CLOWM_DB__USER` | `db.user` | unset | String | `db-user` | Username of the database user |
+| * `CLOWM_DB__PASSWORD` | `db.password` | unset | String | `db-password` | Password of the database user |
+| * `CLOWM_DB__NAME` | `db.name` | unset | String | `db-name` | Name of the database |
+| `CLOWM_DB__VERBOSE` | `db.verbose` | `false` | Boolean | `false` | Enables verbose SQL output.<br>Should be `false` in production |
+
+### S3
+
+| Env variable | Config file key | Default | Value | Example | Description |
+|--------------------------------|-----------------------|---------|----------|--------------------------|----------------------------------------------------------------------------------|
+| * `CLOWM_S3__URI` | `s3.uri` | unset | HTTP URL | `http://localhost` | URI of the S3 Object Storage |
+| * `CLOWM_S3__ACCESS_KEY` | `s3.acess_key` | unset | String | `ZR7U56KMK20VW` | Access key for the S3 that owns the buckets |
+| * `CLOWM_S3__SECRET_KEY` | `s3.secret_key` | unset | String | `9KRUU41EGSCB3H9ODECNHW` | Secret key for the S3 that owns the buckets |
+| * `CLOWM_S3__USERNAME` | `s3.username` | unset | String | `clowm-bucket-manager` | ID of the user in ceph who owns all the buckets. Owner of `CLOWM_S3__ACCESS_KEY` |
+| * `CLOWM_S3__ADMIN_ACCESS_KEY` | `s3.admin_acess_key` | unset | String | `ZR7U56KMK20VW` | Access key for the Ceph Object Gateway user with `user:*` privileges |
+| * `CLOWM_S3__ADMIN_SECRET_KEY` | `s3.admin_secret_key` | unset | String | `9KRUU41EGSCB3H9ODECNHW` | Secret key for the Ceph Object Gateway user with `user:*` privileges. |
+
+### Security
+
+| Env variable | Config file key | Default | Value | Example | Description |
+|------------------------------------------------|----------------------------------|---------|---------------------------------|--------------------|-----------------------------------------------------|
+| * `CLOWM_PUBLIC_KEY` / `CLOWM_PUBLIC_KEY_FILE` | `public_key` / `public_key_file` | unset | Public Key / Path to Public Key | `/path/to/key.pub` | Public part of RSA Key in PEM format to verify JWTs |
+| * `CLOWM_OPA__URI` | `opa.uri` | unset | HTTP URL | `http://localhost` | URI of the OPA Service |
+
+### Monitoring
+
+| Env variable | Config file key | Default | Value | Example | Description |
+|-----------------------------|----------------------|---------|---------|-------------|----------------------------------------------------------------------------------------------|
+| `CLOWM_OTLP__GRPC_ENDPOINT` | `otlp.grpc_endpoint` | unset | String | `localhost` | OTLP compatible endpoint to send traces via gRPC, e.g. Jaeger. If unset, no traces are sent. |
+| `CLOWM_OTLP__SECURE` | `otlp.secure` | `false` | Boolean | `false` | Connection type |
+
## License
-The API is licensed under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license. See the [License](LICENSE) file for more information.
+The API is licensed under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license. See
+the [License](LICENSE) file for more information.
diff --git a/app/api/dependencies.py b/app/api/dependencies.py
index e05eaddec0b7ec691ee6a4aee8fd53896ee9f354..e391753eea990287b98f6fb0a8e181fa50c4a74c 100644
--- a/app/api/dependencies.py
+++ b/app/api/dependencies.py
@@ -55,9 +55,7 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]: # pragma: no cover
db : AsyncGenerator[AsyncSession, None]
Async session object with the database
"""
- async with get_async_session(
- str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
- ) as db:
+ async with get_async_session(str(settings.db.dsn_async), verbose=settings.db.verbose) as db:
yield db
diff --git a/app/api/endpoints/bucket_permissions.py b/app/api/endpoints/bucket_permissions.py
index 56c5424e92e0eb6856dc355e12d97bd1a6b68ddb..f54efbff056682ddf1d6a53cfacfb74269833905 100644
--- a/app/api/endpoints/bucket_permissions.py
+++ b/app/api/endpoints/bucket_permissions.py
@@ -20,9 +20,7 @@ from app.ceph.s3 import get_s3_bucket_policy, put_s3_bucket_policy
from app.crud import DuplicateError
from app.crud.crud_bucket_permission import CRUDBucketPermission
from app.otlp import start_as_current_span_async
-from app.schemas.bucket_permission import BucketPermissionIn as PermissionSchemaIn
-from app.schemas.bucket_permission import BucketPermissionOut as PermissionSchemaOut
-from app.schemas.bucket_permission import BucketPermissionParameters as PermissionParametersSchema
+from app.schemas.bucket_permission import BucketPermissionIn, BucketPermissionOut, BucketPermissionParameters
router = APIRouter(prefix="/permissions", tags=["BucketPermission"])
permission_authorization = AuthorizationDependency(resource="bucket_permission")
@@ -32,7 +30,7 @@ tracer = trace.get_tracer_provider().get_tracer(__name__)
@router.get(
"",
- response_model=list[PermissionSchemaOut],
+ response_model=list[BucketPermissionOut],
summary="Get all permissions.",
response_model_exclude_none=True,
)
@@ -48,7 +46,7 @@ async def list_permissions(
CRUDBucketPermission.PermissionStatus | SkipJsonSchema[None],
Query(description="Status of Bucket Permissions to fetch"),
] = None,
-) -> list[PermissionSchemaOut]:
+) -> list[BucketPermissionOut]:
"""
List all the bucket permissions in the system.\n
Permission `bucket_permission:list_all` required.
@@ -79,12 +77,12 @@ async def list_permissions(
bucket_permissions = await CRUDBucketPermission.list(
db, permission_types=permission_types, permission_status=permission_status
)
- return [PermissionSchemaOut.from_db_model(p) for p in bucket_permissions]
+ return [BucketPermissionOut.from_db_model(p) for p in bucket_permissions]
@router.post(
"",
- response_model=PermissionSchemaOut,
+ response_model=BucketPermissionOut,
status_code=status.HTTP_201_CREATED,
summary="Create a permission.",
response_model_exclude_none=True,
@@ -95,8 +93,8 @@ async def create_permission(
current_user: CurrentUser,
s3: S3Resource,
authorization: Authorization,
- permission: Annotated[PermissionSchemaIn, Body(..., description="Permission to create")],
-) -> PermissionSchemaOut:
+ permission: Annotated[BucketPermissionIn, Body(..., description="Permission to create")],
+) -> BucketPermissionOut:
"""
Create a permission for a bucket and user.\n
Permission `bucket_permission:create` required.
@@ -146,12 +144,12 @@ async def create_permission(
json_policy["Statement"] += permission.map_to_bucket_policy_statement(permission_db.uid)
put_s3_bucket_policy(s3, bucket_name=permission.bucket_name, policy=json.dumps(json_policy))
- return PermissionSchemaOut.from_db_model(permission_db)
+ return BucketPermissionOut.from_db_model(permission_db)
@router.get(
"/user/{uid}",
- response_model=list[PermissionSchemaOut],
+ response_model=list[BucketPermissionOut],
summary="Get all permissions for a user.",
response_model_exclude_none=True,
)
@@ -169,7 +167,7 @@ async def list_permissions_per_user(
CRUDBucketPermission.PermissionStatus | SkipJsonSchema[None],
Query(description="Status of Bucket Permissions to fetch"),
] = None,
-) -> list[PermissionSchemaOut]:
+) -> list[BucketPermissionOut]:
"""
List all the bucket permissions for the given user.\n
Permission `bucket_permission:list_user` required if current user is the target the bucket permission,
@@ -206,12 +204,12 @@ async def list_permissions_per_user(
bucket_permissions = await CRUDBucketPermission.list(
db, uid=user.uid, permission_types=permission_types, permission_status=permission_status
)
- return [PermissionSchemaOut.from_db_model(p) for p in bucket_permissions]
+ return [BucketPermissionOut.from_db_model(p) for p in bucket_permissions]
@router.get(
"/bucket/{bucket_name}",
- response_model=list[PermissionSchemaOut],
+ response_model=list[BucketPermissionOut],
summary="Get all permissions for a bucket.",
response_model_exclude_none=True,
)
@@ -229,7 +227,7 @@ async def list_permissions_per_bucket(
CRUDBucketPermission.PermissionStatus | SkipJsonSchema[None],
Query(description="Status of Bucket Permissions to fetch"),
] = None,
-) -> list[PermissionSchemaOut]:
+) -> list[BucketPermissionOut]:
"""
List all the bucket permissions for the given bucket.\n
Permission `bucket_permission:list_bucket` required if current user is owner of the bucket,
@@ -266,12 +264,12 @@ async def list_permissions_per_bucket(
bucket_permissions = await CRUDBucketPermission.list(
db, bucket_name=bucket.name, permission_types=permission_types, permission_status=permission_status
)
- return [PermissionSchemaOut.from_db_model(p) for p in bucket_permissions]
+ return [BucketPermissionOut.from_db_model(p) for p in bucket_permissions]
@router.get(
"/bucket/{bucket_name}/user/{uid}",
- response_model=PermissionSchemaOut,
+ response_model=BucketPermissionOut,
summary="Get permission for bucket and user combination.",
response_model_exclude_none=True,
)
@@ -282,7 +280,7 @@ async def get_permission_for_bucket(
current_user: CurrentUser,
authorization: Authorization,
user: PathUser,
-) -> PermissionSchemaOut:
+) -> BucketPermissionOut:
"""
Get the bucket permissions for the specific combination of bucket and user.\n
The owner of the bucket and the grantee of the permission can view it.\n
@@ -312,7 +310,7 @@ async def get_permission_for_bucket(
await authorization(rbac_operation)
bucket_permission = await CRUDBucketPermission.get(db, bucket.name, user.uid)
if bucket_permission:
- return PermissionSchemaOut.from_db_model(bucket_permission)
+ return BucketPermissionOut.from_db_model(bucket_permission)
raise HTTPException(
status.HTTP_404_NOT_FOUND,
detail=f"Permission for combination of bucket={bucket.name} and user={user.uid} doesn't exists",
@@ -369,7 +367,7 @@ async def delete_permission(
detail=f"Permission for combination of bucket={bucket.name} and user={str(user.uid)} doesn't exists",
)
await CRUDBucketPermission.delete(db, bucket_name=bucket_permission.bucket_name, uid=bucket_permission.uid)
- bucket_permission_schema = PermissionSchemaOut.from_db_model(bucket_permission)
+ bucket_permission_schema = BucketPermissionOut.from_db_model(bucket_permission)
s3_policy = get_s3_bucket_policy(s3, bucket_name=bucket_permission_schema.bucket_name)
policy = json.loads(s3_policy.policy)
policy["Statement"] = [
@@ -381,7 +379,7 @@ async def delete_permission(
@router.put(
"/bucket/{bucket_name}/user/{uid}",
status_code=status.HTTP_200_OK,
- response_model=PermissionSchemaOut,
+ response_model=BucketPermissionOut,
summary="Update a bucket permission",
response_model_exclude_none=True,
)
@@ -393,8 +391,8 @@ async def update_permission(
s3: S3Resource,
authorization: Authorization,
user: PathUser,
- permission_parameters: Annotated[PermissionParametersSchema, Body(..., description="Permission to create")],
-) -> PermissionSchemaOut:
+ permission_parameters: Annotated[BucketPermissionParameters, Body(..., description="Permission to create")],
+) -> BucketPermissionOut:
"""
Update a permission for a bucket and user.\n
Permission `bucket_permission:update` required.
@@ -433,7 +431,7 @@ async def update_permission(
detail=f"Permission for combination of bucket={bucket.name} and user={user.uid} doesn't exists",
)
updated_permission = await CRUDBucketPermission.update_permission(db, bucket_permission, permission_parameters)
- updated_permission_schema = PermissionSchemaOut.from_db_model(updated_permission)
+ updated_permission_schema = BucketPermissionOut.from_db_model(updated_permission)
s3_policy = get_s3_bucket_policy(s3, bucket_name=bucket.name)
policy = json.loads(s3_policy.policy)
policy["Statement"] = [
diff --git a/app/api/endpoints/buckets.py b/app/api/endpoints/buckets.py
index 96ef449274495a1f283a5a36aea2cbd68d764fb9..90b3f7e1955490a1bd88bedb50f9c165cd3d473d 100644
--- a/app/api/endpoints/buckets.py
+++ b/app/api/endpoints/buckets.py
@@ -1,6 +1,5 @@
import json
-from functools import reduce
-from typing import TYPE_CHECKING, Annotated, Any, Awaitable, Callable
+from typing import Annotated, Any, Awaitable, Callable, Iterable
from uuid import UUID
from botocore.exceptions import ClientError
@@ -10,7 +9,7 @@ from opentelemetry import trace
from pydantic.json_schema import SkipJsonSchema
from app.api.dependencies import AuthorizationDependency, CurrentBucket, CurrentUser, DBSession, S3Resource
-from app.ceph.s3 import get_s3_bucket_objects, put_s3_bucket_policy
+from app.ceph.s3 import get_s3_bucket_objects, get_s3_bucket_policy, put_s3_bucket_policy
from app.core.config import settings
from app.crud import DuplicateError
from app.crud.crud_bucket import CRUDBucket
@@ -19,15 +18,11 @@ from app.otlp import start_as_current_span_async
from app.schemas.bucket import BucketIn as BucketInSchema
from app.schemas.bucket import BucketOut as BucketOutSchema
-if TYPE_CHECKING:
- from mypy_boto3_s3.service_resource import ObjectSummary
-else:
- ObjectSummary = object
-
router = APIRouter(prefix="/buckets", tags=["Bucket"])
bucket_authorization = AuthorizationDependency(resource="bucket")
Authorization = Annotated[Callable[[str], Awaitable[Any]], Depends(bucket_authorization)]
tracer = trace.get_tracer_provider().get_tracer(__name__)
+ANONYMOUS_ACCESS_SID = "AnonymousAccess"
cors_rule = {
"CORSRules": [
@@ -45,7 +40,7 @@ cors_rule = {
"content-md5",
],
"AllowedMethods": ["GET", "PUT", "POST", "DELETE", "HEAD"],
- "AllowedOrigins": [str(settings.CLOWM_URL)[:-1]],
+ "AllowedOrigins": [str(settings.ui_uri)[:-1]],
"ExposeHeaders": [
"Etag",
],
@@ -55,11 +50,29 @@ cors_rule = {
}
+def get_anonymously_bucket_policy(bucket_name: str) -> list[dict[str, Any]]:
+ return [
+ {
+ "Sid": ANONYMOUS_ACCESS_SID,
+ "Effect": "Allow",
+ "Principal": "*",
+ "Resource": f"arn:aws:s3:::{bucket_name}/*",
+ "Action": ["s3:GetObject"],
+ },
+ {
+ "Sid": ANONYMOUS_ACCESS_SID,
+ "Effect": "Allow",
+ "Principal": "*",
+ "Resource": f"arn:aws:s3:::{bucket_name}",
+ "Action": ["s3:ListBucket"],
+ },
+ ]
+
+
@router.get("", response_model=list[BucketOutSchema], summary="List buckets of user")
@start_as_current_span_async("api_list_buckets", tracer=tracer)
async def list_buckets(
db: DBSession,
- s3: S3Resource,
current_user: CurrentUser,
authorization: Authorization,
owner_id: Annotated[
@@ -72,7 +85,7 @@ async def list_buckets(
bucket_type: Annotated[
CRUDBucket.BucketType, Query(description="Type of the bucket to get. Ignored when `user` parameter not set")
] = CRUDBucket.BucketType.ALL,
-) -> list[BucketOutSchema]:
+) -> Iterable[Bucket]:
"""
List all the buckets in the system or of the desired user where the user has READ permissions for.\n
Permission `bucket:list` required if the current user is the owner of the bucket,
@@ -94,7 +107,7 @@ async def list_buckets(
Async function to ask the auth service for authorization. Dependency Injection.
Returns
-------
- buckets : list[app.schemas.bucket.BucketOut]
+ buckets : list[clowmdb.models.Bucket]
All the buckets for which the user has READ permissions.
"""
current_span = trace.get_current_span()
@@ -107,22 +120,7 @@ async def list_buckets(
else:
buckets = await CRUDBucket.get_for_user(db, owner_id, bucket_type)
- def map_buckets(bucket: Bucket) -> BucketOutSchema:
- # define function to fetch objects only one time for each bucket
- objects = list(get_s3_bucket_objects(s3, bucket.name))
- return BucketOutSchema(
- **{
- "description": bucket.description,
- "name": bucket.name,
- "created_at": bucket.created_at,
- "owner_id": bucket.owner_id,
- "num_objects": sum(1 for obj in objects if not obj.key.endswith("/")),
- "size": reduce(lambda x, y: x + y.size, objects, 0),
- "owner_constraint": bucket.owner_constraint,
- }
- )
-
- return list(map(map_buckets, buckets))
+ return buckets
@router.post(
@@ -138,7 +136,7 @@ async def create_bucket(
db: DBSession,
s3: S3Resource,
authorization: Authorization,
-) -> BucketOutSchema:
+) -> Bucket:
"""
Create a bucket for the current user.\n
The name of the bucket has some constraints.
@@ -161,7 +159,7 @@ async def create_bucket(
Returns
-------
- bucket : app.schemas.bucket.BucketOut
+ bucket : clowmdb.models.Bucket
The newly created bucket.
"""
current_span = trace.get_current_span()
@@ -187,7 +185,7 @@ async def create_bucket(
{
"Sid": "ProxyOwnerPerm",
"Effect": "Allow",
- "Principal": {"AWS": [f"arn:aws:iam:::user/{settings.BUCKET_CEPH_USERNAME}"]},
+ "Principal": {"AWS": [f"arn:aws:iam:::user/{settings.s3.username}"]},
"Action": ["s3:GetObject"],
"Resource": [f"arn:aws:s3:::{db_bucket.name}/*"],
},
@@ -205,27 +203,17 @@ async def create_bucket(
with tracer.start_as_current_span("s3_put_bucket_cors_rules") as span:
span.set_attribute("bucket_name", db_bucket.name)
s3_bucket.Cors().put(CORSConfiguration=cors_rule) # type: ignore[arg-type]
- return BucketOutSchema(
- **{
- "description": db_bucket.description,
- "name": db_bucket.name,
- "created_at": db_bucket.created_at,
- "owner_id": db_bucket.owner.uid,
- "num_objects": 0,
- "size": 0,
- }
- )
+ return db_bucket
@router.get("/{bucket_name}", response_model=BucketOutSchema, summary="Get a bucket by its name")
@start_as_current_span_async("api_get_bucket", tracer=tracer)
async def get_bucket(
bucket: CurrentBucket,
- s3: S3Resource,
current_user: CurrentUser,
authorization: Authorization,
db: DBSession,
-) -> BucketOutSchema:
+) -> Bucket:
"""
Get a bucket by its name if the current user has READ permissions for the bucket.\n
Permission `bucket:read` required if the current user is the owner of the bucket,
@@ -235,8 +223,6 @@ async def get_bucket(
----------
bucket : clowmdb.models.Bucket
Bucket with the name provided in the URL path. Dependency Injection.
- s3 : boto3_type_annotations.s3.ServiceResource
- S3 Service to perform operations on buckets in Ceph. Dependency Injection.
authorization : Callable[[str], Awaitable[Any]]
Async function to ask the auth service for authorization. Dependency Injection.
db : sqlalchemy.ext.asyncio.AsyncSession.
@@ -246,7 +232,7 @@ async def get_bucket(
Returns
-------
- bucket : app.schemas.bucket.BucketOut
+ bucket : clowmdb.models.Bucket
Bucket with the provided name.
"""
trace.get_current_span().set_attribute("bucket_name", bucket.name)
@@ -256,18 +242,58 @@ async def get_bucket(
else "read"
)
await authorization(rbac_operation)
- objects: list[ObjectSummary] = list(get_s3_bucket_objects(s3, bucket.name))
- return BucketOutSchema(
- **{
- "description": bucket.description,
- "name": bucket.name,
- "created_at": bucket.created_at,
- "owner_id": bucket.owner_id,
- "num_objects": sum(1 for obj in objects if not obj.key.endswith("/")),
- "size": reduce(lambda x, y: x + y.size, objects, 0),
- "owner_constraint": bucket.owner_constraint,
- }
- )
+ return bucket
+
+
+@router.patch("/{bucket_name}/public", response_model=BucketOutSchema, summary="Toggle public status")
+@start_as_current_span_async("api_toggle_bucket_public_state", tracer=tracer)
+async def update_bucket_public_state(
+ bucket: CurrentBucket,
+ current_user: CurrentUser,
+ authorization: Authorization,
+ db: DBSession,
+ s3: S3Resource,
+) -> Bucket:
+ """
+ Toggle the buckets public state. A bucket with an owner constraint can't be made public.\n
+ Permission `bucket:update` required if the current user is the owner of the bucket,
+ otherwise `bucket:update_any` required.
+ \f
+ Parameters
+ ----------
+ bucket : clowmdb.models.Bucket
+ Bucket with the name provided in the URL path. Dependency Injection.
+ authorization : Callable[[str], Awaitable[Any]]
+ Async function to ask the auth service for authorization. Dependency Injection.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on. Dependency Injection.
+ current_user : clowmdb.models.User
+ Current user. Dependency Injection.
+ s3 : boto3_type_annotations.s3.ServiceResource
+ S3 Service to perform operations on buckets in Ceph. Dependency Injection.
+
+ Returns
+ -------
+ bucket : clowmdb.models.Bucket
+ Bucket with the toggled public state.
+ """
+ trace.get_current_span().set_attributes({"bucket_name": bucket.name, "old_public_state": bucket.public})
+ rbac_operation = "update" if bucket.owner_id == current_user.uid else "update_any"
+ await authorization(rbac_operation)
+ if bucket.owner_constraint is not None:
+ raise HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail=f"can't make a bucket with owner constraint {bucket.owner_constraint} public",
+ )
+ s3_policy = get_s3_bucket_policy(s3, bucket_name=bucket.name)
+ json_policy = json.loads(s3_policy.policy)
+ if bucket.public:
+ json_policy["Statement"] = [stmt for stmt in json_policy["Statement"] if stmt["Sid"] != ANONYMOUS_ACCESS_SID]
+ else:
+ json_policy["Statement"] += get_anonymously_bucket_policy(bucket.name)
+ put_s3_bucket_policy(s3, bucket_name=bucket.name, policy=json.dumps(json_policy))
+ await CRUDBucket.update_public_state(db=db, public=not bucket.public, bucket_name=bucket.name)
+ return bucket
@router.delete("/{bucket_name}", status_code=status.HTTP_204_NO_CONTENT, summary="Delete a bucket")
diff --git a/app/ceph/rgw.py b/app/ceph/rgw.py
index 563cc2bb487b501f0e14f144aef7597ef9af53b3..3b538f06ad5cc22ba40b868f520dadfdb0f48c05 100644
--- a/app/ceph/rgw.py
+++ b/app/ceph/rgw.py
@@ -9,10 +9,10 @@ from app.schemas.s3key import S3Key
tracer = trace.get_tracer_provider().get_tracer(__name__)
rgw = RGWAdmin(
- access_key=settings.USER_CEPH_ACCESS_KEY,
- secret_key=settings.USER_CEPH_SECRET_KEY,
- secure=str(settings.OBJECT_GATEWAY_URI).startswith("https"),
- server=str(settings.OBJECT_GATEWAY_URI).split("://")[-1][:-1],
+ access_key=settings.s3.admin_access_key,
+ secret_key=settings.s3.admin_secret_key.get_secret_value(),
+ secure=settings.s3.uri.scheme == "https",
+ server=str(settings.s3.uri).split("://")[-1][:-1],
)
diff --git a/app/ceph/s3.py b/app/ceph/s3.py
index d41f225ac8396173aa8b392ba776106f4f0d93fb..bb48798f5532388291d1b04fa098d5621b3dfcff 100644
--- a/app/ceph/s3.py
+++ b/app/ceph/s3.py
@@ -17,10 +17,10 @@ tracer = trace.get_tracer_provider().get_tracer(__name__)
s3_resource: S3ServiceResource = resource(
service_name="s3",
- endpoint_url=str(settings.OBJECT_GATEWAY_URI)[:-1],
- aws_access_key_id=settings.BUCKET_CEPH_ACCESS_KEY,
- aws_secret_access_key=settings.BUCKET_CEPH_SECRET_KEY,
- verify=str(settings.OBJECT_GATEWAY_URI).startswith("https"),
+ endpoint_url=str(settings.s3.uri)[:-1],
+ aws_access_key_id=settings.s3.access_key,
+ aws_secret_access_key=settings.s3.secret_key.get_secret_value(),
+ verify=settings.s3.uri.scheme == "https",
)
diff --git a/app/check_ceph_connection.py b/app/check_ceph_connection.py
index 4a3716ea39af355a8183d2441fdf8560aa1409ea..f19bdb9348875e98981e71f22ed0d06d069669ec 100644
--- a/app/check_ceph_connection.py
+++ b/app/check_ceph_connection.py
@@ -20,7 +20,7 @@ wait_seconds = 2
)
def init() -> None:
try:
- httpx.get(str(settings.OBJECT_GATEWAY_URI), timeout=5.0)
+ httpx.get(str(settings.s3.uri), timeout=5.0)
except Exception as e:
logger.error(e)
raise e
diff --git a/app/check_database_connection.py b/app/check_database_connection.py
index 30102f8561f941951fee1bef510f4ab8b2376b81..af929d5f38e963150642eaf2db549b427d8cde54 100644
--- a/app/check_database_connection.py
+++ b/app/check_database_connection.py
@@ -22,7 +22,7 @@ wait_seconds = 3
)
def init() -> None:
try:
- with get_session(url=str(settings.SQLALCHEMY_DATABASE_NORMAL_URI)) as db:
+ with get_session(url=str(settings.db.dsn_sync)) as db:
# Try to create session to check if DB is awake
db_revision = db.execute(text("SELECT version_num FROM alembic_version LIMIT 1")).scalar_one_or_none()
if db_revision != latest_revision:
diff --git a/app/core/config.py b/app/core/config.py
index 4b9a41f7f4fb489fc387b5c6315f82a4a8558085..6b14107965b74695c1ae03775ccab54e47648681 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -1,111 +1,148 @@
-from pathlib import Path
-from typing import Any
+import os
+from functools import cached_property
+from typing import Literal, Type
+
+from pydantic import AnyHttpUrl, BaseModel, Field, FilePath, MySQLDsn, NameEmail, SecretStr
+from pydantic_settings import (
+ BaseSettings,
+ JsonConfigSettingsSource,
+ PydanticBaseSettingsSource,
+ SettingsConfigDict,
+ TomlConfigSettingsSource,
+ YamlConfigSettingsSource,
+)
+
+
+class DBSettings(BaseModel):
+ port: int = Field(3306, description="Port of the database.")
+ host: str = Field("localhost", description="Host of the database.")
+ name: str = Field(..., description="Name of the database.")
+ user: str = Field(..., description="Username in the database.")
+ password: SecretStr = Field(..., description="Password for the database user.")
+ verbose: bool = Field(False, description="Flag whether to print the SQL Queries in the logs.")
+
+ @cached_property
+ def dsn_sync(self) -> MySQLDsn:
+ return MySQLDsn.build(
+ scheme="mysql+pymysql",
+ password=self.password.get_secret_value(),
+ host=self.host,
+ port=self.port,
+ path=self.name,
+ username=self.user,
+ )
-from pydantic import AnyHttpUrl, AnyUrl, Field, computed_field
-from pydantic_settings import BaseSettings, SettingsConfigDict
+ @cached_property
+ def dsn_async(self) -> MySQLDsn:
+ return MySQLDsn.build(
+ scheme="mysql+aiomysql",
+ password=self.password.get_secret_value(),
+ host=self.host,
+ port=self.port,
+ path=self.name,
+ username=self.user,
+ )
-def _assemble_db_uri(values: dict[str, Any], async_flag: bool = True) -> Any:
- return AnyUrl.build(
- scheme=f"mysql+{'aiomysql' if async_flag else 'pymysql'}",
- password=values.get("DB_PASSWORD"),
- username=values.get("DB_USER"),
- port=values.get("DB_PORT"),
- host=values.get("DB_HOST"), # type: ignore[arg-type]
- path=f"{values.get('DB_DATABASE') or ''}",
+class S3Settings(BaseModel):
+ uri: AnyHttpUrl = Field(..., description="URI of the S3 Object Storage.")
+ access_key: str = Field(..., description="Access key for the S3 that owns the buckets.")
+ secret_key: SecretStr = Field(..., description="Secret key for the S3 that owns the buckets.")
+ username: str = Field(
+ ..., description="ID of the user in ceph who owns all the buckets. Owner of 'CLOWM_S3__ACCESS_KEY'"
+ )
+ admin_access_key: str = Field(
+ ..., description="Access key for the Ceph Object Gateway with 'user:read,user:write privileges'."
+ )
+ admin_secret_key: SecretStr = Field(
+ ..., description="Secret key for the Ceph Object Gateway with 'user:read,user:write privileges'."
)
-def _load_public_key(pub_key_val: str | None, pub_key_file: Path | None) -> str:
- pub_key = ""
- if pub_key_val is not None:
- pub_key = pub_key_val
- if pub_key_file is not None:
- with open(pub_key_file) as f:
- pub_key = f.read()
- if len(pub_key) == 0:
- raise ValueError("PUBLIC_KEY_VALUE or PUBLIC_KEY_FILE must be set")
- return pub_key
-
+class OPASettings(BaseModel):
+ uri: AnyHttpUrl = Field(..., description="URI of the OPA Service")
-class Settings(BaseSettings):
- CLOWM_URL: AnyHttpUrl = Field(
- AnyHttpUrl("http://localhost:8080"),
- description="Base HTTP URL where the CloWM service is reachable.",
- examples=["http://localhost:8080"],
- )
- API_PREFIX: str = Field("/api", description="Path Prefix for all API endpoints.")
- public_key_value: str | None = Field(
- None, description="Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_VALUE"
- )
- public_key_file: Path | None = Field(
- None, description="Path to Public RSA Key in PEM format to sign the JWTs.", validation_alias="PUBLIC_KEY_FILE"
+class OTLPSettings(BaseModel):
+ grpc_endpoint: str | None = Field(
+ None, description="OTLP compatible endpoint to send traces via gRPC, e.g. Jaeger", examples=["localhost:8080"]
)
+ secure: bool = Field(False, description="Connection type")
- @computed_field # type: ignore[misc]
- @property
- def PUBLIC_KEY(self) -> str:
- return _load_public_key(self.public_key_value, self.public_key_file)
-
- DB_HOST: str = Field(..., description="Host of the database.")
- DB_USER: str = Field(..., description="Username in the database.")
- DB_PASSWORD: str = Field(..., description="Password for the database user.")
- DB_DATABASE: str = Field(..., description="Name of the database.")
- DB_PORT: int = Field(3306, description="Port of the database.")
- SQLALCHEMY_VERBOSE_LOGGER: bool = Field(False, description="Flag whether to print the SQL Queries in the logs.")
-
- @computed_field # type: ignore[misc]
- @property
- def SQLALCHEMY_DATABASE_ASYNC_URI(self) -> AnyUrl:
- return _assemble_db_uri(
- {
- "DB_HOST": self.DB_HOST,
- "DB_USER": self.DB_USER,
- "DB_PASSWORD": self.DB_PASSWORD,
- "DB_DATABASE": self.DB_DATABASE,
- "DB_PORT": self.DB_PORT,
- },
- async_flag=True,
- )
-
- @computed_field # type: ignore[misc]
- @property
- def SQLALCHEMY_DATABASE_NORMAL_URI(self) -> AnyUrl:
- return _assemble_db_uri(
- {
- "DB_HOST": self.DB_HOST,
- "DB_USER": self.DB_USER,
- "DB_PASSWORD": self.DB_PASSWORD,
- "DB_DATABASE": self.DB_DATABASE,
- "DB_PORT": self.DB_PORT,
- },
- async_flag=False,
- )
- OBJECT_GATEWAY_URI: AnyHttpUrl = Field(..., description="URI of the Ceph Object Gateway.")
- USER_CEPH_ACCESS_KEY: str = Field(
- ..., description="Access key for the Ceph Object Gateway with 'user:read,user:write privileges'."
- )
- USER_CEPH_SECRET_KEY: str = Field(
- ..., description="Secret key for the Ceph Object Gateway with 'user:read,user:write privileges'."
+class EmailSettings(BaseModel):
+ server: str | Literal["console"] | None = Field(
+ None,
+ description="Hostname of SMTP server. If `console`, emails are printed to the console. If None, no emails are sent",
)
- BUCKET_CEPH_ACCESS_KEY: str = Field(
- ..., description="Access key for the Ceph Object Gateway with unlimited buckets."
+ port: int = Field(587, description="Port of the SMTP server")
+ sender_email: NameEmail = Field(
+ NameEmail(email="no-reply@clowm.com", name="CloWM"), description="Email address from which the emails are sent."
)
- BUCKET_CEPH_SECRET_KEY: str = Field(
- ..., description="Secret key for the Ceph Object Gateway with unlimited buckets."
+ reply_email: NameEmail | None = Field(None, description="Email address in the `Reply-To` header.")
+ connection_security: Literal["ssl", "tls"] | Literal["starttls"] | None = Field(
+ None, description="Connection security to the SMTP server."
)
- BUCKET_CEPH_USERNAME: str = Field(
- ..., description="ID of the user in ceph who owns all the buckets. Owner of 'BUCKET_CEPH_ACCESS_KEY'"
+ local_hostname: str | None = Field(None, description="Overwrite the local hostname from which the emails are sent.")
+ ca_path: FilePath | None = Field(None, description="Path to a custom CA certificate.")
+ key_path: FilePath | None = Field(None, description="Path to the CA key.")
+ user: str | None = Field(None, description="Username to use for SMTP login")
+ password: SecretStr | None = Field(None, description="Password to use for SMTP login")
+
+
+class Settings(BaseSettings):
+ api_prefix: str = Field("/api/s3proxy-service", description="Path Prefix for all API endpoints.")
+ public_key: SecretStr | None = Field(None, description="Public RSA Key in PEM format to verify the JWTs.")
+ public_key_file: FilePath | None = Field(
+ None, description="Path to Public RSA Key in PEM format to verify the JWTs."
)
- OPA_URI: AnyHttpUrl = Field(..., description="URI of the OPA Service")
- OPA_POLICY_PATH: str = Field("/clowm/authz/allow", description="Path to the OPA Policy for Authorization")
- OTLP_GRPC_ENDPOINT: str | None = Field(
- None, description="OTLP compatible endpoint to send traces via gRPC, e.g. Jaeger"
+ ui_uri: AnyHttpUrl = Field(..., description="URL of the UI")
+
+ @cached_property
+ def public_key_value(self) -> SecretStr:
+ pub_key = ""
+ if self.public_key is not None:
+ pub_key = self.public_key.get_secret_value()
+ if self.public_key_file is not None:
+ with open(self.public_key_file) as f:
+ pub_key = f.read()
+ if len(pub_key) == 0:
+ raise ValueError("CLOWM_PUBLIC_KEY or CLOWM_PUBLIC_KEY_FILE must be set")
+ return SecretStr(pub_key)
+
+ db: DBSettings
+ smtp: EmailSettings = EmailSettings()
+ s3: S3Settings
+ opa: OPASettings
+ otlp: OTLPSettings | None = None
+
+ model_config = SettingsConfigDict(
+ env_prefix="CLOWM_",
+ env_file=".env",
+ extra="ignore",
+ secrets_dir="/run/secrets" if os.path.isdir("/run/secrets") else None,
+ env_nested_delimiter="__",
+ yaml_file=os.getenv("CLOWM_CONFIG_FILE_YAML", "config.yaml"),
+ toml_file=os.getenv("CLOWM_CONFIG_FILE_TOML", "config.toml"),
+ json_file=os.getenv("CLOWM_CONFIG_FILE_JSON", "config.json"),
)
- model_config = SettingsConfigDict(case_sensitive=True, env_file=".env", secrets_dir="/run/secrets", extra="ignore")
+ @classmethod
+ def settings_customise_sources(
+ cls,
+ settings_cls: Type[BaseSettings],
+ init_settings: PydanticBaseSettingsSource,
+ env_settings: PydanticBaseSettingsSource,
+ dotenv_settings: PydanticBaseSettingsSource,
+ file_secret_settings: PydanticBaseSettingsSource,
+ ) -> tuple[PydanticBaseSettingsSource, ...]:
+ return (
+ env_settings,
+ dotenv_settings,
+ YamlConfigSettingsSource(settings_cls),
+ TomlConfigSettingsSource(settings_cls),
+ JsonConfigSettingsSource(settings_cls),
+ )
settings = Settings()
diff --git a/app/core/security.py b/app/core/security.py
index a592a3e407687320f74bd0e8167d898521a47493..b4f79f766a331d615e93ba8085cbcbc649039156 100644
--- a/app/core/security.py
+++ b/app/core/security.py
@@ -1,11 +1,12 @@
+from typing import Type, TypeVar
+
from authlib.jose import JsonWebToken
from fastapi import HTTPException, status
from httpx import AsyncClient
from opentelemetry import trace
from app.core.config import settings
-from app.otlp import start_as_current_span_async
-from app.schemas.security import AuthzRequest, AuthzResponse
+from app.schemas.security import AuthzRequest, AuthzResponse, OPARequest, OPAResponse
ISSUER = "clowm"
ALGORITHM = "RS256"
@@ -13,6 +14,8 @@ jwt = JsonWebToken([ALGORITHM])
tracer = trace.get_tracer_provider().get_tracer(__name__)
+T = TypeVar("T", bound=OPAResponse)
+
def decode_token(token: str) -> dict[str, str]: # pragma: no cover
"""
@@ -30,7 +33,7 @@ def decode_token(token: str) -> dict[str, str]: # pragma: no cover
"""
claims = jwt.decode(
s=token,
- key=settings.PUBLIC_KEY,
+ key=settings.public_key_value.get_secret_value(),
claims_options={
"iss": {"essential": True},
"sub": {"essential": True},
@@ -41,7 +44,38 @@ def decode_token(token: str) -> dict[str, str]: # pragma: no cover
return claims
-@start_as_current_span_async("authorization", tracer=tracer)
+async def _request_opa(request: OPARequest, response_type: Type[T], *, client: AsyncClient) -> T:
+ """
+ Wrapper to send a generic request to OPA.
+
+ Parameters
+ ----------
+ request : OPARequest
+ The request for OPA.
+ response_type : Type[OPAResponse -> T]
+ The class to which the response will be parsed. T must inherit from OPAResponse.
+ client : httpx.AsyncClient
+ An async http client with an open connection.
+
+ Returns
+ -------
+ response : T
+ The parsed response from OPA.
+ """
+ with tracer.start_as_current_span(
+ "opa_request",
+ attributes={"opa_path": request.opa_path.strip("/"), "body": request.model_dump_json(indent=4)},
+ ) as span:
+ response = await client.post(
+ str(settings.opa.uri) + request.opa_path.strip("/"),
+ content=request.model_dump_json(),
+ headers={"content-type": "application/json"},
+ )
+ parsed_response = response_type.model_validate_json(response.content)
+ span.set_attribute("decision_id", str(parsed_response.decision_id))
+ return parsed_response
+
+
async def request_authorization(request_params: AuthzRequest, client: AsyncClient) -> AuthzResponse:
"""
Send a request to OPA for a policy decision. Raise an HTTPException with status 403 if authorization is denied.
@@ -58,17 +92,23 @@ async def request_authorization(request_params: AuthzRequest, client: AsyncClien
response : app.schemas.security.AuthzResponse
Response by the Auth service about the authorization request
"""
- current_span = trace.get_current_span()
- current_span.set_attributes({"resource": request_params.resource, "operation": request_params.operation})
- response = await client.post(
- f"{settings.OPA_URI}v1/data{settings.OPA_POLICY_PATH}", json={"input": request_params.model_dump()}
- )
- parsed_response = AuthzResponse.model_validate(response.json())
- current_span.set_attribute("decision_id", str(parsed_response.decision_id))
- if not parsed_response.result: # pragma: no cover
+ with tracer.start_as_current_span(
+ "authorization",
+ attributes={
+ "resource": request_params.resource,
+ "operation": request_params.operation,
+ "uid": request_params.uid,
+ },
+ ):
+ response = await _request_opa(
+ request=OPARequest(input=request_params, opa_path="/v1/data/clowm/authz/allow"),
+ response_type=AuthzResponse,
+ client=client,
+ )
+ if not response.result: # pragma: no cover
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
- detail=f"Action forbidden. Decision ID {str(parsed_response.decision_id)}",
+ detail=f"Action forbidden. Decision ID {response.decision_id}",
)
- return parsed_response
+ return response
diff --git a/app/crud/crud_bucket.py b/app/crud/crud_bucket.py
index bdc8ae7e1d37f0f5ade6d1f0be762bbf141daa68..729e030b271ae376a1d6a50a1b2e82b7da49a931 100644
--- a/app/crud/crud_bucket.py
+++ b/app/crud/crud_bucket.py
@@ -5,7 +5,7 @@ from uuid import UUID
from clowmdb.models import Bucket
from clowmdb.models import BucketPermission as BucketPermissionDB
from opentelemetry import trace
-from sqlalchemy import delete, func, or_, select
+from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.crud import DuplicateError
@@ -181,9 +181,30 @@ class CRUDBucket:
raise DuplicateError(f"Bucket {bucket.name} exists already")
db.add(bucket)
await db.commit()
- await db.refresh(bucket)
return bucket
+ @staticmethod
+ async def update_public_state(db: AsyncSession, bucket_name: str, public: bool) -> None:
+ """
+ Update the public state of a bucket
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession
+ Async database session to perform query on.
+ bucket_name : str
+ Name of a bucket.
+ public : bool
+ New public state of the bucket.
+ """
+ stmt = update(Bucket).where(Bucket.name == bucket_name).values(public=public)
+ with tracer.start_as_current_span(
+ "db_update_bucket_public_state",
+ attributes={"bucket_name": bucket_name, "public": public},
+ ):
+ await db.execute(stmt)
+ await db.commit()
+
@staticmethod
async def delete(db: AsyncSession, bucket_name: str) -> None:
"""
diff --git a/app/main.py b/app/main.py
index 0fad3197f0c886c5da38e9e5f986a1dac288bc44..a3212bc5327a108469f8c613ccbd980b6a4fa30a 100644
--- a/app/main.py
+++ b/app/main.py
@@ -49,18 +49,20 @@ app = FastAPI(
},
generate_unique_id_function=custom_generate_unique_id,
license_info={"name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0"},
- root_path=settings.API_PREFIX,
+ root_path=settings.api_prefix,
openapi_url=None, # create it manually to enable caching on client side
lifespan=lifespan,
)
-if settings.API_PREFIX: # pragma: no cover
- app.servers.insert(0, {"url": settings.API_PREFIX})
+if settings.api_prefix: # pragma: no cover
+ app.servers.insert(0, {"url": settings.api_prefix})
-if settings.OTLP_GRPC_ENDPOINT is not None and len(settings.OTLP_GRPC_ENDPOINT) > 0: # pragma: no cover
+if (
+ settings.otlp is not None and settings.otlp.grpc_endpoint is not None and len(settings.otlp.grpc_endpoint) > 0
+): # pragma: no cover
resource = Resource(attributes={SERVICE_NAME: "clowm-s3proxy-service"})
provider = TracerProvider(resource=resource)
provider.add_span_processor(
- BatchSpanProcessor(OTLPSpanExporter(endpoint=settings.OTLP_GRPC_ENDPOINT, insecure=True))
+ BatchSpanProcessor(OTLPSpanExporter(endpoint=settings.otlp.grpc_endpoint, insecure=not settings.otlp.secure))
)
trace.set_tracer_provider(provider)
diff --git a/app/schemas/bucket.py b/app/schemas/bucket.py
index c763f54bca1c3322cd2c31c2b8566e2263a5efe7..568dc201916d3cfe6b133222e1884c896a8648c2 100644
--- a/app/schemas/bucket.py
+++ b/app/schemas/bucket.py
@@ -26,7 +26,7 @@ class _BaseBucket(BaseModel):
...,
examples=["This is a sample description of a bucket"],
description="Description of the bucket",
- min_length=32,
+ min_length=16,
max_length=2**16,
)
@@ -55,11 +55,6 @@ class BucketOut(_BaseBucket):
description="Time when the bucket was created as UNIX timestamp",
)
owner_id: UUID = Field(..., description="UID of the owner", examples=["1d3387f3-95c0-4813-8767-2cad87faeebf"])
- num_objects: int = Field(..., description="Number of Objects in this bucket", examples=[6])
- size: int = Field(..., description="Total size of objects in this bucket in bytes", examples=[3256216])
owner_constraint: Bucket.Constraint | None = Field(None, description="Constraint for the owner of the bucket")
- description: str = Field(
- ...,
- description="Description of the bucket",
- )
+ public: bool = Field(..., description="Flag if the bucket is anonymously readable")
model_config = ConfigDict(from_attributes=True)
diff --git a/app/schemas/security.py b/app/schemas/security.py
index d065001db07cf1c76133a507398889f46f8a0bac..3f8d0f508357faa22f1e208680a2a84a71307ac4 100644
--- a/app/schemas/security.py
+++ b/app/schemas/security.py
@@ -1,25 +1,40 @@
from datetime import datetime
+from typing import Any
from pydantic import BaseModel, Field
from app.schemas import UUID
-class AuthzResponse(BaseModel):
- """Schema for a response from OPA"""
+class OPARequest(BaseModel):
+ opa_path: str = Field(
+ ...,
+ description="URL path where OPA is queried",
+ exclude=True,
+ examples=["/v1/data/clowm/authz/allow"],
+ )
+ input: Any
+
+class OPAResponse(BaseModel):
decision_id: UUID = Field(
...,
description="Decision ID for for the specific decision",
examples=["8851dce0-7546-4e81-a89d-111cbec376c1"],
)
+ result: Any
+
+
+class AuthzResponse(OPAResponse):
+ """Schema for a response from OPA"""
+
result: bool = Field(..., description="Result of the Authz request")
class AuthzRequest(BaseModel):
"""Schema for a Request to OPA"""
- uid: str = Field(..., description="lifescience id of user", examples=["28c5353b8bb34984a8bd4169ba94c606"])
+ uid: str = Field(..., description="UID of user", examples=["28c5353b8bb34984a8bd4169ba94c606"])
operation: str = Field(..., description="Operation the user wants to perform", examples=["read"])
resource: str = Field(..., description="Resource the operation should be performed on", examples=["bucket"])
diff --git a/app/tests/api/test_bucket_permissions.py b/app/tests/api/test_bucket_permissions.py
index 153e0d8b0adee8eebb7ff1cbe2868464c609d5af..001dcd77df6532ad690ab12b8e5af4b634ceeea4 100644
--- a/app/tests/api/test_bucket_permissions.py
+++ b/app/tests/api/test_bucket_permissions.py
@@ -5,6 +5,7 @@ import pytest
from clowmdb.models import Bucket, BucketPermission
from fastapi import status
from httpx import AsyncClient
+from pydantic import TypeAdapter
from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncSession
@@ -33,11 +34,11 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.get(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(random_bucket_permission_schema.uid)}", # noqa:E501
@@ -45,7 +46,7 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
)
assert response.status_code == status.HTTP_200_OK
- permission = BucketPermissionOut.model_validate(response.json())
+ permission = BucketPermissionOut.model_validate_json(response.content)
assert permission
assert permission.uid == random_bucket_permission_schema.uid
@@ -64,11 +65,11 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.get(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(uuid4())}",
@@ -90,13 +91,13 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
response = await client.get(
f"{self.base_path}/bucket/{random_bucket.name}/user/{str(random_second_user.user.uid)}",
@@ -117,18 +118,18 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
response = await client.get(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(random_second_user.user.uid)}",
headers=random_second_user.auth_headers,
)
assert response.status_code == status.HTTP_200_OK
- permission = BucketPermissionOut.model_validate(response.json())
+ permission = BucketPermissionOut.model_validate_json(response.content)
assert permission
assert permission.uid == random_bucket_permission_schema.uid
@@ -148,13 +149,13 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(db, random_bucket_permission_schema.bucket_name, random_third_user.user.uid)
@@ -177,21 +178,23 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.get(
f"{self.base_path}/user/{str(random_bucket_permission_schema.uid)}",
headers=random_second_user.auth_headers,
)
assert response.status_code == status.HTTP_200_OK
- permission_list = response.json()
- assert isinstance(permission_list, list)
- assert len(permission_list) == 1
- permission = BucketPermissionOut.model_validate(permission_list[0])
+
+ ta = TypeAdapter(list[BucketPermissionOut])
+ permissions = ta.validate_json(response.content)
+ assert len(permissions) == 1
+
+ permission = permissions[0]
assert permission.uid == random_bucket_permission_schema.uid
assert permission.bucket_name == random_bucket_permission_schema.bucket_name
@@ -208,18 +211,20 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.get(self.base_path, headers=random_user.auth_headers, params={"allow_admin": True})
assert response.status_code == status.HTTP_200_OK
- permission_list = response.json()
- assert isinstance(permission_list, list)
- assert len(permission_list) == 1
- permission = BucketPermissionOut.model_validate(permission_list[0])
+
+ ta = TypeAdapter(list[BucketPermissionOut])
+ permissions = ta.validate_json(response.content)
+ assert len(permissions) == 1
+
+ permission = permissions[0]
assert permission.uid == random_bucket_permission_schema.uid
assert permission.bucket_name == random_bucket_permission_schema.bucket_name
@@ -236,21 +241,23 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.get(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}",
headers=random_user.auth_headers,
)
assert response.status_code == status.HTTP_200_OK
- permission_list = response.json()
- assert isinstance(permission_list, list)
- assert len(permission_list) == 1
- permission = BucketPermissionOut.model_validate(permission_list[0])
+
+ ta = TypeAdapter(list[BucketPermissionOut])
+ permissions = ta.validate_json(response.content)
+ assert len(permissions) == 1
+
+ permission = permissions[0]
assert permission.uid == random_bucket_permission_schema.uid
assert permission.bucket_name == random_bucket_permission_schema.bucket_name
@@ -267,11 +274,11 @@ class TestBucketPermissionRoutesGet(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.get(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}",
@@ -291,14 +298,16 @@ class TestBucketPermissionRoutesCreate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=uuid4())
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=permission.model_dump())
+ response = await client.post(
+ self.base_path, headers=random_user.auth_headers, content=permission.model_dump_json()
+ )
assert response.status_code == status.HTTP_404_NOT_FOUND
@pytest.mark.asyncio
@@ -311,16 +320,18 @@ class TestBucketPermissionRoutesCreate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing who is owner of the bucket. pytest fixture.
+ Random user for testing who is owner of the bucket.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=random_user.user.uid)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=permission.model_dump())
+ response = await client.post(
+ self.base_path, headers=random_user.auth_headers, content=permission.model_dump_json()
+ )
assert response.status_code == status.HTTP_400_BAD_REQUEST
@pytest.mark.asyncio
@@ -336,16 +347,18 @@ class TestBucketPermissionRoutesCreate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
permission = BucketPermissionSchema(
bucket_name=random_bucket_permission_schema.bucket_name, uid=random_bucket_permission_schema.uid
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=permission.model_dump())
+ response = await client.post(
+ self.base_path, headers=random_user.auth_headers, content=permission.model_dump_json()
+ )
assert response.status_code == status.HTTP_400_BAD_REQUEST
@pytest.mark.asyncio
@@ -361,16 +374,16 @@ class TestBucketPermissionRoutesCreate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=random_second_user.user.uid)
response = await client.post(
- self.base_path, headers=random_second_user.auth_headers, json=permission.model_dump()
+ self.base_path, headers=random_second_user.auth_headers, content=permission.model_dump_json()
)
assert response.status_code == status.HTTP_403_FORBIDDEN
@@ -389,22 +402,24 @@ class TestBucketPermissionRoutesCreate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
"""
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=random_second_user.user.uid)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=permission.model_dump())
+ response = await client.post(
+ self.base_path, headers=random_user.auth_headers, content=permission.model_dump_json()
+ )
assert response.status_code == status.HTTP_201_CREATED
- created_permission = BucketPermissionOut.model_validate(response.json())
+ created_permission = BucketPermissionOut.model_validate_json(response.content)
assert created_permission.uid == random_second_user.user.uid
assert created_permission.bucket_name == random_bucket.name
@@ -423,15 +438,15 @@ class TestBucketPermissionRoutesCreate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
update_stmt = (
update(Bucket).where(Bucket.name == random_bucket.name).values(owner_constraint=Bucket.Constraint.READ)
@@ -440,7 +455,9 @@ class TestBucketPermissionRoutesCreate(_TestBucketPermissionRoutes):
await db.commit()
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=random_second_user.user.uid)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=permission.model_dump())
+ response = await client.post(
+ self.base_path, headers=random_user.auth_headers, content=permission.model_dump_json()
+ )
assert response.status_code == status.HTTP_403_FORBIDDEN
@@ -458,11 +475,11 @@ class TestBucketPermissionRoutesDelete(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.delete(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(random_bucket_permission_schema.uid)}", # noqa:E501
@@ -483,11 +500,11 @@ class TestBucketPermissionRoutesDelete(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.get(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(random_bucket_permission_schema.uid)}", # noqa:E501
@@ -508,9 +525,9 @@ class TestBucketPermissionRoutesDelete(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
response = await client.delete(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(uuid4())}",
@@ -532,11 +549,11 @@ class TestBucketPermissionRoutesDelete(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
response = await client.delete(
f"{self.base_path}/bucket/{random_bucket.name}/user/{str(random_second_user.user.uid)}",
@@ -558,13 +575,13 @@ class TestBucketPermissionRoutesDelete(_TestBucketPermissionRoutes):
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(db, random_bucket_permission_schema.bucket_name, random_third_user.user.uid)
response = await client.delete(
@@ -588,9 +605,9 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
new_from_time = round(datetime(2022, 1, 1, 0, 0).timestamp())
new_params = BucketPermissionParametersSchema(
@@ -602,10 +619,10 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
response = await client.put(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(random_bucket_permission_schema.uid)}", # noqa:E501
headers=random_user.auth_headers,
- json=new_params.model_dump(),
+ content=new_params.model_dump_json(),
)
assert response.status_code == status.HTTP_200_OK
- updated_permission = BucketPermissionOut.model_validate(response.json())
+ updated_permission = BucketPermissionOut.model_validate_json(response.content)
assert updated_permission.uid == random_bucket_permission_schema.uid
assert updated_permission.bucket_name == random_bucket_permission_schema.bucket_name
if new_params.from_timestamp is not None and new_params.to_timestamp is not None:
@@ -627,9 +644,9 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
new_params = BucketPermissionParametersSchema(
permission=BucketPermission.Permission.READWRITE,
@@ -638,7 +655,7 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
response = await client.put(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(uuid4())}",
headers=random_user.auth_headers,
- json=new_params.model_dump(),
+ content=new_params.model_dump_json(),
)
assert response.status_code == status.HTTP_404_NOT_FOUND
@@ -656,11 +673,11 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
new_params = BucketPermissionParametersSchema(
permission=BucketPermission.Permission.READWRITE,
@@ -669,7 +686,7 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
response = await client.put(
f"{self.base_path}/bucket/{random_bucket.name}/user/{str(random_second_user.user.uid)}",
headers=random_user.auth_headers,
- json=new_params.model_dump(),
+ content=new_params.model_dump_json(),
)
assert response.status_code == status.HTTP_404_NOT_FOUND
@@ -686,11 +703,11 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
new_params = BucketPermissionParametersSchema(
permission=BucketPermission.Permission.READWRITE,
@@ -699,7 +716,7 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
response = await client.put(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(random_second_user.user.uid)}",
headers=random_second_user.auth_headers,
- json=new_params.model_dump(),
+ content=new_params.model_dump_json(),
)
assert response.status_code == status.HTTP_403_FORBIDDEN
@@ -716,11 +733,11 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
new_params = BucketPermissionParametersSchema(
permission=BucketPermission.Permission.READWRITE,
@@ -729,6 +746,6 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
response = await client.put(
f"{self.base_path}/bucket/{random_bucket_permission_schema.bucket_name}/user/{str(random_bucket_permission_schema.uid)}", # noqa:E501
headers=random_third_user.auth_headers,
- json=new_params.model_dump(),
+ content=new_params.model_dump_json(),
)
assert response.status_code == status.HTTP_403_FORBIDDEN
diff --git a/app/tests/api/test_buckets.py b/app/tests/api/test_buckets.py
index e629bd74c25fef223b4b3a6fb3be45147a8097bc..10b5f1434c15872e692889231b11218a293754fb 100644
--- a/app/tests/api/test_buckets.py
+++ b/app/tests/api/test_buckets.py
@@ -2,12 +2,15 @@ import pytest
from clowmdb.models import Bucket, BucketPermission
from fastapi import status
from httpx import AsyncClient
+from pydantic import TypeAdapter
+from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncSession
+from app.api.endpoints.buckets import ANONYMOUS_ACCESS_SID
from app.crud.crud_bucket import CRUDBucket
from app.schemas.bucket import BucketIn, BucketOut
from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
-from app.tests.utils.bucket import add_permission_for_bucket, delete_bucket
+from app.tests.utils.bucket import add_permission_for_bucket, delete_bucket, make_bucket_public
from app.tests.utils.cleanup import CleanupList
from app.tests.utils.user import UserWithAuthHeader
from app.tests.utils.utils import random_lower_string
@@ -28,19 +31,20 @@ class TestBucketRoutesGet(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.get(f"{self.base_path}", headers=random_second_user.auth_headers)
assert response.status_code == status.HTTP_200_OK
- buckets = response.json()
+ ta = TypeAdapter(list[BucketOut])
+ buckets = ta.validate_json(response.content)
assert len(buckets) == 1
- bucket = BucketOut.model_validate(buckets[0])
+ bucket = buckets[0]
assert bucket.name == random_bucket.name
assert bucket.owner_id == random_bucket.owner_id
@@ -58,21 +62,22 @@ class TestBucketRoutesGet(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.get(
f"{self.base_path}", params={"owner_id": str(random_bucket.owner_id)}, headers=random_user.auth_headers
)
assert response.status_code == status.HTTP_200_OK
- buckets = response.json()
+ ta = TypeAdapter(list[BucketOut])
+ buckets = ta.validate_json(response.content)
assert len(buckets) == 1
- bucket = BucketOut.model_validate(buckets[0])
+ bucket = buckets[0]
assert bucket.name == random_bucket.name
assert bucket.owner_id == random_bucket.owner_id
@@ -90,16 +95,16 @@ class TestBucketRoutesGet(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.get(f"{self.base_path}/{random_bucket.name}", headers=random_user.auth_headers)
assert response.status_code == status.HTTP_200_OK
- bucket = BucketOut.model_validate(response.json())
+ bucket = BucketOut.model_validate_json(response.content)
assert bucket.name == random_bucket.name
assert bucket.owner_id == random_bucket.owner.uid
@@ -112,9 +117,9 @@ class TestBucketRoutesGet(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.get(f"{self.base_path}/impossible_bucket_name", headers=random_user.auth_headers)
assert response.status_code == status.HTTP_404_NOT_FOUND
@@ -124,20 +129,53 @@ class TestBucketRoutesGet(_TestBucketRoutes):
self, client: AsyncClient, random_bucket: Bucket, random_second_user: UserWithAuthHeader
) -> None:
"""
- Test for getting a foreign bucket with permission by its name.
+ Test for getting a foreign bucket without permission.
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random user which is not the owner of the bucket. pytest fixture.
+ Random user who is not the owner of the bucket.
"""
response = await client.get(f"{self.base_path}/{random_bucket.name}", headers=random_second_user.auth_headers)
assert response.status_code == status.HTTP_403_FORBIDDEN
+ @pytest.mark.asyncio
+ async def test_get_foreign_public_bucket(
+ self,
+ client: AsyncClient,
+ random_bucket: Bucket,
+ random_second_user: UserWithAuthHeader,
+ db: AsyncSession,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for getting a foreign public bucket.
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ random_bucket : clowmdb.models.Bucket
+ Random bucket for testing.
+ random_second_user : app.tests.utils.user.UserWithAuthHeader
+ Random user who is not the owner of the bucket.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects.
+ """
+ await make_bucket_public(db=db, s3=mock_s3_service, bucket_name=random_bucket.name)
+ response = await client.get(f"{self.base_path}/{random_bucket.name}", headers=random_second_user.auth_headers)
+ assert response.status_code == status.HTTP_200_OK
+
+ bucket = BucketOut.model_validate_json(response.content)
+
+ assert bucket.name == random_bucket.name
+
class TestBucketRoutesCreate(_TestBucketRoutes):
@pytest.mark.asyncio
@@ -154,11 +192,11 @@ class TestBucketRoutesCreate(_TestBucketRoutes):
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
cleanup : app.tests.utils.utils.CleanupList
Cleanup object where (async) functions can be registered which get executed after a (failed) test.
"""
@@ -168,18 +206,20 @@ class TestBucketRoutesCreate(_TestBucketRoutes):
db=db,
bucket_name=bucket_info.name,
)
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=bucket_info.model_dump())
+ response = await client.post(
+ self.base_path, headers=random_user.auth_headers, content=bucket_info.model_dump_json()
+ )
assert response.status_code == status.HTTP_201_CREATED
- bucket = BucketOut.model_validate(response.json())
+ bucket = BucketOut.model_validate_json(response.content)
assert bucket
assert bucket.name == bucket_info.name
assert bucket.owner_id == random_user.user.uid
- dbBucket = await CRUDBucket.get(db, bucket_info.name)
- assert dbBucket
- assert dbBucket.name == bucket_info.name
- assert dbBucket.owner_id == random_user.user.uid
+ db_bucket = await CRUDBucket.get(db, bucket_info.name)
+ assert db_bucket
+ assert db_bucket.name == bucket_info.name
+ assert db_bucket.owner_id == random_user.user.uid
@pytest.mark.asyncio
async def test_create_duplicated_bucket(
@@ -194,18 +234,121 @@ class TestBucketRoutesCreate(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
bucket_info = BucketIn(name=random_bucket.name, description=random_lower_string(127))
- response = await client.post(self.base_path, headers=random_user.auth_headers, json=bucket_info.model_dump())
+ response = await client.post(
+ self.base_path, headers=random_user.auth_headers, content=bucket_info.model_dump_json()
+ )
assert response.status_code == status.HTTP_400_BAD_REQUEST
+class TestBucketRoutesUpdate(_TestBucketRoutes):
+ @pytest.mark.asyncio
+ async def test_make_bucket_public(
+ self,
+ client: AsyncClient,
+ random_bucket: Bucket,
+ random_user: UserWithAuthHeader,
+ mock_s3_service: MockS3ServiceResource,
+ ) -> None:
+ """
+ Test for getting a foreign public bucket.
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ random_bucket : clowmdb.models.Bucket
+ Random bucket for testing.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user who is owner of the bucket.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects.
+ """
+ response = await client.patch(f"{self.base_path}/{random_bucket.name}/public", headers=random_user.auth_headers)
+ assert response.status_code == status.HTTP_200_OK
+
+ bucket = BucketOut.model_validate_json(response.content)
+
+ assert bucket.name == random_bucket.name
+ assert bucket.public
+
+ assert ANONYMOUS_ACCESS_SID in mock_s3_service.Bucket(bucket.name).Policy().policy
+
+ @pytest.mark.asyncio
+ async def test_make_bucket_private(
+ self,
+ client: AsyncClient,
+ random_bucket: Bucket,
+ random_user: UserWithAuthHeader,
+ mock_s3_service: MockS3ServiceResource,
+ db: AsyncSession,
+ ) -> None:
+ """
+ Test for getting a foreign public bucket.
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ random_bucket : clowmdb.models.Bucket
+ Random bucket for testing.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user who is owner of the bucket.
+ mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ """
+ await make_bucket_public(db=db, s3=mock_s3_service, bucket_name=random_bucket.name)
+ response = await client.patch(f"{self.base_path}/{random_bucket.name}/public", headers=random_user.auth_headers)
+ assert response.status_code == status.HTTP_200_OK
+
+ bucket = BucketOut.model_validate_json(response.content)
+
+ assert bucket.name == random_bucket.name
+ assert not bucket.public
+
+ assert ANONYMOUS_ACCESS_SID not in mock_s3_service.Bucket(bucket.name).Policy().policy
+
+ @pytest.mark.asyncio
+ async def test_make_bucket_public_with_owner_constraint(
+ self,
+ client: AsyncClient,
+ random_bucket: Bucket,
+ random_user: UserWithAuthHeader,
+ db: AsyncSession,
+ ) -> None:
+ """
+ Test for getting a foreign public bucket.
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ random_bucket : clowmdb.models.Bucket
+ Random bucket for testing.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user who is owner of the bucket.
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ """
+ update_stmt = (
+ update(Bucket).where(Bucket.name == random_bucket.name).values(owner_constraint=Bucket.Constraint.READ)
+ )
+ await db.execute(update_stmt)
+ await db.commit()
+
+ response = await client.patch(f"{self.base_path}/{random_bucket.name}/public", headers=random_user.auth_headers)
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+
+
class TestBucketRoutesDelete(_TestBucketRoutes):
@pytest.mark.asyncio
async def test_delete_empty_bucket(
@@ -220,11 +363,11 @@ class TestBucketRoutesDelete(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.delete(
f"{self.base_path}/{random_bucket.name}", headers=random_user.auth_headers, params={"force_delete": False}
@@ -237,8 +380,8 @@ class TestBucketRoutesDelete(_TestBucketRoutes):
self,
client: AsyncClient,
db: AsyncSession,
- random_user: UserWithAuthHeader,
random_second_user: UserWithAuthHeader,
+ random_third_user: UserWithAuthHeader,
cleanup: CleanupList,
) -> None:
"""
@@ -246,12 +389,14 @@ class TestBucketRoutesDelete(_TestBucketRoutes):
Parameters
----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
- random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ HTTP Client to perform the request on.
+ random_third_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random user which is not the owner of the bucket. pytest fixture.
+ Random user who is not the owner of the bucket.
cleanup : app.tests.utils.utils.CleanupList
Cleanup object where (async) functions can be registered which get executed after a (failed) test.
"""
@@ -268,11 +413,11 @@ class TestBucketRoutesDelete(_TestBucketRoutes):
bucket_name=bucket.name,
)
await add_permission_for_bucket(
- db, bucket.name, random_user.user.uid, permission=BucketPermission.Permission.READWRITE
+ db, bucket.name, random_third_user.user.uid, permission=BucketPermission.Permission.READWRITE
)
response = await client.delete(
- f"{self.base_path}/{bucket.name}", headers=random_user.auth_headers, params={"force_delete": False}
+ f"{self.base_path}/{bucket.name}", headers=random_third_user.auth_headers, params={"force_delete": False}
)
assert response.status_code == status.HTTP_403_FORBIDDEN
@@ -291,13 +436,13 @@ class TestBucketRoutesDelete(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
- Mock S3 Service to manipulate objects. pytest fixture.
+ Mock S3 Service to manipulate objects.
"""
mock_s3_service.create_object_in_bucket(bucket_name=random_bucket.name, key=random_lower_string())
response = await client.delete(
@@ -320,13 +465,13 @@ class TestBucketRoutesDelete(_TestBucketRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
mock_s3_service : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
- Mock S3 Service to manipulate objects. pytest fixture.
+ Mock S3 Service to manipulate objects.
"""
mock_s3_service.create_object_in_bucket(bucket_name=random_bucket.name, key=random_lower_string())
response = await client.delete(
diff --git a/app/tests/api/test_s3_keys.py b/app/tests/api/test_s3_keys.py
index 1413da4aa9de94c44e6ec23ca97c4920e3df9ec5..46c37a2d00da0ad638675d98ffe6f69ea4778024 100644
--- a/app/tests/api/test_s3_keys.py
+++ b/app/tests/api/test_s3_keys.py
@@ -1,6 +1,7 @@
import pytest
from fastapi import status
from httpx import AsyncClient
+from pydantic import TypeAdapter
from app.schemas.s3key import S3Key
from app.tests.mocks.mock_rgw_admin import MockRGWAdmin
@@ -25,11 +26,11 @@ class TestS3KeyRoutesGet(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random foreign user for testing. pytest fixture.
+ Random foreign user for testing.
"""
response = await client.get(
f"{self.base_path}/{str(random_second_user.user.uid)}/keys", headers=random_user.auth_headers
@@ -49,18 +50,19 @@ class TestS3KeyRoutesGet(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.get(
f"{self.base_path}/{str(random_user.user.uid)}/keys", headers=random_user.auth_headers
)
- keys = response.json()
assert response.status_code == status.HTTP_200_OK
- assert isinstance(keys, list)
+
+ ta = TypeAdapter(list[S3Key])
+ keys = ta.validate_json(response.content)
assert len(keys) == 1
- assert S3Key.model_validate(keys[0]).uid == random_user.user.uid
+ assert keys[0].uid == random_user.user.uid
@pytest.mark.asyncio
async def test_get_specific_s3_key_for_user(
@@ -75,18 +77,18 @@ class TestS3KeyRoutesGet(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
mock_rgw_admin : app.tests.mocks.mock_rgw_admin.MockRGWAdmin
- Mock class for rgwadmin package. pytest fixture.
+ Mock class for rgwadmin package.
"""
s3_key = mock_rgw_admin.get_user(uid=str(random_user.user.uid))["keys"][0]
response = await client.get(
f"{self.base_path}/{str(random_user.user.uid)}/keys/{s3_key['access_key']}",
headers=random_user.auth_headers,
)
- response_key = S3Key.model_validate(response.json())
+ response_key = S3Key.model_validate_json(response.content)
assert response.status_code == status.HTTP_200_OK
assert response_key.access_key == s3_key["access_key"]
assert response_key.secret_key == s3_key["secret_key"]
@@ -106,13 +108,13 @@ class TestS3KeyRoutesGet(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
mock_rgw_admin : app.tests.mocks.mock_rgw_admin.MockRGWAdmin
- Mock class for rgwadmin package. pytest fixture.
+ Mock class for rgwadmin package.
"""
s3_key = mock_rgw_admin.get_user(uid=str(random_user.user.uid))["keys"][0]
response = await client.get(
@@ -129,9 +131,9 @@ class TestS3KeyRoutesGet(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.get(
f"{self.base_path}/{str(random_user.user.uid)}/keys/impossible_key", headers=random_user.auth_headers
@@ -153,17 +155,17 @@ class TestS3KeyRoutesCreate(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
mock_rgw_admin : app.tests.mocks.mock_rgw_admin.MockRGWAdmin
- Mock class for rgwadmin package. pytest fixture.
+ Mock class for rgwadmin package.
"""
old_s3_key = mock_rgw_admin.get_user(uid=str(random_user.user.uid))["keys"][0]
response = await client.post(
f"{self.base_path}/{str(random_user.user.uid)}/keys", headers=random_user.auth_headers
)
- new_key = S3Key.model_validate(response.json())
+ new_key = S3Key.model_validate_json(response.content)
assert response.status_code == status.HTTP_201_CREATED
assert new_key.access_key != old_s3_key["access_key"]
@@ -179,11 +181,11 @@ class TestS3KeyRoutesCreate(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
response = await client.post(
f"{self.base_path}/{str(random_second_user.user.uid)}/keys", headers=random_user.auth_headers
@@ -203,11 +205,11 @@ class TestS3KeyRoutesDelete(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
mock_rgw_admin : app.tests.mocks.mock_rgw_admin.MockRGWAdmin
- Mock class for rgwadmin package. pytest fixture.
+ Mock class for rgwadmin package.
"""
new_s3_key = mock_rgw_admin.create_key(uid=str(random_user.user.uid))[-1]
assert len(mock_rgw_admin.get_user(uid=str(random_user.user.uid))["keys"]) == 2
@@ -229,11 +231,11 @@ class TestS3KeyRoutesDelete(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
mock_rgw_admin : app.tests.mocks.mock_rgw_admin.MockRGWAdmin
- Mock class for rgwadmin package. pytest fixture.
+ Mock class for rgwadmin package.
"""
assert len(mock_rgw_admin.get_user(uid=str(random_user.user.uid))["keys"]) == 1
key_id = mock_rgw_admin.get_user(uid=str(random_user.user.uid))["keys"][0]
@@ -257,11 +259,11 @@ class TestS3KeyRoutesDelete(_TestS3KeyRoutes):
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
mock_rgw_admin : app.tests.mocks.mock_rgw_admin.MockRGWAdmin
- Mock class for rgwadmin package. pytest fixture.
+ Mock class for rgwadmin package.
"""
mock_rgw_admin.create_key(uid=str(random_user.user.uid))
response = await client.delete(
diff --git a/app/tests/api/test_security.py b/app/tests/api/test_security.py
index d386176e47f9f5ce3a8727ec046dea34eeda5ecd..9ca034f911675112451f1bda9f0833a445c3c826 100644
--- a/app/tests/api/test_security.py
+++ b/app/tests/api/test_security.py
@@ -3,6 +3,8 @@ from fastapi import status
from httpx import AsyncClient
from sqlalchemy.ext.asyncio import AsyncSession
+from app.tests.mocks.mock_opa_service import MockOpaService
+from app.tests.utils.cleanup import CleanupList
from app.tests.utils.user import UserWithAuthHeader
@@ -17,7 +19,7 @@ class TestJWTProtectedRoutes:
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
"""
response = await client.get(self.protected_route)
error = response.json()
@@ -33,7 +35,7 @@ class TestJWTProtectedRoutes:
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
"""
response = await client.get(self.protected_route, headers={"Authorization": "Bearer not-a-jwt-token"})
error = response.json()
@@ -49,12 +51,12 @@ class TestJWTProtectedRoutes:
Parameters
----------
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
response = await client.get(
- self.protected_route, params={"owner_id": str(random_user.user.uid)}, headers=random_user.auth_headers
+ self.protected_route, params={"maintainer_id": str(random_user.user.uid)}, headers=random_user.auth_headers
)
assert response.status_code == status.HTTP_200_OK
@@ -71,16 +73,48 @@ class TestJWTProtectedRoutes:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
client : httpx.AsyncClient
- HTTP Client to perform the request on. pytest fixture.
+ HTTP Client to perform the request on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
await db.delete(random_user.user)
await db.commit()
response = await client.get(
- self.protected_route, params={"owner_id": str(random_user.user.uid)}, headers=random_user.auth_headers
+ self.protected_route, params={"user": str(random_user.user.uid)}, headers=random_user.auth_headers
)
assert response.status_code == status.HTTP_404_NOT_FOUND
+
+ @pytest.mark.asyncio
+ async def test_routed_with_insufficient_permissions(
+ self,
+ client: AsyncClient,
+ random_user: UserWithAuthHeader,
+ mock_opa_service: MockOpaService,
+ cleanup: CleanupList,
+ ) -> None:
+ """
+ Test with correct authorization header but with insufficient permissions.
+
+ Parameters
+ ----------
+ client : httpx.AsyncClient
+ HTTP Client to perform the request on.
+ random_user : app.tests.utils.user.UserWithAuthHeader
+ Random user for testing.
+ """
+
+ mock_opa_service.send_error = True
+
+ def reset_opa_service() -> None:
+ mock_opa_service.send_error = False
+
+ cleanup.add_task(reset_opa_service)
+
+ response = await client.get(
+ self.protected_route,
+ headers=random_user.auth_headers,
+ )
+ assert response.status_code == status.HTTP_403_FORBIDDEN
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index 1d269342168f2d65a97a5d7bb3140deab3c03f72..9abb2d95de4a23c4c5e3ab029a18fdc2d630caeb 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -2,8 +2,7 @@ import asyncio
import json
from functools import partial
from secrets import token_urlsafe
-from typing import AsyncIterator, Callable, Generator
-from uuid import uuid4
+from typing import AsyncIterator, Generator, Iterator
import httpx
import pytest
@@ -13,17 +12,17 @@ from clowmdb.models import Bucket
from clowmdb.models import BucketPermission as BucketPermissionDB
from sqlalchemy.ext.asyncio import AsyncSession
-from app.api.dependencies import get_decode_jwt_function, get_httpx_client, get_rgw_admin, get_s3_resource
+from app.api.dependencies import get_db, get_decode_jwt_function, get_httpx_client, get_rgw_admin, get_s3_resource
from app.core.config import settings
from app.main import app
from app.schemas.bucket_permission import BucketPermissionOut as BucketPermissionSchema
-from app.schemas.security import AuthzResponse
+from app.tests.mocks import DefaultMockHTTPService, MockHTTPService
+from app.tests.mocks.mock_opa_service import MockOpaService
from app.tests.mocks.mock_rgw_admin import MockRGWAdmin
from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
from app.tests.utils.bucket import create_random_bucket
from app.tests.utils.cleanup import CleanupList
from app.tests.utils.user import UserWithAuthHeader, create_random_user, decode_mock_token, get_authorization_headers
-from app.tests.utils.utils import request_admin_permission
jwt_secret = token_urlsafe(32)
@@ -54,36 +53,49 @@ def mock_s3_service() -> MockS3ServiceResource:
return MockS3ServiceResource()
+@pytest.fixture(scope="session")
+def mock_opa_service() -> Iterator[MockOpaService]:
+ mock_opa = MockOpaService()
+ yield mock_opa
+ mock_opa.reset()
+
+
+@pytest_asyncio.fixture(scope="session")
+async def mock_client(
+ mock_opa_service: MockOpaService,
+) -> AsyncIterator[httpx.AsyncClient]:
+ def mock_request_handler(request: httpx.Request) -> httpx.Response:
+ url = str(request.url)
+ handler: MockHTTPService
+ if url.startswith(str(settings.opa.uri)):
+ handler = mock_opa_service
+ else:
+ handler = DefaultMockHTTPService()
+ return handler.handle_request(request)
+
+ async with httpx.AsyncClient(transport=httpx.MockTransport(mock_request_handler)) as http_client:
+ yield http_client
+
+
@pytest_asyncio.fixture(scope="module")
async def client(
- mock_rgw_admin: MockRGWAdmin, mock_s3_service: MockS3ServiceResource
+ mock_s3_service: MockS3ServiceResource,
+ db: AsyncSession,
+ mock_opa_service: MockOpaService,
+ mock_client: httpx.AsyncClient,
+ mock_rgw_admin: MockRGWAdmin,
) -> AsyncIterator[httpx.AsyncClient]:
"""
Fixture for creating a TestClient and perform HTTP Request on it.
- Overrides the dependency for the RGW admin operations.
+ Overrides several dependencies.
"""
- def get_decode_token_function() -> Callable[[str], dict[str, str]]:
- # Override the decode_jwt function with mock function for tests and inject random shared secret
- return partial(decode_mock_token, secret=jwt_secret)
-
- async def get_mock_httpx_client(allow_admin: bool = False) -> AsyncIterator[httpx.AsyncClient]:
- def mock_request_handler(request: httpx.Request) -> httpx.Response:
- response_body = {}
- if str(request.url).startswith(str(settings.OPA_URI)):
- response_body = AuthzResponse(
- result=not request_admin_permission(request) or allow_admin, decision_id=uuid4()
- ).model_dump()
- return httpx.Response(200, json=response_body)
-
- async with httpx.AsyncClient(transport=httpx.MockTransport(mock_request_handler)) as http_client:
- yield http_client
-
- app.dependency_overrides[get_httpx_client] = get_mock_httpx_client
app.dependency_overrides[get_rgw_admin] = lambda: mock_rgw_admin
+ app.dependency_overrides[get_httpx_client] = lambda: mock_client
app.dependency_overrides[get_s3_resource] = lambda: mock_s3_service
- app.dependency_overrides[get_decode_jwt_function] = get_decode_token_function
- async with httpx.AsyncClient(app=app, base_url="http://localhost") as ac:
+ app.dependency_overrides[get_decode_jwt_function] = lambda: partial(decode_mock_token, secret=jwt_secret)
+ app.dependency_overrides[get_db] = lambda: db
+ async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://localhost") as ac: # type: ignore[arg-type]
yield ac
app.dependency_overrides = {}
@@ -93,46 +105,56 @@ async def db() -> AsyncIterator[AsyncSession]:
"""
Fixture for creating a database session to connect to.
"""
- async with get_async_session(
- url=str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
- ) as dbSession:
+ async with get_async_session(url=str(settings.db.dsn_async), verbose=settings.db.verbose) as dbSession:
yield dbSession
@pytest_asyncio.fixture(scope="function")
-async def random_user(db: AsyncSession, mock_rgw_admin: MockRGWAdmin) -> AsyncIterator[UserWithAuthHeader]:
+async def random_user(
+ db: AsyncSession, mock_rgw_admin: MockRGWAdmin, mock_opa_service: MockOpaService
+) -> AsyncIterator[UserWithAuthHeader]:
"""
Create a random user and deletes him afterwards.
"""
user = await create_random_user(db)
mock_rgw_admin.create_key(uid=str(user.uid))
+ mock_opa_service.add_user(user.lifescience_id, privileged=True)
yield UserWithAuthHeader(user=user, auth_headers=get_authorization_headers(uid=user.uid, secret=jwt_secret))
+ mock_opa_service.delete_user(user.lifescience_id)
mock_rgw_admin.delete_user(uid=str(user.uid))
await db.delete(user)
await db.commit()
@pytest_asyncio.fixture(scope="function")
-async def random_second_user(db: AsyncSession, mock_rgw_admin: MockRGWAdmin) -> AsyncIterator[UserWithAuthHeader]:
+async def random_second_user(
+ db: AsyncSession, mock_rgw_admin: MockRGWAdmin, mock_opa_service: MockOpaService
+) -> AsyncIterator[UserWithAuthHeader]:
"""
Create a random second user and deletes him afterwards.
"""
user = await create_random_user(db)
mock_rgw_admin.create_key(uid=str(user.uid))
+ mock_opa_service.add_user(user.lifescience_id)
yield UserWithAuthHeader(user=user, auth_headers=get_authorization_headers(uid=user.uid, secret=jwt_secret))
+ mock_opa_service.delete_user(user.lifescience_id)
mock_rgw_admin.delete_user(uid=str(user.uid))
await db.delete(user)
await db.commit()
@pytest_asyncio.fixture(scope="function")
-async def random_third_user(db: AsyncSession, mock_rgw_admin: MockRGWAdmin) -> AsyncIterator[UserWithAuthHeader]:
+async def random_third_user(
+ db: AsyncSession, mock_rgw_admin: MockRGWAdmin, mock_opa_service: MockOpaService
+) -> AsyncIterator[UserWithAuthHeader]:
"""
Create a random third user and deletes him afterwards.
"""
user = await create_random_user(db)
mock_rgw_admin.create_key(uid=str(user.uid))
+ mock_opa_service.add_user(user.lifescience_id)
yield UserWithAuthHeader(user=user, auth_headers=get_authorization_headers(uid=user.uid, secret=jwt_secret))
+ mock_opa_service.delete_user(user.lifescience_id)
mock_rgw_admin.delete_user(uid=str(user.uid))
await db.delete(user)
await db.commit()
@@ -143,7 +165,7 @@ async def random_bucket(
db: AsyncSession, random_user: UserWithAuthHeader, mock_s3_service: MockS3ServiceResource
) -> AsyncIterator[Bucket]:
"""
- Create a random user and deletes him afterwards.
+ Create a random bucket and deletes it afterwards.
"""
bucket = await create_random_bucket(db, random_user.user)
mock_s3_service.Bucket(name=bucket.name).create()
diff --git a/app/tests/crud/test_bucket.py b/app/tests/crud/test_bucket.py
index 479feb03a38dc2f7ef04cadcc7109367ad9e842d..91e34361cbb26f47741d877263201d58df1ccbcf 100644
--- a/app/tests/crud/test_bucket.py
+++ b/app/tests/crud/test_bucket.py
@@ -23,9 +23,9 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
buckets = await CRUDBucket.get_all(db)
assert len(buckets) == 1
@@ -42,9 +42,9 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
bucket = await CRUDBucket.get(db, random_bucket.name)
assert bucket
@@ -60,7 +60,7 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
"""
bucket = await CRUDBucket.get(db, "unknown Bucket")
assert bucket is None
@@ -73,9 +73,9 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
buckets = await CRUDBucket.get_for_user(db, random_bucket.owner_id, CRUDBucket.BucketType.OWN)
@@ -96,11 +96,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
cleanup : app.tests.utils.utils.CleanupList
Cleanup object where (async) functions can be registered which get executed after a (failed) test.
"""
@@ -139,11 +139,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
cleanup : app.tests.utils.utils.CleanupList
Cleanup object where (async) functions can be registered which get executed after a (failed) test.
"""
@@ -179,11 +179,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, permission=BucketPermission.Permission.READ
@@ -204,11 +204,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, permission=BucketPermission.Permission.READWRITE
@@ -229,11 +229,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, permission=BucketPermission.Permission.WRITE
@@ -254,11 +254,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
await add_permission_for_bucket(
db,
@@ -283,11 +283,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, from_=datetime.now() + timedelta(days=10)
@@ -307,11 +307,11 @@ class TestBucketCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, to=datetime.now() - timedelta(days=10)
@@ -336,9 +336,9 @@ class TestBucketCRUDCreate:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing. pytest fixture.
+ Random user for testing.
cleanup : app.tests.utils.utils.CleanupList
Cleanup object where (async) functions can be registered which get executed after a (failed) test.
"""
@@ -369,15 +369,39 @@ class TestBucketCRUDCreate:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
bucket_info = BucketIn(name=random_bucket.name, description=random_lower_string(127))
with pytest.raises(DuplicateError):
await CRUDBucket.create(db, bucket_info, random_bucket.owner_id)
+class TestBucketCRUDUpdate:
+ @pytest.mark.asyncio
+ async def test_update_public_state(self, db: AsyncSession, random_bucket: Bucket) -> None:
+ """
+ Test for deleting a bucket with the CRUD Repository.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ random_bucket : clowmdb.models.Bucket
+ Random bucket for testing.
+ """
+ old_public_state = random_bucket.public
+ await CRUDBucket.update_public_state(db=db, bucket_name=random_bucket.name, public=not old_public_state)
+
+ stmt = select(Bucket).where(Bucket.name == random_bucket.name)
+ bucket_db = await db.scalar(stmt)
+
+ assert bucket_db is not None
+ assert bucket_db == random_bucket
+ assert old_public_state != bucket_db.public
+
+
class TestBucketCRUDDelete:
@pytest.mark.asyncio
async def test_delete_bucket(self, db: AsyncSession, random_bucket: Bucket) -> None:
@@ -387,9 +411,9 @@ class TestBucketCRUDDelete:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
await CRUDBucket.delete(db, random_bucket.name)
diff --git a/app/tests/crud/test_bucket_permission.py b/app/tests/crud/test_bucket_permission.py
index b9a3e8ed15e7c2af99a704d940aad3b8fda8017a..36224dd2cde39750bde5466dc310aa9946992ba2 100644
--- a/app/tests/crud/test_bucket_permission.py
+++ b/app/tests/crud/test_bucket_permission.py
@@ -26,9 +26,9 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket_permission : clowmdb.models.BucketPermission
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
bucket_permission = await CRUDBucketPermission.get(
db, bucket_name=random_bucket_permission.bucket_name, uid=random_bucket_permission.uid
@@ -48,9 +48,9 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket_permission : clowmdb.models.BucketPermission
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
bucket_permissions = await CRUDBucketPermission.list(db, bucket_name=random_bucket_permission.bucket_name)
assert len(bucket_permissions) == 1
@@ -73,11 +73,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(db, random_bucket.name, random_second_user.user.uid)
await add_permission_for_bucket(
@@ -103,11 +103,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(db, random_bucket.name, random_second_user.user.uid)
await add_permission_for_bucket(
@@ -137,11 +137,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, from_=datetime.now() + timedelta(weeks=1)
@@ -170,11 +170,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, to=datetime.now() - timedelta(weeks=1)
@@ -203,11 +203,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db,
@@ -244,11 +244,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, from_=datetime.now() + timedelta(weeks=1)
@@ -277,11 +277,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, to=datetime.now() - timedelta(weeks=1)
@@ -310,11 +310,11 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
random_third_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db,
@@ -346,9 +346,9 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket_permission : clowmdb.models.BucketPermission
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
bucket_permissions = await CRUDBucketPermission.list(db, uid=random_bucket_permission.uid)
assert len(bucket_permissions) == 1
@@ -370,9 +370,9 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, permission=BucketPermissionDB.Permission.WRITE
@@ -396,9 +396,9 @@ class TestBucketPermissionCRUDGet:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random third user who has no permissions for the bucket. pytest fixture.
+ Random third user who has no permissions for the bucket.
"""
await add_permission_for_bucket(
db, random_bucket.name, random_second_user.user.uid, from_=datetime.now() + timedelta(weeks=1)
@@ -419,9 +419,9 @@ class TestBucketPermissionCRUDCreate:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=uuid4())
with pytest.raises(KeyError):
@@ -437,11 +437,11 @@ class TestBucketPermissionCRUDCreate:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_user : app.tests.utils.user.UserWithAuthHeader
- Random user for testing who is owner of the bucket. pytest fixture.
+ Random user for testing who is owner of the bucket.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=random_user.user.uid)
with pytest.raises(ValueError):
@@ -457,9 +457,9 @@ class TestBucketPermissionCRUDCreate:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
permission = BucketPermissionSchema(
bucket_name=random_bucket_permission_schema.bucket_name, uid=random_bucket_permission_schema.uid
@@ -477,11 +477,11 @@ class TestBucketPermissionCRUDCreate:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_second_user : app.tests.utils.user.UserWithAuthHeader
- Random second user for testing. pytest fixture.
+ Random second user for testing.
random_bucket : clowmdb.models.Bucket
- Random bucket for testing. pytest fixture.
+ Random bucket for testing.
"""
permission = BucketPermissionSchema(bucket_name=random_bucket.name, uid=random_second_user.user.uid)
created_permission = await CRUDBucketPermission.create(db, permission)
@@ -501,9 +501,9 @@ class TestBucketPermissionCRUDDelete:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket_permission : clowmdb.models.BucketPermission
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
await CRUDBucketPermission.delete(
db, bucket_name=random_bucket_permission.bucket_name, uid=random_bucket_permission.uid
@@ -531,9 +531,9 @@ class TestBucketPermissionCRUDUpdate:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_bucket_permission : clowmdb.models.BucketPermission
- Bucket permission for a random bucket for testing. pytest fixture.
+ Bucket permission for a random bucket for testing.
"""
new_from_time = round(datetime(2022, 1, 1, 0, 0).timestamp())
new_params = BucketPermissionParametersSchema(
diff --git a/app/tests/crud/test_user.py b/app/tests/crud/test_user.py
index c9856e1fb219f1b0d0b6f86718660404d7334fe1..261aa06ed63c9f12ca79ba209888e8390a9aef6a 100644
--- a/app/tests/crud/test_user.py
+++ b/app/tests/crud/test_user.py
@@ -16,9 +16,9 @@ class TestUserCRUD:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
random_user : clowmdb.models.User
- Random user for testing. pytest fixture.
+ Random user for testing.
"""
user = await CRUDUser.get(db, random_user.user.uid)
assert user
@@ -36,7 +36,7 @@ class TestUserCRUD:
Parameters
----------
db : sqlalchemy.ext.asyncio.AsyncSession.
- Async database session to perform query on. pytest fixture.
+ Async database session to perform query on.
"""
user = await CRUDUser.get(db, uuid4())
assert user is None
diff --git a/app/tests/mocks/__init__.py b/app/tests/mocks/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..39fd604074907615c601d3679f21f64c83692e2e 100644
--- a/app/tests/mocks/__init__.py
+++ b/app/tests/mocks/__init__.py
@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+
+from fastapi import status
+from httpx import Request, Response
+
+
+class MockHTTPService(ABC):
+ def __init__(self) -> None:
+ self.send_error = False
+
+ @abstractmethod
+ def handle_request(self, request: Request) -> Response: ...
+
+ def reset(self) -> None:
+ self.send_error = False
+
+
+class DefaultMockHTTPService(MockHTTPService):
+ def handle_request(self, request: Request) -> Response:
+ return Response(
+ status_code=status.HTTP_404_NOT_FOUND if self.send_error else status.HTTP_200_OK,
+ json={},
+ )
diff --git a/app/tests/mocks/mock_opa_service.py b/app/tests/mocks/mock_opa_service.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9eccf47e8f6c5d5c4e62ced45b1d3ae6a7cec84
--- /dev/null
+++ b/app/tests/mocks/mock_opa_service.py
@@ -0,0 +1,103 @@
+import json
+from uuid import uuid4
+
+from fastapi import status
+from httpx import Request, Response
+
+from app.schemas.security import AuthzRequest, AuthzResponse
+from app.tests.mocks import MockHTTPService
+
+_json_header = {"content-type": "application/json"}
+
+
+class MockOpaService(MockHTTPService):
+ """
+ Class to mock the Open Policy Agent service.
+ Has a simplified role management. A user can be either "Admin" or "Normal User".
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._users: dict[str, bool] = {}
+
+ def add_user(self, uid: str, privileged: bool = False) -> None:
+ """
+ Add a user to the mock service.
+
+ Parameters
+ ----------
+ uid : str
+ ID of a user.
+ privileged : bool, default False
+ Flag if the user is an Admin or not.
+ """
+ self._users[uid] = privileged
+
+ def delete_user(self, uid: str) -> None:
+ """
+ Delete a user in the mock service.
+
+ Parameters
+ ----------
+ uid : str
+ ID of the user to delete.
+ """
+ if uid in self._users.keys():
+ del self._users[uid]
+
+ def reset(self) -> None:
+ """
+ Reset the mock service to its initial state.
+ """
+ super().reset()
+ self._users = {}
+
+ def handle_request(self, request: Request, **kwargs: bool) -> Response:
+ """
+ Handle the raw request that is sent to the mock service.
+
+ Parameters
+ ----------
+ request: httpx.Request
+ Raw HTTP request object.
+
+ Returns
+ -------
+ response : httpx.Response
+ Appropriate response to the received request.
+ """
+ if request.url.path == "/v1/data/clowm/authz/allow":
+ authz_request = AuthzRequest.model_validate(json.loads(request.read().decode("utf-8"))["input"])
+ if self.send_error or authz_request.uid not in self._users:
+ result = False
+ else:
+ result = not MockOpaService.request_admin_permission(authz_request) or self._users[authz_request.uid]
+ return Response(
+ status_code=status.HTTP_200_OK,
+ headers=_json_header,
+ content=AuthzResponse(result=result, decision_id=uuid4()).model_dump_json(),
+ )
+ return Response(
+ status_code=status.HTTP_404_NOT_FOUND,
+ json={},
+ )
+
+ @staticmethod
+ def request_admin_permission(authz_request: AuthzRequest) -> bool:
+ """
+ Helper function to determine if the authorization request needs the 'administrator' role.
+
+ Parameters
+ ----------
+ authz_request : app.schemas.security.AuthzRequest
+ Body of the request.
+
+ Returns
+ -------
+ decision : bool
+ Flag if the request needs the 'administrator' role
+ """
+ checks = "any" in authz_request.operation
+ if "bucket_permission" in authz_request.resource:
+ checks = checks or "all" in authz_request.operation
+ return checks
diff --git a/app/tests/unit/test_bucket_permission_scheme.py b/app/tests/unit/test_bucket_permission_scheme.py
index 263d0afc61a8d6eb5bf64f708de42f701fcddbb6..28dec2abce39c41dd6aa17edfc099ae6beac0055 100644
--- a/app/tests/unit/test_bucket_permission_scheme.py
+++ b/app/tests/unit/test_bucket_permission_scheme.py
@@ -29,7 +29,7 @@ class TestPermissionPolicyPermissionType(_TestPermissionPolicy):
Parameters
----------
random_base_permission : app.schemas.bucket_permission.BucketPermissionOut
- Random base bucket permission for testing. pytest fixture.
+ Random base bucket permission for testing.
"""
uid = uuid4()
stmts = random_base_permission.map_to_bucket_policy_statement(uid=uid)
@@ -62,7 +62,7 @@ class TestPermissionPolicyPermissionType(_TestPermissionPolicy):
Parameters
----------
random_base_permission : app.schemas.bucket_permission.BucketPermissionOut
- Random base bucket permission for testing. pytest fixture.
+ Random base bucket permission for testing.
"""
random_base_permission.permission = BucketPermission.Permission.WRITE
stmts = random_base_permission.map_to_bucket_policy_statement(uid=uuid4())
@@ -89,7 +89,7 @@ class TestPermissionPolicyPermissionType(_TestPermissionPolicy):
Parameters
----------
random_base_permission : app.schemas.bucket_permission.BucketPermissionOut
- Random base bucket permission for testing. pytest fixture.
+ Random base bucket permission for testing.
"""
random_base_permission.permission = BucketPermission.Permission.READWRITE
stmts = random_base_permission.map_to_bucket_policy_statement(uid=uuid4())
@@ -119,7 +119,7 @@ class TestPermissionPolicyCondition(_TestPermissionPolicy):
Parameters
----------
random_base_permission : app.schemas.bucket_permission.BucketPermissionOut
- Random base bucket permission for testing. pytest fixture.
+ Random base bucket permission for testing.
"""
timestamp = round(datetime.now().timestamp())
time = datetime.fromtimestamp(timestamp) # avoid rounding error
@@ -147,7 +147,7 @@ class TestPermissionPolicyCondition(_TestPermissionPolicy):
Parameters
----------
random_base_permission : app.schemas.bucket_permission.BucketPermissionOut
- Random base bucket permission for testing. pytest fixture.
+ Random base bucket permission for testing.
"""
time = datetime.now()
timestamp = round(datetime.now().timestamp())
@@ -176,7 +176,7 @@ class TestPermissionPolicyCondition(_TestPermissionPolicy):
Parameters
----------
random_base_permission : app.schemas.bucket_permission.BucketPermissionOut
- Random base bucket permission for testing. pytest fixture.
+ Random base bucket permission for testing.
"""
random_base_permission.file_prefix = random_lower_string(length=8) + "/" + random_lower_string(length=8) + "/"
diff --git a/app/tests/utils/bucket.py b/app/tests/utils/bucket.py
index 98bbfeaceaab770a4ff6f2d6c387744ba0a91623..cb1f01ca57a91714eecb33ad3428288cedc65a5f 100644
--- a/app/tests/utils/bucket.py
+++ b/app/tests/utils/bucket.py
@@ -1,10 +1,14 @@
+import json
from datetime import datetime
from uuid import UUID
from clowmdb.models import Bucket, BucketPermission, User
-from sqlalchemy import delete
+from sqlalchemy import delete, update
from sqlalchemy.ext.asyncio import AsyncSession
+from app.api.endpoints.buckets import get_anonymously_bucket_policy
+from app.tests.mocks.mock_s3_resource import MockS3ServiceResource
+
from .utils import random_lower_string
@@ -78,3 +82,23 @@ async def add_permission_for_bucket(
)
db.add(perm)
await db.commit()
+
+
+async def make_bucket_public(db: AsyncSession, bucket_name: str, s3: MockS3ServiceResource) -> None:
+ """
+ Creates Permission to a bucket for a user in the database.
+
+ Parameters
+ ----------
+ db : sqlalchemy.ext.asyncio.AsyncSession.
+ Async database session to perform query on.
+ bucket_name : str
+ Name of the bucket.
+ s3 : app.tests.mocks.mock_s3_resource.MockS3ServiceResource
+ Mock S3 Service to manipulate objects.
+ """
+ await db.execute(update(Bucket).where(Bucket.name == bucket_name).values(public=True))
+ await db.commit()
+ policy = json.loads(s3.Bucket(bucket_name).Policy().policy)
+ policy["Statement"] += get_anonymously_bucket_policy(bucket_name)
+ s3.Bucket(bucket_name).Policy().put(Policy=json.dumps(policy))
diff --git a/app/tests/utils/utils.py b/app/tests/utils/utils.py
index ba7a1a4fb4a9ccc51cbb1ac6e45c04db0470b6a6..d38ad6f14a161b07f42f9d6cc6bb156d94aca895 100644
--- a/app/tests/utils/utils.py
+++ b/app/tests/utils/utils.py
@@ -1,8 +1,6 @@
import random
import string
-import httpx
-
def random_lower_string(length: int = 32) -> str:
"""
@@ -31,25 +29,3 @@ def random_ipv4_string() -> str:
Random IPv4 address.
"""
return ".".join(str(random.randint(0, 255)) for _ in range(4))
-
-
-def request_admin_permission(request: httpx.Request) -> bool:
- """
- Rudimentary helper function to determine if the authorization request needs the 'administrator' role.
-
- Parameters
- ----------
- request : httpx.Request
- Raw Request Object from httpx
-
- Returns
- -------
- decision : bool
- Flag if the request needs the 'administrator' role
- """
- request_body: dict[str, str] = eval(request.content.decode("utf-8"))["input"]
- operation = request_body["operation"]
- checks = "any" in operation
- if "bucket_permission" in request_body["resource"]:
- checks = checks or "all" in operation
- return checks
diff --git a/example-config/example-config.json b/example-config/example-config.json
new file mode 100644
index 0000000000000000000000000000000000000000..171b9a3d582ededbf4d3521118ba28305ade3982
--- /dev/null
+++ b/example-config/example-config.json
@@ -0,0 +1,27 @@
+{
+ "db": {
+ "port": 3306,
+ "host": "localhost",
+ "user": "db-user",
+ "name": "db-name",
+ "password": "db-password",
+ "verbose": false
+ },
+ "s3": {
+ "uri": "https://s3-staging.bi.denbi.de",
+ "access_key": "xxx",
+ "secret_key": "xxx",
+ "username": "clowm-bucket-manager",
+ "admin_access_key": "xxx",
+ "admin_secret_key": "xxx"
+ },
+ "opa": {
+ "uri": "http://localhost:8181"
+ },
+ "otlp": {
+ "grpc_endpoint": "localhost:4317"
+ },
+ "api_prefix": "/api/s3proxy-service",
+ "public_key_file": "/path/to/key.pub",
+ "ui_uri": "http://localhost:9999"
+}
diff --git a/example-config/example-config.toml b/example-config/example-config.toml
new file mode 100644
index 0000000000000000000000000000000000000000..894951bac907e0325c724492f4ce9af6b5071f2c
--- /dev/null
+++ b/example-config/example-config.toml
@@ -0,0 +1,21 @@
+api_prefix = "/api/s3proxy-service"
+public_key_file= "/path/to/key.pub"
+ui_uri = "http://localhost:9999"
+[db]
+ port = 3306
+ host = "localhost"
+ user = "db-user"
+ name = "db-name"
+ password = "db-password"
+ verbose = false
+[s3]
+ uri = "https://s3-staging.bi.denbi.de"
+ access_key = "xxx"
+ secret_key = "xxx"
+ username = "clowm-bucket-manager"
+ admin_access_key = "xxx"
+ admin_secret_key = "xxx"
+[opa]
+ uri = "http://localhost:8181"
+[otlp]
+ grpc_endpoint = "localhost:4317"
diff --git a/example-config/example-config.yaml b/example-config/example-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2280b5bd1e60e3b65acab82eae1bc39d33aed125
--- /dev/null
+++ b/example-config/example-config.yaml
@@ -0,0 +1,22 @@
+db:
+ port: 3306
+ host: "localhost"
+ user: "db-user"
+ name: "db-name"
+ password: "db-password"
+ verbose: false
+s3:
+ uri: "https://s3-staging.bi.denbi.de"
+ access_key: "xxx"
+ secret_key: "xxx"
+ resource_bucket: "clowm-resources"
+ username: "clowm-bucket-manager"
+ admin_access_key: "xxx"
+ admin_secret_key: "xxx"
+opa:
+ uri: "http://localhost:8181"
+otlp:
+ grpc_endpoint: "localhost:4317"
+api_prefix: "/api/s3proxy-service"
+public_key_file: "/path/to/key.pub"
+ui_uri: "http://localhost:9999"
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 7d331e2bbcfc83d8f42f1d89338d3328b28b60f5..9efc29a1be52a79a2300481986955bbb7b4c380f 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,17 +1,17 @@
# test packages
-pytest>=8.0.0,<8.1.0
+pytest>=8.0.0,<8.2.0
pytest-asyncio>=0.21.0,<0.22.0
-pytest-cov>=4.1.0,<4.2.0
+pytest-cov>=5.0.0,<5.1.0
coverage[toml]>=7.4.0,<7.5.0
# Linters
-ruff<0.3.0
-black>=24.2.0,<24.3.0
+ruff<0.4.0
+black>=24.2.0,<24.5.0
isort>=5.13.0,<5.14.0
-mypy>=1.8.0,<1.9.0
+mypy>=1.8.0,<1.10.0
# stubs for mypy
boto3-stubs-lite[s3]>=1.34.0,<1.35.0
types-requests
# Miscellaneous
-pre-commit>=3.6.0,<3.7.0
+pre-commit>=3.6.0,<3.8.0
python-dotenv
-uvicorn>=0.27.0,<0.28.0
+uvicorn>=0.27.0,<0.30.0
diff --git a/requirements.txt b/requirements.txt
index c334cf154cd215103f4b47de6feea81871d4a234..86c5c650b70353f740cd249e8f6b0edf34870f16 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,9 +2,10 @@
clowmdb>=3.1.0,<3.2.0
# Webserver packages
-fastapi>=0.109.0,<0.110.0
-pydantic<2.7.0
-pydantic-settings<2.3.0
+fastapi>=0.109.0,<0.111.0
+pydantic<2.8.0
+pydantic-settings[yaml, toml]>=2.2.0,<2.3.0
+email-validator>=2.1.0,<2.2.0
# Database packages
PyMySQL>=1.1.0,<1.2.0
SQLAlchemy>=2.0.0,<2.1.0
diff --git a/scripts/lint.sh b/scripts/lint.sh
index 5246a7ff93a6c8e930a3069db441f0b66b2d5504..3ae0ba81d6719f3be5502679b18e0a149209d6f6 100755
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@@ -2,7 +2,14 @@
set -x
+ruff --version
ruff check app
-black app --check
+
+isort --version
isort -c app
+
+black --version
+black app --check
+
+mypy --version
mypy app