Skip to content
Snippets Groups Projects
Commit d38f8b1c authored by Daniel Göbel's avatar Daniel Göbel
Browse files

Merge branch 'development' into 'main' 

Delete dev branch

See merge request cmg/clowm/clowm-s3proxy-service!71
parents c1815139 ac980ae7
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 281 additions and 793 deletions
.dockerignore
+ 3
− 4
View file @ d38f8b1c
......@@ -11,7 +11,6 @@ README.md
htmlcov
app/tests
figures/
oidc_dev_example
oidc_dev/
traefik_dev
ceph
.gitlab-ci.yml
.coverage
.git
.flake8 deleted 100644 → 0
+ 0
− 4
View file @ c1815139
[flake8]
max-line-length = 120
exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache
extend-ignore = E203
.gitlab-ci.yml
+ 77
− 42
View file @ d38f8b1c
image: python:3.10-slim
image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.11-slim
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PYTHONPATH: "$CI_PROJECT_DIR"
OBJECT_GATEWAY_URI: "http://127.0.0.1:8000"
CEPH_ACCESS_KEY: ""
CEPH_SECRET_KEY: ""
CEPH_USERNAME: ""
OIDC_CLIENT_SECRET: ""
OIDC_CLIENT_ID: ""
OIDC_BASE_URI: "http://127.0.0.1:8000"
BUCKET_CEPH_ACCESS_KEY: ""
BUCKET_CEPH_SECRET_KEY: ""
USER_CEPH_ACCESS_KEY: ""
USER_CEPH_SECRET_KEY: ""
BUCKET_CEPH_USERNAME: ""
FF_NETWORK_PER_BUILD: 1
PUBLIC_KEY_VALUE: "empty"
OPA_URI: "http://127.0.0.1:8181"
DB_PASSWORD: "random"
DB_USER: "random"
DB_DATABASE: "random"
DB_HOST: "random"
cache:
paths:
- .cache/pip
- venv/
before_script:
- python --version # For debugging
- pip install virtualenv
- virtualenv venv
- source venv/bin/activate
- python -m pip install -r requirements.txt
- python -m pip install -r requirements-dev.txt
default:
tags:
- docker
before_script:
- python --version # For debugging
- pip install virtualenv
- virtualenv venv
- source venv/bin/activate
- python -m pip install --upgrade -r requirements.txt -r requirements-dev.txt
stages: # List of stages for jobs, and their order of execution
# - build
- test
# - deploy
#build-job: # This job runs in the build stage, which runs first.
# stage: build
# script:
# - echo "Compiling the code..."
# - echo "Compile complete."
- deploy
integration-test-job: # Runs integration tests with the database
stage: test
......@@ -43,17 +44,17 @@ integration-test-job: # Runs integration tests with the database
DB_DATABASE: "integration-test-db"
DB_HOST: "integration-test-db"
services:
- name: mysql:8
- name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/mysql:8
alias: integration-test-db
variables:
MYSQL_RANDOM_ROOT_PASSWORD: "yes"
MYSQL_DATABASE: "$DB_DATABASE"
MYSQL_USER: "$DB_USER"
MYSQL_PASSWORD: "$DB_PASSWORD"
- name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.3
alias: upgrade-db
script:
- python app/check_database_connection.py
- alembic downgrade base
- alembic upgrade head
- pytest --junitxml=integration-report.xml --cov=app --cov-report=term-missing app/tests/crud
- mkdir coverage-integration
- mv .coverage coverage-integration
......@@ -70,27 +71,18 @@ e2e-test-job: # Runs e2e tests on the API endpoints
DB_USER: "test_api_user"
DB_DATABASE: "e2e-test-db"
DB_HOST: "e2e-test-db"
OIDC_CLIENT_SECRET: "$TEST_OIDC_CLIENT_SECRET"
OIDC_CLIENT_ID: "$TEST_OIDC_CLIENT_ID"
OIDC_BASE_URI: "http://mock-oidc-server"
CLIENTS_CONFIGURATION_INLINE: "$TEST_OIDC_CLIENT_CONFIG"
services:
- name: mysql:8
- name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/mysql:8
alias: e2e-test-db
variables:
MYSQL_RANDOM_ROOT_PASSWORD: "yes"
MYSQL_DATABASE: "$DB_DATABASE"
MYSQL_USER: "$DB_USER"
MYSQL_PASSWORD: "$DB_PASSWORD"
- name: ghcr.io/soluto/oidc-server-mock:latest
alias: mock-oidc-server
variables:
ASPNETCORE_ENVIRONMENT: "Development"
- name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.3
alias: upgrade-db
script:
- python app/check_database_connection.py
- python app/check_oidc_connection.py
- alembic downgrade base
- alembic upgrade head
- pytest --junitxml=e2e-report.xml --cov=app --cov-report=term-missing app/tests/api
- mkdir coverage-e2e
- mv .coverage coverage-e2e
......@@ -137,8 +129,51 @@ lint-test-job: # Runs linters checks on code
script:
- ./scripts/lint.sh
#deploy-job: # This job runs in the deploy stage.
# stage: deploy # It only runs when *both* jobs in the test stage complete successfully.
# script:
# - echo "Deploying application..."
# - echo "Application successfully deployed."
build-publish-dev-docker-container-job:
stage: deploy
image:
name: gcr.io/kaniko-project/executor:v1.17.0-debug
entrypoint: [""]
dependencies: []
only:
refs:
- development
before_script:
- echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
--destination "${CI_REGISTRY_IMAGE}:dev-${CI_COMMIT_SHA}"
--destination "${CI_REGISTRY_IMAGE}:dev-latest"
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile-Gunicorn"
--destination "${CI_REGISTRY_IMAGE}:dev-${CI_COMMIT_SHA}-gunicorn"
--destination "${CI_REGISTRY_IMAGE}:dev-latest-gunicorn"
publish-docker-container-job:
stage: deploy
image:
name: gcr.io/kaniko-project/executor:v1.17.0-debug
entrypoint: [""]
dependencies: []
only:
- tags
before_script:
- echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
--destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}"
--destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1-2)"
--destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1)"
--destination "${CI_REGISTRY_IMAGE}:latest"
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/Dockerfile-Gunicorn"
--destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}-gunicorn"
--destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1-2)-gunicorn"
--destination "${CI_REGISTRY_IMAGE}:$(echo ${CI_COMMIT_TAG} | cut -d'.' -f1)-gunicorn"
--destination "${CI_REGISTRY_IMAGE}:latest-gunicorn"
.pre-commit-config.yaml
+ 14
− 17
View file @ d38f8b1c
......@@ -2,7 +2,7 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.2.0
rev: v4.5.0
hooks:
- id: end-of-file-fixer
- id: check-added-large-files
......@@ -15,32 +15,29 @@ repos:
- id: check-merge-conflict
- id: check-ast
- repo: https://github.com/psf/black
rev: 22.3.0
rev: 23.11.0
hooks:
- id: black
files: app
args: [--check]
- repo: https://github.com/PyCQA/flake8
rev: 4.0.1
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: 'v0.1.6'
hooks:
- id: flake8
- id: ruff
- repo: https://github.com/PyCQA/isort
rev: 5.12.0
hooks:
- id: isort
files: app
args: [--config=.flake8]
args: [-c]
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.960
rev: v1.7.1
hooks:
- id: mypy
files: app
args: [--config=pyproject.toml]
additional_dependencies:
- sqlalchemy2-stubs
- boto3-stubs-lite[s3]
- sqlalchemy<2.0.0
- pydantic
- boto3-stubs-lite[s3]>=1.29.0,<1.30.0
- sqlalchemy>=2.0.0,<2.1.0
- pydantic>=2.5.0,<2.6.0
- types-requests
- repo: https://github.com/PyCQA/isort
rev: 5.10.1
hooks:
- id: isort
files: app
args: [-c]
CHANGELOG.md
+ 8
− 0
View file @ d38f8b1c
## Version 1.1.2
### General
* When requesting a list of buckets, all buckets, including WRITE-only buckets, are returned #34
### Fixes
* User with WRITE/READWRITE permission can now delete multiple Objects with a single request to the S3 endpoint #34
## Version 1.1.1
### General
......
DEVELOPING.md deleted 100644 → 0
+ 0
− 155
View file @ c1815139
## Development Setup
### Python Setup 🐍
Currently, only Python version `>=3.10` is supported because it uses its new features for type annotations
to write more compact code. Since FastAPI relies on these type annotations and `mypy` is integrated into
this project, we make heavy use of this feature.
Write
```python
var1: list[int] = [1,2,3]
var2: str | None = None
```
instead of
```python
from typing import List, Optional
var1: List[int] = [1,2,3]
var2: Optional[str] = None
```
### Environment Setup
Create a virtual environment, install the dependencies and install the [pre-commit](https://pre-commit.com/) hooks.<br>
The linters can prevent a commit of the code quality doesn't meet the standard.
```shell
python -m venv venv
source venv/bin/activate
python -m pip install -r requirements.txt
python -m pip install -r requirements-dev.txt
pre-commit install
```
### Ceph Setup
For how to set up a ceph cluster or how to connect to an existing one see
the [documentation in the ceph folder](ceph/README.md).
A user with `user` capabilities should be created, e.g.<br>
`radosgw-admin user create --uid=myadmin --caps="users=*"`
### Database Setup
#### Dev database
The easiest solution is [Docker](https://docs.docker.com/get-docker/) with an attached volume
to set up a MySQL database.
```shell
docker volume create proxyapi_dev_db
docker run --name proxyapi_devdb \
-e MYSQL_RANDOM_ROOT_PASSWORD=yes \
-e MYSQL_DATABASE=<database_name> \
-e MYSQL_USER=<database_user> \
-e MYSQL_PASSWORD=<database_password> \
-p 127.0.0.1:3306:3306 \
-v proxyapi_dev_db:/var/lib/mysql \
-d \
mysql:8
```
When the container stopped just restart it with
```shell
docker start proxyapi_devdb
```
Look at the [Environment Variables](#environment-variables) section to see which env variables have to be set.
#### Test database
Set up a second database on a different port for the integration tests. This database doesn't have to be persistent
because all data will be purged after each test run.
```shell
docker run --name proxyapi_testdb \
-e MYSQL_RANDOM_ROOT_PASSWORD=yes \
-e MYSQL_DATABASE=<database_name> \
-e MYSQL_USER=<database_user> \
-e MYSQL_PASSWORD=<database_password> \
-p 127.0.0.1:8001:3306 \
-d \
mysql:8
```
### Dev OIDC Provider Setup
To avoid the complex process of connecting the local machine with the LifeScience AAI Test server, a simple [OIDC provider](https://github.com/Soluto/oidc-server-mock)
can be setup with Docker.<br>
Copy the `oidc_dev_example` directory to `oidc_dev`
```shell
cp -r oidc_dev_example oidc_dev
```
In the file `oidc_dev/clients_config.json` add a random value to `ClientId` and `ClientSecrets`. These can be generated for example with `openssl`.
```shell
openssl rand -hex 10
```
You can add/delete users in the file `oidc_dev/users_config.json` according the schema that is provided there.<br>
Adjust the volume path and start the docker container
```shell
docker run --name proxyapi_oidc_provider \
-e CLIENTS_CONFIGURATION_PATH=/tmp/config/clients_config.json \
-e IDENTITY_RESOURCES_PATH=/tmp/config/identity_resources.json \
-e USERS_CONFIGURATION_PATH=/tmp/config/users_config.json \
-e SERVER_OPTIONS_PATH=/tmp/config/server_options.json \
-e ASPNETCORE_ENVIRONMENT=Development \
-p 127.0.0.1:8002:80 \
-v /path/to/folder/oidc_dev:/tmp/config:ro \
-d \
ghcr.io/soluto/oidc-server-mock:latest
```
Set the env variables `OIDC_BASE_URI` to `http://localhost:8002` and `OIDC_CLIENT_SECRET` / `OIDC_CLIENT_ID` to their appropriate value.
### Reverse Proxy Setup
The `API_PREFIX` is handles on the level of the reverse proxy. This simplifies the routing in the code and the cooperation with the [Frontend](https://gitlab.ub.uni-bielefeld.de/denbi/object-storage-access-ui).
An simple Traefik reverse proxy configuration is stored in the repository.
[Traefik](https://traefik.io/) is a reverse Proxy written in Go.
To use it, download the [`traefik`](https://github.com/traefik/traefik/releases) binary and start it with
```shell
cd traefik_dev
/path/to/binary/traefik --configFile=traefik.toml
```
The provided configuration does the following things
* It forwards all request to http://localhost:9999/api/* to http://localhost:8080 (this backend)
* It strips the prefix `/api` before it forwards the request to the backend
* All other request will be forwarded to http://localhost:5173, the corresponding dev [Frontend](https://gitlab.ub.uni-bielefeld.de/denbi/object-storage-access-ui)
* Hides all the RADOS Gateways behind http://localhost:9998 and distributes all requests equally to the Gateways
* Takes care of the CORS header for the RADOS Gateway
You don't have to use Traefik for that. You can use any reverse proxy for this task, like [Caddy](https://caddyserver.com/), [HAProxy](https://www.haproxy.org/) or [nginx](https://nginx.org/en/).<br>
### Run Dev Server
Export all necessary environment variables or create a `.env` file.<br>
Run the dev server with live reload after changes
```shell
python app/check_ceph_connection.py && \
python app/check_oidc_connection.py && \
python app/check_database_connection.py && \
alembic upgrade head && \
uvicorn app.main:app --reload
```
You can check your code with linters or even automatically reformat files based on these rules
```shell
./scripts/lint.sh # check code
./scripts/format.sh # reformat code
```
### Run Tests
Export the port and other variables of the database and then start the test script
```shell
export DB_PORT=8001
./tests-start.sh
```
### Common Problems
Q: When I start the server I get the error `ModuleNotFoundError: No module named 'app'`<br>
A: export the `PYTHONPATH` variable with the current working directory
```shell
export PYTHONPATH=$(pwd)
```
Q: When I start the linters `isort`, `black`, etc. cannot be found<br>
A: Prepend every call with `python -m`
```shell
python -m isort
python -m black
...
```
Dockerfile
+ 13
− 8
View file @ d38f8b1c
FROM python:3.10-slim
WORKDIR /code
ENV PYTHONPATH=/code
EXPOSE 80
FROM python:3.11-slim
EXPOSE 8000
# dumb-init forwards the kill signal to the python process
RUN apt-get update && apt-get -y install dumb-init curl
RUN apt-get clean
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
HEALTHCHECK --interval=35s --timeout=4s CMD curl -f http://localhost/health || exit 1
HEALTHCHECK --interval=30s --timeout=2s CMD curl -f http://localhost:8000/health || exit 1
COPY requirements.txt ./requirements.txt
RUN useradd -m worker
USER worker
WORKDIR /home/worker/code
ENV PYTHONPATH=/home/worker/code
ENV PATH="/home/worker/.local/bin:${PATH}"
RUN pip install --no-cache-dir --upgrade -r requirements.txt
COPY --chown=worker:worker requirements.txt ./requirements.txt
COPY . .
RUN pip install --user --no-cache-dir --upgrade -r requirements.txt
COPY --chown=worker:worker . .
CMD ["./start_service.sh"]
Dockerfile-Gunicorn 0 → 100644
+ 14
− 0
View file @ d38f8b1c
FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim
EXPOSE 8000
ENV PORT=8000
RUN pip install --no-cache-dir httpx[cli]
HEALTHCHECK --interval=30s --timeout=4s CMD httpx http://localhost:$PORT/health || exit 1
COPY ./scripts/prestart.sh /app/prestart.sh
COPY ./requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir --upgrade -r requirements.txt
COPY ./app /app/app
README.md
+ 23
− 29
View file @ d38f8b1c
# S3 Proxy API
# CloWM S3Proxy Service
## Description
Openstack is shipping with an integrated UI to access the Object Store provided by Ceph. Unfortunately, this UI does not allow
......@@ -21,34 +21,28 @@ user-friendly manner. 👍
### Mandatory / Recommended Variables
| Variable | Default | Value | Description |
|----------------------|---------|-----------------------|---------------------------------------|
| `SECRET_KEY` | random | \<random key> | Secret key to sign JWT |
| `DB_HOST` | unset | <db hostname / IP> | IP or Hostname Adress of DB |
| `DB_PORT` | 3306 | Number | Port of the database |
| `DB_USER` | unset | \<db username> | Username of the database user |
| `DB_PASSWORD` | unset | \<db password> | Password of the database user |
| `DB_DATABASE` | unset | \<db name> | Name of the database |
| `OBJECT_GATEWAY_URI` | unset | HTTP URL | HTTP URL of the Ceph Object Gateway |
| `CEPH_ACCESS_KEY` | unset | \<access key> | Ceph access key with admin privileges |
| `CEPH_SECRET_KEY` | unset | \<secret key> | Ceph secret key with admin privileges |
| `CEPH_USERNAME` | unset | \<ceph username> | Username in Ceph of the backend user |
| `OIDC_CLIENT_ID` | unset | \<OIDC client id> | Client ID from the OIDC provider |
| `OIDC_CLIENT_SECRET` | unset | \<OIDC client secret> | Client Secret from the OIDC provider |
| `OIDC_BASE_URI` | unset | HTTP URL | HTTP URL of the OIDC Provider |
| Variable | Default | Value | Description |
|----------------------------------------|-------------------------|---------------------------------|------------------------------------------------------------------------------------|
| `DB_HOST` | unset | <db hostname / IP> | IP or Hostname Address of DB |
| `DB_PORT` | 3306 | Number | Port of the database |
| `DB_USER` | unset | \<db username> | Username of the database user |
| `DB_PASSWORD` | unset | \<db password> | Password of the database user |
| `DB_DATABASE` | unset | \<db name> | Name of the database |
| `OBJECT_GATEWAY_URI` | unset | HTTP URL | HTTP URL of the Ceph Object Gateway |
| `BUCKET_CEPH_ACCESS_KEY` | unset | \<access key> | Access key for the Ceph Object Gateway user with unlimited buckets. |
| `BUCKET_CEPH_SECRET_KEY` | unset | \<secret key> | Secret key for the Ceph Object Gateway user with unlimited buckets. |
| `BUCKET_CEPH_USERNAME` | unset | \<ceph username> | ID of the user in ceph who owns all the buckets. Owner of `BUCKET_CEPH_ACCESS_KEY` |
| `USER_CEPH_ACCESS_KEY` | unset | \<access key> | Access key for the Ceph Object Gateway user with `user:*` privileges |
| `USER_CEPH_SECRET_KEY` | unset | \<secret key> | Secret key for the Ceph Object Gateway user with `user:*` privileges. |
| `PUBLIC_KEY_VALUE` / `PUBLIC_KEY_FILE` | randomly generated | Public Key / Path to Public Key | Public part of RSA Key in PEM format to verify JWTs |
| `OPA_URI` | unset | HTTP URL | HTTP URL of the OPA service |
| `CLOWM_URL` | `http://localhost:8080` | HTTP URL | HTTP URL of the CloWM website |
### Optional Variables
| Variable | Default | Value | Description |
|-----------------------------|-------------------------------------|-----------------------------|---------------------------------------------------------------------------------------|
| `DOMAIN` | `localhost` | string | Domain under which the service will be hosted. |
| `SSL_TERMINATION` | `false` | `<"true"&#x7c;"false">` | Flag if the service runs behind a SSL termination proxy |
| `API_PREFIX` | `/api` | URL path | Prefix before every URL path |
| `JWT_TOKEN_EXPIRE_MINUTES` | 8 days | number | Minutes till a JWT expires |
| `BACKEND_CORS_ORIGINS` | `[]` | json formatted list of urls | List of valid CORS origins |
| `SQLALCHEMY_VERBOSE_LOGGER` | `false` | `<"true"&#x7c;"false">` | Enables verbose SQL output.<br>Should be `false` in production |
| `OIDC_META_INFO_PATH` | `/.well-known/openid-configuration` | URL path | Path to the OIDC configuration file<br> Will be concatenated with the `OIDC_BASE_URI` |
## Getting started
This service depends on multiple other services. See [DEVELOPING.md](DEVELOPING.md) how to set these up for developing
on your local machine.
| Variable | Default | Value | Description |
|-----------------------------|----------------------|-----------------------------|----------------------------------------------------------------|
| `API_PREFIX` | `/api` | URL path | Prefix before every URL path |
| `SQLALCHEMY_VERBOSE_LOGGER` | `false` | `<"true"&#x7c;"false">` | Enables verbose SQL output.<br>Should be `false` in production |
| `OPA_POLICY_PATH` | `/clowm/authz/allow` | URL path | Path to the OPA Policy for Authorization |
| `OTLP_GRPC_ENDPOINT` | unset | <hostname / IP> | OTLP compatible endpoint to send traces via gRPC, e.g. Jaeger |
alembic.ini deleted 100644 → 0
+ 0
− 100
View file @ c1815139
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the python-dateutil library that can be
# installed by adding `alembic[tz]` to the pip requirements
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; This defaults
# to alembic/versions. When using multiple version
# directories, initial revisions must be specified with --version-path.
# The path separator used here should be the separator specified by "version_path_separator" below.
# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
# version path separator; As mentioned above, this is the character used to split
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
# Valid values for version_path_separator are:
#
# version_path_separator = :
# version_path_separator = ;
# version_path_separator = space
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
alembic/README deleted 100644 → 0
+ 0
− 1
View file @ c1815139
Generic single-database configuration with an async dbapi.
alembic/env.py deleted 100644 → 0
+ 0
− 93
View file @ c1815139
import asyncio
from logging.config import fileConfig
from sqlalchemy import engine_from_config, pool
from sqlalchemy.ext.asyncio import AsyncEngine
from alembic import context
from app.core.config import settings
from app.db.base import Base
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def get_url() -> str:
return str(settings.SQLALCHEMY_DATABASE_ASYNC_URI)
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = get_url()
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection):
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
url = get_url()
connectable = AsyncEngine(
engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
future=True,
url=url,
)
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
if context.is_offline_mode():
run_migrations_offline()
else:
asyncio.run(run_migrations_online())
alembic/script.py.mako deleted 100644 → 0
+ 0
− 24
View file @ c1815139
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}
alembic/versions/5521b5759004_create_user_and_bucket_table.py deleted 100644 → 0
+ 0
− 51
View file @ c1815139
"""Create user and bucket table
Revision ID: 5521b5759004
Revises:
Create Date: 2022-05-03 14:01:22.154984
"""
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from alembic import op
# revision identifiers, used by Alembic.
revision = "5521b5759004"
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"user",
sa.Column("uid", sa.String(length=64), nullable=False),
sa.Column("name", sa.String(length=256), nullable=False),
sa.PrimaryKeyConstraint("uid"),
)
op.create_index(op.f("ix_user_uid"), "user", ["uid"], unique=True)
op.create_table(
"bucket",
sa.Column("name", sa.String(length=63), nullable=False),
sa.Column("description", mysql.TEXT(), nullable=False),
sa.Column("public", sa.Boolean(), server_default="0", nullable=True),
sa.Column("owner_id", sa.String(length=64), nullable=True),
sa.ForeignKeyConstraint(
["owner_id"],
["user.uid"],
),
sa.PrimaryKeyConstraint("name"),
)
op.create_index(op.f("ix_bucket_name"), "bucket", ["name"], unique=True)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_bucket_name"), table_name="bucket")
op.drop_table("bucket")
op.drop_index(op.f("ix_user_uid"), table_name="user")
op.drop_table("user")
# ### end Alembic commands ###
alembic/versions/6c64f020818b_make_display_name_for_users_mandatory.py deleted 100644 → 0
+ 0
− 28
View file @ c1815139
"""Make display_name for users mandatory
Revision ID: 6c64f020818b
Revises: 9fa582febebe
Create Date: 2022-10-21 13:53:44.446799
"""
from sqlalchemy.dialects import mysql
from alembic import op
# revision identifiers, used by Alembic.
revision = "6c64f020818b"
down_revision = "9fa582febebe"
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column("user", "display_name", existing_type=mysql.VARCHAR(length=256), nullable=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column("user", "display_name", existing_type=mysql.VARCHAR(length=256), nullable=True)
# ### end Alembic commands ###
alembic/versions/83a3a47a6351_add_username_and_display_name_and_drop_.py deleted 100644 → 0
+ 0
− 35
View file @ c1815139
"""Add username and display_name and drop name for user table
Revision ID: 83a3a47a6351
Revises: cafa1e01b782
Create Date: 2022-05-04 13:22:46.317796
"""
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from alembic import op
# revision identifiers, used by Alembic.
revision = "83a3a47a6351"
down_revision = "cafa1e01b782"
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("user", sa.Column("display_name", sa.String(length=256), nullable=True))
op.add_column("user", sa.Column("username", sa.String(length=256), nullable=False))
op.drop_column("user", "name")
op.create_index(op.f("ix_user_username"), "user", ["username"], unique=True)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("user", sa.Column("name", mysql.VARCHAR(length=256), nullable=False))
op.drop_index(op.f("ix_user_username"), table_name="user")
op.drop_column("user", "username")
op.drop_column("user", "display_name")
# ### end Alembic commands ###
alembic/versions/9fa582febebe_delete_username_from_user.py deleted 100644 → 0
+ 0
− 31
View file @ c1815139
"""Delete username from user
Revision ID: 9fa582febebe
Revises: 83a3a47a6351
Create Date: 2022-07-27 11:10:53.440935
"""
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from alembic import op
# revision identifiers, used by Alembic.
revision = "9fa582febebe"
down_revision = "83a3a47a6351"
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_user_username", table_name="user")
op.drop_column("user", "username")
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("user", sa.Column("username", mysql.VARCHAR(length=256), nullable=False))
op.create_index("ix_user_username", "user", ["username"], unique=False)
# ### end Alembic commands ###
alembic/versions/cafa1e01b782_create_permission_table.py deleted 100644 → 0
+ 0
− 54
View file @ c1815139
"""Create Permission table
Revision ID: cafa1e01b782
Revises: 5521b5759004
Create Date: 2022-05-04 11:41:54.470870
"""
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from alembic import op
# revision identifiers, used by Alembic.
revision = "cafa1e01b782"
down_revision = "5521b5759004"
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"bucketpermission",
sa.Column("user_id", sa.String(length=64), nullable=False),
sa.Column("bucket_name", sa.String(length=63), nullable=False),
sa.Column("from", mysql.TIMESTAMP(), nullable=True),
sa.Column("to", mysql.TIMESTAMP(), nullable=True),
sa.Column("file_prefix", sa.String(length=512), nullable=True),
sa.Column("permissions", mysql.ENUM("READ", "WRITE", "READWRITE"), nullable=False),
sa.ForeignKeyConstraint(["bucket_name"], ["bucket.name"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["user_id"], ["user.uid"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("user_id", "bucket_name"),
)
op.alter_column(
"bucket",
"public",
existing_type=mysql.TINYINT(display_width=1),
nullable=False,
existing_server_default=sa.text("'0'"),
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"bucket",
"public",
existing_type=mysql.TINYINT(display_width=1),
nullable=True,
existing_server_default=sa.text("'0'"),
)
op.drop_table("bucketpermission")
# ### end Alembic commands ###
app/api/api.py
+ 4
− 5
View file @ d38f8b1c
from typing import Any
from typing import Any, Dict, Union
from fastapi import APIRouter, Depends, status
from app.api.dependencies import decode_bearer_token
from app.api.endpoints import bucket_permissions, buckets, login, users
from app.api.endpoints import bucket_permissions, buckets, s3key
from app.schemas.security import ErrorDetail
alternative_responses: dict[int | str, dict[str, Any]] = {
alternative_responses: Dict[Union[int, str], Dict[str, Any]] = {
status.HTTP_400_BAD_REQUEST: {
"model": ErrorDetail,
"description": "Error decoding JWT Token",
......@@ -25,14 +25,13 @@ alternative_responses: dict[int | str, dict[str, Any]] = {
}
api_router = APIRouter()
api_router.include_router(login.router)
api_router.include_router(
buckets.router,
dependencies=[Depends(decode_bearer_token)],
responses=alternative_responses,
)
api_router.include_router(
users.router,
s3key.router,
dependencies=[Depends(decode_bearer_token)],
responses=alternative_responses,
)
......
app/api/dependencies.py
+ 125
− 112
View file @ d38f8b1c
from typing import TYPE_CHECKING, Any, AsyncGenerator
from typing import TYPE_CHECKING, Annotated, AsyncGenerator, Awaitable, Callable, Dict
from authlib.integrations.base_client.errors import OAuthError
from authlib.jose.errors import BadSignatureError, DecodeError, ExpiredTokenError
from fastapi import Depends, HTTPException, Path, status
from fastapi.requests import Request
from clowmdb.db.session import get_async_session
from clowmdb.models import Bucket, User
from fastapi import Depends, HTTPException, Path, Request, status
from fastapi.security import HTTPBearer
from fastapi.security.http import HTTPAuthorizationCredentials
from httpx import AsyncClient
from opentelemetry import trace
from rgwadmin import RGWAdmin
from sqlalchemy.ext.asyncio import AsyncSession
from app.ceph.rgw import rgw, s3_resource
from app.core.security import decode_token, oauth
from app.ceph.rgw import rgw
from app.ceph.s3 import s3_resource
from app.core.config import settings
from app.core.security import decode_token, request_authorization
from app.crud.crud_bucket import CRUDBucket
from app.crud.crud_bucket_permission import CRUDBucketPermission
from app.crud.crud_user import CRUDUser
from app.db.session import SessionAsync as Session
from app.models.bucket import Bucket
from app.models.user import User
from app.schemas.security import JWTToken
from app.otlp import start_as_current_span_async
from app.schemas.security import JWT, AuthzRequest, AuthzResponse
if TYPE_CHECKING:
from boto3.resources.base import ServiceResource
from mypy_boto3_s3.service_resource import S3ServiceResource
else:
ServiceResource = object
S3ServiceResource = object
bearer_token = HTTPBearer(description="JWT Token")
tracer = trace.get_tracer_provider().get_tracer(__name__)
bearer_token = HTTPBearer(description="JWT Header")
class LoginException(Exception):
def __init__(self, error_source: str):
self.error_source = error_source
def get_rgw_admin() -> RGWAdmin: # pragma: no cover
return rgw
def get_rgw_admin() -> RGWAdmin:
return rgw # pragma: no cover
RGWAdminResource = Annotated[RGWAdmin, Depends(get_rgw_admin)]
def get_s3_resource() -> ServiceResource:
def get_s3_resource() -> S3ServiceResource:
return s3_resource # pragma: no cover
async def get_db() -> AsyncGenerator:
S3Resource = Annotated[S3ServiceResource, Depends(get_s3_resource)]
async def get_db() -> AsyncGenerator[AsyncSession, None]:
"""
Get a Session with the database.
......@@ -48,16 +51,47 @@ async def get_db() -> AsyncGenerator:
Returns
-------
db : AsyncGenerator
db : AsyncGenerator[AsyncSession, None]
Async session object with the database
"""
async with Session() as db:
async with get_async_session(
str(settings.SQLALCHEMY_DATABASE_ASYNC_URI), verbose=settings.SQLALCHEMY_VERBOSE_LOGGER
) as db:
yield db
def decode_bearer_token(
DBSession = Annotated[AsyncSession, Depends(get_db)]
async def get_httpx_client(request: Request) -> AsyncClient: # pragma: no cover
# Fetch open http client from the app
return request.app.requests_client
HTTPXClient = Annotated[AsyncClient, Depends(get_httpx_client)]
def get_decode_jwt_function() -> Callable[[str], Dict[str, str]]: # pragma: no cover
"""
Get function to decode and verify the JWT.
This will be injected into the function which will handle the JWT. With this approach, the function to decode and
verify the JWT can be overwritten during tests.
Returns
-------
decode : Callable[[str], Dict[str, str]]
Function to decode & verify the token. raw_token -> claims. Dependency Injection
"""
return decode_token
@start_as_current_span_async("decode_jwt", tracer=tracer)
async def decode_bearer_token(
token: HTTPAuthorizationCredentials = Depends(bearer_token),
) -> JWTToken:
decode: Callable[[str], Dict[str, str]] = Depends(get_decode_jwt_function),
db: AsyncSession = Depends(get_db),
) -> JWT:
"""
Get the decoded JWT or reject request if it is not valid.
......@@ -67,21 +101,69 @@ def decode_bearer_token(
----------
token : fastapi.security.http.HTTPAuthorizationCredentials
Bearer token sent with the HTTP request. Dependency Injection.
decode : Callable[[str], Dict[str, str]]
Function to decode & verify the token. raw_token -> claims. Dependency Injection
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
Returns
-------
token : app.schemas.security.JWTToken
token : app.schemas.security.JWT
The verified and decoded JWT.
"""
try:
return JWTToken(**decode_token(token.credentials))
jwt = JWT(**decode(token.credentials), raw_token=token.credentials)
await get_current_user(jwt, db) # make sure the user exists
return jwt
except ExpiredTokenError: # pragma: no cover
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="JWT Signature has expired")
except (DecodeError, BadSignatureError):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Malformed JWT")
async def get_current_user(token: JWTToken = Depends(decode_bearer_token), db: AsyncSession = Depends(get_db)) -> User:
class AuthorizationDependency:
"""
Class to parameterize the authorization request with the resource to perform an operation on.
"""
def __init__(self, resource: str):
"""
Parameters
----------
resource : str
Resource parameter for the authorization requests
"""
self.resource = resource
def __call__(
self,
token: JWT = Depends(decode_bearer_token),
client: AsyncClient = Depends(get_httpx_client),
) -> Callable[[str], Awaitable[AuthzResponse]]:
"""
Get the function to request the authorization service with the resource, JWT and HTTP Client already injected.
Parameters
----------
token : app.schemas.security.JWT
The verified and decoded JWT. Dependency Injection.
client : httpx.AsyncClient
HTTP Client with an open connection. Dependency Injection.
Returns
-------
authorization_function : Callable[[str], Awaitable[app.schemas.security.AuthzResponse]]
Async function which ask the Auth service for authorization. It takes the operation as the only input.
"""
async def authorization_wrapper(operation: str) -> AuthzResponse:
params = AuthzRequest(operation=operation, resource=self.resource, uid=token.sub)
return await request_authorization(request_params=params, client=client)
return authorization_wrapper
async def get_current_user(token: JWT = Depends(decode_bearer_token), db: AsyncSession = Depends(get_db)) -> User:
"""
Get the current user from the database based on the JWT.
......@@ -89,14 +171,14 @@ async def get_current_user(token: JWTToken = Depends(decode_bearer_token), db: A
Parameters
----------
token : app.schemas.security.JWTToken
token : app.schemas.security.JWT
The verified and decoded JWT.
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
Returns
-------
user : app.models.user.User
user : clowmdb.models.User
User associated with the JWT sent with the HTTP request.
"""
user = await CRUDUser.get(db, token.sub)
......@@ -105,12 +187,14 @@ async def get_current_user(token: JWTToken = Depends(decode_bearer_token), db: A
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
CurrentUser = Annotated[User, Depends(get_current_user)]
async def get_user_by_path_uid(
uid: str = Path(
default=..., description="UID of a user", example="28c5353b8bb34984a8bd4169ba94c606", max_length=64
default=..., description="UID of a user", examples=["28c5353b8bb34984a8bd4169ba94c606"], max_length=64
),
db: AsyncSession = Depends(get_db),
token: JWTToken = Depends(decode_bearer_token),
) -> User:
"""
Get the user from the database with the given uid.
......@@ -124,35 +208,29 @@ async def get_user_by_path_uid(
The uid of a user. URL path parameter.
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
token : app.schemas.security.JWTToken
Decoded JWT sent with the HTTP request.
Returns
-------
user : app.models.user.User
user : clowmdb.models.User
User with the given uid.
"""
user = await CRUDUser.get(db, uid)
if user:
if user.uid == token.sub:
return user
else:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="A user can only access himself",
)
return user
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
PathUser = Annotated[User, Depends(get_user_by_path_uid)]
async def get_current_bucket(
bucket_name: str = Path(..., description="Name of bucket", example="test-bucket", max_length=63, min_length=3),
bucket_name: str = Path(..., description="Name of bucket", examples=["test-bucket"], max_length=63, min_length=3),
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user),
) -> Bucket:
"""
Get the Bucket from the database based on the name in the path.
Reject the request if user has no READ permission for this bucket.
Reject the request if user has no permission for this bucket.
FastAPI Dependency Injection Function
......@@ -162,81 +240,16 @@ async def get_current_bucket(
Name of a bucket. URL Path Parameter.
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
current_user : app.models.user.User
User associated with the JWT sent with the HTTP request. Dependency Injection
Returns
-------
bucket : app.models.bucket.Bucket
bucket : clowmdb.models.Bucket
Bucket with the given name.
"""
bucket = await CRUDBucket.get(db, bucket_name)
if bucket is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Bucket not found")
elif not bucket.public and not await CRUDBucketPermission.check_permission(db, bucket_name, current_user.uid):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No rights for this bucket")
return bucket
async def get_authorized_user_for_permission(
bucket: Bucket = Depends(get_current_bucket),
uid: str = Path(
default=..., description="UID of a user", example="28c5353b8bb34984a8bd4169ba94c606", max_length=64
),
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user),
) -> User:
"""
Get the user for viewing and deleting bucket permissions.\n
Only the owner of a bucket and grantee can do this.
Parameters
----------
bucket : app.models.bucket.Bucket
Bucket with the name provided in the URL path. Dependency Injection.
uid : str
The uid of a user. URL path parameter.
db : sqlalchemy.ext.asyncio.AsyncSession.
Async database session to perform query on. Dependency Injection.
current_user : app.models.user.User
Current user. Dependency Injection.
Returns
-------
user : app.models.user.User
Authorized user for bucket permission. Dependency Injection.
"""
user = await CRUDUser.get(db, uid)
if user is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
elif current_user != user and current_user.uid != bucket.owner_id:
raise HTTPException(
status.HTTP_403_FORBIDDEN, detail="Only the owner or the grantee can access a bucket permission"
)
return user
async def get_userinfo_from_access_token(request: Request) -> dict[str, Any]: # pragma: no cover
"""
Get the userinfo from the OAuth2 userinfo endpoint with the access token.
Parameters
----------
request : starlette.requests.Request
Raw Request object.
Returns
-------
userinfo : dict[str, Any]
Info about the corresponding user.
"""
try:
if "error" in request.query_params.keys():
# if there is an error in the login flow, like a canceld login request, then notify the client
raise LoginException(error_source=request.query_params["error"])
claims = await oauth.lifescience.authorize_access_token(request)
# ID token doesn't have all necessary information, call userinfo endpoint
return await oauth.lifescience.userinfo(token=claims)
except OAuthError:
# if there is an error in the oauth flow, like an expired token, then notify the client
raise LoginException(error_source="oidc")
CurrentBucket = Annotated[Bucket, Depends(get_current_bucket)]
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment