Merge branch 'development' into 'main'

First working version

See merge request denbi/object-storage-access!6

.dockerignore

+.idea/
+*/__pycache__/
+.env
+.venv
+env/
+venv/
+ENV/
+README.md
+.pytest_cache
+.mypy_cache
+htmlcov
+app/tests
+figures/
+oidc_dev_example
+oidc_dev/
+traefik_dev
+ceph

.flake8

+[flake8]
+max-line-length = 120
+exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache

.gitignore

+.idea/
+__pycache__/
+.env
+.venv
+env/
+venv/
+ENV/
+.coverage
+oidc_dev/
+traefik

.gitlab-ci.yml

+image: python:3.10-slim
+
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  PYTHONPATH: "$CI_PROJECT_DIR"
+  OBJECT_GATEWAY_URI: "http://127.0.0.1:8000"
+  CEPH_ACCESS_KEY: ""
+  CEPH_SECRET_KEY: ""
+  OIDC_CLIENT_SECRET: ""
+  OIDC_CLIENT_ID: ""
+  OIDC_BASE_URI: "http://127.0.0.1:8000"
+
+cache:
+  paths:
+    - .cache/pip
+    - venv/
+
+before_script:
+  - python --version  # For debugging
+  - pip install virtualenv
+  - virtualenv venv
+  - source venv/bin/activate
+  - python -m pip install -r requirements.txt
+  - python -m pip install -r requirements-dev.txt
+
+stages: # List of stages for jobs, and their order of execution
+#  - build
+  - test
+#  - deploy
+
+#build-job:       # This job runs in the build stage, which runs first.
+#  stage: build
+#  script:
+#    - echo "Compiling the code..."
+#    - echo "Compile complete."
+
+integration-test-job: # Runs integration tests with the database
+  stage: test
+  variables:
+    DB_PASSWORD: "$TEST_DB_PASSWORD"
+    DB_USER: "test_api_user"
+    DB_DATABASE: "integration-test-db"
+    DB_HOST: "integration-test-db"
+  services:
+    - name: mysql:8
+      alias: integration-test-db
+      variables:
+        MYSQL_RANDOM_ROOT_PASSWORD: "yes"
+        MYSQL_DATABASE: "$DB_DATABASE"
+        MYSQL_USER: "$DB_USER"
+        MYSQL_PASSWORD: "$DB_PASSWORD"
+  script:
+    - python app/check_database_connection.py
+    - alembic downgrade base
+    - alembic upgrade head
+    - pytest --junitxml=integration-report.xml --cov=app --cov-report=term-missing app/tests/crud
+    - mkdir coverage-integration
+    - mv .coverage coverage-integration
+  artifacts:
+    paths:
+      - $CI_PROJECT_DIR/coverage-integration/.coverage
+    reports:
+      junit: $CI_PROJECT_DIR/integration-report.xml
+
+e2e-test-job: # Runs e2e tests on the API endpoints
+  stage: test
+  variables:
+    DB_PASSWORD: "$TEST_DB_PASSWORD"
+    DB_USER: "test_api_user"
+    DB_DATABASE: "e2e-test-db"
+    DB_HOST: "e2e-test-db"
+    OIDC_CLIENT_SECRET: "$TEST_OIDC_CLIENT_SECRET"
+    OIDC_CLIENT_ID: "$TEST_OIDC_CLIENT_ID"
+    OIDC_BASE_URI: "http://mock-oidc-server"
+    CLIENTS_CONFIGURATION_INLINE: "$TEST_OIDC_CLIENT_CONFIG"
+  services:
+    - name: mysql:8
+      alias: e2e-test-db
+      variables:
+        MYSQL_RANDOM_ROOT_PASSWORD: "yes"
+        MYSQL_DATABASE: "$DB_DATABASE"
+        MYSQL_USER: "$DB_USER"
+        MYSQL_PASSWORD: "$DB_PASSWORD"
+    - name: ghcr.io/soluto/oidc-server-mock:latest
+      alias: mock-oidc-server
+      variables:
+        ASPNETCORE_ENVIRONMENT: "Development"
+  script:
+    - python app/check_database_connection.py
+    - python app/check_oidc_connection.py
+    - alembic downgrade base
+    - alembic upgrade head
+    - pytest --junitxml=e2e-report.xml --cov=app --cov-report=term-missing app/tests/api
+    - mkdir coverage-e2e
+    - mv .coverage coverage-e2e
+  artifacts:
+    paths:
+      - $CI_PROJECT_DIR/coverage-e2e/.coverage
+    reports:
+      junit: $CI_PROJECT_DIR/e2e-report.xml
+
+unit-test-job: # Runs unit tests
+  stage: test
+  script:
+    - pytest --junitxml=unit-report.xml --noconftest --cov=app --cov-report=term-missing app/tests/unit
+    - mkdir coverage-unit
+    - mv .coverage coverage-unit
+  artifacts:
+    paths:
+      - $CI_PROJECT_DIR/coverage-unit/.coverage
+    reports:
+      junit: $CI_PROJECT_DIR/unit-report.xml
+
+combine-test-coverage-job: # Combine coverage reports from different test jobs
+  stage: test
+  needs:
+    - job: "e2e-test-job"
+      artifacts: true
+    - job: "integration-test-job"
+      artifacts: true
+    - job: "unit-test-job"
+      artifacts: true
+  script:
+    - coverage combine coverage-e2e/.coverage coverage-integration/.coverage coverage-unit/.coverage
+    - coverage report
+    - coverage xml --data-file=$CI_PROJECT_DIR/.coverage -o coverage.xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: $CI_PROJECT_DIR/coverage.xml
+
+lint-test-job: # Runs linters checks on code
+  stage: test
+  script:
+    - ./scripts/lint.sh
+
+#deploy-job:      # This job runs in the deploy stage.
+#  stage: deploy  # It only runs when *both* jobs in the test stage complete successfully.
+#  script:
+#    - echo "Deploying application..."
+#    - echo "Application successfully deployed."

.pre-commit-config.yaml

+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+    -   id: end-of-file-fixer
+    -   id: check-added-large-files
+    -   id: check-toml
+    -   id: check-docstring-first
+    -   id: detect-private-key
+    -   id: trailing-whitespace
+    -   id: check-yaml
+    -   id: debug-statements
+    -   id: check-merge-conflict
+    -   id: check-ast
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
+        files: app
+        args: [--check]
+-   repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+    -   id: flake8
+        files: app
+        args: [--config=.flake8]
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.960
+    hooks:
+    -   id: mypy
+        files: app
+        args: [--config=pyproject.toml]
+        additional_dependencies:
+            - sqlalchemy2-stubs
+            - boto3-stubs-lite[s3]
+            - sqlalchemy<2.0.0
+            - pydantic
+            - types-requests
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort
+        files: app
+        args: [-c]

DEVELOPING.md

+## Development Setup
+
+### Python Setup 🐍
+Currently, only Python version `>=3.10` is supported because it uses its new features for type annotations
+to write more compact code. Since FastAPI relies on these type annotations and `mypy` is integrated into
+this project, we make heavy use of this feature.
+
+Write
+```python
+var1: list[int] = [1,2,3]
+var2: str | None = None
+```
+instead of
+```python
+from typing import List, Union
+
+var1: List[int] = [1,2,3]
+var2: Union[str, None] = None
+```
+### Environment Setup
+Create a virtual environment, install the dependencies and install the [pre-commit](https://pre-commit.com/) hooks.<br>
+The linters can prevent a commit of the code quality doesn't meet the standard.
+```shell
+python -m venv venv
+source venv/bin/activate
+python -m pip install -r requirements.txt
+python -m pip install -r requirements-dev.txt
+pre-commit install
+```
+
+### Ceph Setup
+For how to set up a ceph cluster or how to connect to an existing one see
+the [documentation in the ceph folder](ceph/README.md).
+
+A user with `user` capabilities should be created, e.g.<br>
+`radosgw-admin user create --uid=myadmin --caps="users=*"`
+
+### Database Setup
+#### Dev database
+The easiest solution is [Docker](https://docs.docker.com/get-docker/) with an attached volume
+to set up a MySQL database.
+```shell
+docker volume create proxyapi_dev_db
+docker run --name proxyapi_devdb \
+  -e MYSQL_RANDOM_ROOT_PASSWORD=yes \
+  -e MYSQL_DATABASE=<database_name> \
+  -e MYSQL_USER=<database_user> \
+  -e MYSQL_PASSWORD=<database_password> \
+  -p 127.0.0.1:3306:3306 \
+  -v proxyapi_dev_db:/var/lib/mysql \
+  -d \
+  mysql:8
+```
+When the container stopped just restart it with
+```shell
+docker start proxyapi_devdb
+```
+Look at the [Environment Variables](#environment-variables) section to see which env variables have to be set.
+#### Test database
+Set up a second database on a different port for the integration tests. This database doesn't have to be persistent
+because all data will be purged after each test run.
+```shell
+docker run --name proxyapi_testdb \
+  -e MYSQL_RANDOM_ROOT_PASSWORD=yes \
+  -e MYSQL_DATABASE=<database_name> \
+  -e MYSQL_USER=<database_user> \
+  -e MYSQL_PASSWORD=<database_password> \
+  -p 127.0.0.1:8001:3306 \
+  -d \
+  mysql:8
+```
+
+### Dev OIDC Provider Setup
+To avoid the complex process of connecting the local machine with the LifeScience AAI Test server, a simple [OIDC provider](https://github.com/Soluto/oidc-server-mock)
+can be setup with Docker.<br>
+Copy the `oidc_dev_example` directory to `oidc_dev`
+```shell
+cp -r oidc_dev_example oidc_dev
+```
+In the file `oidc_dev/clients_config.json` add a random value to `ClientId` and `ClientSecrets`. These can be generated for example with `openssl`.
+```shell
+openssl rand -hex 10
+```
+You can add/delete users in the file `oidc_dev/users_config.json` according the schema that is provided there.<br>
+Adjust the volume path and start the docker container
+```shell
+docker run --name proxyapi_oidc_provider \
+  -e CLIENTS_CONFIGURATION_PATH=/tmp/config/clients_config.json \
+  -e IDENTITY_RESOURCES_PATH=/tmp/config/identity_resources.json \
+  -e USERS_CONFIGURATION_PATH=/tmp/config/users_config.json \
+  -e SERVER_OPTIONS_PATH=/tmp/config/server_options.json \
+  -e ASPNETCORE_ENVIRONMENT=Development \
+  -p 127.0.0.1:8002:80 \
+  -v /path/to/folder/oidc_dev:/tmp/config:ro \
+  -d \
+  ghcr.io/soluto/oidc-server-mock:latest
+```
+Set the env variables `OIDC_BASE_URI` to `http://localhost:8002` and `OIDC_CLIENT_SECRET` / `OIDC_CLIENT_ID` to their appropriate value.
+
+### Reverse Proxy Setup
+The `API_PREFIX` is handles on the level of the reverse proxy. This simplifies the routing in the code and the cooperation with the [Frontend](https://gitlab.ub.uni-bielefeld.de/denbi/object-storage-access-ui).
+An simple Traefik reverse proxy configuration is stored in the repository.
+
+[Traefik](https://traefik.io/) is a reverse Proxy written in Go.
+To use it, download the [`traefik`](https://github.com/traefik/traefik/releases) binary and start it with
+```shell
+cd traefik_dev
+/path/to/binary/traefik --configFile=traefik.toml
+```
+The provided configuration does the following things
+ * It forwards all request to http://localhost:9999/api/* to http://localhost:8080 (this backend)
+ * It strips the prefix `/api` before it forwards the request to the backend
+ * All other request will be forwarded to http://localhost:5173, the corresponding dev [Frontend](https://gitlab.ub.uni-bielefeld.de/denbi/object-storage-access-ui)
+
+You don't have to use Traefik for that. You can use any reverse proxy for this task, like [Caddy](https://caddyserver.com/), [HAProxy](https://www.haproxy.org/) or [nginx](https://nginx.org/en/).<br>
+
+### Run Dev Server
+Export all necessary environment variables or create a `.env` file.<br>
+Run the dev server with live reload after changes
+```shell
+python app/check_ceph_connection.py && \
+ python app/check_oidc_connection.py && \
+ python app/check_database_connection.py && \
+ alembic upgrade head && \
+ uvicorn app.main:app --reload
+```
+You can check your code with linters or even automatically reformat files based on these rules
+```shell
+./scripts/lint.sh      # check code
+./scripts/format.sh    # reformat code
+```
+
+### Run Tests
+Export the port and other variables of the database and then start the test script
+```shell
+export DB_PORT=8001
+./tests-start.sh
+```
+
+### Common Problems
+Q: When I start the server I get the error `ModuleNotFoundError: No module named 'app'`<br>
+A: export the `PYTHONPATH` variable with the current working directory
+```shell
+export PYTHONPATH=$(pwd)
+```
+
+Q: When I start the linters `isort`, `black`, etc. cannot be found<br>
+A: Prepend every call with `python -m`
+```shell
+python -m isort
+python -m black
+...
+```

Dockerfile

+FROM python:3.10-slim
+WORKDIR /code
+ENV PYTHONPATH=/code
+EXPOSE 80
+
+# dumb-init forwards the kill signal to the python process
+RUN apt-get update && apt-get -y install dumb-init curl
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+HEALTHCHECK --interval=35s --timeout=4s CMD curl -f http://localhost/health || exit 1
+
+COPY requirements.txt ./requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY . .
+
+CMD ["./start_service.sh"]

README.md

-# Object Storage Access
-
-
-
-## Getting started
-
-To make it easy for you to get started with GitLab, here's a list of recommended next steps.
-
-Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)!
-
-## Add your files
-
-- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files
-- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command:
-
-```
-cd existing_repo
-git remote add origin https://gitlab.ub.uni-bielefeld.de/denbi/object-storage-access.git
-git branch -M main
-git push -uf origin main
-```
-
-## Integrate with your tools
-
-- [ ] [Set up project integrations](https://gitlab.ub.uni-bielefeld.de/denbi/object-storage-access/-/settings/integrations)
-
-## Collaborate with your team
-
-- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/)
-- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html)
-- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)
-- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/)
-- [ ] [Automatically merge when pipeline succeeds](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html)
-
-## Test and Deploy
-
-Use the built-in continuous integration in GitLab.
-
-- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html)
-- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing(SAST)](https://docs.gitlab.com/ee/user/application_security/sast/)
-- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html)
-- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/)
-- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html)
-
-***
-
-# Editing this README
-
-When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!).  Thank you to [makeareadme.com](https://www.makeareadme.com/) for this template.
-
-## Suggestions for a good README
-Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information.
-
-## Name
-Choose a self-explaining name for your project.
+# S3 Proxy API
 
 ## Description
-Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors.
-
-## Badges
-On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge.
-
-## Visuals
-Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method.
-
-## Installation
-Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection.
-
-## Usage
-Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README.
-
-## Support
-Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc.
-
-## Roadmap
-If you have ideas for releases in the future, it is a good idea to list them in the README.
-
-## Contributing
-State if you are open to contributions and what your requirements are for accepting them.
-
-For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self.
-
-You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.
-
-## Authors and acknowledgment
-Show your appreciation to those who have contributed to the project.
-
-## License
-For open source projects, say how it is licensed.
-
-## Project status
-If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers.
+Openstack is shipping with an integrated UI to access the Object Store provided by ceph. Unfortunately, this UI does not allow
+fine-grained control who can access a bucket or object. You can either make it accessible for everyone or nobody, but
+Ceph can do this and much more. 👎
+This is the backend for a new UI which can leverage the additional powerful functionality provided by Ceph in a
+user-friendly manner. 👍
+
+| Feature                     | Openstack Integration | New UI |
+|-----------------------------|:---------------------:|:------:|
+| Create / Delete Buckets UI  |           ✅           |   ✅    |
+| Create / Delete Buckets CLI |           ✅           |   ❌    |
+| Upload / Download Objects   |           ✅           |   ✅    |
+| Fine-grained Access Control |           ❌           |   ✅    |
+
+### Concept
+![Visualization of Concept](figures/cloud_object_storage.svg)
+
+## Environment Variables
+
+### Mandatory / Recommended Variables
+
+| Variable             | Default | Value                 | Description                           |
+|----------------------|---------|-----------------------|---------------------------------------|
+| `SECRET_KEY`         | random  | \<random key>         | Secret key to sign JWT                |
+| `DB_HOST`            | unset   | <db hostname / IP>    | IP or Hostname Adress of DB           |
+| `DB_PORT`            | 3306    | Number                | Port of the database                  |
+| `DB_USER`            | unset   | \<db username>        | Username of the database user         |
+| `DB_PASSWORD`        | unset   | \<db password>        | Password of the database user         |
+| `DB_DATABASE`        | unset   | \<db name>            | Name of the database                  |
+| `OBJECT_GATEWAY_URI` | unset   | HTTP URL              | HTTP URL of the Ceph Object Gateway   |
+| `CEPH_ACCESS_KEY`    | unset   | \<access key>         | Ceph access key with admin privileges |
+| `CEPH_SECRET_KEY`    | unset   | \<secret key>         | Ceph secret key with admin privileges |
+| `OIDC_CLIENT_ID`     | unset   | \<OIDC client id>     | Client ID from the OIDC provider      |
+| `OIDC_CLIENT_SECRET` | unset   | \<OIDC client secret> | Client Secret from the OIDC provider  |
+| `OIDC_BASE_URI`      | unset   | HTTP URL              | HTTP URL of the OIDC Provider         |
+
+### Optional Variables
+
+| Variable                    | Default                             | Value                       | Description                                                                           |
+|-----------------------------|-------------------------------------|-----------------------------|---------------------------------------------------------------------------------------|
+| `DOMAIN`                    | `localhost`                         | string                      | Domain under which the service will be hosted.                                        |
+| `SSL_TERMINATION`           | `false`                             | `<"true"&#x7c;"false">`     | Flag if the service runs behind a SSL termination proxy                               |
+| `API_PREFIX`                | `/api`                              | URL path                    | Prefix before every URL path                                                          |
+| `JWT_TOKEN_EXPIRE_MINUTES`  | 8 days                              | number                      | Minutes till a JWT expires                                                            |
+| `BACKEND_CORS_ORIGINS`      | `[]`                                | json formatted list of urls | List of valid CORS origins                                                            |
+| `SQLALCHEMY_VERBOSE_LOGGER` | `false`                             | `<"true"&#x7c;"false">`     | Enables verbose SQL output.<br>Should be `false` in production                        |
+| `OIDC_META_INFO_PATH`       | `/.well-known/openid-configuration` | URL path                    | Path to the OIDC configuration file<br> Will be concatenated with the `OIDC_BASE_URI` |

alembic.ini

+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

+Generic single-database configuration with an async dbapi.

alembic/env.py

+import asyncio
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config, pool
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from alembic import context
+from app.core.config import settings
+from app.db.base import Base
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+target_metadata = Base.metadata
+
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def get_url() -> str:
+    return str(settings.SQLALCHEMY_DATABASE_ASYNC_URI)
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = get_url()
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection):
+    context.configure(connection=connection, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    url = get_url()
+    connectable = AsyncEngine(
+        engine_from_config(
+            config.get_section(config.config_ini_section),
+            prefix="sqlalchemy.",
+            poolclass=pool.NullPool,
+            future=True,
+            url=url,
+        )
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    asyncio.run(run_migrations_online())

alembic/script.py.mako

+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

alembic/versions/5521b5759004_create_user_and_bucket_table.py

+"""Create user and bucket table
+
+Revision ID: 5521b5759004
+Revises:
+Create Date: 2022-05-03 14:01:22.154984
+
+"""
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "5521b5759004"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "user",
+        sa.Column("uid", sa.String(length=64), nullable=False),
+        sa.Column("name", sa.String(length=256), nullable=False),
+        sa.PrimaryKeyConstraint("uid"),
+    )
+    op.create_index(op.f("ix_user_uid"), "user", ["uid"], unique=True)
+    op.create_table(
+        "bucket",
+        sa.Column("name", sa.String(length=63), nullable=False),
+        sa.Column("description", mysql.TEXT(), nullable=False),
+        sa.Column("public", sa.Boolean(), server_default="0", nullable=True),
+        sa.Column("owner_id", sa.String(length=64), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["owner_id"],
+            ["user.uid"],
+        ),
+        sa.PrimaryKeyConstraint("name"),
+    )
+    op.create_index(op.f("ix_bucket_name"), "bucket", ["name"], unique=True)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_bucket_name"), table_name="bucket")
+    op.drop_table("bucket")
+    op.drop_index(op.f("ix_user_uid"), table_name="user")
+    op.drop_table("user")
+    # ### end Alembic commands ###

alembic/versions/83a3a47a6351_add_username_and_display_name_and_drop_.py

+"""Add username and display_name and drop name for user table
+
+Revision ID: 83a3a47a6351
+Revises: cafa1e01b782
+Create Date: 2022-05-04 13:22:46.317796
+
+"""
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "83a3a47a6351"
+down_revision = "cafa1e01b782"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("user", sa.Column("display_name", sa.String(length=256), nullable=True))
+    op.add_column("user", sa.Column("username", sa.String(length=256), nullable=False))
+    op.drop_column("user", "name")
+    op.create_index(op.f("ix_user_username"), "user", ["username"], unique=True)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("user", sa.Column("name", mysql.VARCHAR(length=256), nullable=False))
+    op.drop_index(op.f("ix_user_username"), table_name="user")
+    op.drop_column("user", "username")
+    op.drop_column("user", "display_name")
+    # ### end Alembic commands ###

alembic/versions/9fa582febebe_delete_username_from_user.py

+"""Delete username from user
+
+Revision ID: 9fa582febebe
+Revises: 83a3a47a6351
+Create Date: 2022-07-27 11:10:53.440935
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "9fa582febebe"
+down_revision = "83a3a47a6351"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("ix_user_username", table_name="user")
+    op.drop_column("user", "username")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("user", sa.Column("username", mysql.VARCHAR(length=256), nullable=False))
+    op.create_index("ix_user_username", "user", ["username"], unique=False)
+    # ### end Alembic commands ###

alembic/versions/cafa1e01b782_create_permission_table.py

+"""Create Permission table
+
+Revision ID: cafa1e01b782
+Revises: 5521b5759004
+Create Date: 2022-05-04 11:41:54.470870
+
+"""
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "cafa1e01b782"
+down_revision = "5521b5759004"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "bucketpermission",
+        sa.Column("user_id", sa.String(length=64), nullable=False),
+        sa.Column("bucket_name", sa.String(length=63), nullable=False),
+        sa.Column("from", mysql.TIMESTAMP(), nullable=True),
+        sa.Column("to", mysql.TIMESTAMP(), nullable=True),
+        sa.Column("file_prefix", sa.String(length=512), nullable=True),
+        sa.Column("permissions", mysql.ENUM("READ", "WRITE", "READWRITE"), nullable=False),
+        sa.ForeignKeyConstraint(["bucket_name"], ["bucket.name"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["user_id"], ["user.uid"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("user_id", "bucket_name"),
+    )
+    op.alter_column(
+        "bucket",
+        "public",
+        existing_type=mysql.TINYINT(display_width=1),
+        nullable=False,
+        existing_server_default=sa.text("'0'"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "bucket",
+        "public",
+        existing_type=mysql.TINYINT(display_width=1),
+        nullable=True,
+        existing_server_default=sa.text("'0'"),
+    )
+    op.drop_table("bucketpermission")
+    # ### end Alembic commands ###

app/api/api.py

+from typing import Any
+
+from fastapi import APIRouter, Depends, status
+
+from app.api.dependencies import decode_bearer_token
+from app.api.endpoints import bucket_permissions, buckets, login, users
+from app.schemas.security import ErrorDetail
+
+alternative_responses: dict[int | str, dict[str, Any]] = {
+    status.HTTP_400_BAD_REQUEST: {
+        "model": ErrorDetail,
+        "description": "Error decoding JWT Token",
+        "content": {"application/json": {"example": {"detail": "Malformed JWT Token"}}},
+    },
+    status.HTTP_403_FORBIDDEN: {
+        "model": ErrorDetail,
+        "description": "Not authenticated",
+        "content": {"application/json": {"example": {"detail": "Not authenticated"}}},
+    },
+    status.HTTP_404_NOT_FOUND: {
+        "model": ErrorDetail,
+        "description": "Entity not Found",
+        "content": {"application/json": {"example": {"detail": "Entity not found."}}},
+    },
+}
+
+api_router = APIRouter()
+api_router.include_router(login.router)
+api_router.include_router(
+    buckets.router,
+    dependencies=[Depends(decode_bearer_token)],
+    responses=alternative_responses,
+)
+api_router.include_router(
+    users.router,
+    dependencies=[Depends(decode_bearer_token)],
+    responses=alternative_responses,
+)
+api_router.include_router(
+    bucket_permissions.router,
+    dependencies=[Depends(decode_bearer_token)],
+    responses=alternative_responses,
+)

app/api/dependencies.py

+from typing import TYPE_CHECKING, Any, AsyncGenerator
+
+from authlib.jose.errors import BadSignatureError, DecodeError, ExpiredTokenError
+from fastapi import Depends, HTTPException, Path, status
+from fastapi.requests import Request
+from fastapi.security import HTTPBearer
+from fastapi.security.http import HTTPAuthorizationCredentials
+from rgwadmin import RGWAdmin
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.ceph.rgw import rgw, s3_resource
+from app.core.security import decode_token, oauth
+from app.crud.crud_bucket import CRUDBucket
+from app.crud.crud_bucket_permission import CRUDBucketPermission
+from app.crud.crud_user import CRUDUser
+from app.db.session import SessionAsync as Session
+from app.models.bucket import Bucket
+from app.models.user import User
+from app.schemas.security import JWTToken
+
+if TYPE_CHECKING:
+    from boto3.resources.base import ServiceResource
+else:
+    ServiceResource = object
+
+bearer_token = HTTPBearer(description="JWT Token")
+
+
+def get_rgw_admin() -> RGWAdmin:
+    return rgw  # pragma: no cover
+
+
+def get_s3_resource() -> ServiceResource:
+    return s3_resource  # pragma: no cover
+
+
+async def get_db() -> AsyncGenerator:
+    """
+    Get a Session with the database.
+
+    FastAPI Dependency Injection Function.
+
+    Returns
+    -------
+    db : AsyncGenerator
+        Async session object with the database
+    """
+    async with Session() as db:
+        yield db
+
+
+def decode_bearer_token(
+    token: HTTPAuthorizationCredentials = Depends(bearer_token),
+) -> JWTToken:
+    """
+    Get the decoded JWT or reject request if it is not valid.
+
+    FastAPI Dependency Injection Function.
+
+    Parameters
+    ----------
+    token : fastapi.security.http.HTTPAuthorizationCredentials
+        Bearer token sent with the HTTP request. Dependency Injection.
+
+    Returns
+    -------
+    token : app.schemas.security.JWTToken
+        The verified and decoded JWT.
+    """
+    try:
+        return JWTToken(**decode_token(token.credentials))
+    except ExpiredTokenError:  # pragma: no cover
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="JWT Signature has expired")
+    except (DecodeError, BadSignatureError):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Malformed JWT")
+
+
+async def get_current_user(token: JWTToken = Depends(decode_bearer_token), db: AsyncSession = Depends(get_db)) -> User:
+    """
+    Get the current user from the database based on the JWT.
+
+    FastAPI Dependency Injection Function.
+
+    Parameters
+    ----------
+    token : app.schemas.security.JWTToken
+        The verified and decoded JWT.
+    db : sqlalchemy.ext.asyncio.AsyncSession.
+        Async database session to perform query on. Dependency Injection.
+
+    Returns
+    -------
+    user : app.models.user.User
+        User associated with the JWT sent with the HTTP request.
+    """
+    user = await CRUDUser.get(db, token.sub)
+    if user:
+        return user
+    raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
+
+
+async def get_user_by_path_uid(
+    uid: str = Path(
+        default=..., description="UID of a user", example="28c5353b8bb34984a8bd4169ba94c606", max_length=64
+    ),
+    db: AsyncSession = Depends(get_db),
+    token: JWTToken = Depends(decode_bearer_token),
+) -> User:
+    """
+    Get the user from the database with the given uid.
+    Reject the request if the current user is not the same as the requested one.
+
+    FastAPI Dependency Injection Function.
+
+    Parameters
+    ----------
+    uid : str
+        The uid of a user. URL path parameter.
+    db : sqlalchemy.ext.asyncio.AsyncSession.
+        Async database session to perform query on. Dependency Injection.
+    token : app.schemas.security.JWTToken
+        Decoded JWT sent with the HTTP request.
+
+    Returns
+    -------
+    user : app.models.user.User
+        User with the given uid.
+
+    """
+    user = await CRUDUser.get(db, uid)
+    if user:
+        if user.uid == token.sub:
+            return user
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail="A user can only access himself",
+            )
+    raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
+
+
+async def get_current_bucket(
+    bucket_name: str = Path(..., description="Name of bucket", example="test-bucket", max_length=63, min_length=3),
+    db: AsyncSession = Depends(get_db),
+    current_user: User = Depends(get_current_user),
+) -> Bucket:
+    """
+    Get the Bucket from the database based on the name in the path.
+    Reject the request if user has no READ permission for this bucket.
+
+    FastAPI Dependency Injection Function
+
+    Parameters
+    ----------
+    bucket_name : str
+        Name of a bucket. URL Path Parameter.
+    db : sqlalchemy.ext.asyncio.AsyncSession.
+        Async database session to perform query on. Dependency Injection.
+    current_user : app.models.user.User
+        User associated with the JWT sent with the HTTP request. Dependency Injection
+
+    Returns
+    -------
+    bucket : app.models.bucket.Bucket
+        Bucket with the given name.
+    """
+    bucket = await CRUDBucket.get(db, bucket_name)
+    if bucket is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Bucket not found")
+    elif not bucket.public and not await CRUDBucketPermission.check_permission(db, bucket_name, current_user.uid):
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No rights for this bucket")
+    return bucket
+
+
+async def get_authorized_user_for_permission(
+    bucket: Bucket = Depends(get_current_bucket),
+    uid: str = Path(
+        default=..., description="UID of a user", example="28c5353b8bb34984a8bd4169ba94c606", max_length=64
+    ),
+    db: AsyncSession = Depends(get_db),
+    current_user: User = Depends(get_current_user),
+) -> User:
+    """
+    Get the user for viewing and deleting bucket permissions.\n
+    Only the owner of a bucket and grantee can do this.
+
+    Parameters
+    ----------
+    bucket : app.models.bucket.Bucket
+        Bucket with the name provided in the URL path. Dependency Injection.
+    uid : str
+        The uid of a user. URL path parameter.
+    db : sqlalchemy.ext.asyncio.AsyncSession.
+        Async database session to perform query on. Dependency Injection.
+    current_user : app.models.user.User
+        Current user. Dependency Injection.
+
+    Returns
+    -------
+    user : app.models.user.User
+        Authorized user for bucket permission. Dependency Injection.
+    """
+    user = await CRUDUser.get(db, uid)
+    if user is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
+    elif current_user != user and current_user.uid != bucket.owner_id:
+        raise HTTPException(
+            status.HTTP_403_FORBIDDEN, detail="Only the owner or the grantee can access a bucket permission"
+        )
+    return user
+
+
+async def get_userinfo_from_access_token(request: Request) -> dict[str, Any]:  # pragma: no cover
+    """
+    Get the userinfo from the OAuth2 userinfo endpoint with the access token.
+
+    Parameters
+    ----------
+    request : starlette.requests.Request
+        Raw Request object.
+
+    Returns
+    -------
+    userinfo : dict[str, Any]
+        Info about the corresponding user.
+    """
+    claims = await oauth.lifescience.authorize_access_token(request)
+    # ID token doesn't have all necessary information, call userinfo endpoint
+    return await oauth.lifescience.userinfo(token=claims)