Use dependency Proxy in CI and cache OpenAPI schema

#67

.gitlab-ci.yml

-image: python:3.11-slim
+image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.11-slim
 
 variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -16,6 +16,7 @@ variables:
   DB_USER: "random"
   DB_DATABASE: "random"
   DB_HOST: "random"
+  OTLP_GRPC_ENDPOINT: ""
 
 cache:
   paths:
@@ -44,7 +45,7 @@ integration-test-job: # Runs integration tests with the database
     DB_DATABASE: "integration-test-db"
     DB_HOST: "integration-test-db"
   services:
-    - name: mysql:8
+    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/mysql:8
       alias: integration-test-db
       variables:
         MYSQL_RANDOM_ROOT_PASSWORD: "yes"
@@ -72,7 +73,7 @@ e2e-test-job: # Runs e2e tests on the API endpoints
     DB_DATABASE: "e2e-test-db"
     DB_HOST: "e2e-test-db"
   services:
-    - name: mysql:8
+    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/mysql:8
       alias: e2e-test-db
       variables:
         MYSQL_RANDOM_ROOT_PASSWORD: "yes"
@@ -132,7 +133,7 @@ lint-test-job: # Runs linters checks on code
 build-publish-dev-docker-container-job:
   stage: deploy
   image:
-    name: gcr.io/kaniko-project/executor:v1.9.1-debug
+    name: gcr.io/kaniko-project/executor:v1.17.0-debug
     entrypoint: [""]
   dependencies: []
   only:
@@ -155,7 +156,7 @@ build-publish-dev-docker-container-job:
 publish-docker-container-job:
   stage: deploy
   image:
-    name: gcr.io/kaniko-project/executor:v1.9.1-debug
+    name: gcr.io/kaniko-project/executor:v1.17.0-debug
     entrypoint: [""]
   dependencies: []
   only:

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
         files: app
         args: [--check]
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.1.3'
+    rev: 'v0.1.4'
     hooks:
     -   id: ruff
 -   repo: https://github.com/PyCQA/isort

app/main.py

-from fastapi import FastAPI, Request, Response
+from hashlib import md5
+
+from fastapi import FastAPI, Request, Response, status
 from fastapi.exception_handlers import http_exception_handler, request_validation_exception_handler
 from fastapi.exceptions import RequestValidationError, StarletteHTTPException
-from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
-from fastapi.responses import JSONResponse, RedirectResponse
+from fastapi.responses import JSONResponse
 from fastapi.routing import APIRoute
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
@@ -38,9 +39,10 @@ app = FastAPI(
     generate_unique_id_function=custom_generate_unique_id,
     # license_info={"name": "MIT", "url": "https://mit-license.org/"},
     root_path=settings.API_PREFIX,
+    openapi_url=None,  # create it manuale to enable caching on client side
 )
 
-if settings.OTLP_GRPC_ENDPOINT is not None and len(settings.OTLP_GRPC_ENDPOINT) > 0:
+if settings.OTLP_GRPC_ENDPOINT is not None and len(settings.OTLP_GRPC_ENDPOINT) > 0:  # pragma: no cover
     resource = Resource(attributes={SERVICE_NAME: "clowm-s3proxy-service"})
     provider = TracerProvider(resource=resource)
     provider.add_span_processor(
@@ -67,14 +69,14 @@ FastAPIInstrumentor.instrument_app(
     app, excluded_urls="health,docs,openapi.json", tracer_provider=trace.get_tracer_provider()
 )
 
-# CORS Settings for the API
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=settings.BACKEND_CORS_ORIGINS,
-    allow_credentials=False,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
+#  CORS Settings for the API (disabled)
+# app.add_middleware(
+#     CORSMiddleware,
+#     allow_origins=settings.BACKEND_CORS_ORIGINS,
+#     allow_credentials=False,
+#     allow_methods=["*"],
+#     allow_headers=["*"],
+# )
 
 # Enable gzip compression for large responses
 app.add_middleware(GZipMiddleware, minimum_size=500)
@@ -83,7 +85,21 @@ app.add_middleware(GZipMiddleware, minimum_size=500)
 app.include_router(api_router)
 app.include_router(miscellaneous_router)
 
+# Create Custom route for OpenAPI schema
+# Hash openapi.json content and save it
+m = md5()
+m.update(JSONResponse(app.openapi()).body)  # ensure the serialization is the same as the one in the route
+openapi_hash = m.hexdigest()
+del m
+
+
+# Route for openapi.json file with ETag header for client cache support
+async def openapi(req: Request) -> Response:
+    # If schema on clients side is still valid, return empty Body with 304 response code (client will use cached body)
+    if req.headers.get("If-None-Match") == openapi_hash:
+        return Response(status_code=status.HTTP_304_NOT_MODIFIED)
+    # Return openapi.json with ETag header
+    return JSONResponse(app.openapi(), headers={"ETag": openapi_hash})
+
 
-@app.get("/", response_class=RedirectResponse, tags=["Miscellaneous"], include_in_schema=False)
-def redirect_docs() -> str:
-    return settings.API_PREFIX + "/docs"
+app.add_route("/openapi.json", openapi, include_in_schema=False)

app/tests/api/test_miscellaneous_enpoints.py

+import pytest
+from fastapi import status
+from httpx import AsyncClient
+
+
+class TestHealthRoute:
+    @pytest.mark.asyncio
+    async def test_health_route(self, client: AsyncClient) -> None:
+        """
+        Test service health route
+
+        Parameters
+        ----------
+        client : httpx.AsyncClient
+            HTTP Client to perform the request on.
+        """
+        response = await client.get("/health")
+        assert response.status_code == status.HTTP_200_OK
+        body = response.json()
+        assert body["status"] == "OK"
+
+
+class TestOpenAPIRoute:
+    @pytest.mark.asyncio
+    async def test_openapi_route(self, client: AsyncClient) -> None:
+        """
+        Test getting the OpenAPI specification and the caching mechanism.
+
+        Parameters
+        ----------
+        client : httpx.AsyncClient
+            HTTP Client to perform the request on.
+        """
+        response1 = await client.get("/openapi.json")
+        assert response1.status_code == status.HTTP_200_OK
+        assert response1.headers.get("ETag") is not None
+
+        response2 = await client.get("/openapi.json", headers={"If-None-Match": response1.headers.get("ETag")})
+        assert response2.status_code == status.HTTP_304_NOT_MODIFIED

pyproject.toml

@@ -25,10 +25,7 @@ omit = [
     "app/check_database_connection.py",
     "app/check_ceph_connection.py",
     "app/check_oidc_connection.py",
-    "app/db/base*",
     "app/core/config.py",
-    "app/main.py",
-    "app/api/miscellaneous_endpoints.py"
 ]
 
 [tool.coverage.report]

requirements.txt

@@ -6,7 +6,7 @@ anyio>=3.7.0,<4.0.0
 fastapi>=0.104.0,<0.105.0
 pydantic>=2.4.0,<2.5.0
 pydantic-settings>=2.0.0,<2.1.0
-uvicorn>=0.23.0,<0.24.0
+uvicorn>=0.24.0,<0.25.0
 # Database packages
 PyMySQL>=1.1.0,<1.2.0
 SQLAlchemy>=2.0.0,<2.1.0