Merge branch 'feature/60-update-clowmdb-to-2-1-0' into 'development'

Resolve "Update clowmdb to 2.1.0"

Closes #60

See merge request cmg/clowm/clowm-s3proxy-service!57

.flake8

-[flake8]
-max-line-length = 120
-exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache
-extend-ignore = E203

.gitlab-ci.yml

@@ -52,7 +52,7 @@ integration-test-job: # Runs integration tests with the database
         MYSQL_DATABASE: "$DB_DATABASE"
         MYSQL_USER: "$DB_USER"
         MYSQL_PASSWORD: "$DB_PASSWORD"
-    - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0
+    - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.1
       alias: upgrade-db
   script:
     - python app/check_database_connection.py
@@ -80,7 +80,7 @@ e2e-test-job: # Runs e2e tests on the API endpoints
         MYSQL_DATABASE: "$DB_DATABASE"
         MYSQL_USER: "$DB_USER"
         MYSQL_PASSWORD: "$DB_PASSWORD"
-    - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0
+    - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.1
       alias: upgrade-db
   script:
     - python app/check_database_connection.py

.pre-commit-config.yaml

@@ -20,14 +20,18 @@ repos:
     -   id: black
         files: app
         args: [--check]
--   repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+-   repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.285'
     hooks:
-    -   id: flake8
+    -   id: ruff
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort
         files: app
-        args: [--config=.flake8]
+        args: [-c]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.4.1
+    rev: v1.5.1
     hooks:
     -   id: mypy
         files: app
@@ -37,9 +41,3 @@ repos:
             - sqlalchemy>=2.0.0,<2.1.0
             - pydantic
             - types-requests
--   repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-    -   id: isort
-        files: app
-        args: [-c]

app/api/endpoints/buckets.py

@@ -95,7 +95,7 @@ async def list_buckets(
             **{
                 "description": bucket.description,
                 "name": bucket.name,
-                "created_at": s3.Bucket(name=bucket.name).creation_date,
+                "created_at": bucket.created_at,
                 "owner": bucket.owner_id,
                 "num_objects": sum(1 for obj in s3.Bucket(name=bucket.name).objects.all() if not obj.key.endswith("/")),
                 "size": reduce(lambda x, y: x + y.size, list(s3.Bucket(name=bucket.name).objects.all()), 0),
@@ -181,7 +181,7 @@ async def create_bucket(
         **{
             "description": db_bucket.description,
             "name": db_bucket.name,
-            "created_at": s3.Bucket(name=db_bucket.name).creation_date,
+            "created_at": db_bucket.created_at,
             "owner": db_bucket.owner.uid,
             "num_objects": 0,
             "size": 0,
@@ -232,7 +232,7 @@ async def get_bucket(
         **{
             "description": bucket.description,
             "name": bucket.name,
-            "created_at": s3bucket.creation_date,
+            "created_at": bucket.created_at,
             "owner": bucket.owner_id,
             "num_objects": sum(1 for obj in objects if not obj.key.endswith("/")),
             "size": reduce(lambda x, y: x + y.size, objects, 0),

app/crud/crud_bucket.py

@@ -87,8 +87,8 @@ class CRUDBucket:
         WHERE bucket.owner_id = %s OR (EXISTS
             (SELECT 1 FROM bucketpermission
             WHERE bucket.name = bucketpermission.bucket_name AND bucketpermission.user_id = %s
-                AND(datediff(now(), bucketpermission.`from`) <= 0 OR bucketpermission.`from` IS NULL)
-                AND(datediff(now(), bucketpermission.`to`) >= 0 OR bucketpermission.`to` IS NULL)))
+                AND(UNIX_TIMESTAMP() >= bucketpermission.`from` 0 OR bucketpermission.`from` IS NULL)
+                AND(UNIX_TIMESTAMP() <= bucketpermission.`to` >= 0 OR bucketpermission.`to` IS NULL)))
         ```
 
         SQL Query only foreign buckets where user has permission to
@@ -98,8 +98,8 @@ class CRUDBucket:
         WHERE (EXISTS
             (SELECT 1 FROM bucketpermission
             WHERE bucket.name = bucketpermission.bucket_name AND bucketpermission.user_id = %s
-                AND(datediff(now(), bucketpermission.`from`) <= 0 OR bucketpermission.`from` IS NULL)
-                AND(datediff(now(), bucketpermission.`to`) >= 0 OR bucketpermission.`to` IS NULL)))
+                AND(UNIX_TIMESTAMP() >=  bucketpermission.`from` <= 0 OR bucketpermission.`from` IS NULL)
+                AND(UNIX_TIMESTAMP() <=  bucketpermission.`to` >= 0 OR bucketpermission.`to` IS NULL)))
         ```
         """
         stmt = select(Bucket)
@@ -110,13 +110,13 @@ class CRUDBucket:
                     Bucket.permissions.any(BucketPermissionDB.user_id == uid)
                     .where(
                         or_(
-                            func.datediff(func.now(), BucketPermissionDB.from_) >= 0,
+                            func.UNIX_TIMESTAMP() >= BucketPermissionDB.from_,
                             BucketPermissionDB.from_ == None,  # noqa:E711
                         )
                     )
                     .where(
                         or_(
-                            func.datediff(func.now(), BucketPermissionDB.to) <= 0,
+                            func.UNIX_TIMESTAMP() <= BucketPermissionDB.to,
                             BucketPermissionDB.to == None,  # noqa:E711
                         )
                     ),
@@ -129,13 +129,13 @@ class CRUDBucket:
                 Bucket.permissions.any(BucketPermissionDB.user_id == uid)
                 .where(
                     or_(
-                        func.datediff(func.now(), BucketPermissionDB.from_) >= 0,
+                        func.UNIX_TIMESTAMP() >= BucketPermissionDB.from_,
                         BucketPermissionDB.from_ == None,  # noqa:E711
                     )
                 )
                 .where(
                     or_(
-                        func.datediff(func.now(), BucketPermissionDB.to) <= 0,
+                        func.UNIX_TIMESTAMP() <= BucketPermissionDB.to,
                         BucketPermissionDB.to == None,  # noqa:E711
                     )
                 ),

app/crud/crud_bucket_permission.py

@@ -262,12 +262,12 @@ class CRUDBucketPermission:
         if permission_status == CRUDBucketPermission.PermissionStatus.ACTIVE:
             return stmt.where(
                 or_(
-                    func.datediff(func.now(), BucketPermissionDB.from_) >= 0,
+                    func.UNIX_TIMESTAMP() >= BucketPermissionDB.from_,
                     BucketPermissionDB.from_ == None,  # noqa:E711
                 )
             ).where(
                 or_(
-                    func.datediff(func.now(), BucketPermissionDB.to) <= 0,
+                    func.UNIX_TIMESTAMP() <= BucketPermissionDB.to,
                     BucketPermissionDB.to == None,  # noqa:E711
                 )
             )
@@ -275,11 +275,11 @@ class CRUDBucketPermission:
             return stmt.where(
                 or_(
                     and_(
-                        func.datediff(func.now(), BucketPermissionDB.from_) < 0,
+                        func.UNIX_TIMESTAMP() <= BucketPermissionDB.from_,
                         BucketPermissionDB.from_ != None,  # noqa:E711
                     ),
                     and_(
-                        func.datediff(func.now(), BucketPermissionDB.to) > 0,
+                        func.UNIX_TIMESTAMP() >= BucketPermissionDB.to,
                         BucketPermissionDB.to != None,  # noqa:E711
                     ),
                 )

app/schemas/bucket.py

@@ -58,10 +58,10 @@ class BucketOut(_BaseBucket):
     Schema for answering a request with a bucket.
     """
 
-    created_at: datetime = Field(
+    created_at: int = Field(
         ...,
-        examples=[datetime(2022, 1, 1, 0, 0)],
-        description="Time when the bucket was created",
+        examples=[1640991600],  # 01.01.2022 00:00
+        description="Time when the bucket was created as UNIX timestamp",
     )
     owner: str = Field(..., description="UID of the owner", examples=["28c5353b8bb34984a8bd4169ba94c606"])
     num_objects: int = Field(..., description="Number of Objects in this bucket", examples=[6])

app/schemas/bucket_permission.py

@@ -11,11 +11,15 @@ class BucketPermissionParameters(BaseModel):
     Schema for the parameters of a bucket permission.
     """
 
-    from_timestamp: Optional[datetime] = Field(
-        None, description="Start date of permission", examples=[datetime(2022, 1, 1, 0, 0)]
+    from_timestamp: Optional[int] = Field(
+        None,
+        description="Start date of permission as UNIX timestamp",
+        examples=[1640991600],  # 01.01.2022 00:00
     )
-    to_timestamp: Optional[datetime] = Field(
-        None, description="End date of permission", examples=[datetime(2023, 1, 1, 0, 0)]
+    to_timestamp: Optional[int] = Field(
+        None,
+        description="End date of permission as UNIX timestamp",
+        examples=[1640991600],  # 01.01.2022 00:00
     )
     file_prefix: Optional[str] = Field(None, description="Prefix of subfolder", examples=["pseudo/sub/folder/"])
     permission: Union[BucketPermissionDB.Permission, str] = Field(
@@ -88,13 +92,15 @@ class BucketPermissionIn(BucketPermissionParameters):
             obj_policy["Action"] += ["s3:DeleteObject", "s3:PutObject"]
             bucket_policy["Action"] += ["s3:DeleteObject"]
         if self.to_timestamp is not None:
+            print(self.to_timestamp)
             obj_policy["Condition"]["DateLessThan"] = {
-                "aws:CurrentTime": self.to_timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")
+                "aws:CurrentTime": datetime.fromtimestamp(self.to_timestamp).strftime("%Y-%m-%dT%H:%M:%SZ")
             }
             bucket_policy["Condition"]["DateLessThan"] = obj_policy["Condition"]["DateLessThan"]
         if self.from_timestamp is not None:
+            print(self.from_timestamp)
             obj_policy["Condition"]["DateGreaterThan"] = {
-                "aws:CurrentTime": self.from_timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")
+                "aws:CurrentTime": datetime.fromtimestamp(self.from_timestamp).strftime("%Y-%m-%dT%H:%M:%SZ")
             }
             bucket_policy["Condition"]["DateGreaterThan"] = obj_policy["Condition"]["DateGreaterThan"]
         if self.file_prefix is not None:

app/tests/api/test_bucket_permissions.py

 import json
-from datetime import datetime, timedelta
+from datetime import datetime
 
 import pytest
 from clowmdb.models import Bucket, BucketPermission
@@ -560,10 +560,10 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
         random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut
             Bucket permission for a random bucket for testing. pytest fixture.
         """
-        new_from_time = datetime(2022, 1, 1, 0, 0)
+        new_from_time = round(datetime(2022, 1, 1, 0, 0).timestamp())
         new_params = BucketPermissionParametersSchema(
             from_timestamp=new_from_time,
-            to_timestamp=new_from_time + timedelta(days=1),
+            to_timestamp=new_from_time + 86400,  # plus one day
             permission=BucketPermission.Permission.READWRITE,
             file_prefix="pseudo/folder/",
         )
@@ -577,8 +577,8 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes):
         assert updated_permission["uid"] == random_bucket_permission_schema.uid
         assert updated_permission["bucket_name"] == random_bucket_permission_schema.bucket_name
         if new_params.from_timestamp is not None and new_params.to_timestamp is not None:
-            assert updated_permission["from_timestamp"] == new_params.from_timestamp.strftime("%Y-%m-%dT%H:%M:%S")
-            assert updated_permission["to_timestamp"] == new_params.to_timestamp.strftime("%Y-%m-%dT%H:%M:%S")
+            assert updated_permission["from_timestamp"] == new_params.from_timestamp
+            assert updated_permission["to_timestamp"] == new_params.to_timestamp
         assert updated_permission["permission"] == new_params.permission
         assert updated_permission["file_prefix"] == new_params.file_prefix
 

app/tests/crud/test_bucket_permission.py

@@ -535,10 +535,10 @@ class TestBucketPermissionCRUDUpdate:
         random_bucket_permission : clowmdb.models.BucketPermission
             Bucket permission for a random bucket for testing. pytest fixture.
         """
-        new_from_time = datetime(2022, 1, 1, 0, 0)
+        new_from_time = round(datetime(2022, 1, 1, 0, 0).timestamp())
         new_params = BucketPermissionParametersSchema(
             from_timestamp=new_from_time,
-            to_timestamp=new_from_time + timedelta(days=1),
+            to_timestamp=new_from_time + 86400,  # plus one day
             permission=BucketPermissionDB.Permission.READWRITE,
             file_prefix="pseudo/folder/",
         )

app/tests/unit/test_bucket_permission_scheme.py

@@ -120,8 +120,9 @@ class TestPermissionPolicyCondition(_TestPermissionPolicy):
         random_base_permission : app.schemas.bucket_permission.BucketPermissionOut
             Random base bucket permission for testing. pytest fixture.
         """
-        time = datetime.now()
-        random_base_permission.to_timestamp = time
+        timestamp = round(datetime.now().timestamp())
+        time = datetime.fromtimestamp(timestamp)  # avoid rounding error
+        random_base_permission.to_timestamp = timestamp
 
         stmts = random_base_permission.map_to_bucket_policy_statement(user_id=random_lower_string())
         assert len(stmts) == 2
@@ -148,7 +149,9 @@ class TestPermissionPolicyCondition(_TestPermissionPolicy):
             Random base bucket permission for testing. pytest fixture.
         """
         time = datetime.now()
-        random_base_permission.from_timestamp = time
+        timestamp = round(datetime.now().timestamp())
+        time = datetime.fromtimestamp(timestamp)  # avoid rounding error
+        random_base_permission.from_timestamp = timestamp
 
         stmts = random_base_permission.map_to_bucket_policy_statement(user_id=random_lower_string())
         assert len(stmts) == 2

app/tests/utils/bucket.py

@@ -65,8 +65,8 @@ async def add_permission_for_bucket(
     perm = BucketPermission(
         user_id=uid,
         bucket_name=bucket_name,
-        from_=from_,
-        to=to,
+        from_=round(from_.timestamp()) if from_ is not None else None,
+        to=round(to.timestamp()) if to is not None else None,
         permissions=permission.name,
     )
     db.add(perm)

pyproject.toml

@@ -6,6 +6,10 @@ balanced_wrapping = true
 [tool.black]
 line-length = 120
 
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
 [tool.mypy]
 plugins = ["pydantic.mypy", "sqlalchemy.ext.mypy.plugin"]
 ignore_missing_imports = true

requirements-dev.txt

@@ -2,13 +2,12 @@
 pytest>=7.4.0,<7.5.0
 pytest-asyncio>=0.21.0,<0.22.0
 pytest-cov>=4.1.0,<4.2.0
-coverage[toml]>=7.2.0,<7.3.0
+coverage[toml]>=7.3.0,<7.4.0
 # Linters
-flake8>=6.1.0,<6.2.0
-autoflake>=2.2.0,<2.3.0
+ruff
 black>=23.07.0,<23.08.0
 isort>=5.12.0,<5.13.0
-mypy>=1.4.0,<1.5.0
+mypy>=1.5.0,<1.6.0
 # stubs for mypy
 boto3-stubs-lite[s3]>=1.28.0,<1.29.0
 types-requests

requirements.txt

 --extra-index-url https://gitlab.ub.uni-bielefeld.de/api/v4/projects/5493/packages/pypi/simple
-clowmdb>=2.0.0,<2.1.0
+clowmdb>=2.1.0,<2.2.0
 
 # Webserver packages
 anyio>=3.7.0,<3.8.0
 fastapi>=0.101.0,<0.102.0
-pydantic>=2.1.0,<2.2.0
-pydantic-settings>=2.0.0
+pydantic>=2.2.0,<2.3.0
+pydantic-settings>=2.0.0,<2.1.0
 uvicorn>=0.23.0,<0.24.0
 # Database packages
 PyMySQL>=1.1.0,<1.2.0

scripts/format.sh

@@ -2,6 +2,6 @@
 set -x
 
 isort --force-single-line-imports app
-autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place app --exclude=__init__.py
+ruff check --fix --show-fixes app
 black app
 isort app

scripts/lint.sh

@@ -2,7 +2,7 @@
 
 set -x
 
-mypy app
+ruff check app
 black app --check
 isort -c app
-flake8 app
+mypy app