diff --git a/.flake8 b/.flake8 deleted file mode 100644 index db234bdd12f196f109da823525b07fd5afe967d1..0000000000000000000000000000000000000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 120 -exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache -extend-ignore = E203 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fbd1cea20033ecdee30bace8ee94f1b80d5eeb6..186cbfe3b99658ab9cdfddd0c2975fbebfe193da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,7 +52,7 @@ integration-test-job: # Runs integration tests with the database MYSQL_DATABASE: "$DB_DATABASE" MYSQL_USER: "$DB_USER" MYSQL_PASSWORD: "$DB_PASSWORD" - - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0 + - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.1 alias: upgrade-db script: - python app/check_database_connection.py @@ -80,7 +80,7 @@ e2e-test-job: # Runs e2e tests on the API endpoints MYSQL_DATABASE: "$DB_DATABASE" MYSQL_USER: "$DB_USER" MYSQL_PASSWORD: "$DB_PASSWORD" - - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.0 + - name: $CI_REGISTRY/cmg/clowm/clowm-database:v2.1 alias: upgrade-db script: - python app/check_database_connection.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18698343b729f362c7000d45a7f24683cbba680e..d495fca0460a9e2437686850f0495a6efc7118ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,14 +20,18 @@ repos: - id: black files: app args: [--check] -- repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.0.285' hooks: - - id: flake8 + - id: ruff +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort files: app - args: [--config=.flake8] + args: [-c] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 + rev: v1.5.1 hooks: - id: mypy files: app @@ -37,9 +41,3 @@ repos: - sqlalchemy>=2.0.0,<2.1.0 - pydantic - types-requests -- repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - files: app - args: [-c] diff --git a/app/api/endpoints/buckets.py b/app/api/endpoints/buckets.py index 6dc2036a14468fbbcca021d9db0914934664dcaf..256b49831675652b8bdcb6500dd6700afb879e0f 100644 --- a/app/api/endpoints/buckets.py +++ b/app/api/endpoints/buckets.py @@ -95,7 +95,7 @@ async def list_buckets( **{ "description": bucket.description, "name": bucket.name, - "created_at": s3.Bucket(name=bucket.name).creation_date, + "created_at": bucket.created_at, "owner": bucket.owner_id, "num_objects": sum(1 for obj in s3.Bucket(name=bucket.name).objects.all() if not obj.key.endswith("/")), "size": reduce(lambda x, y: x + y.size, list(s3.Bucket(name=bucket.name).objects.all()), 0), @@ -181,7 +181,7 @@ async def create_bucket( **{ "description": db_bucket.description, "name": db_bucket.name, - "created_at": s3.Bucket(name=db_bucket.name).creation_date, + "created_at": db_bucket.created_at, "owner": db_bucket.owner.uid, "num_objects": 0, "size": 0, @@ -232,7 +232,7 @@ async def get_bucket( **{ "description": bucket.description, "name": bucket.name, - "created_at": s3bucket.creation_date, + "created_at": bucket.created_at, "owner": bucket.owner_id, "num_objects": sum(1 for obj in objects if not obj.key.endswith("/")), "size": reduce(lambda x, y: x + y.size, objects, 0), diff --git a/app/crud/crud_bucket.py b/app/crud/crud_bucket.py index 7d7460acf247531d424196e0c62d940059dbf279..9bf2121a3aa5d75159a8e5cb29041afdae233c5c 100644 --- a/app/crud/crud_bucket.py +++ b/app/crud/crud_bucket.py @@ -87,8 +87,8 @@ class CRUDBucket: WHERE bucket.owner_id = %s OR (EXISTS (SELECT 1 FROM bucketpermission WHERE bucket.name = bucketpermission.bucket_name AND bucketpermission.user_id = %s - AND(datediff(now(), bucketpermission.`from`) <= 0 OR bucketpermission.`from` IS NULL) - AND(datediff(now(), bucketpermission.`to`) >= 0 OR bucketpermission.`to` IS NULL))) + AND(UNIX_TIMESTAMP() >= bucketpermission.`from` 0 OR bucketpermission.`from` IS NULL) + AND(UNIX_TIMESTAMP() <= bucketpermission.`to` >= 0 OR bucketpermission.`to` IS NULL))) ``` SQL Query only foreign buckets where user has permission to @@ -98,8 +98,8 @@ class CRUDBucket: WHERE (EXISTS (SELECT 1 FROM bucketpermission WHERE bucket.name = bucketpermission.bucket_name AND bucketpermission.user_id = %s - AND(datediff(now(), bucketpermission.`from`) <= 0 OR bucketpermission.`from` IS NULL) - AND(datediff(now(), bucketpermission.`to`) >= 0 OR bucketpermission.`to` IS NULL))) + AND(UNIX_TIMESTAMP() >= bucketpermission.`from` <= 0 OR bucketpermission.`from` IS NULL) + AND(UNIX_TIMESTAMP() <= bucketpermission.`to` >= 0 OR bucketpermission.`to` IS NULL))) ``` """ stmt = select(Bucket) @@ -110,13 +110,13 @@ class CRUDBucket: Bucket.permissions.any(BucketPermissionDB.user_id == uid) .where( or_( - func.datediff(func.now(), BucketPermissionDB.from_) >= 0, + func.UNIX_TIMESTAMP() >= BucketPermissionDB.from_, BucketPermissionDB.from_ == None, # noqa:E711 ) ) .where( or_( - func.datediff(func.now(), BucketPermissionDB.to) <= 0, + func.UNIX_TIMESTAMP() <= BucketPermissionDB.to, BucketPermissionDB.to == None, # noqa:E711 ) ), @@ -129,13 +129,13 @@ class CRUDBucket: Bucket.permissions.any(BucketPermissionDB.user_id == uid) .where( or_( - func.datediff(func.now(), BucketPermissionDB.from_) >= 0, + func.UNIX_TIMESTAMP() >= BucketPermissionDB.from_, BucketPermissionDB.from_ == None, # noqa:E711 ) ) .where( or_( - func.datediff(func.now(), BucketPermissionDB.to) <= 0, + func.UNIX_TIMESTAMP() <= BucketPermissionDB.to, BucketPermissionDB.to == None, # noqa:E711 ) ), diff --git a/app/crud/crud_bucket_permission.py b/app/crud/crud_bucket_permission.py index 1295fe522aefed5206637946a7170d9d1b52e246..31dc7d121b9f1e3a636a79a555861cc327b098dc 100644 --- a/app/crud/crud_bucket_permission.py +++ b/app/crud/crud_bucket_permission.py @@ -262,12 +262,12 @@ class CRUDBucketPermission: if permission_status == CRUDBucketPermission.PermissionStatus.ACTIVE: return stmt.where( or_( - func.datediff(func.now(), BucketPermissionDB.from_) >= 0, + func.UNIX_TIMESTAMP() >= BucketPermissionDB.from_, BucketPermissionDB.from_ == None, # noqa:E711 ) ).where( or_( - func.datediff(func.now(), BucketPermissionDB.to) <= 0, + func.UNIX_TIMESTAMP() <= BucketPermissionDB.to, BucketPermissionDB.to == None, # noqa:E711 ) ) @@ -275,11 +275,11 @@ class CRUDBucketPermission: return stmt.where( or_( and_( - func.datediff(func.now(), BucketPermissionDB.from_) < 0, + func.UNIX_TIMESTAMP() <= BucketPermissionDB.from_, BucketPermissionDB.from_ != None, # noqa:E711 ), and_( - func.datediff(func.now(), BucketPermissionDB.to) > 0, + func.UNIX_TIMESTAMP() >= BucketPermissionDB.to, BucketPermissionDB.to != None, # noqa:E711 ), ) diff --git a/app/schemas/bucket.py b/app/schemas/bucket.py index c860ef12e13fc66de3bee575685ce4e5c77f7316..9325c9a78cd2170fb4c5a60cd0e38336ae951368 100644 --- a/app/schemas/bucket.py +++ b/app/schemas/bucket.py @@ -58,10 +58,10 @@ class BucketOut(_BaseBucket): Schema for answering a request with a bucket. """ - created_at: datetime = Field( + created_at: int = Field( ..., - examples=[datetime(2022, 1, 1, 0, 0)], - description="Time when the bucket was created", + examples=[1640991600], # 01.01.2022 00:00 + description="Time when the bucket was created as UNIX timestamp", ) owner: str = Field(..., description="UID of the owner", examples=["28c5353b8bb34984a8bd4169ba94c606"]) num_objects: int = Field(..., description="Number of Objects in this bucket", examples=[6]) diff --git a/app/schemas/bucket_permission.py b/app/schemas/bucket_permission.py index b384f9f0bfe00a5dcbf549216c452ac7cb3a4989..e4fa93f6a2c275822da96d22b6e80ef7590cb36b 100644 --- a/app/schemas/bucket_permission.py +++ b/app/schemas/bucket_permission.py @@ -11,11 +11,15 @@ class BucketPermissionParameters(BaseModel): Schema for the parameters of a bucket permission. """ - from_timestamp: Optional[datetime] = Field( - None, description="Start date of permission", examples=[datetime(2022, 1, 1, 0, 0)] + from_timestamp: Optional[int] = Field( + None, + description="Start date of permission as UNIX timestamp", + examples=[1640991600], # 01.01.2022 00:00 ) - to_timestamp: Optional[datetime] = Field( - None, description="End date of permission", examples=[datetime(2023, 1, 1, 0, 0)] + to_timestamp: Optional[int] = Field( + None, + description="End date of permission as UNIX timestamp", + examples=[1640991600], # 01.01.2022 00:00 ) file_prefix: Optional[str] = Field(None, description="Prefix of subfolder", examples=["pseudo/sub/folder/"]) permission: Union[BucketPermissionDB.Permission, str] = Field( @@ -88,13 +92,15 @@ class BucketPermissionIn(BucketPermissionParameters): obj_policy["Action"] += ["s3:DeleteObject", "s3:PutObject"] bucket_policy["Action"] += ["s3:DeleteObject"] if self.to_timestamp is not None: + print(self.to_timestamp) obj_policy["Condition"]["DateLessThan"] = { - "aws:CurrentTime": self.to_timestamp.strftime("%Y-%m-%dT%H:%M:%SZ") + "aws:CurrentTime": datetime.fromtimestamp(self.to_timestamp).strftime("%Y-%m-%dT%H:%M:%SZ") } bucket_policy["Condition"]["DateLessThan"] = obj_policy["Condition"]["DateLessThan"] if self.from_timestamp is not None: + print(self.from_timestamp) obj_policy["Condition"]["DateGreaterThan"] = { - "aws:CurrentTime": self.from_timestamp.strftime("%Y-%m-%dT%H:%M:%SZ") + "aws:CurrentTime": datetime.fromtimestamp(self.from_timestamp).strftime("%Y-%m-%dT%H:%M:%SZ") } bucket_policy["Condition"]["DateGreaterThan"] = obj_policy["Condition"]["DateGreaterThan"] if self.file_prefix is not None: diff --git a/app/tests/api/test_bucket_permissions.py b/app/tests/api/test_bucket_permissions.py index 65e3605643aa3058cfb5c2ce69153e29930f6861..e209798f43a5e3629207f47014e90b9078c1384a 100644 --- a/app/tests/api/test_bucket_permissions.py +++ b/app/tests/api/test_bucket_permissions.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timedelta +from datetime import datetime import pytest from clowmdb.models import Bucket, BucketPermission @@ -560,10 +560,10 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes): random_bucket_permission_schema : app.schemas.bucket_permission.BucketPermissionOut Bucket permission for a random bucket for testing. pytest fixture. """ - new_from_time = datetime(2022, 1, 1, 0, 0) + new_from_time = round(datetime(2022, 1, 1, 0, 0).timestamp()) new_params = BucketPermissionParametersSchema( from_timestamp=new_from_time, - to_timestamp=new_from_time + timedelta(days=1), + to_timestamp=new_from_time + 86400, # plus one day permission=BucketPermission.Permission.READWRITE, file_prefix="pseudo/folder/", ) @@ -577,8 +577,8 @@ class TestBucketPermissionRoutesUpdate(_TestBucketPermissionRoutes): assert updated_permission["uid"] == random_bucket_permission_schema.uid assert updated_permission["bucket_name"] == random_bucket_permission_schema.bucket_name if new_params.from_timestamp is not None and new_params.to_timestamp is not None: - assert updated_permission["from_timestamp"] == new_params.from_timestamp.strftime("%Y-%m-%dT%H:%M:%S") - assert updated_permission["to_timestamp"] == new_params.to_timestamp.strftime("%Y-%m-%dT%H:%M:%S") + assert updated_permission["from_timestamp"] == new_params.from_timestamp + assert updated_permission["to_timestamp"] == new_params.to_timestamp assert updated_permission["permission"] == new_params.permission assert updated_permission["file_prefix"] == new_params.file_prefix diff --git a/app/tests/crud/test_bucket_permission.py b/app/tests/crud/test_bucket_permission.py index d94c42baa1387ff6e0e73462da3dd6b8bc8ff299..43b9e1f69b9eb91dd545e120ad233c2fd73bf98e 100644 --- a/app/tests/crud/test_bucket_permission.py +++ b/app/tests/crud/test_bucket_permission.py @@ -535,10 +535,10 @@ class TestBucketPermissionCRUDUpdate: random_bucket_permission : clowmdb.models.BucketPermission Bucket permission for a random bucket for testing. pytest fixture. """ - new_from_time = datetime(2022, 1, 1, 0, 0) + new_from_time = round(datetime(2022, 1, 1, 0, 0).timestamp()) new_params = BucketPermissionParametersSchema( from_timestamp=new_from_time, - to_timestamp=new_from_time + timedelta(days=1), + to_timestamp=new_from_time + 86400, # plus one day permission=BucketPermissionDB.Permission.READWRITE, file_prefix="pseudo/folder/", ) diff --git a/app/tests/unit/test_bucket_permission_scheme.py b/app/tests/unit/test_bucket_permission_scheme.py index 351444c8150e86050bffd5ed64ea6f1184532217..8eb79ec9168f8e802632ffea073f5d12c5f572d3 100644 --- a/app/tests/unit/test_bucket_permission_scheme.py +++ b/app/tests/unit/test_bucket_permission_scheme.py @@ -120,8 +120,9 @@ class TestPermissionPolicyCondition(_TestPermissionPolicy): random_base_permission : app.schemas.bucket_permission.BucketPermissionOut Random base bucket permission for testing. pytest fixture. """ - time = datetime.now() - random_base_permission.to_timestamp = time + timestamp = round(datetime.now().timestamp()) + time = datetime.fromtimestamp(timestamp) # avoid rounding error + random_base_permission.to_timestamp = timestamp stmts = random_base_permission.map_to_bucket_policy_statement(user_id=random_lower_string()) assert len(stmts) == 2 @@ -148,7 +149,9 @@ class TestPermissionPolicyCondition(_TestPermissionPolicy): Random base bucket permission for testing. pytest fixture. """ time = datetime.now() - random_base_permission.from_timestamp = time + timestamp = round(datetime.now().timestamp()) + time = datetime.fromtimestamp(timestamp) # avoid rounding error + random_base_permission.from_timestamp = timestamp stmts = random_base_permission.map_to_bucket_policy_statement(user_id=random_lower_string()) assert len(stmts) == 2 diff --git a/app/tests/utils/bucket.py b/app/tests/utils/bucket.py index a5369a2c238b84d2024d4943104c3f6582575abc..76b5634a4f066010829961ce74aa5172d9319d10 100644 --- a/app/tests/utils/bucket.py +++ b/app/tests/utils/bucket.py @@ -65,8 +65,8 @@ async def add_permission_for_bucket( perm = BucketPermission( user_id=uid, bucket_name=bucket_name, - from_=from_, - to=to, + from_=round(from_.timestamp()) if from_ is not None else None, + to=round(to.timestamp()) if to is not None else None, permissions=permission.name, ) db.add(perm) diff --git a/pyproject.toml b/pyproject.toml index fc944134ce59872070e35728126d8999706a1343..60fd4460d5544be92c5582c5df69cd31957bad57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,10 @@ balanced_wrapping = true [tool.black] line-length = 120 +[tool.ruff] +line-length = 120 +target-version = "py310" + [tool.mypy] plugins = ["pydantic.mypy", "sqlalchemy.ext.mypy.plugin"] ignore_missing_imports = true diff --git a/requirements-dev.txt b/requirements-dev.txt index d7cfa3bfe8150f30e795be49704788c9d6389faf..0733ec9e9d677ca8d78feb950c4fe094273fa886 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,13 +2,12 @@ pytest>=7.4.0,<7.5.0 pytest-asyncio>=0.21.0,<0.22.0 pytest-cov>=4.1.0,<4.2.0 -coverage[toml]>=7.2.0,<7.3.0 +coverage[toml]>=7.3.0,<7.4.0 # Linters -flake8>=6.1.0,<6.2.0 -autoflake>=2.2.0,<2.3.0 +ruff black>=23.07.0,<23.08.0 isort>=5.12.0,<5.13.0 -mypy>=1.4.0,<1.5.0 +mypy>=1.5.0,<1.6.0 # stubs for mypy boto3-stubs-lite[s3]>=1.28.0,<1.29.0 types-requests diff --git a/requirements.txt b/requirements.txt index afa2dba60c0c25992954e8d45154e85ed47bcb90..2268187b74f4611d2ac2249caa25dc068f79aacb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ --extra-index-url https://gitlab.ub.uni-bielefeld.de/api/v4/projects/5493/packages/pypi/simple -clowmdb>=2.0.0,<2.1.0 +clowmdb>=2.1.0,<2.2.0 # Webserver packages anyio>=3.7.0,<3.8.0 fastapi>=0.101.0,<0.102.0 -pydantic>=2.1.0,<2.2.0 -pydantic-settings>=2.0.0 +pydantic>=2.2.0,<2.3.0 +pydantic-settings>=2.0.0,<2.1.0 uvicorn>=0.23.0,<0.24.0 # Database packages PyMySQL>=1.1.0,<1.2.0 diff --git a/scripts/format.sh b/scripts/format.sh index 9498ad35f3fb405a115eb7079c1eb0eb82de2749..1c5f45d44046e7f093cf64e34bdfdb713182731e 100755 --- a/scripts/format.sh +++ b/scripts/format.sh @@ -2,6 +2,6 @@ set -x isort --force-single-line-imports app -autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place app --exclude=__init__.py +ruff check --fix --show-fixes app black app isort app diff --git a/scripts/lint.sh b/scripts/lint.sh index 2ae13873d90b0b4ba889c7de7f290b28def2f579..5246a7ff93a6c8e930a3069db441f0b66b2d5504 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -2,7 +2,7 @@ set -x -mypy app +ruff check app black app --check isort -c app -flake8 app +mypy app