Split requirements.txt into two files and add pre-commit git hooks with sensible defaults.

#7

ProxyAPI/.coveragerc

@@ -14,4 +14,4 @@ exclude_lines =
     def __repr__
     def __eq__
     if TYPE_CHECKING
-    if __name__ == .__main__.:
+    if __name__ == .__main__.:
\ No newline at end of file

ProxyAPI/.dockerignore

@@ -7,3 +7,5 @@ venv/
 ENV/
 README.md
 .pytest_cache
+.mypy_cache
+htmlcov

ProxyAPI/.gitignore

@@ -5,4 +5,4 @@ __pycache__/
 env/
 venv/
 ENV/
-.coverage
+.coverage
\ No newline at end of file

ProxyAPI/.isort.cfg

@@ -4,4 +4,4 @@ multi_line_output = 0
 include_trailing_comma = false
 force_grid_wrap = 0
 line_length = 121
-balanced_wrapping = true
+balanced_wrapping = true
\ No newline at end of file

ProxyAPI/.pre-commit-config.yaml

+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+    -   id: trailing-whitespace
+        files: ProxyAPI
+    -   id: end-of-file-fixer
+        files: ProxyAPI
+    -   id: check-added-large-files
+        files: ProxyAPI
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    -   id: black
+        files: ProxyAPI/app
+        args: [--check]
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.960
+    hooks:
+    -   id: mypy
+        files: ProxyAPI/app
+        args: [--config=ProxyAPI/mypy.ini]
+        additional_dependencies:
+            - sqlalchemy2-stubs
+            - boto3-stubs-lite[s3]
+            - sqlalchemy<2.0.0
+            - pydantic
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort
+        files: ProxyAPI/app
+        args: [-c]

ProxyAPI/Dockerfile

@@ -13,4 +13,4 @@ RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
 COPY . .
 
-CMD ["./start_service.sh"]
+CMD ["./start_service.sh"]
\ No newline at end of file

ProxyAPI/README.md

-# S3 Proxy API
+# S3 Proxy API
\ No newline at end of file

ProxyAPI/alembic/README

-Generic single-database configuration with an async dbapi.
+Generic single-database configuration with an async dbapi.
\ No newline at end of file

ProxyAPI/alembic/env.py

@@ -30,14 +30,16 @@ target_metadata = Base.metadata
 
 
 def get_url() -> str:
-    return str(AnyUrl.build(
-        scheme="mysql+aiomysql",
-        password=os.getenv("DB_PASSWORD", ""),
-        user=os.getenv("DB_USER", ""),
-        port=os.getenv("DB_PORT", "3306"),
-        host=os.getenv("DB_HOST", "localhost"),
-        path=f"/{os.getenv('DB_DATABASE', '')}",
-    ))
+    return str(
+        AnyUrl.build(
+            scheme="mysql+aiomysql",
+            password=os.getenv("DB_PASSWORD", ""),
+            user=os.getenv("DB_USER", ""),
+            port=os.getenv("DB_PORT", "3306"),
+            host=os.getenv("DB_HOST", "localhost"),
+            path=f"/{os.getenv('DB_DATABASE', '')}",
+        )
+    )
 
 
 def run_migrations_offline():
@@ -85,7 +87,7 @@ async def run_migrations_online():
             prefix="sqlalchemy.",
             poolclass=pool.NullPool,
             future=True,
-            url=url
+            url=url,
         )
     )
 

ProxyAPI/alembic/versions/5521b5759004_create_user_and_bucket_table.py

 """Create user and bucket table
 
 Revision ID: 5521b5759004
-Revises: 
+Revises:
 Create Date: 2022-05-03 14:01:22.154984
 
 """
@@ -11,7 +11,7 @@ from sqlalchemy.dialects import mysql
 from alembic import op
 
 # revision identifiers, used by Alembic.
-revision = '5521b5759004'
+revision = "5521b5759004"
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -19,28 +19,33 @@ depends_on = None
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('user',
-                    sa.Column('uid', sa.String(length=64), nullable=False),
-                    sa.Column('name', sa.String(length=256), nullable=False),
-                    sa.PrimaryKeyConstraint('uid')
-                    )
-    op.create_index(op.f('ix_user_uid'), 'user', ['uid'], unique=True)
-    op.create_table('bucket',
-                    sa.Column('name', sa.String(length=256), nullable=False),
-                    sa.Column('description', mysql.TEXT(), nullable=False),
-                    sa.Column('public', sa.Boolean(), server_default='0', nullable=True),
-                    sa.Column('owner_id', sa.String(length=64), nullable=True),
-                    sa.ForeignKeyConstraint(['owner_id'], ['user.uid'], ),
-                    sa.PrimaryKeyConstraint('name')
-                    )
-    op.create_index(op.f('ix_bucket_name'), 'bucket', ['name'], unique=True)
+    op.create_table(
+        "user",
+        sa.Column("uid", sa.String(length=64), nullable=False),
+        sa.Column("name", sa.String(length=256), nullable=False),
+        sa.PrimaryKeyConstraint("uid"),
+    )
+    op.create_index(op.f("ix_user_uid"), "user", ["uid"], unique=True)
+    op.create_table(
+        "bucket",
+        sa.Column("name", sa.String(length=256), nullable=False),
+        sa.Column("description", mysql.TEXT(), nullable=False),
+        sa.Column("public", sa.Boolean(), server_default="0", nullable=True),
+        sa.Column("owner_id", sa.String(length=64), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["owner_id"],
+            ["user.uid"],
+        ),
+        sa.PrimaryKeyConstraint("name"),
+    )
+    op.create_index(op.f("ix_bucket_name"), "bucket", ["name"], unique=True)
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(op.f('ix_bucket_name'), table_name='bucket')
-    op.drop_table('bucket')
-    op.drop_index(op.f('ix_user_uid'), table_name='user')
-    op.drop_table('user')
+    op.drop_index(op.f("ix_bucket_name"), table_name="bucket")
+    op.drop_table("bucket")
+    op.drop_index(op.f("ix_user_uid"), table_name="user")
+    op.drop_table("user")
     # ### end Alembic commands ###

ProxyAPI/alembic/versions/83a3a47a6351_add_username_and_display_name_and_drop_.py

@@ -11,25 +11,27 @@ from sqlalchemy.dialects import mysql
 from alembic import op
 
 # revision identifiers, used by Alembic.
-revision = '83a3a47a6351'
-down_revision = 'cafa1e01b782'
+revision = "83a3a47a6351"
+down_revision = "cafa1e01b782"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('user', sa.Column('display_name', sa.String(length=256), nullable=True))
-    op.add_column('user', sa.Column('username', sa.String(length=256), nullable=False))
-    op.drop_column('user', 'name')
-    op.create_index(op.f('ix_user_username'), 'user', ['username'], unique=True)
+    op.add_column(
+        "user", sa.Column("display_name", sa.String(length=256), nullable=True)
+    )
+    op.add_column("user", sa.Column("username", sa.String(length=256), nullable=False))
+    op.drop_column("user", "name")
+    op.create_index(op.f("ix_user_username"), "user", ["username"], unique=True)
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('user', sa.Column('name', mysql.VARCHAR(length=256), nullable=False))
-    op.drop_index(op.f('ix_user_username'), table_name='user')
-    op.drop_column('user', 'username')
-    op.drop_column('user', 'display_name')
+    op.add_column("user", sa.Column("name", mysql.VARCHAR(length=256), nullable=False))
+    op.drop_index(op.f("ix_user_username"), table_name="user")
+    op.drop_column("user", "username")
+    op.drop_column("user", "display_name")
     # ### end Alembic commands ###

ProxyAPI/alembic/versions/cafa1e01b782_create_permission_table.py

@@ -11,37 +11,46 @@ from sqlalchemy.dialects import mysql
 from alembic import op
 
 # revision identifiers, used by Alembic.
-revision = 'cafa1e01b782'
-down_revision = '5521b5759004'
+revision = "cafa1e01b782"
+down_revision = "5521b5759004"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('bucketpermission',
-                    sa.Column('user_id', sa.String(length=64), nullable=False),
-                    sa.Column('bucket_name', sa.String(length=256), nullable=False),
-                    sa.Column('from', mysql.TIMESTAMP(), nullable=True),
-                    sa.Column('to', mysql.TIMESTAMP(), nullable=True),
-                    sa.Column('file_prefix', sa.String(length=512), nullable=True),
-                    sa.Column('permissions', mysql.ENUM('READ', 'WRITE', 'READWRITE'), nullable=False),
-                    sa.ForeignKeyConstraint(['bucket_name'], ['bucket.name'], ondelete='CASCADE'),
-                    sa.ForeignKeyConstraint(['user_id'], ['user.uid'], ondelete='CASCADE'),
-                    sa.PrimaryKeyConstraint('user_id', 'bucket_name')
-                    )
-    op.alter_column('bucket', 'public',
-                    existing_type=mysql.TINYINT(display_width=1),
-                    nullable=False,
-                    existing_server_default=sa.text("'0'"))
+    op.create_table(
+        "bucketpermission",
+        sa.Column("user_id", sa.String(length=64), nullable=False),
+        sa.Column("bucket_name", sa.String(length=256), nullable=False),
+        sa.Column("from", mysql.TIMESTAMP(), nullable=True),
+        sa.Column("to", mysql.TIMESTAMP(), nullable=True),
+        sa.Column("file_prefix", sa.String(length=512), nullable=True),
+        sa.Column(
+            "permissions", mysql.ENUM("READ", "WRITE", "READWRITE"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(["bucket_name"], ["bucket.name"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["user_id"], ["user.uid"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("user_id", "bucket_name"),
+    )
+    op.alter_column(
+        "bucket",
+        "public",
+        existing_type=mysql.TINYINT(display_width=1),
+        nullable=False,
+        existing_server_default=sa.text("'0'"),
+    )
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.alter_column('bucket', 'public',
-                    existing_type=mysql.TINYINT(display_width=1),
-                    nullable=True,
-                    existing_server_default=sa.text("'0'"))
-    op.drop_table('bucketpermission')
+    op.alter_column(
+        "bucket",
+        "public",
+        existing_type=mysql.TINYINT(display_width=1),
+        nullable=True,
+        existing_server_default=sa.text("'0'"),
+    )
+    op.drop_table("bucketpermission")
     # ### end Alembic commands ###

ProxyAPI/app/tests/pytest.ini

 # pytest.ini
 [pytest]
-asyncio_mode = strict
+asyncio_mode = strict
\ No newline at end of file

ProxyAPI/requirements-dev.txt

+# test packages
+httpx>=0.23.0,<0.24.0
+pytest>=7.1.0,<7.2.0
+pytest-asyncio>=0.18.0,<0.19.0
+pytest-cov>=3.0.0,<3.1.0
+coverage>=6.4.0,<6.5.0
+# Linters
+flake8>=4.0.0,<4.1.0
+autoflake>=1.4.0,<1.5.0
+black>=22.3.0,<22.4.0
+isort>=5.10.0,<5.11.0
+mypy==0.960
+# stubs for mypy
+boto3-stubs-lite[s3]>=1.24.0,<1.25.0
+sqlalchemy2-stubs
+# Miscellaneous
+pre-commit>=2.19.0,<2.20.0
+python-dotenv

ProxyAPI/requirements.txt

@@ -11,22 +11,8 @@ aiomysql>=0.1.0,<0.2.0
 # JWT packages
 cryptography>=37.0.0,<38.0.0
 python-jose[cryptography]>=3.3.0,<3.4.0
-# test packages
-httpx>=0.23.0,<0.24.0
-pytest>=7.1.0,<7.2.0
-pytest-cov>=3.0.0,<3.1.0
-coverage>=6.4.0,<6.5.0
-# Linters
-flake8>=4.0.0,<4.1.0
-autoflake>=1.4.0,<1.5.0
-black>=22.3.0,<22.4.0
-isort>=5.10.0,<5.11.0
-mypy>=0.960
-# stubs for mypy
-boto3-stubs-lite[s3]>=1.24.0,<1.25.0
-sqlalchemy2-stubs
 # Ceph and S3 packages
 boto3>=1.22.0,<1.23.0
 rgwadmin>=2.3.0,<2.4.0
 # Miscellaneous
-tenacity>=8.0.0,<8.1.0
+tenacity>=8.0.0,<8.1.0
\ No newline at end of file

ProxyAPI/scripts/lint.sh

@@ -4,5 +4,5 @@ set -x
 
 mypy app
 black app --check
-isort --recursive --check-only app
+isort -c app
 flake8

ProxyAPI/start_service.sh

@@ -5,4 +5,6 @@ python app/check_database_connection.py
 
 # Run migrations
 alembic upgrade head
-uvicorn app.main:app --host 0.0.0.0 --port 80
+
\ No newline at end of file
+# Start webserver
+uvicorn app.main:app --host 0.0.0.0 --port 80