Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into development

daemon/tasks/__init__.py

-from config import Config
-from sqlalchemy import create_engine
-from sqlalchemy.orm import scoped_session, sessionmaker
-import docker
-
-
-config = Config()
-config.init_app()
-docker_client = docker.from_env()
-engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
-Session = scoped_session(sessionmaker(bind=engine))

db.env.tpl

+POSTGRES_DB_NAME=
+
+POSTGRES_USER=
+
+POSTGRES_PASSWORD=

docker-compose.development.yml

@@ -16,10 +16,8 @@ services:
   nopaqued:
     volumes:
       # Mount code as volumes
+      - "./daemon/app:/home/nopaqued/app"
       - "./daemon/boot.sh:/home/nopaqued/boot.sh"
       - "./daemon/config.py:/home/nopaqued/config.py"
-      - "./daemon/logger:/home/nopaqued/logger"
       - "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py"
-      - "./daemon/notify:/home/nopaqued/notify"
       - "./daemon/requirements.txt:/home/nopaqued/requirements.txt"
-      - "./daemon/tasks:/home/nopaqued/tasks"

docker-compose.traefik.yml

 ################################################################################
-# Don't forget to set the NOPAQUE_NUM_PROXIES variable in your .env            #
+# Don't forget to set the proxy variables in your nopaque.env                  #
+# Traefik sets the X_FOR, X_HOST, X_PORT and X_PROTO headers by default        #
 ################################################################################
 version: "3.5"
 
@@ -17,13 +18,13 @@ services:
       - "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http"
       - "traefik.http.routers.nopaque.entrypoints=web"
       - "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file"
-      - "traefik.http.routers.nopaque.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)"
+      - "traefik.http.routers.nopaque.rule=Host(`<DOMAIN>`)"
       ### </http> ###
       ### <https> ###
       - "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https"
       - "traefik.http.routers.nopaque-secure.entrypoints=web-secure"
       - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header"
-      - "traefik.http.routers.nopaque-secure.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)"
+      - "traefik.http.routers.nopaque-secure.rule=Host(`<DOMAIN>`)"
       - "traefik.http.routers.nopaque-secure.tls.certresolver=<CERTRESOLVER>"
       - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file"
       ### </https> ###

docker-compose.yml

@@ -34,10 +34,7 @@ services:
       - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
       - "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}"
   db:
-    environment:
-      - POSTGRES_DB_NAME=${NOPAQUE_DB_NAME}
-      - POSTGRES_USER=${NOPAQUE_DB_USERNAME}
-      - POSTGRES_PASSWORD=${NOPAQUE_DB_PASSWORD}
+    env_file: db.env
     image: postgres:11
     restart: unless-stopped
     volumes:

web/Dockerfile

@@ -29,8 +29,7 @@ WORKDIR /home/nopaque
 
 COPY --chown=nopaque:nopaque [".", "."]
 RUN python -m venv venv \
- && venv/bin/pip install --requirement requirements.txt \
- && mkdir logs
+ && venv/bin/pip install --requirement requirements.txt
 
 
 ENTRYPOINT ["./boot.sh"]

web/app/__init__.py

-from config import Config
+from config import config
 from flask import Flask
 from flask_login import LoginManager
 from flask_mail import Mail
@@ -7,7 +7,6 @@ from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy
 
 
-config = Config()
 db = SQLAlchemy()
 login_manager = LoginManager()
 login_manager.login_view = 'auth.login'
@@ -17,19 +16,19 @@ paranoid.redirect_view = '/'
 socketio = SocketIO()
 
 
-def create_app():
+def create_app(config_name):
     app = Flask(__name__)
-    app.config.from_object(config)
+    app.config.from_object(config[config_name])
 
-    config.init_app(app)
+    config[config_name].init_app(app)
     db.init_app(app)
     login_manager.init_app(app)
     mail.init_app(app)
     paranoid.init_app(app)
-    socketio.init_app(app, message_queue=config.SOCKETIO_MESSAGE_QUEUE_URI)
+    socketio.init_app(
+        app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
 
     from . import events
-
     from .admin import admin as admin_blueprint
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
     from .auth import auth as auth_blueprint

web/boot.sh

 #!/bin/bash
-echo "Waiting for db..."
-wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0
-echo "Waiting for mq..."
-wait-for-it "${NOPAQUE_MQ_HOST}:${NOPAQUE_MQ_PORT}" --strict --timeout=0
-
 source venv/bin/activate
 if [[ "$#" -eq 0 ]]; then
-    flask deploy
+    while true; do
+        flask deploy
+        if [[ "$?" == "0" ]]; then
+            break
+        fi
+        echo Deploy command failed, retrying in 5 secs...
+        sleep 5
+    done
     python nopaque.py
 elif [[ "$1" == "flask" ]]; then
     exec ${@:1}

web/config.py

@@ -3,100 +3,107 @@ import logging
 import os
 
 
-root_dir = os.path.abspath(os.path.dirname(__file__))
-
-
-DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque')
-DEFAULT_DB_PORT = '5432'
-DEFAULT_DEBUG = 'False'
-DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
-DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaque.log')
-DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s '
-                      '(function: %(funcName)s, line: %(lineno)d): '
-                      '%(message)s')
-DEFAULT_LOG_LEVEL = 'ERROR'
-DEFAULT_SMTP_USE_SSL = 'False'
-DEFAULT_SMTP_USE_TLS = 'False'
-DEFAULT_NUM_PROXIES = '0'
-DEFAULT_PROTOCOL = 'http'
-DEFAULT_REMEMBER_COOKIE_HTTPONLY = 'True'
-DEFAULT_REMEMBER_COOKIE_SECURE = 'False'
-DEFAULT_SECRET_KEY = 'hard to guess string'
-DEFAULT_SESSION_COOKIE_SECURE = 'False'
+ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
 
 
 class Config:
-    ''' ### Database ### '''
-    DB_HOST = os.environ.get('NOPAQUE_DB_HOST')
-    DB_NAME = os.environ.get('NOPAQUE_DB_NAME')
-    DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD')
-    DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT)
-    DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME')
-    SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
-        DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME)
+    ''' # Cookies # '''
+    REMEMBER_COOKIE_HTTPONLY = True
+    REMEMBER_COOKIE_SECURE = os.environ.get(
+        'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
+    SESSION_COOKIE_SECURE = os.environ.get(
+        'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
+
+    ''' # Database # '''
     SQLALCHEMY_RECORD_QUERIES = True
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
-    ''' ### Email ### '''
+    ''' # Email # '''
     MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
     MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
-    MAIL_PORT = os.environ.get('NOPAQUE_SMTP_PORT')
+    MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
     MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
     MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
-    MAIL_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL',
-                                  DEFAULT_SMTP_USE_SSL).lower() == 'true'
-    MAIL_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS',
-                                  DEFAULT_SMTP_USE_TLS).lower() == 'true'
+    MAIL_USE_SSL = os.environ.get(
+        'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
+    MAIL_USE_TLS = os.environ.get(
+        'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
 
-    ''' ### General ### '''
+    ''' # General # '''
     ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
     CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
-    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR)
-    DEBUG = os.environ.get('NOPAQUE_DEBUG', DEFAULT_DEBUG).lower() == 'true'
-    NUM_PROXIES = int(os.environ.get('NOPAQUE_NUM_PROXIES',
-                                     DEFAULT_NUM_PROXIES))
-    PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL)
-    REMEMBER_COOKIE_HTTPONLY = os.environ.get(
-        'NOPAQUE_REMEMBER_COOKIE_HTTPONLY',
-        DEFAULT_REMEMBER_COOKIE_HTTPONLY
-    ).lower() == 'true'
-    REMEMBER_COOKIE_SECURE = os.environ.get(
-        'NOPAQUE_REMEMBER_COOKIE_SECURE',
-        DEFAULT_REMEMBER_COOKIE_SECURE
-    ).lower() == 'true'
-    SECRET_KEY = os.environ.get('RECIPY_SECRET_KEY', DEFAULT_SECRET_KEY)
-    SESSION_COOKIE_SECURE = os.environ.get(
-        'NOPAQUE_SESSION_COOKIE_SECURE',
-        DEFAULT_SESSION_COOKIE_SECURE
-    ).lower() == 'true'
+    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
+    SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
 
-    ''' ### Logging ### '''
+    ''' # Logging # '''
     LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
-                                     DEFAULT_LOG_DATE_FORMAT)
-    LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE', DEFAULT_LOG_FILE)
-    LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT)
-    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL)
-
-    ''' ### Message queue ### '''
-    MQ_HOST = os.environ.get('NOPAQUE_MQ_HOST')
-    MQ_PORT = os.environ.get('NOPAQUE_MQ_PORT')
-    MQ_TYPE = os.environ.get('NOPAQUE_MQ_TYPE')
-    SOCKETIO_MESSAGE_QUEUE_URI = \
-        '{}://{}:{}/'.format(MQ_TYPE, MQ_HOST, MQ_PORT)
-
-    def init_app(self, app):
-        # Configure logging according to the corresponding (LOG_*) config
-        # entries
-        logging.basicConfig(datefmt=self.LOG_DATE_FORMAT,
-                            filename=self.LOG_FILE,
-                            format=self.LOG_FORMAT,
-                            level=self.LOG_LEVEL)
-        # Apply the ProxyFix middleware if nopaque is running behind reverse
-        # proxies. (NUM_PROXIES indicates the number of reverse proxies running
-        # in front of nopaque)
-        if self.NUM_PROXIES > 0:
-            app.wsgi_app = ProxyFix(app.wsgi_app,
-                                    x_for=self.NUM_PROXIES,
-                                    x_host=self.NUM_PROXIES,
-                                    x_port=self.NUM_PROXIES,
-                                    x_proto=self.NUM_PROXIES)
+                                     '%Y-%m-%d %H:%M:%S')
+    LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
+                              os.path.join(ROOT_DIR, 'nopaque.log'))
+    LOG_FORMAT = os.environ.get(
+        'NOPAQUE_LOG_FORMAT',
+        '[%(asctime)s] %(levelname)s in '
+        '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
+    )
+    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
+
+    ''' # Message queue # '''
+    SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
+        'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
+
+    ''' # Proxy fix # '''
+    PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
+    PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
+    PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
+    PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
+    PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
+
+    @classmethod
+    def init_app(cls, app):
+        # Set up logging according to the corresponding (LOG_*) variables
+        logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
+                            filename=cls.LOG_FILE,
+                            format=cls.LOG_FORMAT,
+                            level=cls.LOG_LEVEL)
+        # Set up and apply the ProxyFix middleware according to the
+        # corresponding (PROXY_FIX_*) variables
+        app.wsgi_app = ProxyFix(app.wsgi_app,
+                                x_for=cls.PROXY_FIX_X_FOR,
+                                x_host=cls.PROXY_FIX_X_HOST,
+                                x_port=cls.PROXY_FIX_X_PORT,
+                                x_prefix=cls.PROXY_FIX_X_PREFIX,
+                                x_proto=cls.PROXY_FIX_X_PROTO)
+
+
+class DevelopmentConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DEV_DATABASE_URL',
+        'postgresql://nopaque:nopaque@db/nopaque_dev'
+    )
+
+    ''' # General # '''
+    DEBUG = True
+
+
+class ProductionConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque')
+
+
+class TestingConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_TEST_DATABASE_URL',
+        'postgresql://nopaque:nopaque@db/nopaque_test'
+    )
+
+    ''' # General # '''
+    TESTING = True
+    WTF_CSRF_ENABLED = False
+
+
+config = {'development': DevelopmentConfig,
+          'production': ProductionConfig,
+          'testing': TestingConfig}

web/nopaque.py

+# First things first: apply monkey patch, so that no code gets executed without
+# patched libraries!
 import eventlet
-eventlet.monkey_patch()  # noqa
-from app import create_app, db, socketio
+
+
+eventlet.monkey_patch()
+
+
+from dotenv import load_dotenv  # noqa
+import os  # noqa
+
+# Load environment variables
+DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
+if os.path.exists(DOTENV_FILE):
+    load_dotenv(DOTENV_FILE)
+
+
+from app import create_app, db, socketio  # noqa
 from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
                         NotificationData, NotificationEmailData, QueryResult,
-                        Role, User)
-from flask_migrate import Migrate, upgrade
+                        Role, User)  # noqa
+from flask_migrate import Migrate, upgrade  # noqa
+
 
-app = create_app()
+app = create_app(os.environ.get('NOPAQUE_CONFIG', 'development'))
 migrate = Migrate(app, db, compare_type=True)
 
 

web/requirements.txt

 cqi
 dnspython==1.16.0
-email_validator
 eventlet
 Flask
 Flask-Login
@@ -11,6 +10,8 @@ Flask-SocketIO
 Flask-SQLAlchemy
 Flask-WTF
 jsonpatch
+jsonschema
 psycopg2
+python-dotenv
 redis
-jsonschema
+wtforms[email]