Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into development

.env.tpl

@@ -5,6 +5,10 @@
 # NOTE: Use `.` as <project-root-dir>
 # HOST_DB_DIR=
 
+# DEFAULT: ./mq
+# NOTE: Use `.` as <project-root-dir>
+# HOST_MQ_DIR=
+
 # Example: 999
 # HINT: Use this bash command `getent group docker | cut -d: -f3`
 HOST_DOCKER_GID=
@@ -13,10 +17,6 @@ HOST_DOCKER_GID=
 # HINT: Use this bash command `id -g`
 HOST_GID=
 
-# DEFAULT: ./mq
-# NOTE: Use `.` as <project-root-dir>
-# HOST_MQ_DIR=
-
 # DEFAULT: ./nopaqued.log
 # NOTES: Use `.` as <project-root-dir>,
 #        This file must be present on container startup
@@ -33,22 +33,38 @@ HOST_UID=
 
 
 ################################################################################
-# Database (only PostgreSQL)                                                   #
+# Cookies                                                                      #
 ################################################################################
-NOPAQUE_DB_HOST=
+# CHOOSE ONE: False, True
+# DEFAULT: False
+# HINT: Set to true if you redirect http to https
+# NOPAQUE_REMEMBER_COOKIE_SECURE=
 
-NOPAQUE_DB_NAME=
+# CHOOSE ONE: False, True
+# DEFAULT: False
+# HINT: Set to true if you redirect http to https
+# NOPAQUE_SESSION_COOKIE_SECURE=
 
-NOPAQUE_DB_PASSWORD=
 
-# DEFAULT: 5432
-# NOPAQUE_DB_PORT=
+################################################################################
+# Database                                                                     #
+# DATABASE_URI blueprint:                                                      #
+#   - dialect[+driver]://username:password@host[:port]/database                #
+#     - sqlite is not supported                                                #
+#     - values in square brackets are optional                                 #
+################################################################################
+# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
+# NOPAQUE_DATABASE_URL=
+
+# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
+# NOPAQUE_DEV_DATABASE_URL=
 
-NOPAQUE_DB_USERNAME=
+# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
+# NOPAQUE_TEST_DATABASE_URL=
 
 
 ################################################################################
-# SMTP                                                                         #
+# Email                                                                        #
 ################################################################################
 # EXAMPLE: nopaque Admin <nopaque@example.com>
 NOPAQUE_SMTP_DEFAULT_SENDER=
@@ -61,12 +77,12 @@ NOPAQUE_SMTP_SERVER=
 # EXAMPLE: 587
 NOPAQUE_SMTP_PORT=
 
+# CHOOSE ONE: False, True
 # DEFAULT: False
-# Choose one: False, True
 # NOPAQUE_SMTP_USE_SSL=
 
+# CHOOSE ONE: False, True
 # DEFAULT: False
-# Choose one: False, True
 # NOPAQUE_SMTP_USE_TLS=
 
 # EXAMPLE: nopaque@example.com
@@ -76,78 +92,84 @@ NOPAQUE_SMTP_USERNAME=
 ################################################################################
 # General                                                                      #
 ################################################################################
-# Example: admin.nopaque@example.com
+# EXAMPLE: admin.nopaque@example.com
 NOPAQUE_ADMIN_EMAIL_ADRESS=
 
-# Example: contact.nopaque@example.com
+# DEFAULT: development
+# CHOOSE ONE: development, production, testing
+# NOPAQUE_CONFIG=
+
+# EXAMPLE: contact.nopaque@example.com
 NOPAQUE_CONTACT_EMAIL_ADRESS=
 
 # DEFAULT: /mnt/nopaque
 # NOTE: This must be a network share and it must be available on all Docker Swarm nodes
 # NOPAQUE_DATA_DIR=
 
-# DEFAULT: False
-# Choose one: False, True
-# NOPAQUE_DEBUG=
-
 # DEFAULT: localhost
 # NOPAQUE_DOMAIN=
 
-# DEFAULT: 0
-# NOPAQUE_NUM_PROXIES=
-
+# CHOOSE ONE: http, https
 # DEFAULT: http
-# Choose one: http, https
 # NOPAQUE_PROTOCOL=
 
-# DEFAULT: True
-# Choose one: False, True
-# NOPAQUE_REMEMBER_COOKIE_HTTPONLY=
-
-# DEFAULT: False
-# Choose one: False, True
-# HINT: Set to true if you redirect http to https
-# NOPAQUE_REMEMBER_COOKIE_SECURE=
-
 # DEFAULT: hard to guess string
 # HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
 # NOPAQUE_SECRET_KEY=
 
-# DEFAULT: False
-# Choose one: False, True
-# HINT: Set to true if you redirect http to https
-# NOPAQUE_SESSION_COOKIE_SECURE=
-
 
 ################################################################################
 # Logging                                                                      #
 ################################################################################
-# DEFAULT: <nopaqued-root-dir>/nopaqued.log ~ /home/nopaqued/nopaqued.log
+# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
 # NOTE: Use `.` as <nopaqued-root-dir>
 # NOPAQUE_DAEMON_LOG_FILE=
 
 # DEFAULT: %Y-%m-%d %H:%M:%S
 # NOPAQUE_LOG_DATE_FORMAT=
 
-# DEFAULT: <nopaque-root-dir>/NOPAQUE.log ~ /home/NOPAQUE/NOPAQUE.log
+# DEFAULT: <nopaque-root-dir>/nopaque.log ~ /home/nopaque/nopaque.log
 # NOTE: Use `.` as <nopaque-root-dir>
 # NOPAQUE_LOG_FILE=
 
 # DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
 # NOPAQUE_LOG_FORMAT=
 
-# DEFAULT: ERROR
-# Choose one: CRITICAL, ERROR, WARNING, INFO, DEBUG
+# DEFAULT: WARNING
+# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
 # NOPAQUE_LOG_LEVEL=
 
 
 ################################################################################
 # Message queue                                                                #
+# MESSAGE_QUEUE_URI blueprint:                                                 #
+#   - transport://[userid:password]@hostname[:port]/[virtual_host]             #
+#     - values in square brackets are optional                                 #
 ################################################################################
-NOPAQUE_MQ_HOST=
+# DEFAULT: None
+# HINT: A message queue is not required when using a single server process
+# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
 
-# EXAMPLE: 6379
-NOPAQUE_MQ_PORT=
 
-# Choose one of the supported types by Flask-SocketIO
-NOPAQUE_MQ_TYPE=
+################################################################################
+# Proxy fix                                                                    #
+################################################################################
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-For
+# NOPAQUE_NUM_PROXIES_X_FOR=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Host
+# NOPAQUE_NUM_PROXIES_X_HOST=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Port
+# NOPAQUE_NUM_PROXIES_X_PORT=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Prefix
+# NOPAQUE_NUM_PROXIES_X_PREFIX=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Proto
+# NOPAQUE_NUM_PROXIES_X_PROTO=

.gitignore

-nopaque.log
-nopaqued.log
-
 *.py[cod]
 
 # C extensions
@@ -12,11 +9,14 @@ db
 mq
 
 # Environment files
-.env
+*.env
 
 # Installer logs
 pip-log.txt
 
+# Log files
+*.log
+
 # Packages
 *.egg
 *.egg-info

README.md

@@ -27,7 +27,7 @@ username@hostname:~$ docker run \
                        -p 139:139 \
                        -p 445:445 \
                        dperson/samba \
-                         -p -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
+                         -p -r -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
 
 # Mount the Samba share on all swarm nodes (managers and workers)
 username@hostname:~$ sudo mkdir /mnt/nopaque
@@ -39,9 +39,12 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
 ``` bash
 # Clone the nopaque repository
 username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
+username@hostname:~$ cp db.env.tpl db.env
 username@hostname:~$ cp .env.tpl .env
-# Fill out the variables within this file.
+# Fill out the variables within these files.
+username@hostname:~$ <YOUR EDITOR> db.env
 username@hostname:~$ <YOUR EDITOR> .env
+# Create docker-compose.override.yml file
 username@hostname:~$ touch docker-compose.override.yml
 # Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples in docker-compose.<example>.yml)
 username@hostname:~$ <YOUR EDITOR> docker-compose.override.yml

daemon/Dockerfile

@@ -14,7 +14,6 @@ RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
       build-essential \
       libpq-dev \
-      wait-for-it \
  && rm -r /var/lib/apt/lists/*
 
 
@@ -27,8 +26,7 @@ WORKDIR /home/nopaqued
 
 COPY --chown=nopaqued:nopaqued [".", "."]
 RUN python -m venv venv \
- && venv/bin/pip install --requirement requirements.txt \
- && mkdir logs
+ && venv/bin/pip install --requirement requirements.txt
 
 
 ENTRYPOINT ["./boot.sh"]

daemon/app/__init__.py

+from config import config
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+from time import sleep
+import docker
+import os
+
+
+configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')]
+configuration.init()
+docker_client = docker.from_env()
+engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI)
+Session = scoped_session(sessionmaker(bind=engine))
+
+
+def run():
+    from .tasks.check_corpora import check_corpora
+    check_corpora_thread = check_corpora()
+    from .tasks.check_jobs import check_jobs
+    check_jobs_thread = check_jobs()
+    from .tasks.notify import notify
+    notify_thread = notify()
+
+    while True:
+        if not check_corpora_thread.is_alive():
+            check_corpora_thread = check_corpora()
+        if not check_jobs_thread.is_alive():
+            check_jobs_thread = check_jobs()
+        if not notify_thread.is_alive():
+            notify_thread = notify()
+        sleep(3)

daemon/tasks/check_corpora.py → daemon/app/tasks/check_corpora.py

-from . import config, docker_client, Session
-from .decorators import background
-from .models import Corpus
+from .. import configuration as config
+from .. import docker_client, Session
+from ..decorators import background
+from ..models import Corpus
 import docker
 import logging
 import os

daemon/tasks/check_jobs.py → daemon/app/tasks/check_jobs.py

 from datetime import datetime
-from . import config, docker_client, Session
-from .decorators import background
-from .models import Job, JobResult, NotificationData, NotificationEmailData
+from .. import configuration as config
+from .. import docker_client, Session
+from ..decorators import background
+from ..models import Job, JobResult, NotificationData, NotificationEmailData
 import docker
 import logging
 import json

daemon/tasks/notify.py → daemon/app/tasks/notify.py

-from notify.notification import Notification
-from notify.service import NotificationService
 from sqlalchemy import asc
-from . import config, Session
-from .decorators import background
-from .models import NotificationEmailData
+from .libnotify.notification import Notification
+from .libnotify.service import NotificationService
+from .. import configuration as config
+from .. import Session
+from ..decorators import background
+from ..models import NotificationEmailData
 import logging
+import os
 import smtplib
 
 
+ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
+
+
 @background
 def notify():
     session = Session()
@@ -72,8 +77,10 @@ def __create_mail_notifications(notification_service, session):
                                      'status': data.notify_status,
                                      'time': data.creation_date,
                                      'url': url}
-        txt_tmplt = 'notify/templates/notification_messages/notification.txt'
-        html_tmplt = 'notify/templates/notification_messages/notification.html'
+        txt_tmplt = os.path.join(ROOT_DIR,
+                                 'libnotify/templates/notification.txt')
+        html_tmplt = os.path.join(ROOT_DIR,
+                                  'libnotify/templates/notification.html')
         notification.set_notification_content(subject_template,
                                               subject_template_values_dict,
                                               txt_tmplt,

daemon/boot.sh

 #!/bin/bash
-echo "Waiting for db..."
-wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0
-echo "Waiting for nopaque..."
-wait-for-it nopaque:5000 --strict --timeout=0
-
 source venv/bin/activate
 python nopaqued.py

daemon/config.py

@@ -2,60 +2,70 @@ import logging
 import os
 
 
-root_dir = os.path.abspath(os.path.dirname(__file__))
-
-
-DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque')
-DEFAULT_DB_PORT = '5432'
-DEFAULT_DOMAIN = 'localhost'
-DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
-DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaqued.log')
-DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s '
-                      '(function: %(funcName)s, line: %(lineno)d): '
-                      '%(message)s')
-DEFAULT_LOG_LEVEL = 'ERROR'
-DEFAULT_MAIL_USE_SSL = 'False'
-DEFAULT_MAIL_USE_TLS = 'False'
-DEFAULT_PROTOCOL = 'http'
+ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
 
 
 class Config:
-    ''' ### Database ### '''
-    DB_HOST = os.environ.get('NOPAQUE_DB_HOST')
-    DB_NAME = os.environ.get('NOPAQUE_DB_NAME')
-    DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD')
-    DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT)
-    DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME')
-    SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
-        DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME)
-
-    ''' ### SMTP ### '''
+    ''' # Email # '''
     SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
     SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
-    SMTP_PORT = os.environ.get('NOPAQUE_SMTP_PORT')
+    SMTP_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
     SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
     SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
-    SMTP_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL',
-                                  DEFAULT_MAIL_USE_SSL).lower() == 'true'
-    SMTP_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS',
-                                  DEFAULT_MAIL_USE_TLS).lower() == 'true'
+    SMTP_USE_SSL = os.environ.get(
+        'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
+    SMTP_USE_TLS = os.environ.get(
+        'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
 
-    ''' ### General ### '''
-    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR)
-    DOMAIN = os.environ.get('NOPAQUE_DOMAIN', DEFAULT_DOMAIN)
-    PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL)
+    ''' # General # '''
+    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
+    DOMAIN = os.environ.get('NOPAQUE_DOMAIN', 'localhost')
+    PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', 'http')
+    SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
 
-    ''' ### Logging ### '''
+    ''' # Logging # '''
     LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
-                                     DEFAULT_LOG_DATE_FORMAT)
-    LOG_FILE = os.environ.get('NOPAQUE_DAEMON_LOG_FILE', DEFAULT_LOG_FILE)
-    LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT)
-    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL)
-
-    def init_app(self):
-        # Configure logging according to the corresponding (LOG_*) config
-        # entries
-        logging.basicConfig(datefmt=self.LOG_DATE_FORMAT,
-                            filename=self.LOG_FILE,
-                            format=self.LOG_FORMAT,
-                            level=self.LOG_LEVEL)
+                                     '%Y-%m-%d %H:%M:%S')
+    LOG_FILE = os.environ.get('NOPAQUED_LOG_FILE',
+                              os.path.join(ROOT_DIR, 'nopaqued.log'))
+    LOG_FORMAT = os.environ.get(
+        'NOPAQUE_LOG_FORMAT',
+        '[%(asctime)s] %(levelname)s in '
+        '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
+    )
+    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
+
+    @classmethod
+    def init(cls):
+        # Set up logging according to the corresponding (LOG_*) variables
+        logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
+                            filename=cls.LOG_FILE,
+                            format=cls.LOG_FORMAT,
+                            level=cls.LOG_LEVEL)
+
+
+class DevelopmentConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DEV_DATABASE_URL',
+        'sqlite:///' + os.path.join(ROOT_DIR, 'data-dev.sqlite')
+    )
+
+
+class ProductionConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DATABASE_URL',
+        'sqlite:///' + os.path.join(ROOT_DIR, 'data.sqlite')
+    )
+
+
+class TestingConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_TEST_DATABASE_URL', 'sqlite://')
+
+
+config = {'development': DevelopmentConfig,
+          'production': ProductionConfig,
+          'testing': TestingConfig}

daemon/nopaqued.py

-from tasks.check_corpora import check_corpora
-from tasks.check_jobs import check_jobs
-from tasks.notify import notify
-from time import sleep
+from dotenv import load_dotenv
+from app import run
+import os
 
 
-def nopaqued():
-    check_corpora_thread = check_corpora()
-    check_jobs_thread = check_jobs()
-    notify_thread = notify()
-
-    while True:
-        if not check_corpora_thread.is_alive():
-            check_corpora_thread = check_corpora()
-        if not check_jobs_thread.is_alive():
-            check_jobs_thread = check_jobs()
-        if not notify_thread.is_alive():
-            notify_thread = notify()
-        sleep(3)
+# Load environment variables
+DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
+if os.path.exists(DOTENV_FILE):
+    load_dotenv(DOTENV_FILE)
 
 
 if __name__ == '__main__':
-    nopaqued()
+    run()

daemon/requirements.txt

 docker
 psycopg2
+python-dotenv
 SQLAlchemy