From 5a06a6b241e19e8b3fa33d154740f0da9a631d1b Mon Sep 17 00:00:00 2001
From: Patrick Jentsch <p.jentsch@uni-bielefeld.de>
Date: Fri, 13 Nov 2020 10:01:51 +0100
Subject: [PATCH] More exception handling. Remove unused database models. New
common view structure!
---
.env.tpl | 135 +++---
docker-compose.yml | 1 +
web/Dockerfile | 5 +-
web/app/__init__.py | 2 +-
web/app/admin/__init__.py | 2 +-
web/app/admin/forms.py | 1 -
web/app/admin/views.py | 14 +-
web/app/auth/__init__.py | 2 +-
web/app/auth/forms.py | 2 +-
web/app/auth/views.py | 107 +++--
web/app/corpora/views.py | 390 ++++++++----------
web/app/email.py | 4 +-
web/app/jobs/__init__.py | 2 +-
web/app/jobs/views.py | 30 +-
web/app/main/__init__.py | 2 +-
web/app/main/views.py | 25 +-
web/app/models.py | 181 +++-----
web/app/query_results/views.py | 150 -------
web/app/services/__init__.py | 2 +-
web/app/services/views.py | 61 ++-
web/app/settings/forms.py | 2 +-
web/app/settings/views.py | 19 +-
web/app/tasks/__init__.py | 14 +-
web/app/tasks/corpus_utils.py | 131 +++---
web/app/tasks/job_utils.py | 90 ++--
web/app/templates/auth/login.html.j2 | 10 +-
web/app/templates/auth/register.html.j2 | 12 +-
web/app/templates/auth/reset_password.html.j2 | 8 +-
.../auth/reset_password_request.html.j2 | 6 +-
web/app/templates/corpora/add_corpus.html.j2 | 8 +-
.../templates/corpora/add_corpus_file.html.j2 | 14 +-
.../templates/corpora/analyse_corpus.html.j2 | 2 +-
web/app/templates/corpora/corpus_file.html.j2 | 12 +-
.../templates/corpora/import_corpus.html.j2 | 12 +-
.../query_results/add_query_result.html.j2 | 10 +-
web/app/templates/main/index.html.j2 | 10 +-
web/app/templates/nopaque.html.j2 | 6 +-
web/app/templates/services/file-setup.html.j2 | 12 +-
web/app/templates/services/nlp.html.j2 | 18 +-
web/app/templates/services/ocr.html.j2 | 18 +-
.../tasks/email/notification.html.j2 | 7 +-
.../templates/tasks/email/notification.txt.j2 | 7 +-
web/boot.sh | 1 +
web/config.py | 137 +++---
web/nopaque.py | 11 +-
45 files changed, 691 insertions(+), 1004 deletions(-)
delete mode 100644 web/app/query_results/views.py
diff --git a/.env.tpl b/.env.tpl
index 2c13eb3e..e7c2cea7 100644
--- a/.env.tpl
+++ b/.env.tpl
@@ -9,128 +9,116 @@
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
-# Example: 999
-# HINT: Use this bash command `getent group docker | cut -d: -f3`
-HOST_DOCKER_GID=
+# Example: 1000
+# HINT: Use this bash command `id -u`
+HOST_UID=
# Example: 1000
# HINT: Use this bash command `id -g`
HOST_GID=
-# DEFAULT: ./nopaqued.log
-# NOTES: Use `.` as <project-root-dir>,
-# This file must be present on container startup
-# HOST_NOPAQUE_DAEMON_LOG_FILE=
+# Example: 999
+# HINT: Use this bash command `getent group docker | cut -d: -f3`
+HOST_DOCKER_GID=
# DEFAULT: ./nopaque.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
-# HOST_NOPAQUE_LOG_FILE=
-
-# Example: 1000
-# HINT: Use this bash command `id -u`
-HOST_UID=
+# HOST_LOG_FILE=
################################################################################
-# Cookies #
+# Flask #
+# https://flask.palletsprojects.com/en/1.1.x/config/ #
################################################################################
-# CHOOSE ONE: False, True
-# DEFAULT: False
-# HINT: Set to true if you redirect http to https
-# NOPAQUE_REMEMBER_COOKIE_SECURE=
+# DEFAULT: hard to guess string
+# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
+# SECRET_KEY=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
-# NOPAQUE_SESSION_COOKIE_SECURE=
+# SESSION_COOKIE_SECURE=
################################################################################
-# Database #
-# DATABASE_URI blueprint: #
-# - dialect[+driver]://username:password@host[:port]/database #
-# - sqlite is not supported #
-# - values in square brackets are optional #
+# Flask-Login #
+# https://flask-login.readthedocs.io/en/latest/ #
################################################################################
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
-# NOPAQUE_DATABASE_URL=
-
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
-# NOPAQUE_DEV_DATABASE_URL=
-
-# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
-# NOPAQUE_TEST_DATABASE_URL=
+# CHOOSE ONE: False, True
+# DEFAULT: False
+# HINT: Set to true if you redirect http to https
+# REMEMBER_COOKIE_SECURE=
################################################################################
-# Email #
+# Flask-Mail #
+# https://pythonhosted.org/Flask-Mail/ #
################################################################################
# EXAMPLE: nopaque Admin <nopaque@example.com>
-NOPAQUE_SMTP_DEFAULT_SENDER=
+MAIL_DEFAULT_SENDER=
-NOPAQUE_SMTP_PASSWORD=
+MAIL_PASSWORD=
# EXAMPLE: smtp.example.com
-NOPAQUE_SMTP_SERVER=
+MAIL_SERVER=
# EXAMPLE: 587
-NOPAQUE_SMTP_PORT=
+MAIL_PORT=
# CHOOSE ONE: False, True
# DEFAULT: False
-# NOPAQUE_SMTP_USE_SSL=
+# MAIL_USE_SSL=
# CHOOSE ONE: False, True
# DEFAULT: False
-# NOPAQUE_SMTP_USE_TLS=
+# MAIL_USE_TLS=
# EXAMPLE: nopaque@example.com
-NOPAQUE_SMTP_USERNAME=
+MAIL_USERNAME=
+
+
+################################################################################
+# Flask-SQLAlchemy #
+# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
+################################################################################
+# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
+# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
+# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
+# SQLALCHEMY_DATABASE_URI=
################################################################################
-# General #
+# nopaque #
################################################################################
+# If an account is registered with this email adress gets automatically
+# assigned the administrator role.
# EXAMPLE: admin.nopaque@example.com
-NOPAQUE_ADMIN_EMAIL_ADRESS=
+NOPAQUE_ADMIN=
# DEFAULT: development
# CHOOSE ONE: development, production, testing
# NOPAQUE_CONFIG=
+# This email adress is used for the contact button in the nopaque footer. If
+# not set, no contact button is displayed.
# DEFAULT: None
# EXAMPLE: contact.nopaque@example.com
-# NOPAQUE_CONTACT_EMAIL_ADRESS=
+# NOPAQUE_CONTACT=
# DEFAULT: /mnt/nopaque
-# NOTE: This must be a network share and it must be available on all Docker Swarm nodes
+# NOTE: This must be a network share and it must be available on all Docker
+# Swarm nodes
# NOPAQUE_DATA_DIR=
-# DEFAULT: localhost
-# NOPAQUE_DOMAIN=
-
# DEFAULT: 0.0.0.0
# NOPAQUE_HOST=
# DEFAULT: 5000
# NOPAQUE_PORT=
-# CHOOSE ONE: http, https
-# DEFAULT: http
-# NOPAQUE_PROTOCOL=
-
-# DEFAULT: hard to guess string
-# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
-# NOPAQUE_SECRET_KEY=
-
-
-################################################################################
-# Logging #
-################################################################################
-# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
-# NOTE: Use `.` as <nopaqued-root-dir>
-# NOPAQUE_DAEMON_LOG_FILE=
+# transport://[userid:password]@hostname[:port]/[virtual_host]
+NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT=
@@ -146,37 +134,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL=
-
-################################################################################
-# Message queue #
-# MESSAGE_QUEUE_URI blueprint: #
-# - transport://[userid:password]@hostname[:port]/[virtual_host] #
-# - values in square brackets are optional #
-################################################################################
-# DEFAULT: None
-# HINT: A message queue is not required when using a single server process
-# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
-
-
-################################################################################
-# Proxy fix #
-################################################################################
# DEFAULT: 0
# Number of values to trust for X-Forwarded-For
-# NOPAQUE_NUM_PROXIES_X_FOR=
+# NOPAQUE_PROXY_FIX_X_FOR=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Host
-# NOPAQUE_NUM_PROXIES_X_HOST=
+# NOPAQUE_PROXY_FIX_X_HOST=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Port
-# NOPAQUE_NUM_PROXIES_X_PORT=
+# NOPAQUE_PROXY_FIX_X_PORT=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Prefix
-# NOPAQUE_NUM_PROXIES_X_PREFIX=
+# NOPAQUE_PROXY_FIX_X_PREFIX=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Proto
-# NOPAQUE_NUM_PROXIES_X_PROTO=
+# NOPAQUE_PROXY_FIX_X_PROTO=
diff --git a/docker-compose.yml b/docker-compose.yml
index 57f8b5bd..c1d75151 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -28,5 +28,6 @@ services:
image: nopaque:development
restart: unless-stopped
volumes:
+ - "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
diff --git a/web/Dockerfile b/web/Dockerfile
index 4d8037e1..3681b701 100644
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -21,8 +21,9 @@ RUN apt-get update \
&& rm -r /var/lib/apt/lists/*
-RUN groupadd --gid ${GID} --system nopaque \
- && useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque
+RUN groupadd --gid ${DOCKER_GID} --system docker \
+ && groupadd --gid ${GID} --system nopaque \
+ && useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
USER nopaque
WORKDIR /home/nopaque
diff --git a/web/app/__init__.py b/web/app/__init__.py
index 75108bd3..9a399ddc 100644
--- a/web/app/__init__.py
+++ b/web/app/__init__.py
@@ -26,7 +26,7 @@ def create_app(config_name):
mail.init_app(app)
paranoid.init_app(app)
socketio.init_app(
- app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
+ app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
with app.app_context():
from . import events
diff --git a/web/app/admin/__init__.py b/web/app/admin/__init__.py
index 40fd29a4..9bb011f8 100644
--- a/web/app/admin/__init__.py
+++ b/web/app/admin/__init__.py
@@ -2,4 +2,4 @@ from flask import Blueprint
admin = Blueprint('admin', __name__)
-from . import views # noqa
+from . import views
diff --git a/web/app/admin/forms.py b/web/app/admin/forms.py
index 42706bab..a0f796d6 100644
--- a/web/app/admin/forms.py
+++ b/web/app/admin/forms.py
@@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
super().__init__(*args, user=user, **kwargs)
self.role.choices = [(role.id, role.name)
for role in Role.query.order_by(Role.name).all()]
- self.user = user
diff --git a/web/app/admin/views.py b/web/app/admin/views.py
index c3f4e875..798c0df8 100644
--- a/web/app/admin/views.py
+++ b/web/app/admin/views.py
@@ -29,12 +29,11 @@ def user(user_id):
@admin_required
def delete_user(user_id):
settings_tasks.delete_user(user_id)
- flash('User has been deleted!')
+ flash('User has been marked for deletion!')
return redirect(url_for('.users'))
-@admin.route('/users/<int:user_id>/edit_general_settings',
- methods=['GET', 'POST'])
+@admin.route('/users/<int:user_id>/edit_general_settings', methods=['GET', 'POST']) # noqa
@login_required
@admin_required
def edit_general_settings(user_id):
@@ -46,16 +45,13 @@ def edit_general_settings(user_id):
user.username = form.username.data
user.confirmed = form.confirmed.data
user.role = Role.query.get(form.role.data)
- db.session.add(user)
db.session.commit()
- flash('The profile has been updated.')
- return redirect(url_for('admin.edit_general_settings', user_id=user.id))
+ flash('Settings have been updated.')
+ return redirect(url_for('.edit_general_settings', user_id=user.id))
form.confirmed.data = user.confirmed
form.dark_mode.data = user.setting_dark_mode
form.email.data = user.email
form.role.data = user.role_id
form.username.data = user.username
return render_template('admin/edit_general_settings.html.j2',
- form=form,
- title='General settings',
- user=user)
+ form=form, title='General settings', user=user)
diff --git a/web/app/auth/__init__.py b/web/app/auth/__init__.py
index 893d7071..a45dc3f3 100644
--- a/web/app/auth/__init__.py
+++ b/web/app/auth/__init__.py
@@ -2,4 +2,4 @@ from flask import Blueprint
auth = Blueprint('auth', __name__)
-from . import views # noqa
+from . import views
diff --git a/web/app/auth/forms.py b/web/app/auth/forms.py
index 3344096b..98dab654 100644
--- a/web/app/auth/forms.py
+++ b/web/app/auth/forms.py
@@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
username = StringField(
'Username',
validators=[DataRequired(), Length(1, 64),
- Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
+ Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)
diff --git a/web/app/auth/views.py b/web/app/auth/views.py
index c1fe37ed..b6355642 100644
--- a/web/app/auth/views.py
+++ b/web/app/auth/views.py
@@ -1,5 +1,5 @@
-from flask import (current_app, flash, redirect, render_template, request,
- url_for)
+from datetime import datetime
+from flask import abort, flash, redirect, render_template, request, url_for
from flask_login import current_user, login_user, login_required, logout_user
from . import auth
from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
from .. import db
from ..email import create_message, send
from ..models import User
+import logging
import os
-import shutil
@auth.before_app_request
@@ -18,11 +18,12 @@ def before_request():
unconfirmed view if user is unconfirmed.
"""
if current_user.is_authenticated:
- current_user.ping()
- if not current_user.confirmed \
- and request.endpoint \
- and request.blueprint != 'auth' \
- and request.endpoint != 'static':
+ current_user.last_seen = datetime.utcnow()
+ db.session.commit()
+ if (not current_user.confirmed
+ and request.endpoint
+ and request.blueprint != 'auth'
+ and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed'))
@@ -30,20 +31,19 @@ def before_request():
def login():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
- login_form = LoginForm(prefix='login-form')
- if login_form.validate_on_submit():
- user = User.query.filter_by(username=login_form.user.data).first()
+ form = LoginForm(prefix='login-form')
+ if form.validate_on_submit():
+ user = User.query.filter_by(username=form.user.data).first()
if user is None:
- user = User.query.filter_by(email=login_form.user.data).first()
- if user is not None and user.verify_password(login_form.password.data):
- login_user(user, login_form.remember_me.data)
+ user = User.query.filter_by(email=form.user.data.lower()).first()
+ if user is not None and user.verify_password(form.password.data):
+ login_user(user, form.remember_me.data)
next = request.args.get('next')
if next is None or not next.startswith('/'):
next = url_for('main.dashboard')
return redirect(next)
flash('Invalid email/username or password.')
- return render_template('auth/login.html.j2', login_form=login_form,
- title='Log in')
+ return render_template('auth/login.html.j2', form=form, title='Log in')
@auth.route('/logout')
@@ -58,26 +58,28 @@ def logout():
def register():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
- registration_form = RegistrationForm(prefix='registration-form')
- if registration_form.validate_on_submit():
- user = User(email=registration_form.email.data.lower(),
- password=registration_form.password.data,
- username=registration_form.username.data)
+ form = RegistrationForm(prefix='registration-form')
+ if form.validate_on_submit():
+ user = User(email=form.email.data.lower(),
+ password=form.password.data,
+ username=form.username.data)
db.session.add(user)
db.session.commit()
- user_dir = os.path.join(current_app.config['DATA_DIR'],
- str(user.id))
- if os.path.exists(user_dir):
- shutil.rmtree(user_dir)
- os.mkdir(user_dir)
- token = user.generate_confirmation_token()
- msg = create_message(user.email, 'Confirm Your Account',
- 'auth/email/confirm', token=token, user=user)
- send(msg)
- flash('A confirmation email has been sent to you by email.')
- return redirect(url_for('auth.login'))
- return render_template('auth/register.html.j2',
- registration_form=registration_form,
+ try:
+ os.makedirs(user.path)
+ except OSError:
+ logging.error('Make dir {} led to an OSError!'.format(user.path))
+ db.session.delete(user)
+ db.session.commit()
+ abort(500)
+ else:
+ token = user.generate_confirmation_token()
+ msg = create_message(user.email, 'Confirm Your Account',
+ 'auth/email/confirm', token=token, user=user)
+ send(msg)
+ flash('A confirmation email has been sent to you by email.')
+ return redirect(url_for('.login'))
+ return render_template('auth/register.html.j2', form=form,
title='Register')
@@ -92,7 +94,7 @@ def confirm(token):
return redirect(url_for('main.dashboard'))
else:
flash('The confirmation link is invalid or has expired.')
- return redirect(url_for('auth.unconfirmed'))
+ return redirect(url_for('.unconfirmed'))
@auth.route('/unconfirmed')
@@ -119,39 +121,32 @@ def resend_confirmation():
def reset_password_request():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
- reset_password_request_form = ResetPasswordRequestForm(
- prefix='reset-password-request-form')
- if reset_password_request_form.validate_on_submit():
- submitted_email = reset_password_request_form.email.data
- user = User.query.filter_by(email=submitted_email.lower()).first()
- if user:
+ form = ResetPasswordRequestForm(prefix='reset-password-request-form')
+ if form.validate_on_submit():
+ user = User.query.filter_by(email=form.email.data.lower()).first()
+ if user is not None:
token = user.generate_reset_token()
msg = create_message(user.email, 'Reset Your Password',
'auth/email/reset_password', token=token,
user=user)
send(msg)
- flash('An email with instructions to reset your password has been '
- 'sent to you.')
- return redirect(url_for('auth.login'))
- return render_template(
- 'auth/reset_password_request.html.j2',
- reset_password_request_form=reset_password_request_form,
- title='Password Reset')
+ flash('An email with instructions to reset your password has been sent to you.') # noqa
+ return redirect(url_for('.login'))
+ return render_template('auth/reset_password_request.html.j2', form=form,
+ title='Password Reset')
@auth.route('/reset/<token>', methods=['GET', 'POST'])
def reset_password(token):
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
- reset_password_form = ResetPasswordForm(prefix='reset-password-form')
- if reset_password_form.validate_on_submit():
- if User.reset_password(token, reset_password_form.password.data):
+ form = ResetPasswordForm(prefix='reset-password-form')
+ if form.validate_on_submit():
+ if User.reset_password(token, form.password.data):
db.session.commit()
flash('Your password has been updated.')
- return redirect(url_for('auth.login'))
+ return redirect(url_for('.login'))
else:
return redirect(url_for('main.index'))
- return render_template('auth/reset_password.html.j2',
- reset_password_form=reset_password_form,
- title='Password Reset',
- token=token)
+ return render_template('auth/reset_password.html.j2', form=form,
+ title='Password Reset', token=token)
diff --git a/web/app/corpora/views.py b/web/app/corpora/views.py
index a6bef316..13874243 100644
--- a/web/app/corpora/views.py
+++ b/web/app/corpora/views.py
@@ -1,4 +1,4 @@
-from flask import (abort, current_app, flash, make_response, redirect, request,
+from flask import (abort, flash, make_response, redirect, request,
render_template, url_for, send_from_directory)
from flask_login import current_user, login_required
from . import corpora
@@ -11,6 +11,7 @@ from jsonschema import validate
from .. import db
from ..models import Corpus, CorpusFile, QueryResult
import json
+import logging
import os
import shutil
import glob
@@ -22,106 +23,92 @@ from .import_corpus import check_zip_contents
@corpora.route('/add', methods=['GET', 'POST'])
@login_required
def add_corpus():
- add_corpus_form = AddCorpusForm()
- if add_corpus_form.validate_on_submit():
+ form = AddCorpusForm()
+ if form.validate_on_submit():
corpus = Corpus(creator=current_user,
- description=add_corpus_form.description.data,
- status='unprepared', title=add_corpus_form.title.data)
+ description=form.description.data,
+ title=form.title.data)
db.session.add(corpus)
db.session.commit()
- dir = os.path.join(current_app.config['DATA_DIR'],
- str(corpus.user_id), 'corpora', str(corpus.id))
try:
- os.makedirs(dir)
+ os.makedirs(corpus.path)
except OSError:
- flash('[ERROR]: Could not add corpus!', 'corpus')
- corpus.delete()
- else:
- url = url_for('corpora.corpus', corpus_id=corpus.id)
- flash('[<a href="{}">{}</a>] added'.format(url, corpus.title),
- 'corpus')
- return redirect(url_for('corpora.corpus', corpus_id=corpus.id))
- return render_template('corpora/add_corpus.html.j2',
- add_corpus_form=add_corpus_form,
+ logging.error('Make dir {} led to an OSError!'.format(corpus.path))
+ db.session.delete(corpus)
+ db.session.commit()
+ abort(500)
+ flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
+ return redirect(url_for('.corpus', corpus_id=corpus.id))
+ return render_template('corpora/add_corpus.html.j2', form=form,
title='Add corpus')
@corpora.route('/import', methods=['GET', 'POST'])
@login_required
def import_corpus():
- import_corpus_form = ImportCorpusForm()
- if import_corpus_form.is_submitted():
- if not import_corpus_form.validate():
- return make_response(import_corpus_form.errors, 400)
+ form = ImportCorpusForm()
+ if form.is_submitted():
+ if not form.validate():
+ return make_response(form.errors, 400)
corpus = Corpus(creator=current_user,
- description=import_corpus_form.description.data,
- status='unprepared',
- title=import_corpus_form.title.data)
+ description=form.description.data,
+ title=form.title.data)
db.session.add(corpus)
db.session.commit()
- dir = os.path.join(current_app.config['DATA_DIR'],
- str(corpus.user_id), 'corpora', str(corpus.id))
try:
- os.makedirs(dir)
+ os.makedirs(corpus.path)
except OSError:
- flash('[ERROR]: Could not import corpus!', 'corpus')
- corpus.delete()
+ logging.error('Make dir {} led to an OSError!'.format(corpus.path))
+ db.session.delete(corpus)
+ db.session.commit()
+ flash('Internal Server Error', 'error')
+ return make_response(
+ {'redirect_url': url_for('.import_corpus')}, 500)
+ # Upload zip
+ archive_file = os.path.join(corpus.path, form.file.data.filename)
+ form.file.data.save(archive_file)
+ # Some checks to verify it is a valid exported corpus
+ with ZipFile(archive_file, 'r') as zip:
+ contents = zip.namelist()
+ if set(check_zip_contents).issubset(contents):
+ # Unzip
+ shutil.unpack_archive(archive_file, corpus.path)
+ # Register vrt files to corpus
+ vrts = glob.glob(corpus.path + '/*.vrt')
+ for file in vrts:
+ element_tree = ET.parse(file)
+ text_node = element_tree.find('text')
+ corpus_file = CorpusFile(
+ address=text_node.get('address', 'NULL'),
+ author=text_node.get('author', 'NULL'),
+ booktitle=text_node.get('booktitle', 'NULL'),
+ chapter=text_node.get('chapter', 'NULL'),
+ corpus=corpus,
+ editor=text_node.get('editor', 'NULL'),
+ filename=os.path.basename(file),
+ institution=text_node.get('institution', 'NULL'),
+ journal=text_node.get('journal', 'NULL'),
+ pages=text_node.get('pages', 'NULL'),
+ publisher=text_node.get('publisher', 'NULL'),
+ publishing_year=text_node.get('publishing_year', ''),
+ school=text_node.get('school', 'NULL'),
+ title=text_node.get('title', 'NULL')
+ )
+ db.session.add(corpus_file)
+ # finish import and redirect to imported corpus
+ corpus.status = 'prepared'
+ db.session.commit()
+ os.remove(archive_file)
+ flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
+ return make_response(
+ {'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
else:
- # Upload zip
- archive_file = os.path.join(current_app.config['DATA_DIR'], dir,
- import_corpus_form.file.data.filename)
- corpus_dir = os.path.dirname(archive_file)
- import_corpus_form.file.data.save(archive_file)
- # Some checks to verify it is a valid exported corpus
- with ZipFile(archive_file, 'r') as zip:
- contents = zip.namelist()
- if set(check_zip_contents).issubset(contents):
- # Unzip
- shutil.unpack_archive(archive_file, corpus_dir)
- # Register vrt files to corpus
- vrts = glob.glob(corpus_dir + '/*.vrt')
- for file in vrts:
- element_tree = ET.parse(file)
- text_node = element_tree.find('text')
- corpus_file = CorpusFile(
- address=text_node.get('address', 'NULL'),
- author=text_node.get('author', 'NULL'),
- booktitle=text_node.get('booktitle', 'NULL'),
- chapter=text_node.get('chapter', 'NULL'),
- corpus=corpus,
- dir=dir,
- editor=text_node.get('editor', 'NULL'),
- filename=os.path.basename(file),
- institution=text_node.get('institution', 'NULL'),
- journal=text_node.get('journal', 'NULL'),
- pages=text_node.get('pages', 'NULL'),
- publisher=text_node.get('publisher', 'NULL'),
- publishing_year=text_node.get('publishing_year', ''),
- school=text_node.get('school', 'NULL'),
- title=text_node.get('title', 'NULL'))
- db.session.add(corpus_file)
- # finish import and got to imported corpus
- url = url_for('corpora.corpus', corpus_id=corpus.id)
- corpus.status = 'prepared'
- db.session.commit()
- os.remove(archive_file)
- flash('[<a href="{}">{}</a>] imported'.format(url,
- corpus.title),
- 'corpus')
- return make_response(
- {'redirect_url': url_for('corpora.corpus',
- corpus_id=corpus.id)},
- 201)
- else:
- # If imported zip is not valid delete corpus and give feedback
- corpus.delete()
- db.session.commit()
- flash('Imported corpus is not valid.', 'error')
- return make_response(
- {'redirect_url': url_for('corpora.import_corpus')},
- 201)
- return render_template('corpora/import_corpus.html.j2',
- import_corpus_form=import_corpus_form,
+ # If imported zip is not valid delete corpus and give feedback
+ flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error') # noqa
+ tasks.delete_corpus(corpus.id)
+ return make_response(
+ {'redirect_url': url_for('.import_corpus')}, 201)
+ return render_template('corpora/import_corpus.html.j2', form=form,
title='Import Corpus')
@@ -131,17 +118,9 @@ def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
- corpus_files = [dict(filename=corpus_file.filename,
- author=corpus_file.author,
- title=corpus_file.title,
- publishing_year=corpus_file.publishing_year,
- corpus_id=corpus.id,
- id=corpus_file.id)
- for corpus_file in corpus.files]
- return render_template('corpora/corpus.html.j2',
- corpus=corpus,
- corpus_files=corpus_files,
- title='Corpus')
+ corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
+ return render_template('corpora/corpus.html.j2', corpus=corpus,
+ corpus_files=corpus_files, title='Corpus')
@corpora.route('/<int:corpus_id>/export')
@@ -150,12 +129,11 @@ def export_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
+ # TODO: Check what happens here
dir = os.path.dirname(corpus.archive_file)
filename = os.path.basename(corpus.archive_file)
- return send_from_directory(directory=dir,
- filename=filename,
- mimetype='zip',
- as_attachment=True)
+ return send_from_directory(as_attachment=True, directory=dir,
+ filename=filename, mimetype='zip')
@corpora.route('/<int:corpus_id>/analyse')
@@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
result_context=request.args.get('context', 20),
- results_per_page=request.args.get('results_per_page', 30))
+ results_per_page=request.args.get('results_per_page', 30)
+ )
query_form = QueryForm(prefix='query-form',
query=request.args.get('query'))
query_download_form = QueryDownloadForm(prefix='query-download-form')
@@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
return render_template(
'corpora/analyse_corpus.html.j2',
corpus=corpus,
- corpus_id=corpus_id,
display_options_form=display_options_form,
+ inspect_display_options_form=inspect_display_options_form,
query_form=query_form,
query_download_form=query_download_form,
- inspect_display_options_form=inspect_display_options_form,
- title='Corpus analysis')
+ title='Corpus analysis'
+ )
@corpora.route('/<int:corpus_id>/delete')
@@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
+ flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
tasks.delete_corpus(corpus_id)
- flash('Corpus deleted!', 'corpus')
return redirect(url_for('main.dashboard'))
@@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
- add_corpus_file_form = AddCorpusFileForm(corpus,
- prefix='add-corpus-file-form')
- if add_corpus_file_form.is_submitted():
- if not add_corpus_file_form.validate():
- return make_response(add_corpus_file_form.errors, 400)
+ form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
+ if form.is_submitted():
+ if not form.validate():
+ return make_response(form.errors, 400)
# Save the file
- dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
- add_corpus_file_form.file.data.save(
- os.path.join(current_app.config['DATA_DIR'], dir,
- add_corpus_file_form.file.data.filename))
- corpus_file = CorpusFile(
- address=add_corpus_file_form.address.data,
- author=add_corpus_file_form.author.data,
- booktitle=add_corpus_file_form.booktitle.data,
- chapter=add_corpus_file_form.chapter.data,
- corpus=corpus,
- dir=dir,
- editor=add_corpus_file_form.editor.data,
- filename=add_corpus_file_form.file.data.filename,
- institution=add_corpus_file_form.institution.data,
- journal=add_corpus_file_form.journal.data,
- pages=add_corpus_file_form.pages.data,
- publisher=add_corpus_file_form.publisher.data,
- publishing_year=add_corpus_file_form.publishing_year.data,
- school=add_corpus_file_form.school.data,
- title=add_corpus_file_form.title.data)
+ form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
+ corpus_file = CorpusFile(address=form.address.data,
+ author=form.author.data,
+ booktitle=form.booktitle.data,
+ chapter=form.chapter.data,
+ corpus=corpus,
+ editor=form.editor.data,
+ filename=form.file.data.filename,
+ institution=form.institution.data,
+ journal=form.journal.data,
+ pages=form.pages.data,
+ publisher=form.publisher.data,
+ publishing_year=form.publishing_year.data,
+ school=form.school.data,
+ title=form.title.data)
db.session.add(corpus_file)
corpus.status = 'unprepared'
db.session.commit()
- flash('Corpus file added!', 'corpus')
- return make_response(
- {'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)},
- 201)
- return render_template('corpora/add_corpus_file.html.j2',
- corpus=corpus,
- add_corpus_file_form=add_corpus_file_form,
- title='Add corpus file')
+ flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
+ return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa
+ return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
+ form=form, title='Add corpus file')
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
+ flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus') # noqa
tasks.delete_corpus_file(corpus_file_id)
- flash('Corpus file deleted!', 'corpus')
- return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+ return redirect(url_for('.corpus', corpus_id=corpus_id))
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
- dir = os.path.join(current_app.config['DATA_DIR'],
- corpus_file.dir)
- return send_from_directory(as_attachment=True, directory=dir,
+ return send_from_directory(as_attachment=True,
+ directory=corpus_file.corpus.path,
filename=corpus_file.filename)
@@ -274,48 +242,45 @@ def download_corpus_file(corpus_id, corpus_file_id):
methods=['GET', 'POST'])
@login_required
def corpus_file(corpus_id, corpus_file_id):
- corpus = Corpus.query.get_or_404(corpus_id)
corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
- if not corpus_file.corpus_id == corpus_id:
+ if corpus_file.corpus_id != corpus_id:
abort(404)
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
- edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form')
- if edit_corpus_file_form.validate_on_submit():
- corpus_file.address = edit_corpus_file_form.address.data
- corpus_file.author = edit_corpus_file_form.author.data
- corpus_file.booktitle = edit_corpus_file_form.booktitle.data
- corpus_file.chapter = edit_corpus_file_form.chapter.data
- corpus_file.editor = edit_corpus_file_form.editor.data
- corpus_file.institution = edit_corpus_file_form.institution.data
- corpus_file.journal = edit_corpus_file_form.journal.data
- corpus_file.pages = edit_corpus_file_form.pages.data
- corpus_file.publisher = edit_corpus_file_form.publisher.data
- corpus_file.publishing_year = \
- edit_corpus_file_form.publishing_year.data
- corpus_file.school = edit_corpus_file_form.school.data
- corpus_file.title = edit_corpus_file_form.title.data
+ form = EditCorpusFileForm(prefix='edit-corpus-file-form')
+ if form.validate_on_submit():
+ corpus_file.address = form.address.data
+ corpus_file.author = form.author.data
+ corpus_file.booktitle = form.booktitle.data
+ corpus_file.chapter = form.chapter.data
+ corpus_file.editor = form.editor.data
+ corpus_file.institution = form.institution.data
+ corpus_file.journal = form.journal.data
+ corpus_file.pages = form.pages.data
+ corpus_file.publisher = form.publisher.data
+ corpus_file.publishing_year = form.publishing_year.data
+ corpus_file.school = form.school.data
+ corpus_file.title = form.title.data
corpus.status = 'unprepared'
db.session.commit()
- flash('Corpus file edited!', 'corpus')
- return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+ flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus') # noqa
+ return redirect(url_for('.corpus', corpus_id=corpus_id))
# If no form is submitted or valid, fill out fields with current values
- edit_corpus_file_form.address.data = corpus_file.address
- edit_corpus_file_form.author.data = corpus_file.author
- edit_corpus_file_form.booktitle.data = corpus_file.booktitle
- edit_corpus_file_form.chapter.data = corpus_file.chapter
- edit_corpus_file_form.editor.data = corpus_file.editor
- edit_corpus_file_form.institution.data = corpus_file.institution
- edit_corpus_file_form.journal.data = corpus_file.journal
- edit_corpus_file_form.pages.data = corpus_file.pages
- edit_corpus_file_form.publisher.data = corpus_file.publisher
- edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year
- edit_corpus_file_form.school.data = corpus_file.school
- edit_corpus_file_form.title.data = corpus_file.title
- return render_template('corpora/corpus_file.html.j2',
- corpus_file=corpus_file, corpus=corpus,
- edit_corpus_file_form=edit_corpus_file_form,
+ form.address.data = corpus_file.address
+ form.author.data = corpus_file.author
+ form.booktitle.data = corpus_file.booktitle
+ form.chapter.data = corpus_file.chapter
+ form.editor.data = corpus_file.editor
+ form.institution.data = corpus_file.institution
+ form.journal.data = corpus_file.journal
+ form.pages.data = corpus_file.pages
+ form.publisher.data = corpus_file.publisher
+ form.publishing_year.data = corpus_file.publishing_year
+ form.school.data = corpus_file.school
+ form.title.data = corpus_file.title
+ return render_template('corpora/corpus_file.html.j2', corpus=corpus,
+ corpus_file=corpus_file, form=form,
title='Edit corpus file')
@@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
abort(403)
if corpus.files.all():
tasks.build_corpus(corpus_id)
- flash('Building Corpus...', 'corpus')
+ flash('Corpus "{}" has been marked to get build!', 'corpus')
else:
- flash('Can not build corpus, please add corpus file(s).', 'corpus')
- return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
+ flash('Can not build corpus "{}": No corpus file(s)!', 'error')
+ return redirect(url_for('.corpus', corpus_id=corpus_id))
# Following are view functions to add, view etc. exported results.
@@ -340,35 +305,29 @@ def add_query_result():
'''
View to import a result as a json file.
'''
- add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
- if add_query_result_form.is_submitted():
- if not add_query_result_form.validate():
- return make_response(add_query_result_form.errors, 400)
- query_result = QueryResult(
- creator=current_user,
- description=add_query_result_form.description.data,
- filename=add_query_result_form.file.data.filename,
- title=add_query_result_form.title.data
- )
+ form = AddQueryResultForm(prefix='add-query-result-form')
+ if form.is_submitted():
+ if not form.validate():
+ return make_response(form.errors, 400)
+ query_result = QueryResult(creator=current_user,
+ description=form.description.data,
+ filename=form.file.data.filename,
+ title=form.title.data)
db.session.add(query_result)
db.session.commit()
- # create paths to save the uploaded json file
- query_result_dir = os.path.join(current_app.config['DATA_DIR'],
- str(current_user.id),
- 'query_results',
- str(query_result.id))
try:
- os.makedirs(query_result_dir)
- except Exception:
+ os.makedirs(query_result.path)
+ except OSError:
+ logging.error('Make dir {} led to an OSError!'.format(query_result.path)) # noqa
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
- redirect_url = url_for('corpora.add_query_result')
- return make_response({'redirect_url': redirect_url}, 500)
+ return make_response(
+ {'redirect_url': url_for('.add_query_result')}, 500)
# save the uploaded file
- query_result_file_path = os.path.join(query_result_dir,
+ query_result_file_path = os.path.join(query_result.path,
query_result.filename)
- add_query_result_form.file.data.save(query_result_file_path)
+ form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
@@ -381,19 +340,16 @@ def add_query_result():
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
- redirect_url = url_for('corpora.add_query_result')
- return make_response({'redirect_url': redirect_url}, 201)
+ return make_response(
+ {'redirect_url': url_for('.add_query_result')}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
- redirect_url = url_for('corpora.query_result',
- query_result_id=query_result.id)
- return make_response({'redirect_url': redirect_url}, 201)
+ return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa
return render_template('corpora/query_results/add_query_result.html.j2',
- add_query_result_form=add_query_result_form,
- title='Add query result')
+ form=form, title='Add query result')
@corpora.route('/result/<int:query_result_id>')
@@ -404,8 +360,7 @@ def query_result(query_result_id):
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
- query_result=query_result,
- title='Query result')
+ query_result=query_result, title='Query result')
@corpora.route('/result/<int:query_result_id>/inspect')
@@ -427,13 +382,7 @@ def inspect_query_result(query_result_id):
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
- query_result_file_path = os.path.join(
- current_app.config['DATA_DIR'],
- str(current_user.id),
- 'query_results',
- str(query_result.id),
- query_result.filename
- )
+ query_result_file_path = os.path.join(query_result.path, query_result.filename) # noqa
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
@@ -452,8 +401,8 @@ def delete_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
+ flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result') # noqa
tasks.delete_query_result(query_result_id)
- flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@@ -464,10 +413,5 @@ def download_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
- query_result_dir = os.path.join(current_app.config['DATA_DIR'],
- str(current_user.id),
- 'query_results',
- str(query_result.id))
- return send_from_directory(as_attachment=True,
- directory=query_result_dir,
+ return send_from_directory(as_attachment=True, directory=query_result.path,
filename=query_result.filename)
diff --git a/web/app/email.py b/web/app/email.py
index 4969b05e..4d9f0036 100644
--- a/web/app/email.py
+++ b/web/app/email.py
@@ -1,11 +1,11 @@
-from flask import render_template
+from flask import current_app, render_template
from flask_mail import Message
from . import mail
from .decorators import background
def create_message(recipient, subject, template, **kwargs):
- msg = Message('[nopaque] {}'.format(subject), recipients=[recipient])
+ msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa
msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs)
return msg
diff --git a/web/app/jobs/__init__.py b/web/app/jobs/__init__.py
index 43e34614..07e0e1bb 100644
--- a/web/app/jobs/__init__.py
+++ b/web/app/jobs/__init__.py
@@ -2,4 +2,4 @@ from flask import Blueprint
jobs = Blueprint('jobs', __name__)
-from . import views # noqa
+from . import views
diff --git a/web/app/jobs/views.py b/web/app/jobs/views.py
index a92013f7..739f153c 100644
--- a/web/app/jobs/views.py
+++ b/web/app/jobs/views.py
@@ -1,11 +1,10 @@
-from flask import (abort, current_app, flash, redirect, render_template,
+from flask import (abort, flash, redirect, render_template,
send_from_directory, url_for)
from flask_login import current_user, login_required
from . import jobs
from . import tasks
from ..decorators import admin_required
from ..models import Job, JobInput, JobResult
-import os
@jobs.route('/<int:job_id>')
@@ -14,13 +13,8 @@ def job(job_id):
job = Job.query.get_or_404(job_id)
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
- job_inputs = [dict(filename=input.filename,
- id=input.id,
- job_id=job.id)
- for input in job.inputs]
- return render_template('jobs/job.html.j2',
- job=job,
- job_inputs=job_inputs,
+ job_inputs = [job_input.to_dict() for job_input in job.inputs]
+ return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
title='Job')
@@ -31,7 +25,7 @@ def delete_job(job_id):
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
tasks.delete_job(job_id)
- flash('Job has been deleted!', 'job')
+ flash('Job has been marked for deletion!', 'job')
return redirect(url_for('main.dashboard'))
@@ -44,9 +38,8 @@ def download_job_input(job_id, job_input_id):
if not (job_input.job.creator == current_user
or current_user.is_administrator()):
abort(403)
- dir = os.path.join(current_app.config['DATA_DIR'],
- job_input.dir)
- return send_from_directory(as_attachment=True, directory=dir,
+ return send_from_directory(as_attachment=True,
+ directory=job_input.job.path,
filename=job_input.filename)
@@ -56,11 +49,11 @@ def download_job_input(job_id, job_input_id):
def restart(job_id):
job = Job.query.get_or_404(job_id)
if job.status != 'failed':
- flash('Could not restart job: status is not "failed"', 'error')
+ flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error') # noqa
else:
tasks.restart_job(job_id)
- flash('Job has been restarted!', 'job')
- return redirect(url_for('jobs.job', job_id=job_id))
+ flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job') # noqa
+ return redirect(url_for('.job', job_id=job_id))
@jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
@@ -72,7 +65,6 @@ def download_job_result(job_id, job_result_id):
if not (job_result.job.creator == current_user
or current_user.is_administrator()):
abort(403)
- dir = os.path.join(current_app.config['DATA_DIR'],
- job_result.dir)
- return send_from_directory(as_attachment=True, directory=dir,
+ return send_from_directory(as_attachment=True,
+ directory=job_result.job.path,
filename=job_result.filename)
diff --git a/web/app/main/__init__.py b/web/app/main/__init__.py
index 74430247..d658fca7 100644
--- a/web/app/main/__init__.py
+++ b/web/app/main/__init__.py
@@ -2,4 +2,4 @@ from flask import Blueprint
main = Blueprint('main', __name__)
-from . import views # noqa
+from . import views
diff --git a/web/app/main/views.py b/web/app/main/views.py
index fcc0ed5d..6f3816de 100644
--- a/web/app/main/views.py
+++ b/web/app/main/views.py
@@ -7,17 +7,16 @@ from ..models import User
@main.route('/', methods=['GET', 'POST'])
def index():
- login_form = LoginForm(prefix='login-form')
- if login_form.validate_on_submit():
- user = User.query.filter_by(username=login_form.user.data).first()
+ form = LoginForm(prefix='login-form')
+ if form.validate_on_submit():
+ user = User.query.filter_by(username=form.user.data).first()
if user is None:
- user = User.query.filter_by(email=login_form.user.data).first()
- if user is not None and user.verify_password(login_form.password.data):
- login_user(user, login_form.remember_me.data)
- return redirect(url_for('main.dashboard'))
+ user = User.query.filter_by(email=form.user.data.lower()).first()
+ if user is not None and user.verify_password(form.password.data):
+ login_user(user, form.remember_me.data)
+ return redirect(url_for('.dashboard'))
flash('Invalid email/username or password.')
- return render_template('main/index.html.j2', login_form=login_form,
- title='nopaque')
+ return render_template('main/index.html.j2', form=form, title='nopaque')
@main.route('/about_and_faq')
@@ -31,7 +30,6 @@ def dashboard():
return render_template('main/dashboard.html.j2', title='Dashboard')
-
@main.route('/news')
def news():
return render_template('main/news.html.j2', title='News')
@@ -40,12 +38,9 @@ def news():
@main.route('/privacy_policy')
def privacy_policy():
return render_template('main/privacy_policy.html.j2',
- title=('Information on the processing of personal'
- ' data for the nopaque platform (GDPR)'))
+ title='Privacy statement (GDPR)')
@main.route('/terms_of_use')
def terms_of_use():
- return render_template('main/terms_of_use.html.j2',
- title='General Terms of Use of the platform '
- 'nopaque')
+ return render_template('main/terms_of_use.html.j2', title='Terms of Use')
diff --git a/web/app/models.py b/web/app/models.py
index 4f72960a..6af02a2e 100644
--- a/web/app/models.py
+++ b/web/app/models.py
@@ -7,6 +7,7 @@ from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.utils import secure_filename
import xml.etree.ElementTree as ET
from . import db, login_manager
+import logging
import os
import shutil
@@ -54,7 +55,7 @@ class Role(db.Model):
'''
String representation of the Role. For human readability.
'''
- return '<Role {role_name}>'.format(role_name=self.name)
+ return '<Role {}>'.format(self.name)
def add_permission(self, perm):
'''
@@ -138,6 +139,18 @@ class User(UserMixin, db.Model):
cascade='save-update, merge, delete',
lazy='dynamic')
+ @property
+ def path(self):
+ return os.path.join(current_app.config['NOPAQUE_DATA_DIR'], str(self.id))
+
+ @property
+ def password(self):
+ raise AttributeError('password is not a readable attribute')
+
+ @password.setter
+ def password(self, password):
+ self.password_hash = generate_password_hash(password)
+
def to_dict(self):
return {'id': self.id,
'role_id': self.role_id,
@@ -162,7 +175,7 @@ class User(UserMixin, db.Model):
'''
String representation of the User. For human readability.
'''
- return '<User {username}>'.format(username=self.username)
+ return '<User {}>'.format(self.username)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
@@ -220,14 +233,6 @@ class User(UserMixin, db.Model):
db.session.add(user)
return True
- @property
- def password(self):
- raise AttributeError('password is not a readable attribute')
-
- @password.setter
- def password(self, password):
- self.password_hash = generate_password_hash(password)
-
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
@@ -244,17 +249,11 @@ class User(UserMixin, db.Model):
'''
return self.can(Permission.ADMIN)
- def ping(self):
- self.last_seen = datetime.utcnow()
- db.session.add(self)
-
def delete(self):
'''
Delete the user and its corpora and jobs from database and filesystem.
'''
- user_dir = os.path.join(current_app.config['DATA_DIR'],
- str(self.id))
- shutil.rmtree(user_dir, ignore_errors=True)
+ shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
@@ -280,14 +279,17 @@ class JobInput(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
- dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
+ @property
+ def path(self):
+ return os.path.join(self.job.path, self.filename)
+
def __repr__(self):
'''
String representation of the JobInput. For human readability.
'''
- return '<JobInput {filename}>'.format(filename=self.filename)
+ return '<JobInput {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
@@ -305,14 +307,17 @@ class JobResult(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
- dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
+ @property
+ def path(self):
+ return os.path.join(self.job.path, self.filename)
+
def __repr__(self):
'''
String representation of the JobResult. For human readability.
'''
- return '<JobResult {filename}>'.format(filename=self.filename)
+ return '<JobResult {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
@@ -351,19 +356,16 @@ class Job(db.Model):
cascade='save-update, merge, delete')
results = db.relationship('JobResult', backref='job', lazy='dynamic',
cascade='save-update, merge, delete')
- notification_data = db.relationship('NotificationData',
- cascade='save-update, merge, delete',
- uselist=False,
- back_populates='job') # One-to-One relationship
- notification_email_data = db.relationship('NotificationEmailData',
- cascade='save-update, merge, delete',
- back_populates='job')
+
+ @property
+ def path(self):
+ return os.path.join(self.creator.path, 'jobs', str(self.id))
def __repr__(self):
'''
String representation of the Job. For human readability.
'''
- return '<Job {job_title}>'.format(job_title=self.title)
+ return '<Job {}>'.format(self.title)
def create_secure_filename(self):
'''
@@ -385,11 +387,7 @@ class Job(db.Model):
db.session.commit()
sleep(1)
db.session.refresh(self)
- job_dir = os.path.join(current_app.config['DATA_DIR'],
- str(self.user_id),
- 'jobs',
- str(self.id))
- shutil.rmtree(job_dir, ignore_errors=True)
+ shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def restart(self):
@@ -399,12 +397,8 @@ class Job(db.Model):
if self.status != 'failed':
raise Exception('Could not restart job: status is not "failed"')
- job_dir = os.path.join(current_app.config['DATA_DIR'],
- str(self.user_id),
- 'jobs',
- str(self.id))
- shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
- shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
+ shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
+ shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa
self.end_date = None
self.status = 'submitted'
@@ -425,63 +419,6 @@ class Job(db.Model):
for result in self.results}}
-class NotificationData(db.Model):
- '''
- Class to define notification data used for sending a notification mail with
- nopaque_notify.
- '''
- __tablename__ = 'notification_data'
- # Primary key
- id = db.Column(db.Integer, primary_key=True)
- # Foreign Key
- job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
- # relationships
- job = db.relationship('Job', back_populates='notification_data')
- # Fields
- notified_on = db.Column(db.String(16), default=None)
-
- def __repr__(self):
- '''
- String representation of the NotificationData. For human readability.
- '''
- return '<NotificationData {id}>'.format(id=self.id)
-
- def to_dict(self):
- return {'id': self.id,
- 'job_id': self.job_id,
- 'job': self.job,
- 'notified': self.notified}
-
-
-class NotificationEmailData(db.Model):
- '''
- Class to define data that will be used to send a corresponding Notification
- via email.
- '''
- __tablename__ = 'notification_email_data'
- # Primary Key
- id = db.Column(db.Integer, primary_key=True)
- # Foreign Key
- job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
- # relationships
- job = db.relationship('Job', back_populates='notification_email_data')
- notify_status = db.Column(db.String(16), default=None)
- creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
-
- def __repr__(self):
- '''
- String representation of the NotificationEmailData. For human readability.
- '''
- return '<NotificationData {id}>'.format(id=self.id)
-
- def to_dict(self):
- return {'id': self.id,
- 'job_id': self.job_id,
- 'job': self.job,
- 'notify_status': self.notify_status,
- 'creation_date': self.creation_date}
-
-
class CorpusFile(db.Model):
'''
Class to define Files.
@@ -496,7 +433,6 @@ class CorpusFile(db.Model):
author = db.Column(db.String(255))
booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255))
- dir = db.Column(db.String(255))
editor = db.Column(db.String(255))
filename = db.Column(db.String(255))
institution = db.Column(db.String(255))
@@ -507,15 +443,15 @@ class CorpusFile(db.Model):
school = db.Column(db.String(255))
title = db.Column(db.String(255))
+ @property
+ def path(self):
+ return os.path.join(self.corpus.path, self.filename)
+
def delete(self):
- corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
- str(self.corpus.user_id),
- 'corpora',
- str(self.corpus_id),
- self.filename)
try:
- os.remove(corpus_file_path)
+ os.remove(self.path)
except OSError:
+ logging.error('Removing {} led to an OSError!'.format(self.path))
pass
db.session.delete(self)
self.corpus.status = 'unprepared'
@@ -553,13 +489,17 @@ class Corpus(db.Model):
description = db.Column(db.String(255))
last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
- status = db.Column(db.String(16))
+ status = db.Column(db.String(16), default='unprepared')
title = db.Column(db.String(32))
archive_file = db.Column(db.String(255))
# Relationships
files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
cascade='save-update, merge, delete')
+ @property
+ def path(self):
+ return os.path.join(self.creator.path, 'corpora', str(self.id))
+
def to_dict(self):
return {'id': self.id,
'user_id': self.user_id,
@@ -571,19 +511,14 @@ class Corpus(db.Model):
'files': {file.id: file.to_dict() for file in self.files}}
def build(self):
- corpus_dir = os.path.join(current_app.config['DATA_DIR'],
- str(self.user_id),
- 'corpora',
- str(self.id))
- output_dir = os.path.join(corpus_dir, 'merged')
+ output_dir = os.path.join(self.path, 'merged')
shutil.rmtree(output_dir, ignore_errors=True)
os.mkdir(output_dir)
master_element_tree = ET.ElementTree(
ET.fromstring('<corpus>\n</corpus>')
)
for corpus_file in self.files:
- corpus_file_path = os.path.join(corpus_dir, corpus_file.filename)
- element_tree = ET.parse(corpus_file_path)
+ element_tree = ET.parse(corpus_file.path)
text_node = element_tree.find('text')
text_node.set('address', corpus_file.address or "NULL")
text_node.set('author', corpus_file.author)
@@ -597,7 +532,7 @@ class Corpus(db.Model):
text_node.set('publishing_year', str(corpus_file.publishing_year))
text_node.set('school', corpus_file.school or "NULL")
text_node.set('title', corpus_file.title)
- element_tree.write(corpus_file_path)
+ element_tree.write(corpus_file.path)
master_element_tree.getroot().insert(1, text_node)
output_file = os.path.join(output_dir, 'corpus.vrt')
master_element_tree.write(output_file,
@@ -607,18 +542,14 @@ class Corpus(db.Model):
self.status = 'submitted'
def delete(self):
- corpus_dir = os.path.join(current_app.config['DATA_DIR'],
- str(self.user_id),
- 'corpora',
- str(self.id))
- shutil.rmtree(corpus_dir, ignore_errors=True)
+ shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def __repr__(self):
'''
String representation of the corpus. For human readability.
'''
- return '<Corpus {corpus_title}>'.format(corpus_title=self.title)
+ return '<Corpus {}>'.format(self.title)
class QueryResult(db.Model):
@@ -636,12 +567,12 @@ class QueryResult(db.Model):
query_metadata = db.Column(db.JSON())
title = db.Column(db.String(32))
+ @property
+ def path(self):
+ return os.path.join(self.creator.path, 'query_results', str(self.id))
+
def delete(self):
- query_result_dir = os.path.join(current_app.config['DATA_DIR'],
- str(self.user_id),
- 'query_results',
- str(self.id))
- shutil.rmtree(query_result_dir, ignore_errors=True)
+ shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def to_dict(self):
@@ -654,7 +585,7 @@ class QueryResult(db.Model):
def __repr__(self):
'''
- String representation of the CorpusAnalysisResult. For human readability.
+ String representation of the QueryResult. For human readability.
'''
return '<QueryResult {}>'.format(self.title)
diff --git a/web/app/query_results/views.py b/web/app/query_results/views.py
deleted file mode 100644
index ff6eae5f..00000000
--- a/web/app/query_results/views.py
+++ /dev/null
@@ -1,150 +0,0 @@
-from . import query_results
-from . import tasks
-from .. import db
-from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
-from ..models import QueryResult
-from .forms import AddQueryResultForm
-from flask import (abort, current_app, flash, make_response, redirect,
- render_template, request, send_from_directory, url_for)
-from flask_login import current_user, login_required
-import json
-import os
-from jsonschema import validate
-
-
-@query_results.route('/add', methods=['GET', 'POST'])
-@login_required
-def add_query_result():
- '''
- View to import a result as a json file.
- '''
- add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
- if add_query_result_form.is_submitted():
- if not add_query_result_form.validate():
- return make_response(add_query_result_form.errors, 400)
- query_result = QueryResult(
- creator=current_user,
- description=add_query_result_form.description.data,
- filename=add_query_result_form.file.data.filename,
- title=add_query_result_form.title.data
- )
- db.session.add(query_result)
- db.session.commit()
- # create paths to save the uploaded json file
- query_result_dir = os.path.join(current_app.config['DATA_DIR'],
- str(current_user.id),
- 'query_results',
- str(query_result.id))
- try:
- os.makedirs(query_result_dir)
- except Exception:
- db.session.delete(query_result)
- db.session.commit()
- flash('Internal Server Error', 'error')
- redirect_url = url_for('query_results.add_query_result')
- return make_response({'redirect_url': redirect_url}, 500)
- # save the uploaded file
- query_result_file_path = os.path.join(query_result_dir,
- query_result.filename)
- add_query_result_form.file.data.save(query_result_file_path)
- # parse json from file
- with open(query_result_file_path, 'r') as file:
- query_result_file_content = json.load(file)
- # parse json schema
- with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
- schema = json.load(file)
- try:
- # validate imported json file
- validate(instance=query_result_file_content, schema=schema)
- except Exception:
- tasks.delete_query_result(query_result.id)
- flash('Uploaded file is invalid', 'result')
- redirect_url = url_for('query_results.add_query_result')
- return make_response({'redirect_url': redirect_url}, 201)
- query_result_file_content.pop('matches')
- query_result_file_content.pop('cpos_lookup')
- query_result.query_metadata = query_result_file_content
- db.session.commit()
- flash('Query result added!', 'result')
- redirect_url = url_for('query_results.query_result',
- query_result_id=query_result.id)
- return make_response({'redirect_url': redirect_url}, 201)
- return render_template('corpora/query_results/add_query_result.html.j2',
- add_query_result_form=add_query_result_form,
- title='Add query result')
-
-
-@query_results.route('/<int:query_result_id>')
-@login_required
-def query_result(query_result_id):
- query_result = QueryResult.query.get_or_404(query_result_id)
- if not (query_result.creator == current_user
- or current_user.is_administrator()):
- abort(403)
- return render_template('corpora/query_results/query_result.html.j2',
- query_result=query_result,
- title='Query result')
-
-
-@query_results.route('/<int:query_result_id>/inspect')
-@login_required
-def inspect_query_result(query_result_id):
- '''
- View to inspect imported result file in a corpus analysis like interface
- '''
- query_result = QueryResult.query.get_or_404(query_result_id)
- query_metadata = query_result.query_metadata
- if not (query_result.creator == current_user
- or current_user.is_administrator()):
- abort(403)
- display_options_form = DisplayOptionsForm(
- prefix='display-options-form',
- results_per_page=request.args.get('results_per_page', 30),
- result_context=request.args.get('context', 20)
- )
- inspect_display_options_form = InspectDisplayOptionsForm(
- prefix='inspect-display-options-form'
- )
- query_result_file_path = os.path.join(
- current_app.config['DATA_DIR'],
- str(current_user.id),
- 'query_results',
- str(query_result.id),
- query_result.filename
- )
- with open(query_result_file_path, 'r') as query_result_file:
- query_result_file_content = json.load(query_result_file)
- return render_template('corpora/query_results/inspect.html.j2',
- display_options_form=display_options_form,
- inspect_display_options_form=inspect_display_options_form,
- query_result_file_content=query_result_file_content,
- query_metadata=query_metadata,
- title='Inspect query result')
-
-
-@query_results.route('/<int:query_result_id>/delete')
-@login_required
-def delete_query_result(query_result_id):
- query_result = QueryResult.query.get_or_404(query_result_id)
- if not (query_result.creator == current_user
- or current_user.is_administrator()):
- abort(403)
- tasks.delete_query_result(query_result_id)
- flash('Query result deleted!', 'result')
- return redirect(url_for('services.service', service="corpus_analysis"))
-
-
-@query_results.route('/<int:query_result_id>/download')
-@login_required
-def download_query_result(query_result_id):
- query_result = QueryResult.query.get_or_404(query_result_id)
- if not (query_result.creator == current_user
- or current_user.is_administrator()):
- abort(403)
- query_result_dir = os.path.join(current_app.config['DATA_DIR'],
- str(current_user.id),
- 'query_results',
- str(query_result.id))
- return send_from_directory(as_attachment=True,
- directory=query_result_dir,
- filename=query_result.filename)
diff --git a/web/app/services/__init__.py b/web/app/services/__init__.py
index 0bc0cfb2..ea9a403f 100644
--- a/web/app/services/__init__.py
+++ b/web/app/services/__init__.py
@@ -2,4 +2,4 @@ from flask import Blueprint
services = Blueprint('services', __name__)
-from . import views # noqa
+from . import views
diff --git a/web/app/services/views.py b/web/app/services/views.py
index 6fbf2ef0..a6567985 100644
--- a/web/app/services/views.py
+++ b/web/app/services/views.py
@@ -1,5 +1,4 @@
-from flask import (abort, current_app, flash, make_response, render_template,
- url_for)
+from flask import abort, flash, make_response, render_template, url_for
from flask_login import current_user, login_required
from werkzeug.utils import secure_filename
from . import services
@@ -7,19 +6,20 @@ from .. import db
from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
from ..models import Job, JobInput
import json
+import logging
import os
SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
'file-setup': {'name': 'File setup',
'resources': {'mem_mb': 4096, 'n_cores': 4},
- 'add_job_form': AddFileSetupJobForm},
+ 'form': AddFileSetupJobForm},
'nlp': {'name': 'Natural Language Processing',
'resources': {'mem_mb': 4096, 'n_cores': 2},
- 'add_job_form': AddNLPJobForm},
+ 'form': AddNLPJobForm},
'ocr': {'name': 'Optical Character Recognition',
'resources': {'mem_mb': 8192, 'n_cores': 4},
- 'add_job_form': AddOCRJobForm}}
+ 'form': AddOCRJobForm}}
@services.route('/<service>', methods=['GET', 'POST'])
@@ -30,54 +30,49 @@ def service(service):
if service == 'corpus_analysis':
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'])
- add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form')
- if add_job_form.is_submitted():
- if not add_job_form.validate():
- return make_response(add_job_form.errors, 400)
+ form = SERVICES[service]['form'](prefix='add-job-form')
+ if form.is_submitted():
+ if not form.validate():
+ return make_response(form.errors, 400)
service_args = []
if service == 'nlp':
- service_args.append('-l {}'.format(add_job_form.language.data))
- if add_job_form.check_encoding.data:
+ service_args.append('-l {}'.format(form.language.data))
+ if form.check_encoding.data:
service_args.append('--check-encoding')
if service == 'ocr':
- service_args.append('-l {}'.format(add_job_form.language.data))
- if add_job_form.binarization.data:
+ service_args.append('-l {}'.format(form.language.data))
+ if form.binarization.data:
service_args.append('--binarize')
job = Job(creator=current_user,
- description=add_job_form.description.data,
+ description=form.description.data,
mem_mb=SERVICES[service]['resources']['mem_mb'],
n_cores=SERVICES[service]['resources']['n_cores'],
service=service, service_args=json.dumps(service_args),
- service_version=add_job_form.version.data,
- status='preparing', title=add_job_form.title.data)
+ service_version=form.version.data,
+ status='preparing', title=form.title.data)
if job.service != 'corpus_analysis':
job.create_secure_filename()
db.session.add(job)
db.session.commit()
- relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
- absolut_dir = os.path.join(current_app.config['DATA_DIR'],
- relative_dir)
try:
- os.makedirs(absolut_dir)
+ os.makedirs(job.path)
except OSError:
- job.delete()
- flash('Internal Server Error', 'job')
- return make_response({'redirect_url': url_for('services.service',
- service=service)},
- 500)
+ logging.error('Make dir {} led to an OSError!'.format(job.path))
+ db.session.delete(job)
+ db.session.commit()
+ flash('Internal Server Error', 'error')
+ return make_response(
+ {'redirect_url': url_for('.service', service=service)}, 500)
else:
- for file in add_job_form.files.data:
+ for file in form.files.data:
filename = secure_filename(file.filename)
- file.save(os.path.join(absolut_dir, filename))
- job_input = JobInput(dir=relative_dir, filename=filename,
- job=job)
+ job_input = JobInput(dir=job.path, filename=filename, job=job)
+ file.save(job_input.path)
db.session.add(job_input)
job.status = 'submitted'
db.session.commit()
- url = url_for('jobs.job', job_id=job.id)
- flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
+ flash('Job "{}" added'.format(job.title), 'job')
return make_response(
{'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
return render_template('services/{}.html.j2'.format(service),
- title=SERVICES[service]['name'],
- add_job_form=add_job_form)
+ form=form, title=SERVICES[service]['name'])
diff --git a/web/app/settings/forms.py b/web/app/settings/forms.py
index 6f7abeef..5c822fd9 100644
--- a/web/app/settings/forms.py
+++ b/web/app/settings/forms.py
@@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
'Benutzername',
validators=[DataRequired(),
Length(1, 64),
- Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
+ Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)
diff --git a/web/app/settings/views.py b/web/app/settings/views.py
index 1bd4a07f..a7fc0b38 100644
--- a/web/app/settings/views.py
+++ b/web/app/settings/views.py
@@ -1,13 +1,9 @@
-from flask import current_app, flash, redirect, render_template, url_for
+from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required, logout_user
from . import settings, tasks
from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
EditNotificationSettingsForm)
from .. import db
-from ..decorators import admin_required
-from ..models import Role, User
-import os
-import uuid
@settings.route('/')
@@ -26,8 +22,7 @@ def change_password():
flash('Your password has been updated.')
return redirect(url_for('.change_password'))
return render_template('settings/change_password.html.j2',
- form=form,
- title='Change password')
+ form=form, title='Change password')
@settings.route('/edit_general_settings', methods=['GET', 'POST'])
@@ -40,12 +35,12 @@ def edit_general_settings():
current_user.username = form.username.data
db.session.commit()
flash('Your changes have been saved.')
+ return redirect(url_for('.edit_general_settings'))
form.dark_mode.data = current_user.setting_dark_mode
form.email.data = current_user.email
form.username.data = current_user.username
return render_template('settings/edit_general_settings.html.j2',
- form=form,
- title='General settings')
+ form=form, title='General settings')
@settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@@ -59,13 +54,13 @@ def edit_notification_settings():
form.job_status_site_notifications.data
db.session.commit()
flash('Your changes have been saved.')
+ return redirect(url_for('.edit_notification_settings'))
form.job_status_mail_notifications.data = \
current_user.setting_job_status_mail_notifications
form.job_status_site_notifications.data = \
current_user.setting_job_status_site_notifications
return render_template('settings/edit_notification_settings.html.j2',
- form=form,
- title='Notification settings')
+ form=form, title='Notification settings')
@settings.route('/delete')
@@ -76,5 +71,5 @@ def delete():
"""
tasks.delete_user(current_user.id)
logout_user()
- flash('Your account has been deleted!')
+ flash('Your account has been marked for deletion!')
return redirect(url_for('main.index'))
diff --git a/web/app/tasks/__init__.py b/web/app/tasks/__init__.py
index ba33a1fe..9bd21af6 100644
--- a/web/app/tasks/__init__.py
+++ b/web/app/tasks/__init__.py
@@ -11,15 +11,11 @@ def check_corpora():
corpora = Corpus.query.all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
corpus_utils.create_build_corpus_service(corpus)
- for corpus in filter(lambda corpus: (corpus.status == 'queued'
- or corpus.status == 'running'),
- corpora):
+ for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora): # noqa
corpus_utils.checkout_build_corpus_service(corpus)
- for corpus in filter(lambda corpus: corpus.status == 'start analysis',
- corpora):
+ for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora): # noqa
corpus_utils.create_cqpserver_container(corpus)
- for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
- corpora):
+ for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora): # noqa
corpus_utils.remove_cqpserver_container(corpus)
db.session.commit()
@@ -28,8 +24,6 @@ def check_jobs():
jobs = Job.query.all()
for job in filter(lambda job: job.status == 'submitted', jobs):
job_utils.create_job_service(job)
- for job in filter(lambda job: job.status == 'queued', jobs):
- job_utils.checkout_job_service(job)
- for job in filter(lambda job: job.status == 'running', jobs):
+ for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
job_utils.checkout_job_service(job)
db.session.commit()
diff --git a/web/app/tasks/corpus_utils.py b/web/app/tasks/corpus_utils.py
index c06b19ac..dd37ad62 100644
--- a/web/app/tasks/corpus_utils.py
+++ b/web/app/tasks/corpus_utils.py
@@ -1,4 +1,3 @@
-from flask import current_app
from . import docker_client
import docker
import logging
@@ -7,20 +6,14 @@ import shutil
def create_build_corpus_service(corpus):
- corpus_dir = os.path.join(current_app.config['DATA_DIR'],
- str(corpus.user_id),
- 'corpora',
- str(corpus.id))
- corpus_data_dir = os.path.join(corpus_dir, 'data')
- corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
- corpus_registry_dir = os.path.join(corpus_dir, 'registry')
- if os.path.exists(corpus_data_dir):
- shutil.rmtree(corpus_data_dir)
- if os.path.exists(corpus_registry_dir):
- shutil.rmtree(corpus_registry_dir)
+ corpus_data_dir = os.path.join(corpus.path, 'data')
+ shutil.rmtree(corpus_data_dir, ignore_errors=True)
os.mkdir(corpus_data_dir)
+ corpus_registry_dir = os.path.join(corpus.path, 'registry')
+ shutil.rmtree(corpus_registry_dir, ignore_errors=True)
os.mkdir(corpus_registry_dir)
- service_args = {
+ corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
+ service_kwargs = {
'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
@@ -32,30 +25,34 @@ def create_build_corpus_service(corpus):
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()
}
- service_image = \
- 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
+ service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
try:
- docker_client.services.create(service_image, **service_args)
+ docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
- logging.error('create_build_corpus_service({}): '.format(corpus.id)
- + '{} (status: {} -> failed)'.format(e, corpus.status))
- corpus.status = 'failed'
+ logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
else:
corpus.status = 'queued'
- finally:
- # TODO: send email
- pass
def checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id)
try:
service = docker_client.services.get(service_name)
- except docker.errors.NotFound as e:
- logging.error('checkout_build_corpus_service({}):'.format(corpus.id)
- + ' {} (stauts: {} -> failed)'.format(e, corpus.status))
+ except docker.errors.NotFound:
+ logging.error('Get "{}" service raised '.format(service_name)
+ + '[docker-NotFound] The service does not exist. '
+ + '(corpus.status: {} -> failed)'.format(corpus.status))
corpus.status = 'failed'
- # TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
+ except docker.errors.APIError as e:
+ logging.error('Get "{}" service raised '.format(service_name)
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
+ except docker.errors.InvalidVersion:
+ logging.error('Get "{}" service raised '.format(service_name)
+ + '[docker-InvalidVersion] One of the arguments is '
+ + 'not supported with the current API version.')
else:
service_tasks = service.tasks()
if not service_tasks:
@@ -63,25 +60,23 @@ def checkout_build_corpus_service(corpus):
task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running'
- elif corpus.status == 'running' and task_state == 'complete':
- service.remove()
- corpus.status = 'prepared'
- elif corpus.status == 'running' and task_state == 'failed':
- service.remove()
- corpus.status = task_state
- finally:
- # TODO: send email
- pass
+ elif corpus.status == 'running' and task_state in ['complete', 'failed']: # noqa
+ try:
+ service.remove()
+ except docker.errors.APIError as e:
+ logging.error('Remove "{}" service raised '.format(service_name) # noqa
+ + '[docker-APIError] The server returned an error. ' # noqa
+ + 'Details: {}'.format(e))
+ return
+ else:
+ corpus.status = 'prepared' if task_state == 'complete' \
+ else 'failed'
def create_cqpserver_container(corpus):
- corpus_dir = os.path.join(current_app.config['DATA_DIR'],
- str(corpus.user_id),
- 'corpora',
- str(corpus.id))
- corpus_data_dir = os.path.join(corpus_dir, 'data')
- corpus_registry_dir = os.path.join(corpus_dir, 'registry')
- container_args = {
+ corpus_data_dir = os.path.join(corpus.path, 'data')
+ corpus_registry_dir = os.path.join(corpus.path, 'registry')
+ container_kwargs = {
'command': 'cqpserver',
'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw',
@@ -89,20 +84,43 @@ def create_cqpserver_container(corpus):
'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default'
}
- container_image = \
- 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
+ container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa
+ # Check if a cqpserver container already exists. If this is the case,
+ # remove it and create a new one
try:
- container = docker_client.containers.get(container_args['name'])
+ container = docker_client.containers.get(container_kwargs['name'])
except docker.errors.NotFound:
pass
- except docker.errors.DockerException:
+ except docker.errors.APIError as e:
+ logging.error('Get "{}" container raised '.format(container_kwargs['name'])
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
return
else:
- container.remove(force=True)
+ try:
+ container.remove(force=True)
+ except docker.errors.APIError as e:
+ logging.error('Remove "{}" container raised '.format(container_kwargs['name'])
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
+ return
try:
- docker_client.containers.run(container_image, **container_args)
- except docker.errors.DockerException:
- return
+ docker_client.containers.run(container_image, **container_kwargs)
+ except docker.errors.ContainerError:
+ # This case should not occur, because detach is True.
+ logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ + '[docker-ContainerError] The container exits with a '
+ + 'non-zero exit code and detach is False.')
+ corpus.status = 'failed'
+ except docker.errors.ImageNotFound:
+ logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ + '[docker-ImageNotFound] The specified image does not '
+ + 'exist.')
+ corpus.status = 'failed'
+ except docker.errors.APIError as e:
+ logging.error('Run "{}" container raised '.format(container_kwargs['name'])
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
else:
corpus.status = 'analysing'
@@ -113,8 +131,17 @@ def remove_cqpserver_container(corpus):
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:
pass
- except docker.errors.DockerException:
+ except docker.errors.APIError as e:
+ logging.error('Get "{}" container raised '.format(container_name)
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
return
else:
- container.remove(force=True)
+ try:
+ container.remove(force=True)
+ except docker.errors.APIError as e:
+ logging.error('Remove "{}" container raised '.format(container_name)
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
+ return
corpus.status = 'prepared'
diff --git a/web/app/tasks/job_utils.py b/web/app/tasks/job_utils.py
index 2094e7cd..68db9507 100644
--- a/web/app/tasks/job_utils.py
+++ b/web/app/tasks/job_utils.py
@@ -1,7 +1,7 @@
from datetime import datetime
-from flask import current_app
from . import docker_client
from .. import db
+from ..email import create_message, send
from ..models import JobResult
import docker
import logging
@@ -10,51 +10,60 @@ import os
def create_job_service(job):
- job_dir = os.path.join(current_app.config['DATA_DIR'],
- str(job.user_id),
- 'jobs',
- str(job.id))
cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup':
cmd += ' -f {}'.format(job.secure_filename)
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
cmd += ' ' + ' '.join(json.loads(job.service_args))
- service_args = {'command': cmd,
- 'constraints': ['node.role==worker'],
- 'labels': {'origin': 'nopaque',
- 'type': 'service.{}'.format(job.service),
- 'job_id': str(job.id)},
- 'mounts': [job_dir + ':/files:rw'],
- 'name': 'job_{}'.format(job.id),
- 'resources': docker.types.Resources(
- cpu_reservation=job.n_cores * (10 ** 9),
- mem_reservation=job.mem_mb * (10 ** 6)),
- 'restart_policy': docker.types.RestartPolicy()}
+ service_kwargs = {'command': cmd,
+ 'constraints': ['node.role==worker'],
+ 'labels': {'origin': 'nopaque',
+ 'type': 'service.{}'.format(job.service),
+ 'job_id': str(job.id)},
+ 'mounts': [job.path + ':/files:rw'],
+ 'name': 'job_{}'.format(job.id),
+ 'resources': docker.types.Resources(
+ cpu_reservation=job.n_cores * (10 ** 9),
+ mem_reservation=job.mem_mb * (10 ** 6)
+ ),
+ 'restart_policy': docker.types.RestartPolicy()}
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
+ job.service + ':' + job.service_version)
try:
- docker_client.services.create(service_image, **service_args)
+ docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
- logging.error('create_job_service({}): {} '.format(job.id, e)
- + '(status: {} -> failed)'.format(job.status))
- job.status = 'failed'
+ logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
else:
job.status = 'queued'
- finally:
- # TODO: send email
- pass
+ msg = create_message(
+ job.creator.email,
+ 'Status update for your Job "{}"'.format(job.title),
+ 'tasks/email/notification',
+ job=job
+ )
+ send(msg)
def checkout_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
- except docker.errors.NotFound as e:
- logging.error('checkout_job_service({}): {} '.format(job.id, e)
- + '(status: {} -> submitted)'.format(job.status))
- job.status = 'submitted'
- # TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
+ except docker.errors.NotFound:
+ logging.error('Get "{}" service raised '.format(service_name)
+ + '[docker-NotFound] The service does not exist. '
+ + '(job.status: {} -> failed)'.format(job.status))
+ job.status = 'failed'
+ except docker.errors.APIError as e:
+ logging.error('Get "{}" service raised '.format(service_name)
+ + '[docker-APIError] The server returned an error. '
+ + 'Details: {}'.format(e))
+ except docker.errors.InvalidVersion:
+ logging.error('Get "{}" service raised '.format(service_name)
+ + '[docker-InvalidVersion] One of the arguments is '
+ + 'not supported with the current API version.')
else:
service_tasks = service.tasks()
if not service_tasks:
@@ -62,22 +71,16 @@ def checkout_job_service(job):
task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending':
job.status = 'running'
- elif job.status == 'queued' and task_state == 'complete':
+ elif job.status == 'running' and task_state == 'complete':
service.remove()
job.end_date = datetime.utcnow()
job.status = task_state
if task_state == 'complete':
- results_dir = os.path.join(current_app.config['DATA_DIR'],
- str(job.user_id),
- 'jobs',
- str(job.id),
- 'output')
- results = filter(lambda x: x.endswith('.zip'),
- os.listdir(results_dir))
- for result in results:
- job_result = JobResult(dir=results_dir,
- filename=result,
- job_id=job.id)
+ job_results_dir = os.path.join(job.path, 'output')
+ job_results = filter(lambda x: x.endswith('.zip'),
+ os.listdir(job_results_dir))
+ for job_result in job_results:
+ job_result = JobResult(filename=job_result, job=job)
db.session.add(job_result)
elif job.status == 'running' and task_state == 'failed':
service.remove()
@@ -85,6 +88,13 @@ def checkout_job_service(job):
job.status = task_state
finally:
# TODO: send email
+ msg = create_message(
+ job.creator.email,
+ '[nopaque] Status update for your Job "{}"'.format(job.title),
+ 'tasks/email/notification',
+ job=job
+ )
+ send(msg)
pass
diff --git a/web/app/templates/auth/login.html.j2 b/web/app/templates/auth/login.html.j2
index fa99f31a..db83e8b0 100644
--- a/web/app/templates/auth/login.html.j2
+++ b/web/app/templates/auth/login.html.j2
@@ -35,20 +35,20 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
- {{ login_form.hidden_tag() }}
- {{ wtf.render_field(login_form.user, material_icon='person') }}
- {{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.user, material_icon='person') }}
+ {{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
- {{ wtf.render_field(login_form.remember_me) }}
+ {{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(login_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/auth/register.html.j2 b/web/app/templates/auth/register.html.j2
index e41990b3..7b8db08d 100644
--- a/web/app/templates/auth/register.html.j2
+++ b/web/app/templates/auth/register.html.j2
@@ -34,14 +34,14 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
- {{ registration_form.hidden_tag() }}
- {{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }}
- {{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }}
- {{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }}
- {{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }}
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
+ {{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
+ {{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
+ {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
- {{ wtf.render_field(registration_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/auth/reset_password.html.j2 b/web/app/templates/auth/reset_password.html.j2
index 21da27db..4002c158 100644
--- a/web/app/templates/auth/reset_password.html.j2
+++ b/web/app/templates/auth/reset_password.html.j2
@@ -20,12 +20,12 @@
<div class="card">
<form method="POST">
<div class="card-content">
- {{ reset_password_form.hidden_tag() }}
- {{ wtf.render_field(reset_password_form.password, data_length='128') }}
- {{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }}
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.password, data_length='128') }}
+ {{ wtf.render_field(form.password_confirmation, data_length='128') }}
</div>
<div class="card-action right-align">
- {{ wtf.render_field(reset_password_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/auth/reset_password_request.html.j2 b/web/app/templates/auth/reset_password_request.html.j2
index 0a2baf9f..07a0808e 100644
--- a/web/app/templates/auth/reset_password_request.html.j2
+++ b/web/app/templates/auth/reset_password_request.html.j2
@@ -20,11 +20,11 @@
<div class="card">
<form method="POST">
<div class="card-content">
- {{ reset_password_request_form.hidden_tag() }}
- {{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }}
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
- {{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/corpora/add_corpus.html.j2 b/web/app/templates/corpora/add_corpus.html.j2
index 219b098f..c5e5e3b9 100644
--- a/web/app/templates/corpora/add_corpus.html.j2
+++ b/web/app/templates/corpora/add_corpus.html.j2
@@ -27,18 +27,18 @@
<div class="card">
<form method="POST">
<div class="card-content">
- {{ add_corpus_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
- {{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
- {{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }}
+ {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(add_corpus_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/corpora/add_corpus_file.html.j2 b/web/app/templates/corpora/add_corpus_file.html.j2
index 4ca54776..14b09304 100644
--- a/web/app/templates/corpora/add_corpus_file.html.j2
+++ b/web/app/templates/corpora/add_corpus_file.html.j2
@@ -27,24 +27,24 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
- {{ add_corpus_file_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
- {{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }}
+ {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
- {{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
- {{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }}
+ {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
<div class="col s12">
- {{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
+ {{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@@ -52,7 +52,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
<div class="collapsible-body">
- {% for field in add_corpus_file_form
+ {% for field in form
if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}
diff --git a/web/app/templates/corpora/analyse_corpus.html.j2 b/web/app/templates/corpora/analyse_corpus.html.j2
index 29dba027..af44f86f 100644
--- a/web/app/templates/corpora/analyse_corpus.html.j2
+++ b/web/app/templates/corpora/analyse_corpus.html.j2
@@ -155,7 +155,7 @@ import {
*/
document.addEventListener("DOMContentLoaded", () => {
// Initialize the client for server client communication in dynamic mode
- let corpusId = {{ corpus_id }}
+ let corpusId = {{ corpus.id }}
const client = new Client({'corpusId': corpusId,
'socket': nopaque.socket,
'logging': true,
diff --git a/web/app/templates/corpora/corpus_file.html.j2 b/web/app/templates/corpora/corpus_file.html.j2
index 7548604b..d8022303 100644
--- a/web/app/templates/corpora/corpus_file.html.j2
+++ b/web/app/templates/corpora/corpus_file.html.j2
@@ -20,23 +20,23 @@
<div class="col s12">
<form method="POST">
- {{ edit_corpus_file_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="card">
<div class="card-content">
<div class="row">
<div class="col s12 m4">
- {{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }}
+ {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
- {{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
- {{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }}
+ {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@@ -44,7 +44,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
<div class="collapsible-body">
- {% for field in edit_corpus_file_form
+ {% for field in form
if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}
diff --git a/web/app/templates/corpora/import_corpus.html.j2 b/web/app/templates/corpora/import_corpus.html.j2
index 0bc47d8e..1a8caf08 100644
--- a/web/app/templates/corpora/import_corpus.html.j2
+++ b/web/app/templates/corpora/import_corpus.html.j2
@@ -1,4 +1,4 @@
-{% extends "nopaque.html.j2" %}
+ {% extends "nopaque.html.j2" %}
{% from '_colors.html.j2' import colors %}
{% import 'materialize/wtf.html.j2' as wtf %}
@@ -27,23 +27,23 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
- {{ import_corpus_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
- {{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
- {{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }}
+ {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
<div class="row">
<div class="col s12">
- {{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
+ {{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(import_corpus_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/corpora/query_results/add_query_result.html.j2 b/web/app/templates/corpora/query_results/add_query_result.html.j2
index 97a83983..6ad4ebd6 100644
--- a/web/app/templates/corpora/query_results/add_query_result.html.j2
+++ b/web/app/templates/corpora/query_results/add_query_result.html.j2
@@ -27,21 +27,21 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
- {{ add_query_result_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
- {{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
- {{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }}
+ {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
- {{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }}
+ {{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(add_query_result_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
</form>
diff --git a/web/app/templates/main/index.html.j2 b/web/app/templates/main/index.html.j2
index 8268de6a..0ded9824 100644
--- a/web/app/templates/main/index.html.j2
+++ b/web/app/templates/main/index.html.j2
@@ -159,20 +159,20 @@
<form method="POST">
<div class="card-content">
<span class="card-title">Log in</span>
- {{ login_form.hidden_tag() }}
- {{ wtf.render_field(login_form.user, material_icon='person') }}
- {{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.user, material_icon='person') }}
+ {{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
- {{ wtf.render_field(login_form.remember_me) }}
+ {{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(login_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/nopaque.html.j2 b/web/app/templates/nopaque.html.j2
index 81eadb6f..a54fda33 100644
--- a/web/app/templates/nopaque.html.j2
+++ b/web/app/templates/nopaque.html.j2
@@ -231,9 +231,9 @@
</div>
<div class="col s12 m9 right-align">
<a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
- {% if config.CONTACT_EMAIL_ADRESS %}
- <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a>
- <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a>
+ {% if config.NOPAQUE_CONTACT %}
+ <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
+ <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% endif %}
<a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
</div>
diff --git a/web/app/templates/services/file-setup.html.j2 b/web/app/templates/services/file-setup.html.j2
index 31f5e824..2674545c 100644
--- a/web/app/templates/services/file-setup.html.j2
+++ b/web/app/templates/services/file-setup.html.j2
@@ -48,24 +48,24 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
- {{ add_job_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
- {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
- {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+ {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
- {{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
+ {{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
</div>
<div class="col s12 hide">
- {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+ {{ wtf.render_field(form.version, material_icon='apps') }}
</div>
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(add_job_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/services/nlp.html.j2 b/web/app/templates/services/nlp.html.j2
index 83d83396..4c5018bc 100644
--- a/web/app/templates/services/nlp.html.j2
+++ b/web/app/templates/services/nlp.html.j2
@@ -66,34 +66,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
- {{ add_job_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
- {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
- {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+ {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
- {{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }}
+ {{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
</div>
<div class="col s12 l4">
- {{ wtf.render_field(add_job_form.language, material_icon='language') }}
+ {{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
- {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+ {{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
- <p>{{ add_job_form.check_encoding.label.text }}</p>
+ <p>{{ form.check_encoding.label.text }}</p>
<p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
- {{ add_job_form.check_encoding() }}
+ {{ form.check_encoding() }}
<span class="lever"></span>
</label>
</div>
@@ -107,7 +107,7 @@
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(add_job_form.submit, material_icon='send') }}
+ {{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/services/ocr.html.j2 b/web/app/templates/services/ocr.html.j2
index 00608e0d..09759e0c 100644
--- a/web/app/templates/services/ocr.html.j2
+++ b/web/app/templates/services/ocr.html.j2
@@ -48,34 +48,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
- {{ add_job_form.hidden_tag() }}
+ {{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
- {{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
+ {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
- {{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
+ {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
- {{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
+ {{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
</div>
<div class="col s12 l4">
- {{ wtf.render_field(add_job_form.language, material_icon='language') }}
+ {{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
- {{ wtf.render_field(add_job_form.version, material_icon='apps') }}
+ {{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
- <p>{{ add_job_form.binarization.label.text }}</p>
+ <p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
- {{ add_job_form.binarization() }}
+ {{ form.binarization() }}
<span class="lever"></span>
</label>
</div>
@@ -134,7 +134,7 @@
</div>
</div>
<div class="card-action right-align">
- {{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }}
+ {{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
</div>
</form>
</div>
diff --git a/web/app/templates/tasks/email/notification.html.j2 b/web/app/templates/tasks/email/notification.html.j2
index 79f0e2dd..1aac0bf7 100644
--- a/web/app/templates/tasks/email/notification.html.j2
+++ b/web/app/templates/tasks/email/notification.html.j2
@@ -1,9 +1,8 @@
-<p>Dear <b>{{ user.username }}</b>,</p>
+<p>Dear <b>{{ job.creator.username }}</b>,</p>
-<p>The status of your Job/Corpus({{ job.id }}) with the title <b>"{{ job.title }}"</b> has changed!</p>
+<p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
<p>It is now <b>{{ job.status }}</b>!</p>
-<p>Time of this status update was: <b>{time} UTC</b></p>
-<p>You can access your Job/Corpus here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
+<p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>Kind regards!<br>Your nopaque team</p>
diff --git a/web/app/templates/tasks/email/notification.txt.j2 b/web/app/templates/tasks/email/notification.txt.j2
index 25d797c8..03012b3e 100644
--- a/web/app/templates/tasks/email/notification.txt.j2
+++ b/web/app/templates/tasks/email/notification.txt.j2
@@ -1,10 +1,9 @@
-Dear {{ user.username }},
+Dear {{ job.creator.username }},
-The status of your Job/Corpus({{ job.id }}) with the title "{{ job.title }}" has changed!
+The status of your Job "{{ job.title }}" has changed!
It is now {{ job.status }}!
-Time of this status update was: {time} UTC
-You can access your Job/Corpus here: {{ url_for('jobs.job', job_id=job.id) }}
+You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
Kind regards!
Your nopaque team
diff --git a/web/boot.sh b/web/boot.sh
index f39bb4c8..9c87cfd1 100755
--- a/web/boot.sh
+++ b/web/boot.sh
@@ -1,5 +1,6 @@
#!/bin/bash
source venv/bin/activate
+
while true; do
flask deploy
if [[ "$?" == "0" ]]; then
diff --git a/web/config.py b/web/config.py
index 4ca3704f..97e07697 100644
--- a/web/config.py
+++ b/web/config.py
@@ -7,103 +7,96 @@ ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
- ''' # Cookies # '''
- REMEMBER_COOKIE_HTTPONLY = True
- REMEMBER_COOKIE_SECURE = os.environ.get(
- 'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
- SESSION_COOKIE_SECURE = os.environ.get(
- 'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
+ ''' # Flask # '''
+ SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')
+ SESSION_COOKIE_SECURE = \
+ os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true'
- ''' # Database # '''
+ ''' # Flask-Login # '''
+ REMEMBER_COOKIE_HTTPONLY = True
+ REMEMBER_COOKIE_SECURE = \
+ os.environ.get('REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
+
+ ''' # Flask-Mail # '''
+ MAIL_DEFAULT_SENDER = os.environ.get('MAIL_DEFAULT_SENDER')
+ MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
+ MAIL_PORT = int(os.environ.get('MAIL_PORT'))
+ MAIL_SERVER = os.environ.get('MAIL_SERVER')
+ MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
+ MAIL_USE_SSL = os.environ.get('MAIL_USE_SSL', 'false').lower() == 'true'
+ MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', 'false').lower() == 'true'
+
+ ''' # Flask-SQLAlchemy # '''
SQLALCHEMY_RECORD_QUERIES = True
SQLALCHEMY_TRACK_MODIFICATIONS = False
- ''' # Email # '''
- MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
- MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
- MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
- MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
- MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
- MAIL_USE_SSL = os.environ.get(
- 'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
- MAIL_USE_TLS = os.environ.get(
- 'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
-
- ''' # General # '''
- ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
- ALLOWED_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
- CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
- DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
- SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
-
- ''' # Logging # '''
- LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
- '%Y-%m-%d %H:%M:%S')
- LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
- os.path.join(ROOT_DIR, 'nopaque.log'))
- LOG_FORMAT = os.environ.get(
- 'NOPAQUE_LOG_FORMAT',
- '[%(asctime)s] %(levelname)s in '
- '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
- )
- LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
-
- ''' # Message queue # '''
- SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
- 'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
-
- ''' # Proxy fix # '''
- PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
- PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
- PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
- PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
- PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
+ ''' # nopaque # '''
+ NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN')
+ NOPAQUE_CONTACT = os.environ.get('NOPAQUE_CONTACT')
+ NOPAQUE_DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
+ NOPAQUE_MAIL_SUBJECT_PREFIX = '[nopaque]'
+ NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
+ os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
+ NOPAQUE_USERNAME_REGEX = '^[A-Za-zÄÖÜäöüß0-9_.]*$'
@classmethod
def init_app(cls, app):
- # Set up logging according to the corresponding (LOG_*) variables
- logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
- filename=cls.LOG_FILE,
- format=cls.LOG_FORMAT,
- level=cls.LOG_LEVEL)
+ # Set up logging according to the corresponding (NOPAQUE_LOG_*)
+ # environment variables
+ basic_config_kwargs = {
+ 'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
+ '%Y-%m-%d %H:%M:%S'),
+ 'filename': os.environ.get('NOPAQUE_LOG_FILE',
+ os.path.join(ROOT_DIR, 'nopaque.log')),
+ 'format': os.environ.get(
+ 'NOPAQUE_LOG_FORMAT',
+ '[%(asctime)s] %(levelname)s in '
+ '%(pathname)s (function: %(funcName)s, line: %(lineno)d): '
+ '%(message)s'
+ ),
+ 'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
+ }
+ logging.basicConfig(**basic_config_kwargs)
# Set up and apply the ProxyFix middleware according to the
- # corresponding (PROXY_FIX_*) variables
- app.wsgi_app = ProxyFix(app.wsgi_app,
- x_for=cls.PROXY_FIX_X_FOR,
- x_host=cls.PROXY_FIX_X_HOST,
- x_port=cls.PROXY_FIX_X_PORT,
- x_prefix=cls.PROXY_FIX_X_PREFIX,
- x_proto=cls.PROXY_FIX_X_PROTO)
+ # corresponding (NOPAQUE_PROXY_FIX_*) environment variables
+ proxy_fix_kwargs = {
+ 'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')),
+ 'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')),
+ 'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')),
+ 'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')),
+ 'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
+ }
+ app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
class DevelopmentConfig(Config):
- ''' # Database # '''
+ ''' # Flask # '''
+ DEBUG = True
+
+ ''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
- 'NOPAQUE_DEV_DATABASE_URL',
+ 'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_dev'
)
- ''' # General # '''
- DEBUG = True
-
class ProductionConfig(Config):
- ''' # Database # '''
+ ''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
- 'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque')
+ 'SQLALCHEMY_DATABASE_URI', 'postgresql://nopaque:nopaque@db/nopaque')
class TestingConfig(Config):
- ''' # Database # '''
+ ''' # Flask # '''
+ TESTING = True
+ WTF_CSRF_ENABLED = False
+
+ ''' # Flask-SQLAlchemy # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
- 'NOPAQUE_TEST_DATABASE_URL',
+ 'SQLALCHEMY_DATABASE_URI',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
- ''' # General # '''
- TESTING = True
- WTF_CSRF_ENABLED = False
-
config = {'development': DevelopmentConfig,
'production': ProductionConfig,
diff --git a/web/nopaque.py b/web/nopaque.py
index 43d69c38..5c5c5af5 100644
--- a/web/nopaque.py
+++ b/web/nopaque.py
@@ -17,8 +17,7 @@ if os.path.exists(DOTENV_FILE):
from app import create_app, db, socketio # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
- NotificationData, NotificationEmailData, QueryResult,
- Role, User) # noqa
+ QueryResult, Role, User) # noqa
from flask_migrate import Migrate, upgrade # noqa
@@ -34,8 +33,6 @@ def make_shell_context():
'Job': Job,
'JobInput': JobInput,
'JobResult': JobResult,
- 'NotificationData': NotificationData,
- 'NotificationEmailData': NotificationEmailData,
'QueryResult': QueryResult,
'Role': Role,
'User': User}
@@ -53,9 +50,9 @@ def deploy():
@app.cli.command()
def tasks():
- from app.tasks import process_corpora, process_jobs
- process_corpora()
- process_jobs()
+ from app.tasks import check_corpora, check_jobs
+ check_corpora()
+ check_jobs()
@app.cli.command()
--
GitLab