diff --git a/.dockerignore b/.dockerignore
index 1269488f7fb1f4b56a8c0e5eb48cecbfadfa9219..07d50b4bec6da994020d9a0c82a855d6b7395df2 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1 +1,29 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+README.md
+
+
data
diff --git a/Dockerfile b/Dockerfile
index 6ec450b5d0cccdea5b364151c1c5f3f4adb68794..0a2309f574ed575c34ed0c14353a9b87065f5ba0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.8.13-slim-buster
+FROM python:3.8.10-slim-buster
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
@@ -7,10 +7,12 @@ LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
ARG DOCKER_GID
ARG UID
ARG GID
-ENV LANG=C.UTF-8
ENV FLASK_APP nopaque.py
+ENV LANG=C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
RUN apt-get update \
diff --git a/README.md b/README.md
index 3b45225d35f0c22890e3cdb73363368e83973bbb..30c84ae244c07b3bb18ad14bb3ac4f35b704e452 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ username@hostname:~$ <YOUR EDITOR> db.env
username@hostname:~$ <YOUR EDITOR> .env
# Create docker-compose.override.yml file
username@hostname:~$ touch docker-compose.override.yml
-# Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples in docker-compose.<example>.yml)
+# Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples inside the docker-compose directory)
username@hostname:~$ <YOUR EDITOR> docker-compose.override.yml
# Build docker images
username@hostname:~$ docker-compose build
diff --git a/app/__init__.py b/app/__init__.py
index 145f2b3eef971455974eef321c849b398f8c25c3..294421dc7012d195cf295da7cfa300a6d75aa370 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -1,9 +1,12 @@
+from apifairy import APIFairy
from config import Config
+from docker import DockerClient
from flask import Flask
from flask_apscheduler import APScheduler
from flask_assets import Environment
from flask_login import LoginManager
from flask_mail import Mail
+from flask_marshmallow import Marshmallow
from flask_migrate import Migrate
from flask_paranoid import Paranoid
from flask_socketio import SocketIO
@@ -11,18 +14,21 @@ from flask_sqlalchemy import SQLAlchemy
from flask_hashids import Hashids
-assets: Environment = Environment()
-db: SQLAlchemy = SQLAlchemy()
-hashids: Hashids = Hashids()
-login: LoginManager = LoginManager()
+apifairy = APIFairy()
+assets = Environment()
+db = SQLAlchemy()
+docker_client = DockerClient()
+hashids = Hashids()
+login = LoginManager()
login.login_view = 'auth.login'
login.login_message = 'Please log in to access this page.'
-mail: Mail = Mail()
-migrate: Migrate = Migrate()
-paranoid: Paranoid = Paranoid()
+ma = Marshmallow()
+mail = Mail()
+migrate = Migrate()
+paranoid = Paranoid()
paranoid.redirect_view = '/'
-scheduler: APScheduler = APScheduler() # TODO: Use this!
-socketio: SocketIO = SocketIO()
+scheduler = APScheduler()
+socketio = SocketIO()
def create_app(config: Config = Config) -> Flask:
@@ -30,19 +36,24 @@ def create_app(config: Config = Config) -> Flask:
app: Flask = Flask(__name__)
app.config.from_object(config)
config.init_app(app)
+ docker_client.login(
+ app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'],
+ password=app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'],
+ registry=app.config['NOPAQUE_DOCKER_REGISTRY']
+ )
+ apifairy.init_app(app)
assets.init_app(app)
db.init_app(app)
hashids.init_app(app)
login.init_app(app)
+ ma.init_app(app)
mail.init_app(app)
migrate.init_app(app, db)
paranoid.init_app(app)
+ scheduler.init_app(app)
socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI']) # noqa
- from app import socketio_event_listeners
- from app import sqlalchemy_event_listeners
-
from .admin import bp as admin_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin')
@@ -52,8 +63,8 @@ def create_app(config: Config = Config) -> Flask:
from .auth import bp as auth_blueprint
app.register_blueprint(auth_blueprint, url_prefix='/auth')
- from .contribute import bp as contribute_blueprint
- app.register_blueprint(contribute_blueprint, url_prefix='/contribute')
+ from .contributions import bp as contributions_blueprint
+ app.register_blueprint(contributions_blueprint, url_prefix='/contributions')
from .corpora import bp as corpora_blueprint
app.register_blueprint(corpora_blueprint, url_prefix='/corpora')
@@ -65,7 +76,7 @@ def create_app(config: Config = Config) -> Flask:
app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
from .main import bp as main_blueprint
- app.register_blueprint(main_blueprint)
+ app.register_blueprint(main_blueprint, url_prefix='/')
from .services import bp as services_blueprint
app.register_blueprint(services_blueprint, url_prefix='/services')
@@ -73,6 +84,9 @@ def create_app(config: Config = Config) -> Flask:
from .settings import bp as settings_blueprint
app.register_blueprint(settings_blueprint, url_prefix='/settings')
+ from .users import bp as users_blueprint
+ app.register_blueprint(users_blueprint, url_prefix='/users')
+
from .test import bp as test_blueprint
app.register_blueprint(test_blueprint, url_prefix='/test')
diff --git a/app/admin/forms.py b/app/admin/forms.py
index 30d994481c61e4cd38510454a828996e5766b8ac..d73d389215a630216092955b05e9181129a7f2dd 100644
--- a/app/admin/forms.py
+++ b/app/admin/forms.py
@@ -10,7 +10,9 @@ class AdminEditUserForm(FlaskForm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.role.choices = [
- (role.hashid, role.name)
- for role in Role.query.order_by(Role.name).all()
- ]
+ self.role.choices = [(x.hashid, x.name) for x in Role.query.all()]
+
+ def prefill(self, user):
+ ''' Pre-fill the form with data of an exististing user '''
+ self.confirmed.data = user.confirmed
+ self.role.data = user.role.hashid
diff --git a/app/admin/routes.py b/app/admin/routes.py
index f117839892593e52a22abad56d3b1253aed1cace..011de1bb0dabd490a8369dd0eddb07146794055b 100644
--- a/app/admin/routes.py
+++ b/app/admin/routes.py
@@ -1,14 +1,14 @@
+from flask import current_app, flash, redirect, render_template, url_for
+from flask_login import login_required
+from threading import Thread
from app import db, hashids
from app.decorators import admin_required
from app.models import Role, User, UserSettingJobStatusMailNotificationLevel
-from app.settings import tasks as settings_tasks
from app.settings.forms import (
EditGeneralSettingsForm,
EditInterfaceSettingsForm,
EditNotificationSettingsForm
)
-from flask import flash, redirect, render_template, url_for
-from flask_login import login_required
from . import bp
from .forms import AdminEditUserForm
@@ -24,20 +24,17 @@ def before_request():
pass
-@bp.route('/')
+@bp.route('')
def index():
return redirect(url_for('.users'))
@bp.route('/users')
def users():
- dict_users = {
- user.id: user.to_dict(backrefs=True, relationships=False)
- for user in User.query.all()
- }
+ json_users = [x.to_json(backrefs=True) for x in User.query.all()]
return render_template(
'admin/users.html.j2',
- dict_users=dict_users,
+ json_users=json_users,
title='Users'
)
@@ -48,59 +45,45 @@ def user(user_id):
return render_template('admin/user.html.j2', title='User', user=user)
-@bp.route('/users/<hashid:user_id>/delete')
-def delete_user(user_id):
- settings_tasks.delete_user(user_id)
- flash('User has been marked for deletion')
- return redirect(url_for('.users'))
-
-
@bp.route('/users/<hashid:user_id>/edit', methods=['GET', 'POST'])
def edit_user(user_id):
user = User.query.get_or_404(user_id)
admin_edit_user_form = AdminEditUserForm(
- prefix='admin_edit_user_form'
+ prefix='admin-edit-user-form'
)
edit_general_settings_form = EditGeneralSettingsForm(
user,
- prefix='edit_general_settings_form'
+ prefix='edit-general-settings-form'
)
edit_interface_settings_form = EditInterfaceSettingsForm(
- prefix='edit_interface_settings_form'
+ prefix='edit-interface-settings-form'
)
edit_notification_settings_form = EditNotificationSettingsForm(
- prefix='edit_notification_settings_form'
+ prefix='edit-notification-settings-form'
)
- if (
- admin_edit_user_form.submit.data
- and admin_edit_user_form.validate()
- ):
+ if (admin_edit_user_form.submit.data
+ and admin_edit_user_form.validate()):
user.confirmed = admin_edit_user_form.confirmed.data
role_id = hashids.decode(admin_edit_user_form.role.data)
user.role = Role.query.get(role_id)
+ db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.edit_user', user_id=user.id))
- if (
- edit_general_settings_form.submit.data
- and edit_general_settings_form.validate()
- ):
+ if (edit_general_settings_form.submit.data
+ and edit_general_settings_form.validate()):
user.email = edit_general_settings_form.email.data
user.username = edit_general_settings_form.username.data
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.edit_user', user_id=user.id))
- if (
- edit_interface_settings_form.submit.data
- and edit_interface_settings_form.validate()
- ):
+ if (edit_interface_settings_form.submit.data
+ and edit_interface_settings_form.validate()):
user.setting_dark_mode = edit_interface_settings_form.dark_mode.data
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.edit_user', user_id=user.id))
- if (
- edit_notification_settings_form.submit.data
- and edit_notification_settings_form.validate()
- ):
+ if (edit_notification_settings_form.submit.data
+ and edit_notification_settings_form.validate()):
user.setting_job_status_mail_notification_level = \
UserSettingJobStatusMailNotificationLevel[
edit_notification_settings_form.job_status_mail_notification_level.data # noqa
@@ -108,13 +91,10 @@ def edit_user(user_id):
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.edit_user', user_id=user.id))
- admin_edit_user_form.confirmed.data = user.confirmed
- admin_edit_user_form.role.data = user.role.hashid
- edit_general_settings_form.email.data = user.email
- edit_general_settings_form.username.data = user.username
- edit_interface_settings_form.dark_mode.data = user.setting_dark_mode
- edit_notification_settings_form.job_status_mail_notification_level.data = \
- user.setting_job_status_mail_notification_level.name
+ admin_edit_user_form.prefill(user)
+ edit_general_settings_form.prefill(user)
+ edit_interface_settings_form.prefill(user)
+ edit_notification_settings_form.prefill(user)
return render_template(
'admin/edit_user.html.j2',
admin_edit_user_form=admin_edit_user_form,
@@ -124,3 +104,20 @@ def edit_user(user_id):
title='Edit user',
user=user
)
+
+
+@bp.route('/users/<hashid:user_id>/delete', methods=['DELETE'])
+def delete_user(user_id):
+ def _delete_user(app, user_id):
+ with app.app_context():
+ user = User.query.get(user_id)
+ user.delete()
+ db.session.commit()
+
+ User.query.get_or_404(user_id)
+ thread = Thread(
+ target=_delete_user,
+ args=(current_app._get_current_object(), user_id)
+ )
+ thread.start()
+ return {}, 202
diff --git a/app/api/__init__.py b/app/api/__init__.py
index e7674c87d8a298b4d3917e54690051add2257911..39e40db183f0afc2f79d0c0d16e413e03a6870f4 100644
--- a/app/api/__init__.py
+++ b/app/api/__init__.py
@@ -1,25 +1,14 @@
from flask import Blueprint
-from flask_restx import Api
-from .tokens import ns as tokens_ns
bp = Blueprint('api', __name__)
-authorizations = {
- 'basicAuth': {
- 'type': 'basic'
- },
- 'apiKey': {
- 'type': 'apiKey',
- 'in': 'header',
- 'name': 'Authorization'
- }
-}
-api = Api(
- bp,
- authorizations=authorizations,
- description='An API to interact with nopaque',
- title='nopaque API',
- version='1.0'
-)
-api.add_namespace(tokens_ns)
+
+from .tokens import bp as tokens_blueprint
+bp.register_blueprint(tokens_blueprint, url_prefix='/tokens')
+
+from .users import bp as users_blueprint
+bp.register_blueprint(users_blueprint, url_prefix='/users')
+
+from .jobs import bp as jobs_blueprint
+bp.register_blueprint(jobs_blueprint, url_prefix='/jobs')
diff --git a/app/api/auth.py b/app/api/auth.py
index 4c6a3dd959477e2dab3ee8adf0f939344371404d..afda3a30699ebb21f46b3f2b7a448c63c5fcef93 100644
--- a/app/api/auth.py
+++ b/app/api/auth.py
@@ -1,34 +1,49 @@
-from app.models import User
+from flask import current_app
from flask_httpauth import HTTPBasicAuth, HTTPTokenAuth
-from sqlalchemy import or_
-from werkzeug.http import HTTP_STATUS_CODES
+from werkzeug.exceptions import Forbidden, Unauthorized
+from app.models import User
+
basic_auth = HTTPBasicAuth()
token_auth = HTTPTokenAuth()
-
+auth_error_responses = {
+ Unauthorized.code: Unauthorized.description,
+ Forbidden.code: Forbidden.description
+}
@basic_auth.verify_password
def verify_password(email_or_username, password):
- user = User.query.filter(
- or_(
- User.username == email_or_username,
- User.email == email_or_username.lower()
- )
- ).first()
- if user and user.verify_password(password):
+ user = User.query.filter((User.email == email_or_username.lower()) | (User.username == email_or_username)).first()
+ if user is not None and user.verify_password(password):
return user
@basic_auth.error_handler
def basic_auth_error(status):
- return {'error': HTTP_STATUS_CODES.get(status, 'Unknown error')}, status
+ error = (Forbidden if status == 403 else Unauthorized)()
+ return {
+ 'code': error.code,
+ 'message': error.name,
+ 'description': error.description,
+ }, error.code, {'WWW-Authenticate': 'Form'}
@token_auth.verify_token
def verify_token(token):
- return User.check_token(token) if token else None
+ return User.verify_access_token(token) if token else None
@token_auth.error_handler
def token_auth_error(status):
- return {'error': HTTP_STATUS_CODES.get(status, 'Unknown error')}, status
+ error = (Forbidden if status == 403 else Unauthorized)()
+ return {
+ 'code': error.code,
+ 'message': error.name,
+ 'description': error.description,
+ }, error.code
+
+
+@basic_auth.get_user_roles
+@token_auth.get_user_roles
+def get_user_roles(user):
+ return [user.role.name]
diff --git a/app/api/jobs.py b/app/api/jobs.py
new file mode 100644
index 0000000000000000000000000000000000000000..e730f2e60b3c85cacb151cd673a59b6448201de3
--- /dev/null
+++ b/app/api/jobs.py
@@ -0,0 +1,102 @@
+
+from apifairy import authenticate, response
+from apifairy.decorators import body, other_responses
+from flask import abort, Blueprint
+from werkzeug.exceptions import InternalServerError
+from app import db, hashids
+from app.models import Job, JobInput, JobStatus, TesseractOCRModel
+from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRModelSchema
+from .auth import auth_error_responses, token_auth
+
+
+bp = Blueprint('jobs', __name__)
+job_schema = JobSchema()
+jobs_schema = JobSchema(many=True)
+spacy_nlp_pipeline_job_schema = SpaCyNLPPipelineJobSchema()
+tesseract_ocr_pipeline_job_schema = TesseractOCRPipelineJobSchema()
+tesseract_ocr_model_schema = TesseractOCRModelSchema()
+tesseract_ocr_models_schema = TesseractOCRModelSchema(many=True)
+
+
+@bp.route('', methods=['GET'])
+@authenticate(token_auth, role='Administrator')
+@response(jobs_schema)
+@other_responses(auth_error_responses)
+def get_jobs():
+ """Get all jobs"""
+ return Job.query.all()
+
+
+@bp.route('/tesseract-ocr-pipeline', methods=['POST'])
+@authenticate(token_auth)
+@body(tesseract_ocr_pipeline_job_schema, location='form')
+@response(job_schema)
+@other_responses({**auth_error_responses, InternalServerError.code: InternalServerError.description})
+def create_tesseract_ocr_pipeline_job(args):
+ """Create a new Tesseract OCR Pipeline job"""
+ current_user = token_auth.current_user()
+ try:
+ job = Job.create(
+ title=args['title'],
+ description=args['description'],
+ service='tesseract-ocr-pipeline',
+ service_args={
+ 'model': hashids.decode(args['model_id']),
+ 'binarization': args['binarization']
+ },
+ service_version=args['service_version'],
+ user=current_user
+ )
+ except OSError:
+ abort(500)
+ try:
+ JobInput.create(args['pdf'], job=job)
+ except OSError:
+ abort(500)
+ job.status = JobStatus.SUBMITTED
+ db.session.commit()
+ return job, 201
+
+
+@bp.route('/tesseract-ocr-pipeline/models', methods=['GET'])
+@authenticate(token_auth)
+@response(tesseract_ocr_models_schema)
+@other_responses(auth_error_responses)
+def get_tesseract_ocr_models():
+ """Get all Tesseract OCR Models"""
+ return TesseractOCRModel.query.all()
+
+
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
+@authenticate(token_auth)
+@response(EmptySchema, status_code=204)
+@other_responses(auth_error_responses)
+def delete_job(job_id):
+ """Delete a job by id"""
+ current_user = token_auth.current_user()
+ job = Job.query.get(job_id)
+ if job is None:
+ abort(404)
+ if not (job.user == current_user or current_user.is_administrator()):
+ abort(403)
+ try:
+ job.delete()
+ except OSError as e:
+ abort(500)
+ db.session.commit()
+ return {}, 204
+
+
+@bp.route('/<hashid:job_id>', methods=['GET'])
+@authenticate(token_auth)
+@response(job_schema)
+@other_responses(auth_error_responses)
+def get_job(job_id):
+ """Get a job by id"""
+ current_user = token_auth.current_user()
+ job = Job.query.get(job_id)
+ if job is None:
+ abort(404)
+ if not (job.user == current_user or current_user.is_administrator()):
+ abort(403)
+ return job
diff --git a/app/api/schemas.py b/app/api/schemas.py
new file mode 100644
index 0000000000000000000000000000000000000000..394b1ebb6e2f29468477bd3fdc9a54ad36165f90
--- /dev/null
+++ b/app/api/schemas.py
@@ -0,0 +1,165 @@
+from apifairy.fields import FileField
+from marshmallow import validate, validates, ValidationError
+from marshmallow.decorators import post_dump
+from app import ma
+from app.auth import USERNAME_REGEX
+from app.models import Job, JobStatus, TesseractOCRModel, Token, User, UserSettingJobStatusMailNotificationLevel
+from app.services import SERVICES
+
+
+
+class EmptySchema(ma.Schema):
+ pass
+
+
+class TokenSchema(ma.SQLAlchemySchema):
+ class Meta:
+ model = Token
+ ordered = True
+
+ access_token = ma.String(required=True)
+ refresh_token = ma.String()
+
+
+class TesseractOCRModelSchema(ma.SQLAlchemySchema):
+ class Meta:
+ model = TesseractOCRModel
+ ordered = True
+
+ hashid = ma.String(data_key='id', dump_only=True)
+ user_hashid = ma.String(data_key='user_id', dump_only=True)
+ title = ma.auto_field(
+ required=True,
+ validate=validate.Length(min=1, max=64)
+ )
+ description = ma.auto_field(
+ required=True,
+ validate=validate.Length(min=1, max=255)
+ )
+ version = ma.String(
+ required=True,
+ validate=validate.Length(min=1, max=16)
+ )
+ compatible_service_versions = ma.List(
+ ma.String(required=True, validate=validate.Length(min=1, max=16)),
+ required=True,
+ validate=validate.Length(min=1, max=255)
+ )
+ publisher = ma.String(
+ required=True,
+ validate=validate.Length(min=1, max=128)
+ )
+ publisher_url = ma.String(
+ validate=[validate.URL(), validate.Length(min=1, max=512)]
+ )
+ publishing_url = ma.String(
+ required=True,
+ validate=[validate.URL(), validate.Length(min=1, max=512)]
+ )
+ publishing_year = ma.Int(
+ required=True
+ )
+ shared = ma.Boolean(required=True)
+
+
+class JobSchema(ma.SQLAlchemySchema):
+ class Meta:
+ model = Job
+ ordered = True
+
+ hashid = ma.String(data_key='id', dump_only=True)
+ user_hashid = ma.String(data_key='user_id', dump_only=True)
+ title = ma.auto_field(
+ required=True,
+ validate=validate.Length(min=1, max=32)
+ )
+ description = ma.auto_field(
+ required=True,
+ validate=validate.Length(min=1, max=255)
+ )
+ creation_date = ma.auto_field(dump_only=True)
+ end_date = ma.auto_field(dump_only=True)
+ service = ma.String(
+ dump_only=True,
+ validate=validate.OneOf(SERVICES.keys())
+ )
+ service_args = ma.Dict(dump_only=True)
+ service_version = ma.String(dump_only=True)
+ status = ma.String(
+ dump_only=True,
+ validate=validate.OneOf(list(JobStatus.__members__.keys()))
+ )
+
+ @post_dump(pass_original=True)
+ def post_dump(self, serialized_job, job, **kwargs):
+ serialized_job['status'] = job.status.name
+ return serialized_job
+
+
+class TesseractOCRPipelineJobSchema(JobSchema):
+ binarization = ma.Boolean(load_only=True, missing=False)
+ model_id = ma.String(required=True, load_only=True)
+ service_version = ma.auto_field(
+ required=True,
+ validate=[validate.Length(min=1, max=16), validate.OneOf(list(SERVICES['tesseract-ocr-pipeline']['versions'].keys()))]
+ )
+ pdf = FileField()
+
+ @validates('pdf')
+ def validate_pdf(self, value):
+ if value.mimetype != 'application/pdf':
+ raise ValidationError('PDF files only!')
+
+
+class SpaCyNLPPipelineJobSchema(JobSchema):
+ binarization = ma.Boolean(load_only=True, missing=False)
+ model_id = ma.String(required=True, load_only=True)
+ service_version = ma.auto_field(
+ required=True,
+ validate=[validate.Length(min=1, max=16), validate.OneOf(list(SERVICES['tesseract-ocr-pipeline']['versions'].keys()))]
+ )
+ txt = FileField(required=True)
+
+ @validates('txt')
+ def validate_txt(self, value):
+ if value.mimetype != 'text/plain':
+ raise ValidationError('Plain text files only!')
+
+
+class UserSchema(ma.SQLAlchemySchema):
+ class Meta:
+ model = User
+ ordered = True
+
+ hashid = ma.String(data_key='id', dump_only=True)
+ username = ma.auto_field(
+ validate=[
+ validate.Length(min=1, max=64),
+ validate.Regexp(USERNAME_REGEX, error='Usernames must have only letters, numbers, dots or underscores')
+ ]
+ )
+ email = ma.auto_field(validate=validate.Email())
+ member_since = ma.auto_field(dump_only=True)
+ last_seen = ma.auto_field(dump_only=True)
+ password = ma.String(load_only=True)
+ last_seen = ma.auto_field(dump_only=True)
+ setting_dark_mode = ma.auto_field()
+ setting_job_status_mail_notification_level = ma.String(
+ validate=validate.OneOf(list(UserSettingJobStatusMailNotificationLevel.__members__.keys()))
+ )
+
+ @validates('email')
+ def validate_email(self, email):
+ if User.query.filter(User.email == email).first():
+ raise ValidationError('Email already registered')
+
+ @validates('username')
+ def validate_username(self, username):
+ if User.query.filter(User.username == username).first():
+ raise ValidationError('Username already in use')
+
+ @post_dump(pass_original=True)
+ def post_dump(self, serialized_user, user, **kwargs):
+ serialized_user['setting_job_status_mail_notification_level'] = \
+ user.setting_job_status_mail_notification_level.name
+ return serialized_user
diff --git a/app/api/tokens.py b/app/api/tokens.py
index e1b55527ddc3a41534b21a3344db52c1a5a5dcc0..0aaa3415c3cf0635a2f07bfc55a504ffc6edabcd 100644
--- a/app/api/tokens.py
+++ b/app/api/tokens.py
@@ -1,27 +1,58 @@
+from apifairy import authenticate, body, response, other_responses
+from flask import Blueprint, request, abort
from app import db
-from flask_restx import Namespace, Resource
-from .auth import basic_auth, token_auth
+from app.models import Token, User
+from .auth import basic_auth
+from .schemas import EmptySchema, TokenSchema
-ns = Namespace('tokens', description='Token operations')
+bp = Blueprint('tokens', __name__)
+token_schema = TokenSchema()
-@ns.route('')
-class API_Tokens(Resource):
- '''Get or revoke a user authentication token'''
+@bp.route('', methods=['DELETE'])
+@response(EmptySchema, status_code=204, description='Token revoked')
+@other_responses({401: 'Invalid access token'})
+def delete_token():
+ """Revoke an access token"""
+ access_token = request.headers['Authorization'].split()[1]
+ token = Token.query.filter(Token.access_token == access_token).first()
+ if token is None: # pragma: no cover
+ abort(401)
+ token.expire()
+ db.session.commit()
+ return {}
- @ns.doc(security='basicAuth')
- @basic_auth.login_required
- def post(self):
- '''Get a user token'''
- token = basic_auth.current_user().get_token()
- db.session.commit()
- return {'token': 'Bearer ' + token}
- @ns.doc(security='apiKey')
- @token_auth.login_required
- def delete(self):
- '''Revoke a user token'''
- token_auth.current_user().revoke_token()
- db.session.commit()
- return '', 204
+@bp.route('', methods=['POST'])
+@authenticate(basic_auth)
+@response(token_schema)
+@other_responses({401: 'Invalid username or password'})
+def create_token():
+ """Create new access and refresh tokens"""
+ user = basic_auth.current_user()
+ token = user.generate_auth_token()
+ db.session.add(token)
+ Token.clean() # keep token table clean of old tokens
+ db.session.commit()
+ return token, 200
+
+
+@bp.route('', methods=['PUT'])
+@body(token_schema)
+@response(token_schema, description='Newly issued access and refresh tokens')
+@other_responses({401: 'Invalid access or refresh token'})
+def refresh_token(args):
+ """Refresh an access token"""
+ access_token = args.get('access_token')
+ refresh_token = args.get('refresh_token')
+ if access_token is None or refresh_token is None:
+ abort(401)
+ token = User.verify_refresh_token(refresh_token, access_token)
+ if token is None:
+ abort(401)
+ token.expire()
+ new_token = token.user.generate_auth_token()
+ db.session.add_all([token, new_token])
+ db.session.commit()
+ return new_token, 200
diff --git a/app/api/users.py b/app/api/users.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc180df0b62419dd623f23ce2251fcbde7822942
--- /dev/null
+++ b/app/api/users.py
@@ -0,0 +1,99 @@
+
+from apifairy import authenticate, body, response
+from apifairy.decorators import other_responses
+from flask import abort, Blueprint, current_app
+from werkzeug.exceptions import InternalServerError
+from app import db
+from app.email import create_message, send
+from app.models import User
+from .schemas import EmptySchema, UserSchema
+from .auth import auth_error_responses, token_auth
+
+
+bp = Blueprint('users', __name__)
+user_schema = UserSchema()
+users_schema = UserSchema(many=True)
+
+
+@bp.route('', methods=['GET'])
+@authenticate(token_auth, role='Administrator')
+@response(users_schema)
+@other_responses(auth_error_responses)
+def get_users():
+ """Get all users"""
+ return User.query.all()
+
+
+@bp.route('', methods=['POST'])
+@body(user_schema)
+@response(user_schema, 201)
+@other_responses({InternalServerError.code: InternalServerError.description})
+def create_user(args):
+ """Create a new user"""
+ try:
+ user = User.create(
+ email=args['email'].lower(),
+ password=args['password'],
+ username=args['username']
+ )
+ except OSError:
+ abort(500)
+ msg = create_message(
+ user.email,
+ 'Confirm Your Account',
+ 'auth/email/confirm',
+ token=user.generate_confirm_token(),
+ user=user
+ )
+ send(msg)
+ db.session.commit()
+ return user, 201
+
+
+@bp.route('/<hashid:user_id>', methods=['DELETE'])
+@authenticate(token_auth)
+@response(EmptySchema, status_code=204)
+@other_responses(auth_error_responses)
+def delete_user(user_id):
+ """Delete a user by id"""
+ current_user = token_auth.current_user()
+ user = User.query.get(user_id)
+ if user is None:
+ abort(404)
+ if not (user == current_user or current_user.is_administrator()):
+ abort(403)
+ user.delete()
+ db.session.commit()
+ return {}, 204
+
+
+@bp.route('/<hashid:user_id>', methods=['GET'])
+@authenticate(token_auth)
+@response(user_schema)
+@other_responses(auth_error_responses)
+@other_responses({404: 'User not found'})
+def get_user(user_id):
+ """Retrieve a user by id"""
+ current_user = token_auth.current_user()
+ user = User.query.get(user_id)
+ if user is None:
+ abort(404)
+ if not (user == current_user or current_user.is_administrator()):
+ abort(403)
+ return user
+
+
+@bp.route('/<username>', methods=['GET'])
+@authenticate(token_auth)
+@response(user_schema)
+@other_responses(auth_error_responses)
+@other_responses({404: 'User not found'})
+def get_user_by_username(username):
+ """Retrieve a user by username"""
+ current_user = token_auth.current_user()
+ user = User.query.filter(User.username == username).first()
+ if user is None:
+ abort(404)
+ if not (user == current_user or current_user.is_administrator()):
+ abort(403)
+ return user
diff --git a/app/auth/forms.py b/app/auth/forms.py
index 8d47e6b1a7b395c1864bbf60e93e4944ad774126..6917b78baf12fe2028a5c2bd27bdcda87d8ac605 100644
--- a/app/auth/forms.py
+++ b/app/auth/forms.py
@@ -1,4 +1,3 @@
-from app.models import User
from flask_wtf import FlaskForm
from wtforms import (
BooleanField,
@@ -7,32 +6,45 @@ from wtforms import (
SubmitField,
ValidationError
)
-from wtforms.validators import DataRequired, InputRequired, Email, EqualTo, Length, Regexp
+from wtforms.validators import InputRequired, Email, EqualTo, Length, Regexp
+from app.models import User
from . import USERNAME_REGEX
-class LoginForm(FlaskForm):
- user = StringField('Email or username', validators=[DataRequired()])
- password = PasswordField('Password', validators=[DataRequired()])
- remember_me = BooleanField('Keep me logged in')
- submit = SubmitField('Log In')
-
-
class RegistrationForm(FlaskForm):
- email = StringField('Email', validators=[DataRequired(), Email()])
- username = StringField('Username',
+ email = StringField(
+ 'Email',
+ validators=[InputRequired(), Email(), Length(max=254)]
+ )
+ username = StringField(
+ 'Username',
validators=[
InputRequired(),
- Length(1, 64),
+ Length(max=64),
Regexp(
USERNAME_REGEX,
- message='Usernames must have only letters, numbers, dots or underscores' # noqa
- )
+ message=(
+ 'Usernames must have only letters, numbers, dots or '
+ 'underscores'
+ )
+ )
]
)
- password = PasswordField('Password', validators=[DataRequired(), EqualTo('password_confirmation', message='Passwords must match')])
- password_confirmation = PasswordField('Password confirmation', validators=[DataRequired(), EqualTo('password', message='Passwords must match')])
- submit = SubmitField('Register')
+ password = PasswordField(
+ 'Password',
+ validators=[
+ InputRequired(),
+ EqualTo('password_2', message='Passwords must match')
+ ]
+ )
+ password_2 = PasswordField(
+ 'Password confirmation',
+ validators=[
+ InputRequired(),
+ EqualTo('password', message='Passwords must match')
+ ]
+ )
+ submit = SubmitField()
def validate_email(self, field):
if User.query.filter_by(email=field.data.lower()).first():
@@ -43,12 +55,31 @@ class RegistrationForm(FlaskForm):
raise ValidationError('Username already in use')
-class ResetPasswordForm(FlaskForm):
- password = PasswordField('New password', validators=[DataRequired(), EqualTo('password_confirmation', message='Passwords must match')])
- password_confirmation = PasswordField('Password confirmation', validators=[DataRequired(), EqualTo('password', message='Passwords must match')])
- submit = SubmitField('Reset Password')
+class LoginForm(FlaskForm):
+ user = StringField('Email or username', validators=[InputRequired()])
+ password = PasswordField('Password', validators=[InputRequired()])
+ remember_me = BooleanField('Keep me logged in')
+ submit = SubmitField()
class ResetPasswordRequestForm(FlaskForm):
- email = StringField('Email', validators=[DataRequired(), Email()])
- submit = SubmitField('Reset Password')
+ email = StringField('Email', validators=[InputRequired(), Email()])
+ submit = SubmitField()
+
+
+class ResetPasswordForm(FlaskForm):
+ password = PasswordField(
+ 'New password',
+ validators=[
+ InputRequired(),
+ EqualTo('password_2', message='Passwords must match')
+ ]
+ )
+ password_2 = PasswordField(
+ 'New password confirmation',
+ validators=[
+ InputRequired(),
+ EqualTo('password', message='Passwords must match')
+ ]
+ )
+ submit = SubmitField()
diff --git a/app/auth/routes.py b/app/auth/routes.py
index 045d0bafcc5efce3372b6a9bdb9f48b5d220d7d2..5655d0dcf59fddf1482cd7d5bcd978758d1d2cb8 100644
--- a/app/auth/routes.py
+++ b/app/auth/routes.py
@@ -1,10 +1,5 @@
-from app import db
-from app.email import create_message, send
-from app.models import User
-from datetime import datetime
from flask import (
abort,
- current_app,
flash,
redirect,
render_template,
@@ -12,7 +7,9 @@ from flask import (
url_for
)
from flask_login import current_user, login_user, login_required, logout_user
-from sqlalchemy import or_
+from app import db
+from app.email import create_message, send
+from app.models import User
from . import bp
from .forms import (
LoginForm,
@@ -29,69 +26,32 @@ def before_request():
unconfirmed view if user is unconfirmed.
"""
if current_user.is_authenticated:
- current_user.last_seen = datetime.utcnow()
+ current_user.ping()
db.session.commit()
- if (
- not current_user.confirmed
- and request.endpoint
- and request.blueprint != 'auth'
- and request.endpoint != 'static'
- ):
+ if (not current_user.confirmed
+ and request.endpoint
+ and request.blueprint != 'auth'
+ and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed'))
-@bp.route('/login', methods=['GET', 'POST'])
-def login():
- if current_user.is_authenticated:
- return redirect(url_for('main.dashboard'))
- form = LoginForm(prefix='login-form')
- if form.validate_on_submit():
- user = User.query.filter(
- or_(
- User.username == form.user.data,
- User.email == form.user.data.lower()
- )
- ).first()
- if user and user.verify_password(form.password.data):
- login_user(user, form.remember_me.data)
- next = request.args.get('next')
- if next is None or not next.startswith('/'):
- next = url_for('main.dashboard')
- return redirect(next)
- flash('Invalid email/username or password', category='error')
- return render_template('auth/login.html.j2', form=form, title='Log in')
-
-
-@bp.route('/logout')
-@login_required
-def logout():
- logout_user()
- flash('You have been logged out')
- return redirect(url_for('main.index'))
-
-
@bp.route('/register', methods=['GET', 'POST'])
def register():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
form = RegistrationForm(prefix='registration-form')
if form.validate_on_submit():
- user = User(
- email=form.email.data.lower(),
- password=form.password.data,
- username=form.username.data
- )
- db.session.add(user)
- db.session.flush(objects=[user])
- db.session.refresh(user)
try:
- user.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
+ user = User.create(
+ email=form.email.data.lower(),
+ password=form.password.data,
+ username=form.username.data
+ )
+ except OSError:
flash('Internal Server Error', category='error')
abort(500)
- token = user.generate_confirmation_token()
+ flash(f'User "{user.username}" created')
+ token = user.generate_confirm_token()
msg = create_message(
user.email,
'Confirm Your Account',
@@ -110,36 +70,46 @@ def register():
)
-@bp.route('/confirm/<token>')
-@login_required
-def confirm(token):
- if current_user.confirmed:
- return redirect(url_for('main.dashboard'))
- if current_user.confirm(token):
- db.session.commit()
- flash('You have confirmed your account')
+@bp.route('/login', methods=['GET', 'POST'])
+def login():
+ if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
- else:
- flash(
- 'The confirmation link is invalid or has expired',
- category='error'
- )
- return redirect(url_for('.unconfirmed'))
+ form = LoginForm(prefix='login-form')
+ if form.validate_on_submit():
+ user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
+ if user and user.verify_password(form.password.data):
+ login_user(user, form.remember_me.data)
+ next = request.args.get('next')
+ if next is None or not next.startswith('/'):
+ next = url_for('main.dashboard')
+ flash('You have been logged in')
+ return redirect(next)
+ flash('Invalid email/username or password', category='error')
+ return render_template('auth/login.html.j2', form=form, title='Log in')
+
+
+@bp.route('/logout')
+@login_required
+def logout():
+ logout_user()
+ flash('You have been logged out')
+ return redirect(url_for('main.index'))
@bp.route('/unconfirmed')
+@login_required
def unconfirmed():
- if current_user.is_anonymous:
- return redirect(url_for('main.index'))
- elif current_user.confirmed:
+ if current_user.confirmed:
return redirect(url_for('main.dashboard'))
return render_template('auth/unconfirmed.html.j2', title='Unconfirmed')
@bp.route('/confirm')
@login_required
-def resend_confirmation():
- token = current_user.generate_confirmation_token()
+def confirm_request():
+ if current_user.confirmed:
+ return redirect(url_for('main.dashboard'))
+ token = current_user.generate_confirm_token()
msg = create_message(
current_user.email,
'Confirm Your Account',
@@ -149,10 +119,23 @@ def resend_confirmation():
)
send(msg)
flash('A new confirmation email has been sent to you by email')
- return redirect(url_for('auth.unconfirmed'))
+ return redirect(url_for('.unconfirmed'))
+
+
+@bp.route('/confirm/<token>')
+@login_required
+def confirm(token):
+ if current_user.confirmed:
+ return redirect(url_for('main.dashboard'))
+ if current_user.confirm(token):
+ db.session.commit()
+ flash('You have confirmed your account')
+ return redirect(url_for('main.dashboard'))
+ flash('The confirmation link is invalid or has expired', category='error')
+ return redirect(url_for('.unconfirmed'))
-@bp.route('/reset', methods=['GET', 'POST'])
+@bp.route('/reset_password', methods=['GET', 'POST'])
def reset_password_request():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
@@ -160,7 +143,7 @@ def reset_password_request():
if form.validate_on_submit():
user = User.query.filter_by(email=form.email.data.lower()).first()
if user is not None:
- token = user.generate_reset_token()
+ token = user.generate_reset_password_token()
msg = create_message(
user.email,
'Reset Your Password',
@@ -170,7 +153,8 @@ def reset_password_request():
)
send(msg)
flash(
- 'An email with instructions to reset your password has been sent to you' # noqa
+ 'An email with instructions to reset your password has been sent '
+ 'to you'
)
return redirect(url_for('.login'))
return render_template(
@@ -180,7 +164,7 @@ def reset_password_request():
)
-@bp.route('/reset/<token>', methods=['GET', 'POST'])
+@bp.route('/reset_password/<token>', methods=['GET', 'POST'])
def reset_password(token):
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
@@ -190,8 +174,7 @@ def reset_password(token):
db.session.commit()
flash('Your password has been updated')
return redirect(url_for('.login'))
- else:
- return redirect(url_for('main.index'))
+ return redirect(url_for('main.index'))
return render_template(
'auth/reset_password.html.j2',
form=form,
diff --git a/app/cli.py b/app/cli.py
index 64cf4fb7a12a29c869a82142ebfb8b8d3dc63145..d9b4fdf0637ad2aec9615eac1b248745d0aa775f 100644
--- a/app/cli.py
+++ b/app/cli.py
@@ -1,9 +1,8 @@
from flask import current_app
from flask_migrate import upgrade
-from . import db
-from .models import Corpus, Role, User, TesseractOCRModel, TranskribusHTRModel
import click
import os
+from app.models import Role, User, TesseractOCRModel, TranskribusHTRModel
def _make_default_dirs():
@@ -41,22 +40,6 @@ def register(app):
current_app.logger.info('Insert/Update default TranskribusHTRModels')
TranskribusHTRModel.insert_defaults()
- @app.cli.group()
- def daemon():
- ''' Daemon commands. '''
- pass
-
- @daemon.command('run')
- def run_daemon():
- ''' Run daemon '''
- corpus: Corpus
- for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0):
- corpus.num_analysis_sessions = 0
- db.session.commit()
- from app.daemon import Daemon
- daemon: Daemon = Daemon()
- daemon.run()
-
@app.cli.group()
def converter():
''' Converter commands. '''
diff --git a/app/contribute/__init__.py b/app/contributions/__init__.py
similarity index 54%
rename from app/contribute/__init__.py
rename to app/contributions/__init__.py
index 15d172ecacca63634bafe62e3c2af0a79671dc8d..af9747a67b39250b7dd0a857948fe915462af504 100644
--- a/app/contribute/__init__.py
+++ b/app/contributions/__init__.py
@@ -1,5 +1,5 @@
from flask import Blueprint
-bp = Blueprint('contribute', __name__)
+bp = Blueprint('contributions', __name__)
from . import routes
diff --git a/app/contribute/routes.py b/app/contributions/routes.py
similarity index 52%
rename from app/contribute/routes.py
rename to app/contributions/routes.py
index 8bfd9bd84666d54b9449378030025a67944937b8..80c6a82ded1202f3b9dd14d0173065f47905c9a4 100644
--- a/app/contribute/routes.py
+++ b/app/contributions/routes.py
@@ -1,9 +1,6 @@
-from app import db
-from app.decorators import permission_required
-from app.models import Permission, Role, User
-from app.settings import tasks as settings_tasks
-from flask import flash, redirect, render_template, url_for
from flask_login import login_required
+from app.decorators import permission_required
+from app.models import Permission
from . import bp
@@ -14,6 +11,6 @@ def before_request():
pass
-@bp.route('/')
-def index():
+@bp.route('')
+def contributions():
pass
diff --git a/app/corpora/cqi_over_socketio/__init__.py b/app/corpora/cqi_over_socketio/__init__.py
index f1f5f8265007245a3a9e5ba2ba0228e48fbe620c..c122e12e3901d3fb11caf88029e5c08a7e187e6a 100644
--- a/app/corpora/cqi_over_socketio/__init__.py
+++ b/app/corpora/cqi_over_socketio/__init__.py
@@ -1,11 +1,11 @@
-from app import db, hashids, socketio
-from app.decorators import socketio_login_required
-from app.models import Corpus, CorpusStatus
from flask import session
from flask_login import current_user
from flask_socketio import ConnectionRefusedError
from threading import Lock
import cqi
+from app import db, hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus, CorpusStatus
'''
diff --git a/app/corpora/cqi_over_socketio/cqi.py b/app/corpora/cqi_over_socketio/cqi.py
index f6edb5fe35cdb7dae4d5870c5fcf82be5c2c8433..9d0fbfd6334392bff6a710642ccac0c226bff884 100644
--- a/app/corpora/cqi_over_socketio/cqi.py
+++ b/app/corpora/cqi_over_socketio/cqi.py
@@ -1,9 +1,9 @@
+from socket import gaierror
+import cqi
from app import socketio
from app.decorators import socketio_login_required
-from socket import gaierror
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
-import cqi
@socketio.on('cqi.connect', namespace=ns)
diff --git a/app/corpora/cqi_over_socketio/cqi_corpora.py b/app/corpora/cqi_over_socketio/cqi_corpora.py
index d0f82e96cc013d039f8c9298f788d09c37a84cd8..7b73429a0c30895c60fd001c0fec5173d63d7954 100644
--- a/app/corpora/cqi_over_socketio/cqi_corpora.py
+++ b/app/corpora/cqi_over_socketio/cqi_corpora.py
@@ -1,8 +1,8 @@
+import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
-import cqi
@socketio.on('cqi.corpora.get', namespace=ns)
diff --git a/app/corpora/cqi_over_socketio/cqi_corpora_corpus.py b/app/corpora/cqi_over_socketio/cqi_corpora_corpus.py
index bfe8437cf70b26bcec32bd88b53837029df897cb..5332aade455d53b0dd5860a1b93b3dfde25f39b0 100644
--- a/app/corpora/cqi_over_socketio/cqi_corpora_corpus.py
+++ b/app/corpora/cqi_over_socketio/cqi_corpora_corpus.py
@@ -1,11 +1,11 @@
+from flask import session
+import cqi
+import math
from app import db, socketio
from app.decorators import socketio_login_required
from app.models import Corpus
-from flask import session
from . import NAMESPACE as ns
from .utils import cqi_over_socketio, lookups_by_cpos
-import cqi
-import math
@socketio.on('cqi.corpora.corpus.drop', namespace=ns)
diff --git a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py
index 95be6771440db4700ca4064490eafea885dd6985..d6382eaa8a43d2c641c58dd0ad0a50186496c641 100644
--- a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py
+++ b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_alignment_attributes.py
@@ -1,8 +1,8 @@
+import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
-import cqi
@socketio.on('cqi.corpora.corpus.alignment_attributes.get', namespace=ns)
diff --git a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py
index e8c11677d2ffeaa6a76708352009ccd12cc5e2ef..41d55bb370824b224c0011ad70fc5870b20781a5 100644
--- a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py
+++ b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_positional_attributes.py
@@ -1,8 +1,8 @@
+import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
-import cqi
@socketio.on('cqi.corpora.corpus.positional_attributes.get', namespace=ns)
diff --git a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py
index 2b1559f6883161ae1e0a8d28fd8a77a457954ad2..111789066522c79d85cbe65536856eb281b9ef3a 100644
--- a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py
+++ b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_structural_attributes.py
@@ -1,8 +1,8 @@
+import cqi
from app import socketio
from app.decorators import socketio_login_required
from . import NAMESPACE as ns
from .utils import cqi_over_socketio
-import cqi
@socketio.on('cqi.corpora.corpus.structural_attributes.get', namespace=ns)
diff --git a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py
index 419be16b62f46636dabe52dd9a39bd3eec9e9f39..3c8cbdd4e5ee873a740c6c091d6ddd16b473f7d6 100644
--- a/app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py
+++ b/app/corpora/cqi_over_socketio/cqi_corpora_corpus_subcorpora.py
@@ -1,13 +1,13 @@
-from app import socketio
-from app.decorators import socketio_login_required
-from app.models import Corpus
from flask import session
-from . import NAMESPACE as ns
-from .utils import cqi_over_socketio, export_subcorpus
import cqi
import json
import math
import os
+from app import socketio
+from app.decorators import socketio_login_required
+from app.models import Corpus
+from . import NAMESPACE as ns
+from .utils import cqi_over_socketio, export_subcorpus
@socketio.on('cqi.corpora.corpus.subcorpora.get', namespace=ns)
diff --git a/app/corpora/forms.py b/app/corpora/forms.py
index 26105a13d4e5242ae4a74ed7b40d6d0aa896acba..73002edc9c2e2c3a67d9f003b6049ef2225b03c3 100644
--- a/app/corpora/forms.py
+++ b/app/corpora/forms.py
@@ -1,80 +1,67 @@
from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired
-from werkzeug.utils import secure_filename
-from wtforms import (
- StringField,
- SubmitField,
- ValidationError,
- IntegerField
-)
-from wtforms.validators import DataRequired, InputRequired, Length
+from wtforms import StringField, SubmitField, ValidationError, IntegerField
+from wtforms.validators import InputRequired, Length
-class AddCorpusFileForm(FlaskForm):
- '''
- Form to add a .vrt corpus file to the current corpus.
- '''
- # Required fields
- author = StringField('Author', validators=[InputRequired(), Length(1, 255)])
- publishing_year = IntegerField('Publishing year', validators=[InputRequired()])
- title = StringField('Title', validators=[InputRequired(), Length(1, 255)])
- vrt = FileField('File', validators=[FileRequired()])
- # Optional fields
- address = StringField('Adress', validators=[Length(0, 255)])
- booktitle = StringField('Booktitle', validators=[Length(0, 255)])
- chapter = StringField('Chapter', validators=[Length(0, 255)])
- editor = StringField('Editor', validators=[Length(0, 255)])
- institution = StringField('Institution', validators=[Length(0, 255)])
- journal = StringField('Journal', validators=[Length(0, 255)])
- pages = StringField('Pages', validators=[Length(0, 255)])
- publisher = StringField('Publisher', validators=[Length(0, 255)])
- school = StringField('School', validators=[Length(0, 255)])
+class CreateCorpusForm(FlaskForm):
+ description = StringField(
+ 'Description',
+ validators=[InputRequired(), Length(max=255)]
+ )
+ title = StringField('Title', validators=[InputRequired(), Length(max=32)])
+ submit = SubmitField()
+
+
+class CorpusFileBaseForm(FlaskForm):
+ author = StringField(
+ 'Author',
+ validators=[InputRequired(), Length(max=255)]
+ )
+ publishing_year = IntegerField(
+ 'Publishing year',
+ validators=[InputRequired()]
+ )
+ title = StringField(
+ 'Title',
+ validators=[InputRequired(), Length(max=255)]
+ )
+ address = StringField('Adress', validators=[Length(max=255)])
+ booktitle = StringField('Booktitle', validators=[Length(max=255)])
+ chapter = StringField('Chapter', validators=[Length(max=255)])
+ editor = StringField('Editor', validators=[Length(max=255)])
+ institution = StringField('Institution', validators=[Length(max=255)])
+ journal = StringField('Journal', validators=[Length(max=255)])
+ pages = StringField('Pages', validators=[Length(max=255)])
+ publisher = StringField('Publisher', validators=[Length(max=255)])
+ school = StringField('School', validators=[Length(max=255)])
submit = SubmitField()
+
+class CreateCorpusFileForm(CorpusFileBaseForm):
+ vrt = FileField('File', validators=[FileRequired()])
+
def validate_vrt(self, field):
if not field.data.filename.lower().endswith('.vrt'):
raise ValidationError('VRT files only!')
-class EditCorpusFileForm(FlaskForm):
- '''
- Form to edit meta data of one corpus file.
- '''
- # Required fields
- author = StringField('Author', validators=[InputRequired(), Length(1, 255)])
- publishing_year = IntegerField('Publishing year', validators=[InputRequired()])
- title = StringField('Title', validators=[InputRequired(), Length(1, 255)])
- # Optional fields
- address = StringField('Adress', validators=[Length(0, 255)])
- booktitle = StringField('Booktitle', validators=[Length(0, 255)])
- chapter = StringField('Chapter', validators=[Length(0, 255)])
- editor = StringField('Editor', validators=[Length(0, 255)])
- institution = StringField('Institution', validators=[Length(0, 255)])
- journal = StringField('Journal', validators=[Length(0, 255)])
- pages = StringField('Pages', validators=[Length(0, 255)])
- publisher = StringField('Publisher', validators=[Length(0, 255)])
- school = StringField('School', validators=[Length(0, 255)])
- submit = SubmitField()
-
-class AddCorpusForm(FlaskForm):
- '''
- Form to add a a new corpus.
- '''
- description = StringField('Description', validators=[InputRequired(), Length(1, 255)])
- title = StringField('Title', validators=[InputRequired(), Length(1, 32)])
- submit = SubmitField()
+class EditCorpusFileForm(CorpusFileBaseForm):
+ def prefill(self, corpus_file):
+ ''' Pre-fill the form with data of an exististing corpus file '''
+ self.address.data = corpus_file.address
+ self.author.data = corpus_file.author
+ self.booktitle.data = corpus_file.booktitle
+ self.chapter.data = corpus_file.chapter
+ self.editor.data = corpus_file.editor
+ self.institution.data = corpus_file.institution
+ self.journal.data = corpus_file.journal
+ self.pages.data = corpus_file.pages
+ self.publisher.data = corpus_file.publisher
+ self.publishing_year.data = corpus_file.publishing_year
+ self.school.data = corpus_file.school
+ self.title.data = corpus_file.title
class ImportCorpusForm(FlaskForm):
- '''
- Form to import a corpus.
- '''
- description = StringField('Description', validators=[InputRequired(), Length(1, 255)])
- archive = FileField('File', validators=[FileRequired()])
- title = StringField('Title', validators=[InputRequired(), Length(1, 32)])
- submit = SubmitField()
-
- def validate_archive(self, field):
- valid_mimetypes = ['application/zip', 'application/x-zip', 'application/x-zip-compressed']
- if field.data.mimetype not in valid_mimetypes:
- raise ValidationError('ZIP files only!')
+ pass
diff --git a/app/corpora/routes.py b/app/corpora/routes.py
index 3a3c76e39721f7203b849bb5f69df81dadba1b73..36e19e2b1807d8bda9f86ca6606e06e223604d0f 100644
--- a/app/corpora/routes.py
+++ b/app/corpora/routes.py
@@ -1,143 +1,44 @@
-from app import db
-from app.models import Corpus, CorpusFile, CorpusStatus
from flask import (
abort,
current_app,
flash,
- make_response,
+ Markup,
redirect,
render_template,
- url_for,
send_from_directory
)
from flask_login import current_user, login_required
-from werkzeug.utils import secure_filename
-from zipfile import ZipFile
-from . import bp
-from . import tasks
-from .forms import (
- AddCorpusFileForm,
- AddCorpusForm,
- EditCorpusFileForm,
- ImportCorpusForm
-)
+from threading import Thread
import os
-import shutil
-import tempfile
-import glob
-import xml.etree.ElementTree as ET
+from app import db
+from app.models import Corpus, CorpusFile, CorpusStatus
+from . import bp
+from .forms import CreateCorpusFileForm, CreateCorpusForm, EditCorpusFileForm
-@bp.route('/add', methods=['GET', 'POST'])
+@bp.route('/create', methods=['GET', 'POST'])
@login_required
-def add_corpus():
- form = AddCorpusForm(prefix='add-corpus-form')
+def create_corpus():
+ form = CreateCorpusForm(prefix='create-corpus-form')
if form.validate_on_submit():
- corpus = Corpus(
- user=current_user,
- description=form.description.data,
- title=form.title.data
- )
- db.session.add(corpus)
- db.session.flush()
- db.session.refresh(corpus)
try:
- corpus.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', category='error')
+ corpus = Corpus.create(
+ title=form.title.data,
+ description=form.description.data,
+ user=current_user
+ )
+ except OSError:
abort(500)
db.session.commit()
- flash(f'Corpus "{corpus.title}" added', category='corpus')
- return redirect(url_for('.corpus', corpus_id=corpus.id))
- return render_template(
- 'corpora/add_corpus.html.j2',
- form=form,
- title='Add corpus'
- )
-
-
-@bp.route('/import', methods=['GET', 'POST'])
-@login_required
-def import_corpus():
- form = ImportCorpusForm(prefix='import-corpus-form')
- if form.is_submitted():
- if not form.validate():
- return make_response(form.errors, 400)
- corpus = Corpus(
- user=current_user,
- description=form.description.data,
- title=form.title.data
+ message = Markup(
+ f'Corpus "<a href="{corpus.url}">{corpus.title}</a>" created'
)
- db.session.add(corpus)
- db.session.flush(objects=[corpus])
- db.session.refresh(corpus)
- try:
- corpus.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', category='error')
- return make_response({'redirect_url': url_for('.import_corpus')}, 500) # noqa
- # Save the uploaded zip file in a temporary directory
- tmp_dir_base = os.path.join(current_app.config['NOPAQUE_DATA_DIR'], 'tmp') # noqa
- with tempfile.TemporaryDirectory(dir=tmp_dir_base) as tmp_dir:
- archive_file = os.path.join(tmp_dir, 'corpus.zip')
- try:
- form.archive.data.save(archive_file)
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error1', category='error')
- return make_response({'redirect_url': url_for('.import_corpus')}, 500) # noqa
- shutil.unpack_archive(archive_file, extract_dir=tmp_dir)
- for vrt_filename in [x for x in os.listdir(tmp_dir) if x.endswith('.vrt')]:
- vrt_file = os.path.join(tmp_dir, vrt_filename)
- element_tree = ET.parse(vrt_file)
- text_node = element_tree.find('text')
- corpus_file = CorpusFile(
- author=text_node.get('author'),
- corpus=corpus,
- filename=vrt_filename,
- mimetype='application/vrt+xml',
- publishing_year=int(text_node.get('publishing_year')),
- title=text_node.get('title')
- )
- if 'address' not in text_node.attrib:
- corpus_file.address = text_node.get('address')
- if 'booktitle' not in text_node.attrib:
- corpus_file.booktitle = text_node.get('booktitle')
- if 'chapter' not in text_node.attrib:
- corpus_file.chapter = text_node.get('chapter')
- if 'editor' not in text_node.attrib:
- corpus_file.editor = text_node.get('editor')
- if 'institution' not in text_node.attrib:
- corpus_file.institution = text_node.get('institution')
- if 'journal' not in text_node.attrib:
- corpus_file.journal = text_node.get('journal')
- if 'pages' not in text_node.attrib:
- corpus_file.pages = text_node.get('pages')
- if 'publisher' not in text_node.attrib:
- corpus_file.publisher = text_node.get('publisher')
- if 'school' not in text_node.attrib:
- corpus_file.school = text_node.get('school')
- db.session.add(corpus_file)
- db.session.flush(objects=[corpus_file])
- db.session.refresh(corpus)
- try:
- shutil.copy2(vrt_file, corpus_file.path)
- except Exception as e:
- db.session.rollback()
- flash('Internal Server Error2', category='error')
- return make_response({'redirect_url': url_for('.import_corpus')}, 500) # noqa
- db.session.commit()
- flash(f'Corpus "{corpus.title}" imported', 'corpus')
- return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
+ flash(message, 'corpus')
+ return redirect(corpus.url)
return render_template(
- 'corpora/import_corpus.html.j2',
+ 'corpora/create_corpus.html.j2',
form=form,
- title='Import Corpus'
+ title='Create corpus'
)
@@ -154,6 +55,26 @@ def corpus(corpus_id):
)
+@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
+@login_required
+def delete_corpus(corpus_id):
+ def _delete_corpus(app, corpus_id):
+ with app.app_context():
+ corpus = Corpus.query.get(corpus_id)
+ corpus.delete()
+ db.session.commit()
+
+ corpus = Corpus.query.get_or_404(corpus_id)
+ if not (corpus.user == current_user or current_user.is_administrator()):
+ abort(403)
+ thread = Thread(
+ target=_delete_corpus,
+ args=(current_app._get_current_object(), corpus_id)
+ )
+ thread.start()
+ return {}, 202
+
+
@bp.route('/<hashid:corpus_id>/analyse')
@login_required
def analyse_corpus(corpus_id):
@@ -165,95 +86,132 @@ def analyse_corpus(corpus_id):
)
-@bp.route('/<hashid:corpus_id>/build')
+@bp.route('/<hashid:corpus_id>/build', methods=['POST'])
@login_required
def build_corpus(corpus_id):
- corpus = Corpus.query.get_or_404(corpus_id)
- if not (corpus.user == current_user or current_user.is_administrator()):
- abort(403)
- if corpus.files.all():
- tasks.build_corpus(corpus_id)
- flash(
- f'Corpus "{corpus.title}" marked for building',
- category='corpus'
- )
- else:
- flash(
- f'Can\'t build corpus "{corpus.title}": No corpus file(s)',
- category='error'
- )
- return redirect(url_for('.corpus', corpus_id=corpus_id))
-
+ def _build_corpus(app, corpus_id):
+ with app.app_context():
+ corpus = Corpus.query.get(corpus_id)
+ corpus.build()
+ db.session.commit()
-@bp.route('/<hashid:corpus_id>/delete')
-@login_required
-def delete_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.user == current_user or current_user.is_administrator()):
abort(403)
- flash(f'Corpus "{corpus.title}" marked for deletion', 'corpus')
- tasks.delete_corpus(corpus_id)
- return redirect(url_for('main.dashboard'))
+ # Check if the corpus has corpus files
+ if not corpus.files.all():
+ response = {'errors': {'message': 'Corpus file(s) required'}}
+ return response, 409
+ thread = Thread(
+ target=_build_corpus,
+ args=(current_app._get_current_object(), corpus_id)
+ )
+ thread.start()
+ return {}, 202
-@bp.route('/<hashid:corpus_id>/export')
+@bp.route('/<hashid:corpus_id>/files/create', methods=['GET', 'POST'])
@login_required
-def export_corpus(corpus_id):
- abort(503)
+def create_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.user == current_user or current_user.is_administrator()):
abort(403)
- return send_from_directory(
- as_attachment=True,
- directory=os.path.join(corpus.user.path, 'corpora'),
- filename=corpus.archive_file,
- mimetype='zip'
+ form = CreateCorpusFileForm(prefix='create-corpus-file-form')
+ if form.is_submitted():
+ if not form.validate():
+ response = {'errors': form.errors}
+ return response, 400
+ try:
+ corpus_file = CorpusFile.create(
+ form.vrt.data,
+ address=form.address.data,
+ author=form.author.data,
+ booktitle=form.booktitle.data,
+ chapter=form.chapter.data,
+ editor=form.editor.data,
+ institution=form.institution.data,
+ journal=form.journal.data,
+ pages=form.pages.data,
+ publisher=form.publisher.data,
+ publishing_year=form.publishing_year.data,
+ school=form.school.data,
+ title=form.title.data,
+ mimetype='application/vrt+xml',
+ corpus=corpus
+ )
+ except OSError:
+ abort(500)
+ corpus.status = CorpusStatus.UNPREPARED
+ db.session.commit()
+ message = Markup(
+ 'Corpus file'
+ f'"<a href="{corpus_file.url}">{corpus_file.filename}</a>" added'
+ )
+ flash(message, category='corpus')
+ return {}, 201, {'Location': corpus.url}
+ return render_template(
+ 'corpora/create_corpus_file.html.j2',
+ corpus=corpus,
+ form=form,
+ title='Add corpus file'
)
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['GET', 'POST']) # noqa
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>',
+ methods=['GET', 'POST'])
@login_required
def corpus_file(corpus_id, corpus_file_id):
- corpus_file = CorpusFile.query.filter(
- CorpusFile.corpus_id == corpus_id,
- CorpusFile.id == corpus_file_id
- ).first_or_404()
- if not (
- corpus_file.corpus.user == current_user
- or current_user.is_administrator()
- ):
+ corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
+ if corpus_file.corpus.id != corpus_id:
+ abort(404)
+ if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
abort(403)
form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if form.validate_on_submit():
- corpus_file.address = form.address.data
- corpus_file.author = form.author.data
- corpus_file.booktitle = form.booktitle.data
- corpus_file.chapter = form.chapter.data
- corpus_file.editor = form.editor.data
- corpus_file.institution = form.institution.data
- corpus_file.journal = form.journal.data
- corpus_file.pages = form.pages.data
- corpus_file.publisher = form.publisher.data
- corpus_file.publishing_year = form.publishing_year.data
- corpus_file.school = form.school.data
- corpus_file.title = form.title.data
- corpus_file.corpus.status = CorpusStatus.UNPREPARED
+ has_changes = False
+ if corpus_file.address != form.address.data:
+ corpus_file.address = form.address.data
+ has_changes = True
+ if corpus_file.author != form.author.data:
+ corpus_file.author = form.author.data
+ has_changes = True
+ if corpus_file.booktitle != form.booktitle.data:
+ corpus_file.booktitle = form.booktitle.data
+ has_changes = True
+ if corpus_file.chapter != form.chapter.data:
+ corpus_file.chapter = form.chapter.data
+ has_changes = True
+ if corpus_file.editor != form.editor.data:
+ corpus_file.editor = form.editor.data
+ has_changes = True
+ if corpus_file.institution != form.institution.data:
+ corpus_file.institution = form.institution.data
+ has_changes = True
+ if corpus_file.journal != form.journal.data:
+ corpus_file.journal = form.journal.data
+ has_changes = True
+ if corpus_file.pages != form.pages.data:
+ corpus_file.pages = form.pages.data
+ has_changes = True
+ if corpus_file.publisher != form.publisher.data:
+ corpus_file.publisher = form.publisher.data
+ has_changes = True
+ if corpus_file.publishing_year != form.publishing_year.data:
+ corpus_file.publishing_year = form.publishing_year.data
+ has_changes = True
+ if corpus_file.school != form.school.data:
+ corpus_file.school = form.school.data
+ has_changes = True
+ if corpus_file.title != form.title.data:
+ corpus_file.title = form.title.data
+ has_changes = True
+ if has_changes:
+ corpus_file.corpus.status = CorpusStatus.UNPREPARED
db.session.commit()
- flash(f'Corpus file "{corpus_file.filename}" edited', category='corpus') # noqa
- return redirect(url_for('.corpus', corpus_id=corpus_id))
- # If no form is submitted or valid, fill out fields with current values
- form.address.data = corpus_file.address
- form.author.data = corpus_file.author
- form.booktitle.data = corpus_file.booktitle
- form.chapter.data = corpus_file.chapter
- form.editor.data = corpus_file.editor
- form.institution.data = corpus_file.institution
- form.journal.data = corpus_file.journal
- form.pages.data = corpus_file.pages
- form.publisher.data = corpus_file.publisher
- form.publishing_year.data = corpus_file.publishing_year
- form.school.data = corpus_file.school
- form.title.data = corpus_file.title
+ message = Markup(f'Corpus file "<a href="{corpus_file.url}">{corpus_file.filename}</a>" updated')
+ flash(message, category='corpus')
+ return redirect(corpus_file.corpus.url)
+ form.prefill(corpus_file)
return render_template(
'corpora/corpus_file.html.j2',
corpus=corpus_file.corpus,
@@ -263,91 +221,52 @@ def corpus_file(corpus_id, corpus_file_id):
)
-@bp.route('/<hashid:corpus_id>/files/add', methods=['GET', 'POST'])
-@login_required
-def add_corpus_file(corpus_id):
- corpus = Corpus.query.get_or_404(corpus_id)
- if not (corpus.user == current_user or current_user.is_administrator()):
- abort(403)
- form = AddCorpusFileForm(prefix='add-corpus-file-form')
- if form.is_submitted():
- if not form.validate():
- return make_response(form.errors, 400)
- # Save the file
- corpus_file = CorpusFile(
- address=form.address.data,
- author=form.author.data,
- booktitle=form.booktitle.data,
- chapter=form.chapter.data,
- corpus=corpus,
- editor=form.editor.data,
- filename=form.vrt.data.filename,
- institution=form.institution.data,
- journal=form.journal.data,
- mimetype='application/vrt+xml',
- pages=form.pages.data,
- publisher=form.publisher.data,
- publishing_year=form.publishing_year.data,
- school=form.school.data,
- title=form.title.data
- )
- db.session.add(corpus_file)
- db.session.flush(objects=[corpus_file])
- db.session.refresh(corpus_file)
- try:
- form.vrt.data.save(corpus_file.path)
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', category='error')
- return make_response({'redirect_url': url_for('.add_corpus_file', corpus_id=corpus.id)}, 500) # noqa
- corpus.status = CorpusStatus.UNPREPARED
- db.session.commit()
- flash(f'Corpus file "{corpus_file.filename}" added', category='corpus')
- return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa
- return render_template(
- 'corpora/add_corpus_file.html.j2',
- corpus=corpus,
- form=form,
- title='Add corpus file'
- )
-
-
-@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/delete')
+@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>', methods=['DELETE'])
@login_required
def delete_corpus_file(corpus_id, corpus_file_id):
- corpus_file = CorpusFile.query.filter(
- CorpusFile.corpus_id == corpus_id,
- CorpusFile.id == corpus_file_id
- ).first_or_404()
- if not (
- corpus_file.corpus.user == current_user
- or current_user.is_administrator()
- ):
+ def _delete_corpus_file(app, corpus_file_id):
+ with app.app_context():
+ corpus_file = CorpusFile.query.get(corpus_file_id)
+ corpus_file.delete()
+ db.session.commit()
+
+ corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
+ if corpus_file.corpus.id != corpus_id:
+ abort(404)
+ if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
abort(403)
- flash(
- f'Corpus file "{corpus_file.filename}" marked for deletion',
- category='corpus'
+ thread = Thread(
+ target=_delete_corpus_file,
+ args=(current_app._get_current_object(), corpus_file_id)
)
- tasks.delete_corpus_file(corpus_file_id)
- return redirect(url_for('.corpus', corpus_id=corpus_id))
+ thread.start()
+ return {}, 202
@bp.route('/<hashid:corpus_id>/files/<hashid:corpus_file_id>/download')
@login_required
def download_corpus_file(corpus_id, corpus_file_id):
- corpus_file = CorpusFile.query.filter(
- CorpusFile.corpus_id == corpus_id,
- CorpusFile.id == corpus_file_id
- ).first_or_404()
- if not (
- corpus_file.corpus.user == current_user
- or current_user.is_administrator()
- ):
+ corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
+ if corpus_file.corpus.id != corpus_id:
+ abort(404)
+ if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
abort(403)
return send_from_directory(
+ os.path.dirname(corpus_file.path),
+ os.path.basename(corpus_file.path),
as_attachment=True,
attachment_filename=corpus_file.filename,
- directory=os.path.dirname(corpus_file.path),
- filename=os.path.basename(corpus_file.path)
- )
\ No newline at end of file
+ mimetype=corpus_file.mimetype
+ )
+
+
+@bp.route('/import', methods=['GET', 'POST'])
+@login_required
+def import_corpus():
+ abort(503)
+
+
+@bp.route('/<hashid:corpus_id>/export')
+@login_required
+def export_corpus(corpus_id):
+ abort(503)
diff --git a/app/corpora/tasks.py b/app/corpora/tasks.py
deleted file mode 100644
index c914a25a05053d1cc1ae879fbe0bd59d066447c9..0000000000000000000000000000000000000000
--- a/app/corpora/tasks.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from app import db
-from app.decorators import background
-from app.models import Corpus, CorpusFile
-
-
-@background
-def build_corpus(corpus_id, *args, **kwargs):
- app = kwargs['app']
- with app.app_context():
- corpus = Corpus.query.get(corpus_id)
- if corpus is None:
- raise Exception(f'Corpus {corpus_id} not found')
- corpus.build()
- db.session.commit()
-
-
-@background
-def delete_corpus(corpus_id, *args, **kwargs):
- with kwargs['app'].app_context():
- corpus = Corpus.query.get(corpus_id)
- if corpus is None:
- raise Exception(f'Corpus {corpus_id} not found')
- corpus.delete()
- db.session.commit()
-
-
-@background
-def delete_corpus_file(corpus_file_id, *args, **kwargs):
- with kwargs['app'].app_context():
- corpus_file = CorpusFile.query.get(corpus_file_id)
- if corpus_file is None:
- raise Exception(f'Corpus file {corpus_file_id} not found')
- corpus_file.delete()
- db.session.commit()
diff --git a/app/daemon/__init__.py b/app/daemon/__init__.py
index 84ed0efee60ad60c3c82f0623c4aab816fad6a45..9cf16bc8df044ec721cb4aeb2615d35c923f08e8 100644
--- a/app/daemon/__init__.py
+++ b/app/daemon/__init__.py
@@ -1,23 +1,11 @@
from app import db
-from flask import current_app
-from time import sleep
-from .corpus_utils import CheckCorporaMixin
-from .job_utils import CheckJobsMixin
-import docker
+from flask import Flask
+from .corpus_utils import check_corpora
+from .job_utils import check_jobs
-class Daemon(CheckCorporaMixin, CheckJobsMixin):
- def __init__(self):
- self.docker = docker.from_env()
- self.docker.login(
- username=current_app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'],
- password=current_app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'],
- registry=current_app.config['NOPAQUE_DOCKER_REGISTRY']
- )
-
- def run(self):
- while True:
- self.check_corpora()
- self.check_jobs()
- db.session.commit()
- sleep(1.5)
+def daemon(app: Flask):
+ with app.app_context():
+ check_corpora()
+ check_jobs()
+ db.session.commit()
diff --git a/app/daemon/corpus_utils.py b/app/daemon/corpus_utils.py
index 228eb64e3b927abeb1d1cf9ffec6a10b1d189baa..1703521a5f96afa1084ec5f3d17768e4279fefcf 100644
--- a/app/daemon/corpus_utils.py
+++ b/app/daemon/corpus_utils.py
@@ -1,3 +1,4 @@
+from app import docker_client
from app.models import Corpus, CorpusStatus
from flask import current_app
import docker
@@ -5,250 +6,216 @@ import os
import shutil
-class CheckCorporaMixin:
- def check_corpora(self):
- corpora = Corpus.query.all()
- for corpus in (x for x in corpora if x.status == CorpusStatus.SUBMITTED): # noqa
- self.create_build_corpus_service(corpus)
- for corpus in (x for x in corpora if x.status == CorpusStatus.QUEUED or x.status == CorpusStatus.BUILDING): # noqa
- self.checkout_build_corpus_service(corpus)
- for corpus in (x for x in corpora if x.status == CorpusStatus.BUILT and x.num_analysis_sessions > 0): # noqa
- corpus.status = CorpusStatus.STARTING_ANALYSIS_SESSION
- for corpus in (x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0): # noqa
- corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
- for corpus in (x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION): # noqa
- self.checkout_analysing_corpus_container(corpus)
- for corpus in (x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION): # noqa
- self.create_cqpserver_container(corpus)
- for corpus in (x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION): # noqa
- self.remove_cqpserver_container(corpus)
+def check_corpora():
+ corpora = Corpus.query.all()
+ for corpus in [x for x in corpora if x.status == CorpusStatus.SUBMITTED]:
+ _create_build_corpus_service(corpus)
+ for corpus in [x for x in corpora if x.status in [CorpusStatus.QUEUED, CorpusStatus.BUILDING]]:
+ _checkout_build_corpus_service(corpus)
+ for corpus in [x for x in corpora if x.status == CorpusStatus.BUILT and x.num_analysis_sessions > 0]:
+ corpus.status = CorpusStatus.STARTING_ANALYSIS_SESSION
+ for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]:
+ corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
+ for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]:
+ _checkout_analysing_corpus_container(corpus)
+ for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]:
+ _create_cqpserver_container(corpus)
+ for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]:
+ _remove_cqpserver_container(corpus)
- def create_build_corpus_service(self, corpus):
- ''' # Docker service settings # '''
- ''' ## Command ## '''
- command = ['bash', '-c']
- command.append(
- f'mkdir /corpora/data/nopaque_{corpus.id}'
- ' && '
- 'cwb-encode'
- ' -c utf8'
- f' -d /corpora/data/nopaque_{corpus.id}'
- ' -f /root/files/corpus.vrt'
- f' -R /usr/local/share/cwb/registry/nopaque_{corpus.id}'
- ' -P pos -P lemma -P simple_pos'
- ' -S ent:0+type -S s:0'
- ' -S text:0+address+author+booktitle+chapter+editor+institution+journal+pages+publisher+publishing_year+school+title' # noqa
- ' -xsB -9'
- ' && '
- f'cwb-make -V NOPAQUE_{corpus.id}'
+def _create_build_corpus_service(corpus):
+ ''' # Docker service settings # '''
+ ''' ## Command ## '''
+ command = ['bash', '-c']
+ command.append(
+ f'mkdir /corpora/data/nopaque_{corpus.id}'
+ ' && '
+ 'cwb-encode'
+ ' -c utf8'
+ f' -d /corpora/data/nopaque_{corpus.id}'
+ ' -f /root/files/corpus.vrt'
+ f' -R /usr/local/share/cwb/registry/nopaque_{corpus.id}'
+ ' -P pos -P lemma -P simple_pos'
+ ' -S ent:0+type -S s:0'
+ ' -S text:0+address+author+booktitle+chapter+editor+institution+journal+pages+publisher+publishing_year+school+title'
+ ' -xsB -9'
+ ' && '
+ f'cwb-make -V NOPAQUE_{corpus.id}'
+ )
+ ''' ## Constraints ## '''
+ constraints = ['node.role==worker']
+ ''' ## Image ## '''
+ image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
+ ''' ## Labels ## '''
+ labels = {
+ 'origin': current_app.config['SERVER_NAME'],
+ 'type': 'corpus.build',
+ 'corpus_id': str(corpus.id)
+ }
+ ''' ## Mounts ## '''
+ mounts = []
+ ''' ### Data mount ### '''
+ data_mount_source = os.path.join(corpus.path, 'cwb', 'data')
+ data_mount_target = '/corpora/data'
+ data_mount = f'{data_mount_source}:{data_mount_target}:rw'
+ # Make sure that their is no data in the data directory
+ shutil.rmtree(data_mount_source, ignore_errors=True)
+ os.makedirs(data_mount_source)
+ mounts.append(data_mount)
+ ''' ### File mount ### '''
+ file_mount_source = os.path.join(corpus.path, 'cwb', 'corpus.vrt')
+ file_mount_target = '/root/files/corpus.vrt'
+ file_mount = f'{file_mount_source}:{file_mount_target}:ro'
+ mounts.append(file_mount)
+ ''' ### Registry mount ### '''
+ registry_mount_source = os.path.join(corpus.path, 'cwb', 'registry')
+ registry_mount_target = '/usr/local/share/cwb/registry'
+ registry_mount = f'{registry_mount_source}:{registry_mount_target}:rw'
+ # Make sure that their is no data in the registry directory
+ shutil.rmtree(registry_mount_source, ignore_errors=True)
+ os.makedirs(registry_mount_source)
+ mounts.append(registry_mount)
+ ''' ## Name ## '''
+ name = f'build-corpus_{corpus.id}'
+ ''' ## Restart policy ## '''
+ restart_policy = docker.types.RestartPolicy()
+ try:
+ docker_client.services.create(
+ image,
+ command=command,
+ constraints=constraints,
+ labels=labels,
+ mounts=mounts,
+ name=name,
+ restart_policy=restart_policy,
+ user='0:0'
)
- ''' ## Constraints ## '''
- constraints = ['node.role==worker']
- ''' ## Image ## '''
- image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
- ''' ## Labels ## '''
- labels = {
- 'origin': current_app.config['SERVER_NAME'],
- 'type': 'corpus.build',
- 'corpus_id': str(corpus.id)
- }
- ''' ## Mounts ## '''
- mounts = []
- ''' ### Data mount ### '''
- data_mount_source = os.path.join(corpus.path, 'cwb', 'data')
- data_mount_target = '/corpora/data'
- data_mount = f'{data_mount_source}:{data_mount_target}:rw'
- # Make sure that their is no data in the data directory
- shutil.rmtree(data_mount_source, ignore_errors=True)
- os.makedirs(data_mount_source)
- mounts.append(data_mount)
- ''' ### File mount ### '''
- file_mount_source = os.path.join(corpus.path, 'cwb', 'corpus.vrt')
- file_mount_target = '/root/files/corpus.vrt'
- file_mount = f'{file_mount_source}:{file_mount_target}:ro'
- mounts.append(file_mount)
- ''' ### Registry mount ### '''
- registry_mount_source = os.path.join(corpus.path, 'cwb', 'registry')
- registry_mount_target = '/usr/local/share/cwb/registry'
- registry_mount = f'{registry_mount_source}:{registry_mount_target}:rw'
- # Make sure that their is no data in the registry directory
- shutil.rmtree(registry_mount_source, ignore_errors=True)
- os.makedirs(registry_mount_source)
- mounts.append(registry_mount)
- ''' ## Name ## '''
- name = f'build-corpus_{corpus.id}'
- ''' ## Restart policy ## '''
- restart_policy = docker.types.RestartPolicy()
- try:
- self.docker.services.create(
- image,
- command=command,
- constraints=constraints,
- labels=labels,
- mounts=mounts,
- name=name,
- restart_policy=restart_policy,
- user='0:0'
- )
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Create service "{name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- return
- corpus.status = CorpusStatus.QUEUED
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Create service "{name}" failed: {e}')
+ return
+ corpus.status = CorpusStatus.QUEUED
- def checkout_build_corpus_service(self, corpus):
- service_name = f'build-corpus_{corpus.id}'
- try:
- service = self.docker.services.get(service_name)
- except docker.errors.NotFound as e:
- current_app.logger.error(
- f'Get service "{service_name}" failed '
- f'due to "docker.errors.NotFound": {e}'
- )
- corpus.status = CorpusStatus.FAILED
- return
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Get service "{service_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- service_tasks = service.tasks()
- if not service_tasks:
- return
- task_state = service_tasks[0].get('Status').get('State')
- if corpus.status == CorpusStatus.QUEUED and task_state != 'pending': # noqa
- corpus.status = CorpusStatus.BUILDING
- return
- elif corpus.status == CorpusStatus.BUILDING and task_state == 'complete': # noqa
- corpus.status = CorpusStatus.BUILT
- elif corpus.status == CorpusStatus.BUILDING and task_state == 'failed': # noqa
- corpus.status = CorpusStatus.FAILED
- else:
- return
- try:
- service.remove()
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Remove service "{service_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
+def _checkout_build_corpus_service(corpus):
+ service_name = f'build-corpus_{corpus.id}'
+ try:
+ service = docker_client.services.get(service_name)
+ except docker.errors.NotFound as e:
+ current_app.logger.error(f'Get service "{service_name}" failed: {e}')
+ corpus.status = CorpusStatus.FAILED
+ return
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Get service "{service_name}" failed: {e}')
+ service_tasks = service.tasks()
+ if not service_tasks:
+ return
+ task_state = service_tasks[0].get('Status').get('State')
+ if corpus.status == CorpusStatus.QUEUED and task_state != 'pending':
+ corpus.status = CorpusStatus.BUILDING
+ return
+ elif corpus.status == CorpusStatus.BUILDING and task_state == 'complete':
+ corpus.status = CorpusStatus.BUILT
+ elif corpus.status == CorpusStatus.BUILDING and task_state == 'failed':
+ corpus.status = CorpusStatus.FAILED
+ else:
+ return
+ try:
+ service.remove()
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
- def create_cqpserver_container(self, corpus):
- ''' # Docker container settings # '''
- ''' ## Command ## '''
- command = []
- command.append(
- 'echo "host *;" > cqpserver.init'
- ' && '
- 'echo "user anonymous \\"\\";" >> cqpserver.init'
- ' && '
- 'cqpserver -I cqpserver.init'
- )
- ''' ## Detach ## '''
- detach = True
- ''' ## Entrypoint ## '''
- entrypoint = ['bash', '-c']
- ''' ## Image ## '''
- image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
- ''' ## Name ## '''
- name = f'cqpserver_{corpus.id}'
- ''' ## Network ## '''
- network = 'nopaque_default'
- ''' ## Volumes ## '''
- volumes = []
- ''' ### Corpus data volume ### '''
- data_volume_source = os.path.join(corpus.path, 'cwb', 'data')
- data_volume_target = '/corpora/data'
- data_volume = f'{data_volume_source}:{data_volume_target}:rw'
- volumes.append(data_volume)
- ''' ### Corpus registry volume ### '''
- registry_volume_source = os.path.join(corpus.path, 'cwb', 'registry')
- registry_volume_target = '/usr/local/share/cwb/registry'
- registry_volume = f'{registry_volume_source}:{registry_volume_target}:rw' # noqa
- volumes.append(registry_volume)
- # Check if a cqpserver container already exists. If this is the case,
- # remove it and create a new one
- try:
- container = self.docker.containers.get(name)
- except docker.errors.NotFound:
- pass
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Get container "{name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- return
- else:
- try:
- container.remove(force=True)
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Remove container "{name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- return
+def _create_cqpserver_container(corpus):
+ ''' # Docker container settings # '''
+ ''' ## Command ## '''
+ command = []
+ command.append(
+ 'echo "host *;" > cqpserver.init'
+ ' && '
+ 'echo "user anonymous \\"\\";" >> cqpserver.init'
+ ' && '
+ 'cqpserver -I cqpserver.init'
+ )
+ ''' ## Detach ## '''
+ detach = True
+ ''' ## Entrypoint ## '''
+ entrypoint = ['bash', '-c']
+ ''' ## Image ## '''
+ image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702'
+ ''' ## Name ## '''
+ name = f'cqpserver_{corpus.id}'
+ ''' ## Network ## '''
+ network = 'nopaque_default'
+ ''' ## Volumes ## '''
+ volumes = []
+ ''' ### Corpus data volume ### '''
+ data_volume_source = os.path.join(corpus.path, 'cwb', 'data')
+ data_volume_target = '/corpora/data'
+ data_volume = f'{data_volume_source}:{data_volume_target}:rw'
+ volumes.append(data_volume)
+ ''' ### Corpus registry volume ### '''
+ registry_volume_source = os.path.join(corpus.path, 'cwb', 'registry')
+ registry_volume_target = '/usr/local/share/cwb/registry'
+ registry_volume = f'{registry_volume_source}:{registry_volume_target}:rw'
+ volumes.append(registry_volume)
+ # Check if a cqpserver container already exists. If this is the case,
+ # remove it and create a new one
+ try:
+ container = docker_client.containers.get(name)
+ except docker.errors.NotFound:
+ pass
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Get container "{name}" failed: {e}')
+ return
+ else:
try:
- self.docker.containers.run(
- image,
- command=command,
- detach=detach,
- entrypoint=entrypoint,
- name=name,
- network=network,
- user='0:0',
- volumes=volumes
- )
- except docker.errors.ImageNotFound as e:
- current_app.logger.error(
- f'Run container "{name}" failed '
- f'due to "docker.errors.ImageNotFound" error: {e}'
- )
- corpus.status = CorpusStatus.FAILED
- return
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Run container "{name}" failed '
- f'due to "docker.errors.APIError" error: {e}'
- )
+ container.remove(force=True)
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Remove container "{name}" failed: {e}')
return
- corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
+ try:
+ docker_client.containers.run(
+ image,
+ command=command,
+ detach=detach,
+ entrypoint=entrypoint,
+ name=name,
+ network=network,
+ user='0:0',
+ volumes=volumes
+ )
+ except docker.errors.ImageNotFound as e:
+ current_app.logger.error(
+ f'Run container "{name}" failed '
+ f'due to "docker.errors.ImageNotFound" error: {e}'
+ )
+ corpus.status = CorpusStatus.FAILED
+ return
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Run container "{name}" failed: {e}')
+ return
+ corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
- def checkout_analysing_corpus_container(self, corpus):
- container_name = f'cqpserver_{corpus.id}'
- try:
- self.docker.containers.get(container_name)
- except docker.errors.NotFound as e:
- current_app.logger.error(
- f'Get container "{container_name}" failed '
- f'due to "docker.errors.NotFound": {e}'
- )
- corpus.num_analysis_sessions = 0
- corpus.status = CorpusStatus.BUILT
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Get container "{container_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
+def _checkout_analysing_corpus_container(corpus):
+ container_name = f'cqpserver_{corpus.id}'
+ try:
+ docker_client.containers.get(container_name)
+ except docker.errors.NotFound as e:
+ current_app.logger.error(f'Get container "{container_name}" failed: {e}')
+ corpus.num_analysis_sessions = 0
+ corpus.status = CorpusStatus.BUILT
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Get container "{container_name}" failed: {e}')
- def remove_cqpserver_container(self, corpus):
- container_name = f'cqpserver_{corpus.id}'
- try:
- container = self.docker.containers.get(container_name)
- except docker.errors.NotFound:
- corpus.status = CorpusStatus.BUILT
- return
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Get container "{container_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- return
- try:
- container.remove(force=True)
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Remove container "{container_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
+def _remove_cqpserver_container(corpus):
+ container_name = f'cqpserver_{corpus.id}'
+ try:
+ container = docker_client.containers.get(container_name)
+ except docker.errors.NotFound:
+ corpus.status = CorpusStatus.BUILT
+ return
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Get container "{container_name}" failed: {e}')
+ return
+ try:
+ container.remove(force=True)
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Remove container "{container_name}" failed: {e}')
diff --git a/app/daemon/job_utils.py b/app/daemon/job_utils.py
index 5f05681f6af9ee7959680b428e578770a3a2de75..38d6c48b05c418f3ac9d91e2eca6f8146cef14d3 100644
--- a/app/daemon/job_utils.py
+++ b/app/daemon/job_utils.py
@@ -1,4 +1,4 @@
-from app import db
+from app import db, docker_client, hashids
from app.models import (
Job,
JobResult,
@@ -15,217 +15,202 @@ import os
import shutil
-class CheckJobsMixin:
- def check_jobs(self):
- jobs = Job.query.all()
- for job in (x for x in jobs if x.status == JobStatus.SUBMITTED):
- self.create_job_service(job)
- for job in (x for x in jobs if x.status in [JobStatus.QUEUED, JobStatus.RUNNING]): # noqa
- self.checkout_job_service(job)
- for job in (x for x in jobs if x.status == JobStatus.CANCELING):
- self.remove_job_service(job)
+def check_jobs():
+ jobs = Job.query.all()
+ for job in [x for x in jobs if x.status == JobStatus.SUBMITTED]:
+ _create_job_service(job)
+ for job in [x for x in jobs if x.status in [JobStatus.QUEUED, JobStatus.RUNNING]]:
+ _checkout_job_service(job)
+ for job in [x for x in jobs if x.status == JobStatus.CANCELING]:
+ _remove_job_service(job)
- def create_job_service(self, job):
- ''' # Docker service settings # '''
- ''' ## Service specific settings ## '''
- if job.service == 'file-setup-pipeline':
- mem_mb = 512
- n_cores = 2
- executable = 'file-setup-pipeline'
- image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}file-setup-pipeline:v{job.service_version}' # noqa
- elif job.service == 'tesseract-ocr-pipeline':
- mem_mb = 1024
- n_cores = 4
- executable = 'tesseract-ocr-pipeline'
- image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}tesseract-ocr-pipeline:v{job.service_version}' # noqa
- elif job.service == 'transkribus-htr-pipeline':
- mem_mb = 1024
- n_cores = 4
- executable = 'transkribus-htr-pipeline'
- image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}transkribus-htr-pipeline:v{job.service_version}' # noqa
- elif job.service == 'spacy-nlp-pipeline':
- mem_mb = 1024
- n_cores = 1
- executable = 'spacy-nlp-pipeline'
- image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}spacy-nlp-pipeline:v{job.service_version}' # noqa
- ''' ## Command ## '''
- command = f'{executable} -i /input -o /output'
- command += ' --log-dir /logs'
- command += f' --mem-mb {mem_mb}'
- command += f' --n-cores {n_cores}'
- if job.service == 'spacy-nlp-pipeline':
- command += f' -m {job.service_args["model"]}'
- if 'encoding_detection' in job.service_args and job.service_args['encoding_detection']: # noqa
- command += ' --check-encoding'
- elif job.service == 'tesseract-ocr-pipeline':
- command += f' -m {job.service_args["model"]}'
- if 'binarization' in job.service_args and job.service_args['binarization']:
- command += ' --binarize'
- elif job.service == 'transkribus-htr-pipeline':
- transkribus_htr_model = TranskribusHTRModel.query.get(job.service_args['model'])
- command += f' -m {transkribus_htr_model.transkribus_model_id}'
- readcoop_username = current_app.config.get('NOPAQUE_READCOOP_USERNAME')
- command += f' --readcoop-username "{readcoop_username}"'
- readcoop_password = current_app.config.get('NOPAQUE_READCOOP_PASSWORD')
- command += f' --readcoop-password "{readcoop_password}"'
- if 'binarization' in job.service_args and job.service_args['binarization']:
- command += ' --binarize'
- ''' ## Constraints ## '''
- constraints = ['node.role==worker']
- ''' ## Labels ## '''
- labels = {
- 'origin': current_app.config['SERVER_NAME'],
- 'type': 'job',
- 'job_id': str(job.id)
- }
- ''' ## Mounts ## '''
- mounts = []
- ''' ### Input mount(s) ### '''
- input_mount_target_base = '/input'
- if job.service == 'file-setup-pipeline':
- input_mount_target_base += f'/{secure_filename(job.title)}'
- for job_input in job.inputs:
- input_mount_source = job_input.path
- input_mount_target = f'{input_mount_target_base}/{job_input.filename}' # noqa
- input_mount = f'{input_mount_source}:{input_mount_target}:ro'
- mounts.append(input_mount)
- if job.service == 'tesseract-ocr-pipeline':
- model = TesseractOCRModel.query.get(job.service_args['model'])
- if model is None:
- job.status = JobStatus.FAILED
- return
- models_mount_source = model.path
- models_mount_target = f'/usr/local/share/tessdata/{model.filename}'
- models_mount = f'{models_mount_source}:{models_mount_target}:ro'
- mounts.append(models_mount)
- ''' ### Output mount ### '''
- output_mount_source = os.path.join(job.path, 'results')
- output_mount_target = '/output'
- output_mount = f'{output_mount_source}:{output_mount_target}:rw'
- # Make sure that their is no data in the output directory
- shutil.rmtree(output_mount_source, ignore_errors=True)
- os.makedirs(output_mount_source)
- mounts.append(output_mount)
- ''' ### Pipeline data mount ### '''
- pyflow_data_mount_source = os.path.join(job.path, 'pipeline_data')
- pyflow_data_mount_target = '/logs/pyflow.data'
- pyflow_data_mount = f'{pyflow_data_mount_source}:{pyflow_data_mount_target}:rw' # noqa
- # Make sure that their is no data in the output directory
- shutil.rmtree(pyflow_data_mount_source, ignore_errors=True)
- os.makedirs(pyflow_data_mount_source)
- mounts.append(pyflow_data_mount)
- ''' ## Name ## '''
- name = f'job_{job.id}'
- ''' ## Resources ## '''
- resources = docker.types.Resources(
- cpu_reservation=n_cores * (10 ** 9),
- mem_reservation=mem_mb * (10 ** 6)
- )
- ''' ## Restart policy ## '''
- restart_policy = docker.types.RestartPolicy()
- try:
- self.docker.services.create(
- image,
- command=command,
- constraints=constraints,
- labels=labels,
- mounts=mounts,
- name=name,
- resources=resources,
- restart_policy=restart_policy,
- user='0:0'
- )
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Create service "{name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- return
- job.status = JobStatus.QUEUED
-
- def checkout_job_service(self, job):
- service_name = f'job_{job.id}'
- try:
- service = self.docker.services.get(service_name)
- except docker.errors.NotFound as e:
- current_app.logger.error(
- f'Get service "{service_name}" failed '
- f'due to "docker.errors.NotFound": {e}'
- )
+def _create_job_service(job):
+ ''' # Docker service settings # '''
+ ''' ## Service specific settings ## '''
+ if job.service == 'file-setup-pipeline':
+ mem_mb = 512
+ n_cores = 2
+ executable = 'file-setup-pipeline'
+ image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}file-setup-pipeline:v{job.service_version}'
+ elif job.service == 'tesseract-ocr-pipeline':
+ mem_mb = 1024
+ n_cores = 4
+ executable = 'tesseract-ocr-pipeline'
+ image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}tesseract-ocr-pipeline:v{job.service_version}'
+ elif job.service == 'transkribus-htr-pipeline':
+ mem_mb = 1024
+ n_cores = 4
+ executable = 'transkribus-htr-pipeline'
+ image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}transkribus-htr-pipeline:v{job.service_version}'
+ elif job.service == 'spacy-nlp-pipeline':
+ mem_mb = 1024
+ n_cores = 1
+ executable = 'spacy-nlp-pipeline'
+ image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}spacy-nlp-pipeline:v{job.service_version}'
+ ''' ## Command ## '''
+ command = f'{executable} -i /input -o /output'
+ command += ' --log-dir /logs'
+ command += f' --mem-mb {mem_mb}'
+ command += f' --n-cores {n_cores}'
+ if job.service == 'spacy-nlp-pipeline':
+ command += f' -m {job.service_args["model"]}'
+ if 'encoding_detection' in job.service_args and job.service_args['encoding_detection']:
+ command += ' --check-encoding'
+ elif job.service == 'tesseract-ocr-pipeline':
+ command += f' -m {job.service_args["model"]}'
+ if 'binarization' in job.service_args and job.service_args['binarization']:
+ command += ' --binarize'
+ elif job.service == 'transkribus-htr-pipeline':
+ transkribus_htr_model = TranskribusHTRModel.query.get(job.service_args['model'])
+ command += f' -m {transkribus_htr_model.transkribus_model_id}'
+ readcoop_username = current_app.config.get('NOPAQUE_READCOOP_USERNAME')
+ command += f' --readcoop-username "{readcoop_username}"'
+ readcoop_password = current_app.config.get('NOPAQUE_READCOOP_PASSWORD')
+ command += f' --readcoop-password "{readcoop_password}"'
+ if 'binarization' in job.service_args and job.service_args['binarization']:
+ command += ' --binarize'
+ ''' ## Constraints ## '''
+ constraints = ['node.role==worker']
+ ''' ## Labels ## '''
+ labels = {
+ 'origin': current_app.config['SERVER_NAME'],
+ 'type': 'job',
+ 'job_id': str(job.id)
+ }
+ ''' ## Mounts ## '''
+ mounts = []
+ ''' ### Input mount(s) ### '''
+ input_mount_target_base = '/input'
+ if job.service == 'file-setup-pipeline':
+ input_mount_target_base += f'/{secure_filename(job.title)}'
+ for job_input in job.inputs:
+ input_mount_source = job_input.path
+ input_mount_target = f'{input_mount_target_base}/{job_input.filename}'
+ input_mount = f'{input_mount_source}:{input_mount_target}:ro'
+ mounts.append(input_mount)
+ if job.service == 'tesseract-ocr-pipeline':
+ if isinstance(job.service_args['model'], str):
+ model_id = hashids.decode(job.service_args['model'])
+ elif isinstance(job.service_args['model'], int):
+ model_id = job.service_args['model']
+ else:
job.status = JobStatus.FAILED
return
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Get service "{service_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- return
- service_tasks = service.tasks()
- if not service_tasks:
- return
- task_state = service_tasks[0].get('Status').get('State')
- if job.status == JobStatus.QUEUED and task_state != 'pending':
- job.status = JobStatus.RUNNING
- return
- elif job.status == JobStatus.RUNNING and task_state == 'complete': # noqa
- job.status = JobStatus.COMPLETED
- results_dir = os.path.join(job.path, 'results')
- with open(os.path.join(results_dir, 'outputs.json')) as f:
- outputs = json.load(f)
- for output in outputs:
- filename = os.path.basename(output['file'])
- job_result = JobResult(
- filename=filename,
- job=job,
- mimetype=output['mimetype']
- )
- if 'description' in output:
- job_result.description = output['description']
- db.session.add(job_result)
- db.session.flush(objects=[job_result])
- db.session.refresh(job_result)
- os.rename(
- os.path.join(results_dir, output['file']),
- job_result.path
- )
- elif job.status == JobStatus.RUNNING and task_state == 'failed':
+ model = TesseractOCRModel.query.get(model_id)
+ if model is None:
job.status = JobStatus.FAILED
- else:
return
- job.end_date = datetime.utcnow()
- try:
- service.remove()
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Remove service "{service_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
+ models_mount_source = model.path
+ models_mount_target = f'/usr/local/share/tessdata/{model.filename}'
+ models_mount = f'{models_mount_source}:{models_mount_target}:ro'
+ mounts.append(models_mount)
+ ''' ### Output mount ### '''
+ output_mount_source = os.path.join(job.path, 'results')
+ output_mount_target = '/output'
+ output_mount = f'{output_mount_source}:{output_mount_target}:rw'
+ # Make sure that their is no data in the output directory
+ shutil.rmtree(output_mount_source, ignore_errors=True)
+ os.makedirs(output_mount_source)
+ mounts.append(output_mount)
+ ''' ### Pipeline data mount ### '''
+ pyflow_data_mount_source = os.path.join(job.path, 'pipeline_data')
+ pyflow_data_mount_target = '/logs/pyflow.data'
+ pyflow_data_mount = f'{pyflow_data_mount_source}:{pyflow_data_mount_target}:rw'
+ # Make sure that their is no data in the output directory
+ shutil.rmtree(pyflow_data_mount_source, ignore_errors=True)
+ os.makedirs(pyflow_data_mount_source)
+ mounts.append(pyflow_data_mount)
+ ''' ## Name ## '''
+ name = f'job_{job.id}'
+ ''' ## Resources ## '''
+ resources = docker.types.Resources(
+ cpu_reservation=n_cores * (10 ** 9),
+ mem_reservation=mem_mb * (10 ** 6)
+ )
+ ''' ## Restart policy ## '''
+ restart_policy = docker.types.RestartPolicy()
+ try:
+ docker_client.services.create(
+ image,
+ command=command,
+ constraints=constraints,
+ labels=labels,
+ mounts=mounts,
+ name=name,
+ resources=resources,
+ restart_policy=restart_policy,
+ user='0:0'
+ )
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Create service "{name}" failed: {e}')
+ return
+ job.status = JobStatus.QUEUED
- def remove_job_service(self, job):
- service_name = f'job_{job.id}'
- try:
- service = self.docker.services.get(service_name)
- except docker.errors.NotFound:
- job.status = JobStatus.CANCELED
- return
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Get service "{service_name}" failed '
- f'due to "docker.errors.APIError": {e}'
+def _checkout_job_service(job):
+ service_name = f'job_{job.id}'
+ try:
+ service = docker_client.services.get(service_name)
+ except docker.errors.NotFound as e:
+ current_app.logger.error(f'Get service "{service_name}" failed: {e}')
+ job.status = JobStatus.FAILED
+ return
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Get service "{service_name}" failed: {e}')
+ return
+ service_tasks = service.tasks()
+ if not service_tasks:
+ return
+ task_state = service_tasks[0].get('Status').get('State')
+ if job.status == JobStatus.QUEUED and task_state != 'pending':
+ job.status = JobStatus.RUNNING
+ return
+ elif job.status == JobStatus.RUNNING and task_state == 'complete':
+ job.status = JobStatus.COMPLETED
+ results_dir = os.path.join(job.path, 'results')
+ with open(os.path.join(results_dir, 'outputs.json')) as f:
+ outputs = json.load(f)
+ for output in outputs:
+ filename = os.path.basename(output['file'])
+ job_result = JobResult(
+ filename=filename,
+ job=job,
+ mimetype=output['mimetype']
)
- return
- try:
- service.update(mounts=None)
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Update service "{service_name}" failed '
- f'due to "docker.errors.APIError": {e}'
- )
- return
- try:
- service.remove()
- except docker.errors.APIError as e:
- current_app.logger.error(
- f'Remove "{service_name}" service failed '
- f'due to "docker.errors.APIError": {e}'
+ if 'description' in output:
+ job_result.description = output['description']
+ db.session.add(job_result)
+ db.session.flush(objects=[job_result])
+ db.session.refresh(job_result)
+ os.rename(
+ os.path.join(results_dir, output['file']),
+ job_result.path
)
+ elif job.status == JobStatus.RUNNING and task_state == 'failed':
+ job.status = JobStatus.FAILED
+ else:
+ return
+ job.end_date = datetime.utcnow()
+ try:
+ service.remove()
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
+
+def _remove_job_service(job):
+ service_name = f'job_{job.id}'
+ try:
+ service = docker_client.services.get(service_name)
+ except docker.errors.NotFound:
+ job.status = JobStatus.CANCELED
+ return
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Get service "{service_name}" failed: {e}')
+ return
+ try:
+ service.update(mounts=None)
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Update service "{service_name}" failed: {e}')
+ return
+ try:
+ service.remove()
+ except docker.errors.DockerException as e:
+ current_app.logger.error(f'Remove "{service_name}" service failed: {e}')
diff --git a/app/decorators.py b/app/decorators.py
index 8c1ba90a351c7c026ec494545d5f92c80b0795be..47e6d7491aa1d18eaf1255a80a4f590f1dffa257 100644
--- a/app/decorators.py
+++ b/app/decorators.py
@@ -2,7 +2,7 @@ from flask import abort, current_app
from flask_login import current_user
from functools import wraps
from threading import Thread
-from .models import Permission
+from app.models import Permission
def permission_required(permission):
diff --git a/app/email.py b/app/email.py
index 50c41caafae177b4be5aca3141e4f53d488bf109..a853e1d7ab84af9ecad7f2eceba64de0b7738a2f 100644
--- a/app/email.py
+++ b/app/email.py
@@ -1,27 +1,25 @@
from flask import current_app, render_template
from flask_mail import Message
-from typing import Any, Text
-from . import mail
-from .decorators import background
+from threading import Thread
+from app import mail
-def create_message(
- recipient: str,
- subject: str,
- template: str,
- **kwargs: Any
-) -> Message:
+def create_message(recipient, subject, template, **kwargs):
subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
msg: Message = Message(
- f'{subject_prefix} {subject}',
- recipients=[recipient]
+ body=render_template(f'{template}.txt.j2', **kwargs),
+ html=render_template(f'{template}.html.j2', **kwargs),
+ recipients=[recipient],
+ subject=f'{subject_prefix} {subject}'
)
- msg.body = render_template(f'{template}.txt.j2', **kwargs)
- msg.html = render_template(f'{template}.html.j2', **kwargs)
return msg
-@background
-def send(msg: Message, *args, **kwargs):
- with kwargs['app'].app_context():
- mail.send(msg)
+def send(msg, *args, **kwargs):
+ def _send(app, msg):
+ with app.app_context():
+ mail.send(msg)
+
+ thread = Thread(target=_send, args=[current_app._get_current_object(), msg])
+ thread.start()
+ return thread
diff --git a/app/errors/handlers.py b/app/errors/handlers.py
index a5e49f901ee3ce5f0247eea1ac4abb27c9fead16..cc6c926869536362764b0cab2ac8d75eed1dc0ea 100644
--- a/app/errors/handlers.py
+++ b/app/errors/handlers.py
@@ -1,52 +1,11 @@
-from flask import render_template, request, jsonify
+from flask import render_template, request
+from werkzeug.exceptions import HTTPException
from . import bp
-@bp.app_errorhandler(403)
-def forbidden(e):
+@bp.errorhandler(HTTPException)
+def generic_error_handler(e):
if (request.accept_mimetypes.accept_json
and not request.accept_mimetypes.accept_html):
- response = jsonify({'error': 'forbidden'})
- response.status_code = 403
- return response
- return render_template('errors/403.html.j2', title='Forbidden'), 403
-
-
-@bp.app_errorhandler(404)
-def not_found(e):
- if (request.accept_mimetypes.accept_json
- and not request.accept_mimetypes.accept_html):
- response = jsonify({'error': 'not found'})
- response.status_code = 404
- return response
- return render_template('errors/404.html.j2', title='Not Found'), 404
-
-
-@bp.app_errorhandler(413)
-def payload_too_large(e):
- if (request.accept_mimetypes.accept_json
- and not request.accept_mimetypes.accept_html):
- response = jsonify({'error': 'payload too large'})
- response.status_code = 413
- return response
- return render_template('errors/413.html.j2', title='Payload Too Large'), 413
-
-
-@bp.app_errorhandler(500)
-def internal_server_error(e):
- if (request.accept_mimetypes.accept_json
- and not request.accept_mimetypes.accept_html):
- response = jsonify({'error': 'internal server error'})
- response.status_code = 500
- return response
- return render_template('errors/500.html.j2', title='Internal Server Error'), 500
-
-
-@bp.app_errorhandler(503)
-def service_unavailable_error(e):
- if (request.accept_mimetypes.accept_json
- and not request.accept_mimetypes.accept_html):
- response = jsonify({'error': 'service unavailable'})
- response.status_code = 503
- return response
- return render_template('errors/503.html.j2', title='Service Unavailable'), 503
+ return {'errors': {'message': e.description}}, e.code
+ return render_template('errors/error.html.j2', error=e), e.code
diff --git a/app/jobs/routes.py b/app/jobs/routes.py
index ac484958282075e4e49950ee45f1b30ff3fe3987..7dae80e1725dcc3b7a77def1c294bb6fcd21b01c 100644
--- a/app/jobs/routes.py
+++ b/app/jobs/routes.py
@@ -1,17 +1,16 @@
-from app.decorators import admin_required
-from app.models import Job, JobInput, JobResult, JobStatus
from flask import (
abort,
- flash,
- redirect,
+ current_app,
render_template,
- send_from_directory,
- url_for
+ send_from_directory
)
from flask_login import current_user, login_required
-from . import bp
-from . import tasks
+from threading import Thread
import os
+from app import db
+from app.decorators import admin_required
+from app.models import Job, JobInput, JobResult, JobStatus
+from . import bp
@bp.route('/<hashid:job_id>')
@@ -27,68 +26,91 @@ def job(job_id):
)
-@bp.route('/<hashid:job_id>/delete')
+@bp.route('/<hashid:job_id>', methods=['DELETE'])
@login_required
def delete_job(job_id):
+ def _delete_job(app, job_id):
+ with app.app_context():
+ job = Job.query.get(job_id)
+ job.delete()
+ db.session.commit()
+
+ job = Job.query.get_or_404(job_id)
+ if not (job.user == current_user or current_user.is_administrator()):
+ abort(403)
+ thread = Thread(
+ target=_delete_job,
+ args=(current_app._get_current_object(), job_id)
+ )
+ thread.start()
+ return {}, 202
+
+
+@bp.route('/<hashid:job_id>/log')
+@login_required
+@admin_required
+def job_log(job_id):
+ job = Job.query.get_or_404(job_id)
+ if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+ response = {'errors': {'message': 'Job status is not completed or failed'}}
+ return response, 409
+ with open(os.path.join(job.path, 'pipeline_data', 'logs', 'pyflow_log.txt')) as log_file:
+ log = log_file.read()
+ return log, 200, {'Content-Type': 'text/plain; charset=utf-8'}
+
+
+@bp.route('/<hashid:job_id>/restart', methods=['POST'])
+@login_required
+def restart_job(job_id):
+ def _restart_job(app, job_id):
+ with app.app_context():
+ job = Job.query.get(job_id)
+ job.restart()
+ db.session.commit()
+
job = Job.query.get_or_404(job_id)
if not (job.user == current_user or current_user.is_administrator()):
abort(403)
- tasks.delete_job(job_id)
- flash(f'Job "{job.title}" marked for deletion', 'job')
- return redirect(url_for('main.dashboard'))
+ if job.status == JobStatus.FAILED:
+ response = {'errors': {'message': 'Job status is not "failed"'}}
+ return response, 409
+ thread = Thread(
+ target=_restart_job,
+ args=(current_app._get_current_object(), job_id)
+ )
+ thread.start()
+ return {}, 202
@bp.route('/<hashid:job_id>/inputs/<hashid:job_input_id>/download')
@login_required
def download_job_input(job_id, job_input_id):
- job_input = JobInput.query.filter(
- JobInput.job_id == job_id,
- JobInput.id == job_input_id
- ).first_or_404()
- if not (
- job_input.job.user == current_user
- or current_user.is_administrator()
- ):
+ job_input = JobInput.query.get_or_404(job_input_id)
+ if job_input.job.id != job_id:
+ abort(404)
+ if not (job_input.job.user == current_user or current_user.is_administrator()):
abort(403)
return send_from_directory(
+ os.path.dirname(job_input.path),
+ os.path.basename(job_input.path),
as_attachment=True,
attachment_filename=job_input.filename,
- directory=os.path.dirname(job_input.path),
- filename=os.path.basename(job_input.path)
+ mimetype=job_input.mimetype
)
-@bp.route('/<hashid:job_id>/restart')
-@login_required
-@admin_required
-def restart(job_id):
- job = Job.query.get_or_404(job_id)
- if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
- flash(
- f'Can\'t restart job "{job.title}": Status is not "Completed/Failed"', # noqa
- category='error'
- )
- else:
- tasks.restart_job(job_id)
- flash(f'Job "{job.title}" marked to get restarted', category='job')
- return redirect(url_for('.job', job_id=job_id))
-
-
@bp.route('/<hashid:job_id>/results/<hashid:job_result_id>/download')
@login_required
def download_job_result(job_id, job_result_id):
- job_result = JobResult.query.filter(
- JobResult.job_id == job_id,
- JobResult.id == job_result_id
- ).first_or_404()
- if not (
- job_result.job.user == current_user
- or current_user.is_administrator()
- ):
+ job_result = JobResult.query.get_or_404(job_result_id)
+ if job_result.job.id != job_id:
+ abort(404)
+ if not (job_result.job.user == current_user or current_user.is_administrator()):
abort(403)
return send_from_directory(
+ os.path.dirname(job_result.path),
+ os.path.basename(job_result.path),
as_attachment=True,
attachment_filename=job_result.filename,
- directory=os.path.dirname(job_result.path),
- filename=os.path.basename(job_result.path)
+ mimetype=job_result.mimetype
)
diff --git a/app/jobs/tasks.py b/app/jobs/tasks.py
deleted file mode 100644
index 1738b0cd77099dd7ccde8e5189911ac90b73f465..0000000000000000000000000000000000000000
--- a/app/jobs/tasks.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from app import db
-from app.decorators import background
-from app.models import Job
-
-
-@background
-def delete_job(job_id, *args, **kwargs):
- with kwargs['app'].app_context():
- job = Job.query.get(job_id)
- if job is None:
- raise Exception(f'Job {job_id} not found')
- job.delete()
- db.session.commit()
-
-
-@background
-def restart_job(job_id, *args, **kwargs):
- with kwargs['app'].app_context():
- job = Job.query.get(job_id)
- if job is None:
- raise Exception(f'Job {job_id} not found')
- try:
- job.restart()
- except Exception:
- pass
- else:
- db.session.commit()
diff --git a/app/main/routes.py b/app/main/routes.py
index cf87f0b5c4cc3871ff470bcb2b6b9a61a24fb1dc..1e7665a38d6fc2a177b386ad36d60d29d433d9ab 100644
--- a/app/main/routes.py
+++ b/app/main/routes.py
@@ -1,30 +1,27 @@
-from app.auth.forms import LoginForm
-from app.models import User
from flask import flash, redirect, render_template, url_for
from flask_login import login_required, login_user
+from app.auth.forms import LoginForm
+from app.models import User
from . import bp
-@bp.route('/', methods=['GET', 'POST'])
+@bp.route('', methods=['GET', 'POST'])
def index():
form = LoginForm(prefix='login-form')
if form.validate_on_submit():
- user = User.query.filter_by(username=form.user.data).first()
- if user is None:
- user = User.query.filter_by(email=form.user.data.lower()).first()
- if user is not None and user.verify_password(form.password.data):
+ user = User.query.filter((User.email == form.user.data.lower()) | (User.username == form.user.data)).first()
+ if user and user.verify_password(form.password.data):
login_user(user, form.remember_me.data)
+ flash('You have been logged in')
return redirect(url_for('.dashboard'))
- flash('Invalid email/username or password.')
+ flash('Invalid email/username or password', category='error')
+ redirect(url_for('.index'))
return render_template('main/index.html.j2', form=form, title='nopaque')
@bp.route('/faq')
def faq():
- return render_template(
- 'main/faq.html.j2',
- title='Frequently Asked Questions'
- )
+ return render_template('main/faq.html.j2', title='Frequently Asked Questions')
@bp.route('/dashboard')
@@ -45,10 +42,7 @@ def news():
@bp.route('/privacy_policy')
def privacy_policy():
- return render_template(
- 'main/privacy_policy.html.j2',
- title='Privacy statement (GDPR)'
- )
+ return render_template('main/privacy_policy.html.j2', title='Privacy statement (GDPR)')
@bp.route('/terms_of_use')
diff --git a/app/models.py b/app/models.py
index f5cd0009fe066936c03fefbecd676670a03c7a26..8efc4bd2d1ce674bc3e086223048c2d99a492619 100644
--- a/app/models.py
+++ b/app/models.py
@@ -1,72 +1,158 @@
-from app import db, login
-from app.converters.vrt import normalize_vrt_file
-from app.sqlalchemy_type_decorators import ContainerColumn, IntEnumColumn
from datetime import datetime, timedelta
-from enum import IntEnum
+from enum import Enum, IntEnum
from flask import current_app, url_for
from flask_hashids import HashidMixin
from flask_login import UserMixin
-from itsdangerous import BadSignature, TimedJSONWebSignatureSerializer
from time import sleep
from tqdm import tqdm
from werkzeug.security import generate_password_hash, check_password_hash
-import base64
+from werkzeug.utils import secure_filename
import json
+import jwt
import os
import requests
+import secrets
import shutil
import xml.etree.ElementTree as ET
import yaml
+from app import db, hashids, login, mail, socketio
+from app.converters.vrt import normalize_vrt_file
+from app.email import create_message
TRANSKRIBUS_HTR_MODELS = \
- json.loads(requests.get('https://transkribus.eu/TrpServer/rest/models/text').content)['trpModelMetadata'] # noqa
+ json.loads(requests.get('https://transkribus.eu/TrpServer/rest/models/text', params={'docType': 'handwritten'}).content)['trpModelMetadata'] # noqa
+
+
+##############################################################################
+# enums #
+##############################################################################
+# region enums
+class CorpusStatus(IntEnum):
+ UNPREPARED = 1
+ SUBMITTED = 2
+ QUEUED = 3
+ BUILDING = 4
+ BUILT = 5
+ FAILED = 6
+ STARTING_ANALYSIS_SESSION = 7
+ RUNNING_ANALYSIS_SESSION = 8
+ CANCELING_ANALYSIS_SESSION = 9
+
+
+class JobStatus(IntEnum):
+ INITIALIZING = 1
+ SUBMITTED = 2
+ QUEUED = 3
+ RUNNING = 4
+ CANCELING = 5
+ CANCELED = 6
+ COMPLETED = 7
+ FAILED = 8
+
+
+class Permission(IntEnum):
+ '''
+ Defines User permissions as integers by the power of 2. User permission
+ can be evaluated using the bitwise operator &.
+ '''
+ ADMINISTRATE = 1
+ CONTRIBUTE = 2
+ USE_API = 4
+class UserSettingJobStatusMailNotificationLevel(IntEnum):
+ NONE = 1
+ END = 2
+ ALL = 3
+# endregion enums
+
+
+##############################################################################
+# mixins #
+##############################################################################
+# region mixins
class FileMixin:
'''
Mixin for db.Model classes. All file related models should use this.
'''
creation_date = db.Column(db.DateTime, default=datetime.utcnow)
filename = db.Column(db.String(255))
- last_edited_date = db.Column(db.DateTime, default=datetime.utcnow)
+ last_edited_date = db.Column(db.DateTime)
mimetype = db.Column(db.String(255))
- def file_mixin_to_dict(self, backrefs=False, relationships=False):
+ def file_mixin_to_json(self, backrefs=False, relationships=False):
return {
- 'creation_date': self.creation_date.isoformat() + 'Z',
+ 'creation_date': f'{self.creation_date.isoformat()}Z',
'filename': self.filename,
- 'last_edited_date': self.last_edited_date.isoformat() + 'Z',
+ 'last_edited_date': (
+ None if self.last_edited_date is None
+ else f'{self.last_edited_date.isoformat()}Z'
+ ),
'mimetype': self.mimetype
}
+# endregion mixins
-class Permission(IntEnum):
- '''
- Defines User permissions as integers by the power of 2. User permission
- can be evaluated using the bitwise operator &.
- '''
- ADMINISTRATE = 1
- CONTRIBUTE = 2
- USE_API = 4
+##############################################################################
+# type_decorators #
+##############################################################################
+# region type_decorators
+class IntEnumColumn(db.TypeDecorator):
+ impl = db.Integer
+
+ def __init__(self, enum_type, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.enum_type = enum_type
+ def process_bind_param(self, value, dialect):
+ if isinstance(value, self.enum_type) and isinstance(value.value, int):
+ return value.value
+ elif isinstance(value, int):
+ return self.enum_type(value).value
+ else:
+ return TypeError()
+
+ def process_result_value(self, value, dialect):
+ return self.enum_type(value)
+
+
+class ContainerColumn(db.TypeDecorator):
+ impl = db.String
+
+ def __init__(self, container_type, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.container_type = container_type
+ def process_bind_param(self, value, dialect):
+ if isinstance(value, self.container_type):
+ return json.dumps(value)
+ elif (isinstance(value, str)
+ and isinstance(json.loads(value), self.container_type)):
+ return value
+ else:
+ return TypeError()
+
+ def process_result_value(self, value, dialect):
+ return json.loads(value)
+# endregion type_decorators
+
+
+##############################################################################
+# Models #
+##############################################################################
+# region models
class Role(HashidMixin, db.Model):
__tablename__ = 'roles'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Fields
- default = db.Column(db.Boolean, default=False, index=True)
name = db.Column(db.String(64), unique=True)
- permissions = db.Column(db.Integer)
+ default = db.Column(db.Boolean, default=False, index=True)
+ permissions = db.Column(db.Integer, default=0)
# Relationships
users = db.relationship('User', backref='role', lazy='dynamic')
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
- if self.permissions is None:
- self.permissions = 0
-
def __repr__(self):
return f'<Role {self.name}>'
@@ -84,19 +170,19 @@ class Role(HashidMixin, db.Model):
def reset_permissions(self):
self.permissions = 0
- def to_dict(self, backrefs=False, relationships=False):
- dict_role = {
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
'id': self.hashid,
'default': self.default,
'name': self.name,
'permissions': self.permissions
}
if relationships:
- dict_role['users'] = {
- x.hashid: x.to_dict(backrefs=False, relationships=True)
+ _json['users'] = {
+ x.hashid: x.to_json(relationships=True)
for x in self.users
}
- return dict_role
+ return _json
@staticmethod
def insert_defaults():
@@ -108,7 +194,8 @@ class Role(HashidMixin, db.Model):
Permission.ADMINISTRATE,
Permission.CONTRIBUTE,
Permission.USE_API
- ]
+ ],
+ 'System user': []
}
default_role_name = 'User'
for role_name, permissions in roles.items():
@@ -123,10 +210,28 @@ class Role(HashidMixin, db.Model):
db.session.commit()
-class UserSettingJobStatusMailNotificationLevel(IntEnum):
- NONE = 1
- END = 2
- ALL = 3
+class Token(db.Model):
+ __tablename__ = 'tokens'
+ # Primary key
+ id = db.Column(db.Integer, primary_key=True)
+ # Foreign keys
+ user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
+ # Fields
+ access_token = db.Column(db.String(64), index=True)
+ access_expiration = db.Column(db.DateTime)
+ refresh_token = db.Column(db.String(64), index=True)
+ refresh_expiration = db.Column(db.DateTime)
+ # Backrefs: user: User
+
+ def expire(self):
+ self.access_expiration = datetime.utcnow()
+ self.refresh_expiration = datetime.utcnow()
+
+ @staticmethod
+ def clean():
+ """Remove any tokens that have been expired for more than a day."""
+ yesterday = datetime.utcnow() - timedelta(days=1)
+ Token.query.filter(Token.refresh_expiration < yesterday).delete()
class User(HashidMixin, UserMixin, db.Model):
@@ -136,19 +241,17 @@ class User(HashidMixin, UserMixin, db.Model):
# Foreign keys
role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
# Fields
+ email = db.Column(db.String(254), index=True, unique=True)
+ username = db.Column(db.String(64), index=True, unique=True)
+ password_hash = db.Column(db.String(128))
confirmed = db.Column(db.Boolean, default=False)
- email = db.Column(db.String(254), unique=True, index=True)
- last_seen = db.Column(db.DateTime(), default=datetime.utcnow)
member_since = db.Column(db.DateTime(), default=datetime.utcnow)
- password_hash = db.Column(db.String(128))
- token = db.Column(db.String(32), index=True, unique=True)
- token_expiration = db.Column(db.DateTime)
- username = db.Column(db.String(64), unique=True, index=True)
setting_dark_mode = db.Column(db.Boolean, default=False)
setting_job_status_mail_notification_level = db.Column(
IntEnumColumn(UserSettingJobStatusMailNotificationLevel),
default=UserSettingJobStatusMailNotificationLevel.END
)
+ last_seen = db.Column(db.DateTime())
# Backrefs: role: Role
# Relationships
tesseract_ocr_models = db.relationship(
@@ -175,6 +278,12 @@ class User(HashidMixin, UserMixin, db.Model):
cascade='all, delete-orphan',
lazy='dynamic'
)
+ tokens = db.relationship(
+ 'Token',
+ backref='user',
+ cascade='all, delete-orphan',
+ lazy='dynamic'
+ )
def __init__(self, **kwargs):
super().__init__(**kwargs)
@@ -205,16 +314,99 @@ class User(HashidMixin, UserMixin, db.Model):
return os.path.join(
current_app.config.get('NOPAQUE_DATA_DIR'), 'users', str(self.id))
+ @staticmethod
+ def create(**kwargs):
+ user = User(**kwargs)
+ db.session.add(user)
+ db.session.flush(objects=[user])
+ db.session.refresh(user)
+ try:
+ os.mkdir(user.path)
+ os.mkdir(os.path.join(user.path, 'tesseract_ocr_models'))
+ os.mkdir(os.path.join(user.path, 'corpora'))
+ os.mkdir(os.path.join(user.path, 'jobs'))
+ except OSError as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+ raise e
+ return user
+
+ @staticmethod
+ def insert_defaults():
+ nopaque_user = User.query.filter_by(username='nopaque').first()
+ system_user_role = Role.query.filter_by(name='System user').first()
+ if nopaque_user is None:
+ nopaque_user = User.create(
+ username='nopaque',
+ role=system_user_role
+ )
+ db.session.add(nopaque_user)
+ elif nopaque_user.role != system_user_role:
+ nopaque_user.role = system_user_role
+ db.session.commit()
+
+ @staticmethod
+ def reset_password(token, new_password):
+ try:
+ payload = jwt.decode(
+ token,
+ current_app.config['SECRET_KEY'],
+ algorithms=['HS256'],
+ issuer=current_app.config['SERVER_NAME'],
+ options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
+ )
+ except jwt.PyJWTError:
+ return False
+ if payload.get('purpose') != 'User.reset_password':
+ return False
+ user_hashid = payload.get('sub')
+ user_id = hashids.decode(user_hashid)
+ user = User.query.get(user_id)
+ if user is None:
+ return False
+ user.password = new_password
+ db.session.add(user)
+ return True
+
+ @staticmethod
+ def verify_access_token(access_token, refresh_token=None):
+ token = Token.query.filter(Token.access_token == access_token).first()
+ if token is not None:
+ if token.access_expiration > datetime.utcnow():
+ token.user.ping()
+ db.session.commit()
+ if token.user.role.name != 'System user':
+ return token.user
+
+ @staticmethod
+ def verify_refresh_token(refresh_token, access_token):
+ token = Token.query.filter((Token.refresh_token == refresh_token) & (Token.access_token == access_token)).first()
+ if token is not None:
+ if token.refresh_expiration > datetime.utcnow():
+ return token
+ # someone tried to refresh with an expired token
+ # revoke all tokens from this user as a precaution
+ token.user.revoke_auth_tokens()
+ db.session.commit()
+
def can(self, permission):
return self.role.has_permission(permission)
- def confirm(self, token):
- s = TimedJSONWebSignatureSerializer(current_app.config['SECRET_KEY'])
+ def confirm(self, confirmation_token):
try:
- data = s.loads(token.encode('utf-8'))
- except BadSignature:
+ payload = jwt.decode(
+ confirmation_token,
+ current_app.config['SECRET_KEY'],
+ algorithms=['HS256'],
+ issuer=current_app.config['SERVER_NAME'],
+ options={'require': ['exp', 'iat', 'iss', 'purpose', 'sub']}
+ )
+ current_app.logger.warning(payload)
+ except jwt.PyJWTError:
+ return False
+ if payload.get('purpose') != 'user.confirm':
return False
- if data.get('confirm') != self.hashid:
+ if payload.get('sub') != self.hashid:
return False
self.confirmed = True
db.session.add(self)
@@ -224,109 +416,93 @@ class User(HashidMixin, UserMixin, db.Model):
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
- def generate_confirmation_token(self, expiration=3600):
- s = TimedJSONWebSignatureSerializer(
- current_app.config['SECRET_KEY'], expiration)
- return s.dumps({'confirm': self.hashid}).decode('utf-8')
+ def generate_auth_token(self):
+ return Token(
+ access_token=secrets.token_urlsafe(),
+ access_expiration=datetime.utcnow() + timedelta(minutes=15),
+ refresh_token=secrets.token_urlsafe(),
+ refresh_expiration=datetime.utcnow() + timedelta(days=7),
+ user=self
+ )
- def generate_reset_token(self, expiration=3600):
- s = TimedJSONWebSignatureSerializer(
- current_app.config['SECRET_KEY'], expiration)
- return s.dumps({'reset': self.hashid}).decode('utf-8')
+ def generate_confirm_token(self, expiration=3600):
+ now = datetime.utcnow()
+ payload = {
+ 'exp': now + timedelta(seconds=expiration),
+ 'iat': now,
+ 'iss': current_app.config['SERVER_NAME'],
+ 'purpose': 'user.confirm',
+ 'sub': self.hashid
+ }
+ return jwt.encode(
+ payload,
+ current_app.config['SECRET_KEY'],
+ algorithm='HS256'
+ )
- def get_token(self, expires_in=3600):
+ def generate_reset_password_token(self, expiration=3600):
now = datetime.utcnow()
- if self.token and self.token_expiration > now + timedelta(seconds=60):
- return self.token
- self.token = base64.b64encode(os.urandom(24)).decode('utf-8')
- self.token_expiration = now + timedelta(seconds=expires_in)
- db.session.add(self)
- return self.token
+ payload = {
+ 'exp': now + timedelta(seconds=expiration),
+ 'iat': now,
+ 'iss': current_app.config['SERVER_NAME'],
+ 'purpose': 'User.reset_password',
+ 'sub': self.hashid
+ }
+ return jwt.encode(
+ payload,
+ current_app.config['SECRET_KEY'],
+ algorithm='HS256'
+ )
def is_administrator(self):
return self.can(Permission.ADMINISTRATE)
- def makedirs(self):
- os.mkdir(self.path)
- os.mkdir(os.path.join(self.path, 'tesseract_ocr_models'))
- os.mkdir(os.path.join(self.path, 'corpora'))
- os.mkdir(os.path.join(self.path, 'jobs'))
+ def ping(self):
+ self.last_seen = datetime.utcnow()
- def revoke_token(self):
- self.token_expiration = datetime.utcnow() - timedelta(seconds=1)
+ def revoke_auth_tokens(self):
+ for token in self.tokens:
+ db.session.delete(token)
- def to_dict(self, backrefs=False, relationships=False):
- dict_user = {
+ def verify_password(self, password):
+ if self.role.name == 'System user':
+ return False
+ return check_password_hash(self.password_hash, password)
+
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
'id': self.hashid,
- 'role_id': self.role.hashid,
'confirmed': self.confirmed,
'email': self.email,
- 'last_seen': self.last_seen.isoformat() + 'Z',
- 'member_since': self.member_since.isoformat() + 'Z',
+ 'last_seen': (
+ None if self.last_seen is None
+ else f'{self.last_seen.isoformat()}Z'
+ ),
+ 'member_since': f'{self.member_since.isoformat()}Z',
'username': self.username,
'settings': {
'dark_mode': self.setting_dark_mode,
- 'job_status_mail_notification_level':
+ 'job_status_mail_notification_level': \
self.setting_job_status_mail_notification_level.name
}
}
if backrefs:
- dict_user['role'] = self.role.to_dict(
- backrefs=True, relationships=False)
+ _json['role'] = self.role.to_json(backrefs=True)
if relationships:
- dict_user['corpora'] = {
- x.hashid: x.to_dict(backrefs=False, relationships=True)
+ _json['corpora'] = {
+ x.hashid: x.to_json(relationships=True)
for x in self.corpora
}
- dict_user['jobs'] = {
- x.hashid: x.to_dict(backrefs=False, relationships=True)
+ _json['jobs'] = {
+ x.hashid: x.to_json(relationships=True)
for x in self.jobs
}
- dict_user['tesseract_ocr_models'] = {
- x.hashid: x.to_dict(backrefs=False, relationships=True)
+ _json['tesseract_ocr_models'] = {
+ x.hashid: x.to_json(relationships=True)
for x in self.tesseract_ocr_models
}
- return dict_user
-
- def verify_password(self, password):
- return check_password_hash(self.password_hash, password)
-
- @staticmethod
- def check_token(token):
- user = User.query.filter_by(token=token).first()
- if user is None or user.token_expiration < datetime.utcnow():
- return None
- return user
-
- @staticmethod
- def insert_defaults():
- if User.query.filter_by(username='nopaque').first() is not None:
- return
- user = User(username='nopaque')
- db.session.add(user)
- db.session.flush(objects=[user])
- db.session.refresh(user)
- try:
- user.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- db.session.commit()
-
- @staticmethod
- def reset_password(token, new_password):
- s = TimedJSONWebSignatureSerializer(current_app.config['SECRET_KEY'])
- try:
- data = s.loads(token.encode('utf-8'))
- except BadSignature:
- return False
- user = User.query.get(data.get('reset'))
- if user is None:
- return False
- user.password = new_password
- db.session.add(user)
- return True
-
+ return _json
class TesseractOCRModel(FileMixin, HashidMixin, db.Model):
__tablename__ = 'tesseract_ocr_models'
@@ -335,15 +511,15 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model):
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
- compatible_service_versions = db.Column(ContainerColumn(list, 255))
+ title = db.Column(db.String(64))
description = db.Column(db.String(255))
+ version = db.Column(db.String(16))
+ compatible_service_versions = db.Column(ContainerColumn(list, 255))
publisher = db.Column(db.String(128))
publisher_url = db.Column(db.String(512))
publishing_url = db.Column(db.String(512))
publishing_year = db.Column(db.Integer)
shared = db.Column(db.Boolean, default=False)
- title = db.Column(db.String(64))
- version = db.Column(db.String(16))
# Backrefs: user: User
@property
@@ -354,30 +530,9 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model):
str(self.id)
)
- def to_dict(self, backrefs=False, relationships=False):
- dict_tesseract_ocr_model = {
- 'id': self.hashid,
- 'user_id': self.user.hashid,
- 'compatible_service_versions': self.compatible_service_versions,
- 'description': self.description,
- 'publisher': self.publisher,
- 'publisher_url': self.publisher_url,
- 'publishing_url': self.publishing_url,
- 'publishing_year': self.publishing_year,
- 'shared': self.shared,
- 'title': self.title,
- **self.file_mixin_to_dict()
- }
- if backrefs:
- dict_tesseract_ocr_model['user'] = self.user.to_dict(
- backrefs=True, relationships=False)
- if relationships:
- pass
- return dict_tesseract_ocr_model
-
@staticmethod
def insert_defaults():
- user = User.query.filter_by(username='nopaque').first()
+ nopaque_user = User.query.filter_by(username='nopaque').first()
defaults_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'TesseractOCRModel.defaults.yml'
@@ -406,7 +561,7 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model):
publishing_year=m['publishing_year'],
shared=True,
title=m['title'],
- user=user,
+ user=nopaque_user,
version=m['version']
)
db.session.add(model)
@@ -430,6 +585,23 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model):
pbar.close()
db.session.commit()
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
+ 'id': self.hashid,
+ 'compatible_service_versions': self.compatible_service_versions,
+ 'description': self.description,
+ 'publisher': self.publisher,
+ 'publisher_url': self.publisher_url,
+ 'publishing_url': self.publishing_url,
+ 'publishing_year': self.publishing_year,
+ 'shared': self.shared,
+ 'title': self.title,
+ **self.file_mixin_to_json()
+ }
+ if backrefs:
+ _json['user'] = self.user.to_json(backrefs=True)
+ return _json
+
class TranskribusHTRModel(HashidMixin, db.Model):
__tablename__ = 'transkribus_htr_models'
@@ -443,49 +615,39 @@ class TranskribusHTRModel(HashidMixin, db.Model):
transkribus_name = db.Column(db.String(64))
# Backrefs: user: User
- def to_dict(self, backrefs=False, relationships=False):
- dict_tesseract_ocr_model = {
- 'id': self.hashid,
- 'user_id': self.user.hashid,
- 'shared': self.shared,
- 'transkribus_model_id': self.transkribus_model_id,
- 'transkribus_name': self.transkribus_name
- }
- if backrefs:
- dict_tesseract_ocr_model['user'] = \
- self.user.to_dict(backrefs=True, relationships=False)
- if relationships:
- pass
- return dict_tesseract_ocr_model
-
@staticmethod
def insert_defaults():
- user = User.query.filter_by(username='nopaque').first()
+ nopaque_user = User.query.filter_by(username='nopaque').first()
# models = [
# m for m in TRANSKRIBUS_HTR_MODELS if True
# and 'creator' in m and m['creator'] == 'Transkribus Team'
# and 'docType' in m and m['docType'] == 'handwritten'
# ]
- models = [
- m for m in TRANSKRIBUS_HTR_MODELS
- if m['modelId'] in [35909, 33744, 33597, 29820, 37789, 13685, 37855, 26124, 37738, 30919, 34763]
- ]
- for m in models:
+ for m in TRANSKRIBUS_HTR_MODELS:
model = TranskribusHTRModel.query.filter_by(transkribus_model_id=m['modelId']).first() # noqa
if model is not None:
model.shared = True
model.transkribus_model_id = m['modelId']
- model.transkribus_name = m['name']
continue
model = TranskribusHTRModel(
shared=True,
- transkribus_name=m['name'],
transkribus_model_id=m['modelId'],
- user=user,
+ user=nopaque_user,
)
db.session.add(model)
db.session.commit()
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
+ 'id': self.hashid,
+ 'user_id': self.user.hashid,
+ 'shared': self.shared,
+ 'transkribus_model_id': self.transkribus_model_id,
+ }
+ if backrefs:
+ _json['user'] = self.user.to_json(backrefs=True)
+ return _json
+
class JobInput(FileMixin, HashidMixin, db.Model):
__tablename__ = 'job_inputs'
@@ -499,7 +661,7 @@ class JobInput(FileMixin, HashidMixin, db.Model):
return f'<JobInput {self.filename}>'
@property
- def download_url(self):
+ def content_url(self):
return url_for(
'jobs.download_job_input',
job_id=self.job.id,
@@ -514,19 +676,6 @@ class JobInput(FileMixin, HashidMixin, db.Model):
def path(self):
return os.path.join(self.job.path, 'inputs', str(self.id))
- def to_dict(self, backrefs=False, relationships=False):
- dict_job_input = {
- 'id': self.hashid,
- 'job_id': self.job.hashid,
- 'download_url': self.download_url,
- 'url': self.url,
- **self.file_mixin_to_dict()
- }
- if backrefs:
- dict_job_input['job'] = self.job.to_dict(
- backrefs=True, relationships=False)
- return dict_job_input
-
@property
def url(self):
return url_for(
@@ -543,6 +692,35 @@ class JobInput(FileMixin, HashidMixin, db.Model):
def user_id(self):
return self.job.user_id
+ @staticmethod
+ def create(input_file, **kwargs):
+ filename = kwargs.get('filename', input_file.filename)
+ mimetype = kwargs.get('mimetype', input_file.mimetype)
+ job_input = JobInput(
+ filename=secure_filename(filename),
+ mimetype=mimetype,
+ **kwargs
+ )
+ db.session.add(job_input)
+ db.session.flush(objects=[job_input])
+ db.session.refresh(job_input)
+ try:
+ input_file.save(job_input.path)
+ except OSError as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+ raise e
+ return job_input
+
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
+ 'id': self.hashid,
+ **self.file_mixin_to_json()
+ }
+ if backrefs:
+ _json['job'] = self.job.to_json(backrefs=True)
+ return _json
+
class JobResult(FileMixin, HashidMixin, db.Model):
__tablename__ = 'job_results'
@@ -573,21 +751,6 @@ class JobResult(FileMixin, HashidMixin, db.Model):
def path(self):
return os.path.join(self.job.path, 'results', str(self.id))
- def to_dict(self, backrefs=False, relationships=False):
- dict_job_result = {
- 'id': self.hashid,
- 'job_id': self.job.hashid,
- 'description': self.description,
- 'download_url': self.download_url,
- 'url': self.url,
- **self.file_mixin_to_dict(
- backrefs=backrefs, relationships=relationships)
- }
- if backrefs:
- dict_job_result['job'] = self.job.to_dict(
- backrefs=True, relationships=False)
- return dict_job_result
-
@property
def url(self):
return url_for(
@@ -604,16 +767,38 @@ class JobResult(FileMixin, HashidMixin, db.Model):
def user_id(self):
return self.job.user_id
+ @staticmethod
+ def create(input_file, **kwargs):
+ filename = kwargs.get('filename', input_file.filename)
+ mimetype = kwargs.get('mimetype', input_file.mimetype)
+ job_result = JobResult(
+ filename=secure_filename(filename),
+ mimetype=mimetype,
+ **kwargs
+ )
+ db.session.add(job_result)
+ db.session.flush(objects=[job_result])
+ db.session.refresh(job_result)
+ try:
+ input_file.save(job_result.path)
+ except OSError as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+ raise e
+ return job_result
-class JobStatus(IntEnum):
- INITIALIZING = 1
- SUBMITTED = 2
- QUEUED = 3
- RUNNING = 4
- CANCELING = 5
- CANCELED = 6
- COMPLETED = 7
- FAILED = 8
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
+ 'id': self.hashid,
+ 'description': self.description,
+ **self.file_mixin_to_json(
+ backrefs=backrefs,
+ relationships=relationships
+ )
+ }
+ if backrefs:
+ _json['job'] = self.job.to_json(backrefs=True)
+ return _json
class Job(HashidMixin, db.Model):
@@ -626,7 +811,8 @@ class Job(HashidMixin, db.Model):
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
- creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
+ creation_date = \
+ db.Column(db.DateTime(), default=datetime.utcnow)
description = db.Column(db.String(255))
end_date = db.Column(db.DateTime())
service = db.Column(db.String(64))
@@ -671,10 +857,26 @@ class Job(HashidMixin, db.Model):
def user_hashid(self):
return self.user.hashid
+ @staticmethod
+ def create(**kwargs):
+ job = Job(**kwargs)
+ db.session.add(job)
+ db.session.flush(objects=[job])
+ db.session.refresh(job)
+ try:
+ os.mkdir(job.path)
+ os.mkdir(os.path.join(job.path, 'inputs'))
+ os.mkdir(os.path.join(job.path, 'pipeline_data'))
+ os.mkdir(os.path.join(job.path, 'results'))
+ except OSError as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+ raise e
+ return job
+
+
def delete(self):
- '''
- Delete the job and its inputs and results from the database.
- '''
+ ''' Delete the job and its inputs and results from the database. '''
if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]: # noqa
self.status = JobStatus.CANCELING
db.session.commit()
@@ -685,36 +887,34 @@ class Job(HashidMixin, db.Model):
db.session.commit()
sleep(1)
db.session.refresh(self)
- shutil.rmtree(self.path, ignore_errors=True)
+ try:
+ shutil.rmtree(self.path)
+ except OSError as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+ raise e
db.session.delete(self)
- def makedirs(self):
- os.mkdir(self.path)
- os.mkdir(os.path.join(self.path, 'inputs'))
- os.mkdir(os.path.join(self.path, 'pipeline_data'))
- os.mkdir(os.path.join(self.path, 'results'))
-
def restart(self):
- '''
- Restart a job - only if the status is complete or failed
- '''
-
- if self.status not in [JobStatus.COMPLETED, JobStatus.FAILED]: # noqa
- raise Exception('Could not restart job: status is not "completed/failed"') # noqa
+ ''' Restart a job - only if the status is failed '''
+ if self.status != JobStatus.FAILED:
+ raise Exception('Job status is not "failed"')
shutil.rmtree(os.path.join(self.path, 'results'), ignore_errors=True)
- shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa
+ shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True)
for result in self.results:
db.session.delete(result)
self.end_date = None
self.status = JobStatus.SUBMITTED
- def to_dict(self, backrefs=False, relationships=False):
- dict_job = {
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
'id': self.hashid,
- 'user_id': self.user.hashid,
- 'creation_date': self.creation_date.isoformat() + 'Z',
+ 'creation_date': f'{self.creation_date.isoformat()}Z',
'description': self.description,
- 'end_date': None if self.end_date is None else f'{self.end_date.isoformat()}Z', # noqa
+ 'end_date': (
+ None if self.end_date is None
+ else f'{self.end_date.isoformat()}Z'
+ ),
'service': self.service,
'service_args': self.service_args,
'service_version': self.service_version,
@@ -723,18 +923,17 @@ class Job(HashidMixin, db.Model):
'url': self.url
}
if backrefs:
- dict_job['user'] = self.user.to_dict(
- backrefs=True, relationships=False)
+ _json['user'] = self.user.to_json(backrefs=True)
if relationships:
- dict_job['inputs'] = {
- x.hashid: x.to_dict(backrefs=False, relationships=True)
+ _json['inputs'] = {
+ x.hashid: x.to_json(relationships=True)
for x in self.inputs
}
- dict_job['results'] = {
- x.hashid: x.to_dict(backrefs=False, relationships=True)
+ _json['results'] = {
+ x.hashid: x.to_json(relationships=True)
for x in self.results
}
- return dict_job
+ return _json
class CorpusFile(FileMixin, HashidMixin, db.Model):
@@ -744,8 +943,10 @@ class CorpusFile(FileMixin, HashidMixin, db.Model):
# Foreign keys
corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
# Fields
- address = db.Column(db.String(255))
author = db.Column(db.String(255))
+ publishing_year = db.Column(db.Integer)
+ title = db.Column(db.String(255))
+ address = db.Column(db.String(255))
booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255))
editor = db.Column(db.String(255))
@@ -753,9 +954,7 @@ class CorpusFile(FileMixin, HashidMixin, db.Model):
journal = db.Column(db.String(255))
pages = db.Column(db.String(255))
publisher = db.Column(db.String(255))
- publishing_year = db.Column(db.Integer)
school = db.Column(db.String(255))
- title = db.Column(db.String(255))
# Backrefs: corpus: Corpus
@property
@@ -801,11 +1000,9 @@ class CorpusFile(FileMixin, HashidMixin, db.Model):
db.session.delete(self)
self.corpus.status = CorpusStatus.UNPREPARED
- def to_dict(self, backrefs=False, relationships=False):
- dict_corpus_file = {
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
'id': self.hashid,
- 'corpus_id': self.corpus.hashid,
- 'download_url': self.download_url,
'url': self.url,
'address': self.address,
'author': self.author,
@@ -819,26 +1016,34 @@ class CorpusFile(FileMixin, HashidMixin, db.Model):
'publishing_year': self.publishing_year,
'school': self.school,
'title': self.title,
- **self.file_mixin_to_dict(
- backrefs=backrefs, relationships=relationships)
+ **self.file_mixin_to_json(
+ backrefs=backrefs,
+ relationships=relationships
+ )
}
if backrefs:
- dict_corpus_file['corpus'] = self.corpus.to_dict(
- backrefs=True, relationships=False)
- return dict_corpus_file
-
-
-class CorpusStatus(IntEnum):
- UNPREPARED = 1
- SUBMITTED = 2
- QUEUED = 3
- BUILDING = 4
- BUILT = 5
- FAILED = 6
- STARTING_ANALYSIS_SESSION = 7
- RUNNING_ANALYSIS_SESSION = 8
- CANCELING_ANALYSIS_SESSION = 9
+ _json['corpus'] = self.corpus.to_json(backrefs=True)
+ return _json
+ @staticmethod
+ def create(input_file, **kwargs):
+ filename = kwargs.pop('filename', input_file.filename)
+ mimetype = kwargs.pop('mimetype', input_file.mimetype)
+ corpus_file = CorpusFile(
+ filename=secure_filename(filename),
+ mimetype=mimetype,
+ **kwargs,
+ )
+ db.session.add(corpus_file)
+ db.session.flush(objects=[corpus_file])
+ db.session.refresh(corpus_file)
+ try:
+ input_file.save(corpus_file.path)
+ except OSError as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+ raise e
+ return corpus_file
class Corpus(HashidMixin, db.Model):
'''
@@ -852,7 +1057,7 @@ class Corpus(HashidMixin, db.Model):
# Fields
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
description = db.Column(db.String(255))
- last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
+ last_edited_date = db.Column(db.DateTime())
status = db.Column(
IntEnumColumn(CorpusStatus),
default=CorpusStatus.UNPREPARED
@@ -894,6 +1099,24 @@ class Corpus(HashidMixin, db.Model):
def user_hashid(self):
return self.user.hashid
+ @staticmethod
+ def create(**kwargs):
+ corpus = Corpus(**kwargs)
+ db.session.add(corpus)
+ db.session.flush(objects=[corpus])
+ db.session.refresh(corpus)
+ try:
+ os.mkdir(corpus.path)
+ os.mkdir(os.path.join(corpus.path, 'files'))
+ os.mkdir(os.path.join(corpus.path, 'cwb'))
+ os.mkdir(os.path.join(corpus.path, 'cwb', 'data'))
+ os.mkdir(os.path.join(corpus.path, 'cwb', 'registry'))
+ except OSError as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+ raise e
+ return corpus
+
def build(self):
corpus_element = ET.fromstring('<corpus>\n</corpus>')
for corpus_file in self.files:
@@ -905,18 +1128,21 @@ class Corpus(HashidMixin, db.Model):
return
element_tree = ET.parse(normalized_vrt_path)
text_element = element_tree.getroot()
- text_element.set('address', corpus_file.address or 'NULL')
text_element.set('author', corpus_file.author)
+ text_element.set('title', corpus_file.title)
+ text_element.set(
+ 'publishing_year',
+ f'{corpus_file.publishing_year}'
+ )
+ text_element.set('address', corpus_file.address or 'NULL')
text_element.set('booktitle', corpus_file.booktitle or 'NULL')
text_element.set('chapter', corpus_file.chapter or 'NULL')
text_element.set('editor', corpus_file.editor or 'NULL')
text_element.set('institution', corpus_file.institution or 'NULL')
text_element.set('journal', corpus_file.journal or 'NULL')
- text_element.set('pages', corpus_file.pages or 'NULL')
+ text_element.set('pages', f'{corpus_file.pages}' or 'NULL')
text_element.set('publisher', corpus_file.publisher or 'NULL')
- text_element.set('publishing_year', str(corpus_file.publishing_year)) # noqa
text_element.set('school', corpus_file.school or 'NULL')
- text_element.set('title', corpus_file.title)
text_element.tail = '\n'
# corpus_element.insert(1, text_element)
corpus_element.append(text_element)
@@ -931,41 +1157,124 @@ class Corpus(HashidMixin, db.Model):
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
- def makedirs(self):
- os.mkdir(self.path)
- os.mkdir(os.path.join(self.path, 'files'))
- os.mkdir(os.path.join(self.path, 'cwb'))
- os.mkdir(os.path.join(self.path, 'cwb', 'data'))
- os.mkdir(os.path.join(self.path, 'cwb', 'registry'))
-
- def to_dict(self, backrefs=False, relationships=False):
- dict_corpus = {
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
'id': self.hashid,
- 'user_id': self.user.hashid,
- 'analysis_url': self.analysis_url,
- 'url': self.url,
- 'creation_date': self.creation_date.isoformat() + 'Z',
+ 'creation_date': f'{self.creation_date.isoformat()}Z',
'description': self.description,
'max_num_tokens': self.max_num_tokens,
'num_analysis_sessions': self.num_analysis_sessions,
'num_tokens': self.num_tokens,
'status': self.status.name,
- 'last_edited_date': self.last_edited_date.isoformat() + 'Z',
+ 'last_edited_date': (
+ None if self.last_edited_date is None
+ else f'{self.last_edited_date.isoformat()}Z'
+ ),
'title': self.title
}
if backrefs:
- dict_corpus['user'] = self.user.to_dict(
- backrefs=True,
- relationships=False
- )
+ _json['user'] = self.user.to_json(backrefs=True)
if relationships:
- dict_corpus['files'] = {
- x.hashid: x.to_dict(backrefs=False, relationships=True)
+ _json['files'] = {
+ x.hashid: x.to_json(relationships=True)
for x in self.files
}
- return dict_corpus
+ return _json
+# endregion models
+
+
+##############################################################################
+# event_handlers #
+##############################################################################
+# region event_handlers
+
+
+@db.event.listens_for(Corpus, 'after_delete')
+@db.event.listens_for(CorpusFile, 'after_delete')
+@db.event.listens_for(Job, 'after_delete')
+@db.event.listens_for(JobInput, 'after_delete')
+@db.event.listens_for(JobResult, 'after_delete')
+def ressource_after_delete(mapper, connection, ressource):
+ jsonpatch = [{'op': 'remove', 'path': ressource.jsonpatch_path}]
+ room = f'users.{ressource.user_hashid}'
+ socketio.emit('users.patch', jsonpatch, room=room)
+ room = f'/users/{ressource.user_hashid}'
+ socketio.emit('PATCH', jsonpatch, room=room)
+
+
+@db.event.listens_for(Corpus, 'after_insert')
+@db.event.listens_for(CorpusFile, 'after_insert')
+@db.event.listens_for(Job, 'after_insert')
+@db.event.listens_for(JobInput, 'after_insert')
+@db.event.listens_for(JobResult, 'after_insert')
+def ressource_after_insert_handler(mapper, connection, ressource):
+ value = ressource.to_json()
+ for attr in mapper.relationships:
+ value[attr.key] = {}
+ jsonpatch = [
+ {'op': 'add', 'path': ressource.jsonpatch_path, 'value': value}
+ ]
+ room = f'/users/{ressource.user_hashid}'
+ socketio.emit('PATCH', jsonpatch, room=room)
+
+
+@db.event.listens_for(Corpus, 'after_update')
+@db.event.listens_for(CorpusFile, 'after_update')
+@db.event.listens_for(Job, 'after_update')
+@db.event.listens_for(JobInput, 'after_update')
+@db.event.listens_for(JobResult, 'after_update')
+def ressource_after_update_handler(mapper, connection, ressource):
+ jsonpatch = []
+ for attr in db.inspect(ressource).attrs:
+ if attr.key in mapper.relationships:
+ continue
+ if not attr.load_history().has_changes():
+ continue
+ if isinstance(attr.value, datetime):
+ value = f'{attr.value.isoformat()}Z'
+ elif isinstance(attr.value, Enum):
+ value = attr.value.name
+ else:
+ value = attr.value
+ jsonpatch.append(
+ {
+ 'op': 'replace',
+ 'path': f'{ressource.jsonpatch_path}/{attr.key}',
+ 'value': value
+ }
+ )
+ if jsonpatch:
+ room = f'/users/{ressource.user_hashid}'
+ socketio.emit('PATCH', jsonpatch, room=room)
+
+
+@db.event.listens_for(Job, 'after_update')
+def job_after_update_handler(mapper, connection, job):
+ for attr in db.inspect(job).attrs:
+ if attr.key != 'status':
+ continue
+ if not attr.load_history().has_changes():
+ return
+ if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.NONE:
+ return
+ if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.END:
+ if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
+ return
+ msg = create_message(
+ job.user.email,
+ f'Status update for your Job "{job.title}"',
+ 'tasks/email/notification',
+ job=job
+ )
+ mail.send(msg)
+# endregion event_handlers
+##############################################################################
+# misc #
+##############################################################################
+# region misc
@login.user_loader
def load_user(user_id):
return User.query.get(int(user_id))
+# endregion misc
diff --git a/app/query_results_models.py b/app/query_results_models.py
index 102d2825d1aa43ea6f9e151f402708dd4627c3c6..132a4cc32158a121b94d3d9df0908471a7fa6896 100644
--- a/app/query_results_models.py
+++ b/app/query_results_models.py
@@ -42,21 +42,17 @@ class QueryResult(FileMixin, HashidMixin, db.Model):
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
- def to_dict(self, backrefs=False, relationships=False):
- dict_query_result = {
+ def to_json(self, backrefs=False, relationships=False):
+ _json = {
'id': self.hashid,
- 'user_id': self.user.hashid,
- 'download_url': self.download_url,
- 'url': self.url,
'corpus_title': self.query_metadata['corpus_name'],
'description': self.description,
'filename': self.filename,
'query': self.query_metadata['query'],
'query_metadata': self.query_metadata,
'title': self.title,
- **self.file_mixin_to_dict(
+ **self.file_mixin_to_json(
backrefs=backrefs, relationships=relationships)
}
if backrefs:
- dict_query_result['user'] = self.user.to_dict(
- backrefs=True, relationships=False)
+ _json['user'] = self.user.to_json(backrefs=True, relationships=False)
diff --git a/app/services/forms.py b/app/services/forms.py
index 8261d2ab9e0ba7827e210d8e041c64f957a8cd60..106c0f7fd559d0db777c0098c2e555f104886c45 100644
--- a/app/services/forms.py
+++ b/app/services/forms.py
@@ -1,4 +1,3 @@
-from app.models import TesseractOCRModel, TranskribusHTRModel
from flask_login import current_user
from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired
@@ -10,19 +9,26 @@ from wtforms import (
SubmitField,
ValidationError
)
-from wtforms.validators import DataRequired, InputRequired, Length
+from wtforms.validators import InputRequired, Length
+from app.models import TesseractOCRModel, TranskribusHTRModel
from . import SERVICES
-class AddJobForm(FlaskForm):
- description = StringField('Description', validators=[InputRequired(), Length(1, 255)])
- title = StringField('Title', validators=[InputRequired(), Length(1, 32)])
- version = SelectField('Version', validators=[DataRequired()])
+class CreateJobBaseForm(FlaskForm):
+ description = StringField(
+ 'Description',
+ validators=[InputRequired(), Length(max=255)]
+ )
+ title = StringField(
+ 'Title',
+ validators=[InputRequired(), Length(max=32)]
+ )
+ version = SelectField('Version', validators=[InputRequired()])
submit = SubmitField()
-class AddFileSetupPipelineJobForm(AddJobForm):
- images = MultipleFileField('File(s)', validators=[DataRequired()])
+class CreateFileSetupPipelineJobForm(CreateJobBaseForm):
+ images = MultipleFileField('File(s)', validators=[InputRequired()])
def validate_images(form, field):
valid_mimetypes = ['image/jpeg', 'image/png', 'image/tiff']
@@ -39,18 +45,15 @@ class AddFileSetupPipelineJobForm(AddJobForm):
self.version.default = service_manifest['latest_version']
-class AddTesseractOCRPipelineJobForm(AddJobForm):
+class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
binarization = BooleanField('Binarization')
pdf = FileField('File', validators=[FileRequired()])
- model = SelectField('Model', validators=[DataRequired()])
+ model = SelectField('Model', validators=[InputRequired()])
def validate_binarization(self, field):
service_info = SERVICES['tesseract-ocr-pipeline']['versions'][self.version.data]
if field.data:
- if(
- 'methods' not in service_info
- or 'binarization' not in service_info['methods']
- ):
+ if not('methods' in service_info and 'binarization' in service_info['methods']):
raise ValidationError('Binarization is not available')
def validate_pdf(self, field):
@@ -81,10 +84,10 @@ class AddTesseractOCRPipelineJobForm(AddJobForm):
self.version.default = service_manifest['latest_version']
-class AddTranskribusHTRPipelineJobForm(AddJobForm):
+class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm):
binarization = BooleanField('Binarization')
pdf = FileField('File', validators=[FileRequired()])
- model = SelectField('Model', validators=[DataRequired()])
+ model = SelectField('Model', validators=[InputRequired()])
def validate_binarization(self, field):
service_info = SERVICES['transkribus-htr-pipeline']['versions'][self.version.data]
@@ -123,10 +126,10 @@ class AddTranskribusHTRPipelineJobForm(AddJobForm):
self.version.default = service_manifest['latest_version']
-class AddSpacyNLPPipelineJobForm(AddJobForm):
+class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
encoding_detection = BooleanField('Encoding detection', render_kw={'disabled': True})
txt = FileField('File', validators=[FileRequired()])
- model = SelectField('Model', validators=[DataRequired()])
+ model = SelectField('Model', validators=[InputRequired()])
def validate_encoding_detection(self, field):
service_info = SERVICES['spacy-nlp-pipeline']['versions'][self.version.data]
diff --git a/app/services/routes.py b/app/services/routes.py
index a33860917cc16603739c6965bf8810e4af8fdcef..9f5c81ef228676291bb525375a33b386eb9f2fb6 100644
--- a/app/services/routes.py
+++ b/app/services/routes.py
@@ -1,3 +1,5 @@
+from flask import abort, current_app, flash, Markup, render_template, request
+from flask_login import current_user, login_required
from app import db, hashids
from app.models import (
Job,
@@ -7,26 +9,13 @@ from app.models import (
TRANSKRIBUS_HTR_MODELS,
TranskribusHTRModel
)
-from flask import (
- abort,
- current_app,
- flash,
- make_response,
- render_template,
- request,
- url_for
-)
-from flask_login import current_user, login_required
-from werkzeug.utils import secure_filename
-from . import bp
-from . import SERVICES
+from . import bp, SERVICES
from .forms import (
- AddFileSetupPipelineJobForm,
- AddTesseractOCRPipelineJobForm,
- AddTranskribusHTRPipelineJobForm,
- AddSpacyNLPPipelineJobForm
+ CreateFileSetupPipelineJobForm,
+ CreateTesseractOCRPipelineJobForm,
+ CreateTranskribusHTRPipelineJobForm,
+ CreateSpacyNLPPipelineJobForm
)
-import json
@bp.route('/file-setup-pipeline', methods=['GET', 'POST'])
@@ -37,49 +26,32 @@ def file_setup_pipeline():
version = request.args.get('version', service_manifest['latest_version'])
if version not in service_manifest['versions']:
abort(404)
- form = AddFileSetupPipelineJobForm(prefix='add-job-form', version=version)
+ form = CreateFileSetupPipelineJobForm(prefix='create-job-form', version=version)
if form.is_submitted():
if not form.validate():
- return make_response(form.errors, 400)
- service_args = {}
- job = Job(
- user=current_user,
- description=form.description.data,
- service=service,
- service_args=service_args,
- service_version=form.version.data,
- title=form.title.data
- )
- db.session.add(job)
- db.session.flush(objects=[job])
- db.session.refresh(job)
+ response = {'errors': form.errors}
+ return response, 400
try:
- job.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
- for image_file in form.images.data:
- job_input = JobInput(
- filename=secure_filename(image_file.filename),
- job=job,
- mimetype=image_file.mimetype
+ job = Job.create(
+ title=form.title.data,
+ description=form.description.data,
+ service=service,
+ service_args={},
+ service_version=form.version.data,
+ user=current_user
)
- db.session.add(job_input)
- db.session.flush(objects=[job_input])
- db.session.refresh(job_input)
+ except OSError:
+ abort(500)
+ for input_file in form.images.data:
try:
- image_file.save(job_input.path)
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
+ JobInput.create(input_file, job=job)
+ except OSError:
+ abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
- flash(f'Job "{job.title}" added', 'job')
- return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201) # noqa
+ message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+ flash(message, 'job')
+ return {}, 201, {'Location': job.url}
return render_template(
'services/file_setup_pipeline.html.j2',
form=form,
@@ -90,61 +62,43 @@ def file_setup_pipeline():
@bp.route('/tesseract-ocr-pipeline', methods=['GET', 'POST'])
@login_required
def tesseract_ocr_pipeline():
- service = 'tesseract-ocr-pipeline'
- service_manifest = SERVICES[service]
+ service_name = 'tesseract-ocr-pipeline'
+ service_manifest = SERVICES[service_name]
version = request.args.get('version', service_manifest['latest_version'])
if version not in service_manifest['versions']:
abort(404)
- form = AddTesseractOCRPipelineJobForm(prefix='add-job-form', version=version)
+ form = CreateTesseractOCRPipelineJobForm(prefix='create-job-form', version=version)
if form.is_submitted():
if not form.validate():
- return make_response(form.errors, 400)
- service_args = {}
- service_args['model'] = hashids.decode(form.model.data)
- if form.binarization.data:
- service_args['binarization'] = True
- job = Job(
- user=current_user,
- description=form.description.data,
- service=service,
- service_args=service_args,
- service_version=form.version.data,
- title=form.title.data
- )
- db.session.add(job)
- db.session.flush(objects=[job])
- db.session.refresh(job)
+ response = {'errors': form.errors}
+ return response, 400
try:
- job.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
- job_input = JobInput(
- filename=secure_filename(form.pdf.data.filename),
- job=job,
- mimetype=form.pdf.data.mimetype
- )
- db.session.add(job_input)
- db.session.flush(objects=[job_input])
- db.session.refresh(job_input)
+ job = Job.create(
+ title=form.title.data,
+ description=form.description.data,
+ service=service_name,
+ service_args={
+ 'binarization': form.binarization.data,
+ 'model': hashids.decode(form.model.data)
+ },
+ service_version=form.version.data,
+ user=current_user
+ )
+ except OSError:
+ abort(500)
try:
- form.pdf.data.save(job_input.path)
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
+ JobInput.create(form.pdf.data, job=job)
+ except OSError:
+ abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
- flash(f'Job "{job.title}" added', 'job')
- return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201) # noqa
+ message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+ flash(message, 'job')
+ return {}, 201, {'Location': job.url}
tesseract_ocr_models = [
- x for x in TesseractOCRModel.query.filter().all()
+ x for x in TesseractOCRModel.query.all()
if version in x.compatible_service_versions and (x.shared == True or x.user == current_user)
]
- current_app.logger.warning(tesseract_ocr_models)
return render_template(
'services/tesseract_ocr_pipeline.html.j2',
form=form,
@@ -163,57 +117,40 @@ def transkribus_htr_pipeline():
version = request.args.get('version', service_manifest['latest_version'])
if version not in service_manifest['versions']:
abort(404)
- form = AddTranskribusHTRPipelineJobForm(prefix='add-job-form', version=version)
+ form = CreateTranskribusHTRPipelineJobForm(prefix='create-job-form', version=version)
if form.is_submitted():
if not form.validate():
- return make_response(form.errors, 400)
- service_args = {}
- service_args['model'] = hashids.decode(form.model.data)
- if form.binarization.data:
- service_args['binarization'] = True
- job = Job(
- user=current_user,
- description=form.description.data,
- service=service,
- service_args=service_args,
- service_version=form.version.data,
- title=form.title.data
- )
- db.session.add(job)
- db.session.flush(objects=[job])
- db.session.refresh(job)
+ response = {'errors': form.errors}
+ return response, 400
try:
- job.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
- job_input = JobInput(
- filename=secure_filename(form.pdf.data.filename),
- job=job,
- mimetype=form.pdf.data.mimetype
- )
- db.session.add(job_input)
- db.session.flush(objects=[job_input])
- db.session.refresh(job_input)
+ job = Job.create(
+ title=form.title.data,
+ description=form.description.data,
+ service=service,
+ service_args={
+ 'binarization': form.binarization.data,
+ 'model': hashids.decode(form.model.data)
+ },
+ service_version=form.version.data,
+ user=current_user
+ )
+ except OSError:
+ abort(500)
try:
- form.pdf.data.save(job_input.path)
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
+ JobInput.create(form.pdf.data, job=job)
+ except OSError:
+ abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
- flash(f'Job "{job.title}" added', 'job')
- return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201) # noqa
+ message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+ flash(message, 'job')
+ return {}, 201, {'Location': job.url}
transkribus_htr_models = [
- x for x in TranskribusHTRModel.query.filter().all()
+ x for x in TranskribusHTRModel.query.all()
if x.shared == True or x.user == current_user
]
return render_template(
- f'services/transkribus_htr_pipeline.html.j2',
+ 'services/transkribus_htr_pipeline.html.j2',
form=form,
title=service_manifest['name'],
TRANSKRIBUS_HTR_MODELS=TRANSKRIBUS_HTR_MODELS,
@@ -229,51 +166,34 @@ def spacy_nlp_pipeline():
version = request.args.get('version', SERVICES[service]['latest_version'])
if version not in service_manifest['versions']:
abort(404)
- form = AddSpacyNLPPipelineJobForm(prefix='add-job-form', version=version)
+ form = CreateSpacyNLPPipelineJobForm(prefix='create-job-form', version=version)
if form.is_submitted():
if not form.validate():
- return make_response(form.errors, 400)
- service_args = {}
- service_args['model'] = form.model.data
- if form.encoding_detection.data:
- service_args['encoding_detection'] = True
- job = Job(
- user=current_user,
- description=form.description.data,
- service=service,
- service_args=service_args,
- service_version=form.version.data,
- title=form.title.data
- )
- db.session.add(job)
- db.session.flush(objects=[job])
- db.session.refresh(job)
+ response = {'errors': form.errors}
+ return response, 400
try:
- job.makedirs()
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
- job_input = JobInput(
- filename=secure_filename(form.txt.data.filename),
- job=job,
- mimetype=form.txt.data.mimetype
- )
- db.session.add(job_input)
- db.session.flush(objects=[job_input])
- db.session.refresh(job_input)
+ job = Job.create(
+ title=form.title.data,
+ description=form.description.data,
+ service=service,
+ service_args={
+ 'encoding_detection': form.encoding_detection.data,
+ 'model': form.model.data
+ },
+ service_version=form.version.data,
+ user=current_user
+ )
+ except OSError:
+ abort(500)
try:
- form.txt.data.save(job_input.path)
- except OSError as e:
- current_app.logger.error(e)
- db.session.rollback()
- flash('Internal Server Error', 'error')
- return make_response({'redirect_url': url_for('.service', service=service)}, 500) # noqa
+ JobInput.create(form.txt.data, job=job)
+ except OSError:
+ abort(500)
job.status = JobStatus.SUBMITTED
db.session.commit()
- flash(f'Job "{job.title}" added', 'job')
- return make_response({'redirect_url': url_for('jobs.job', job_id=job.id)}, 201) # noqa
+ message = Markup(f'Job "<a href="{job.url}">{job.title}</a>" created')
+ flash(message, 'job')
+ return {}, 201, {'Location': job.url}
return render_template(
'services/spacy_nlp_pipeline.html.j2',
form=form,
diff --git a/app/services/services.yml b/app/services/services.yml
index 2979539c2ca9d1ff837ac2c2e5e0b990cb94c211..b7a4947319ea81d5e4c74d462d047dc413d4aafa 100644
--- a/app/services/services.yml
+++ b/app/services/services.yml
@@ -17,6 +17,11 @@ tesseract-ocr-pipeline:
- 'binarization'
publishing_year: 2022
url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.0'
+ 0.1.1:
+ methods:
+ - 'binarization'
+ publishing_year: 2022
+ url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1'
transkribus-htr-pipeline:
name: 'Transkribus HTR Pipeline'
publisher: 'Bielefeld University - CRC 1288 - INF'
@@ -47,4 +52,4 @@ spacy-nlp-pipeline:
ru: 'Russian'
zh: 'Chinese'
publishing_year: 2022
- url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.0'
\ No newline at end of file
+ url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.0'
diff --git a/app/settings/forms.py b/app/settings/forms.py
index cfc6918c3192aa13389a274b6da0c3c309447b0f..3bd3b5ab108a3a5320e06ad234d8539db0470f40 100644
--- a/app/settings/forms.py
+++ b/app/settings/forms.py
@@ -1,5 +1,3 @@
-from app.auth import USERNAME_REGEX
-from app.models import User, UserSettingJobStatusMailNotificationLevel
from flask_wtf import FlaskForm
from wtforms import (
BooleanField,
@@ -9,14 +7,35 @@ from wtforms import (
SubmitField,
ValidationError
)
-from wtforms.validators import DataRequired, InputRequired, Email, EqualTo, Length, Regexp
+from wtforms.validators import (
+ DataRequired,
+ InputRequired,
+ Email,
+ EqualTo,
+ Length,
+ Regexp
+)
+from app.models import User, UserSettingJobStatusMailNotificationLevel
+from app.auth import USERNAME_REGEX
class ChangePasswordForm(FlaskForm):
password = PasswordField('Old password', validators=[DataRequired()])
- new_password = PasswordField('New password', validators=[DataRequired(), EqualTo('new_password_confirmation', message='Passwords must match')])
- new_password_confirmation = PasswordField('Confirm new password', validators=[DataRequired(), EqualTo('new_password', message='Passwords must match')])
- submit = SubmitField('Submit')
+ new_password = PasswordField(
+ 'New password',
+ validators=[
+ DataRequired(),
+ EqualTo('new_password_2', message='Passwords must match')
+ ]
+ )
+ new_password_2 = PasswordField(
+ 'New password confirmation',
+ validators=[
+ DataRequired(),
+ EqualTo('new_password', message='Passwords must match')
+ ]
+ )
+ submit = SubmitField()
def __init__(self, user, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -28,43 +47,51 @@ class ChangePasswordForm(FlaskForm):
class EditGeneralSettingsForm(FlaskForm):
- email = StringField('E-Mail', validators=[DataRequired(), Length(1, 254), Email()])
+ email = StringField(
+ 'E-Mail',
+ validators=[InputRequired(), Length(max=254), Email()]
+ )
username = StringField(
'Username',
validators=[
InputRequired(),
- Length(1, 64),
+ Length(max=64),
Regexp(
USERNAME_REGEX,
- message='Usernames must have only letters, numbers, dots or underscores' # noqa
+ message=(
+ 'Usernames must have only letters, numbers, dots or '
+ 'underscores'
+ )
)
]
)
- submit = SubmitField('Submit')
+ submit = SubmitField()
def __init__(self, user, *args, **kwargs):
super().__init__(*args, **kwargs)
self.user = user
+ def prefill(self, user):
+ self.email.data = user.email
+ self.username.data = user.username
+
def validate_email(self, field):
- if (
- field.data != self.user.email
- and User.query.filter_by(email=field.data).first()
- ):
+ if (field.data != self.user.email
+ and User.query.filter_by(email=field.data).first()):
raise ValidationError('Email already registered')
def validate_username(self, field):
- if (
- field.data != self.user.username
- and User.query.filter_by(username=field.data).first()
- ):
+ if (field.data != self.user.username
+ and User.query.filter_by(username=field.data).first()):
raise ValidationError('Username already in use')
class EditInterfaceSettingsForm(FlaskForm):
dark_mode = BooleanField('Dark mode')
- submit = SubmitField('Submit')
+ submit = SubmitField()
+ def prefill(self, user):
+ self.dark_mode.data = user.setting_dark_mode
class EditNotificationSettingsForm(FlaskForm):
job_status_mail_notification_level = SelectField(
@@ -72,11 +99,15 @@ class EditNotificationSettingsForm(FlaskForm):
choices=[('', 'Choose your option')],
validators=[DataRequired()]
)
- submit = SubmitField('Submit')
+ submit = SubmitField()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.job_status_mail_notification_level.choices += [
- (enum_member.name, enum_member.name.capitalize())
- for enum_member in UserSettingJobStatusMailNotificationLevel
+ (x.name, x.name.capitalize())
+ for x in UserSettingJobStatusMailNotificationLevel
]
+
+ def prefill(self, user):
+ self.job_status_mail_notification_level.data = \
+ user.setting_job_status_mail_notification_level.name
diff --git a/app/settings/routes.py b/app/settings/routes.py
index 8d828e3337d016f258b7ab1a92ff6ec3b08eeb10..eb2636e815f96b29252aeeefdba22bdae026ec99 100644
--- a/app/settings/routes.py
+++ b/app/settings/routes.py
@@ -1,32 +1,32 @@
from flask import flash, redirect, render_template, url_for
-from flask_login import current_user, login_required, logout_user
-from . import bp, tasks
+from flask_login import current_user, login_required
+from app import db
+from app.models import UserSettingJobStatusMailNotificationLevel
+from . import bp
from .forms import (
ChangePasswordForm,
EditGeneralSettingsForm,
EditInterfaceSettingsForm,
EditNotificationSettingsForm
)
-from .. import db
-from ..models import UserSettingJobStatusMailNotificationLevel
@bp.route('', methods=['GET', 'POST'])
@login_required
-def index():
+def settings():
change_password_form = ChangePasswordForm(
- current_user._get_current_object(),
- prefix='change_password_form'
+ current_user,
+ prefix='change-password-form'
)
edit_general_settings_form = EditGeneralSettingsForm(
- current_user._get_current_object(),
- prefix='edit_general_settings_form'
+ current_user,
+ prefix='edit-general-settings-form'
)
edit_interface_settings_form = EditInterfaceSettingsForm(
- prefix='edit_interface_settings_form'
+ prefix='edit-interface-settings-form'
)
edit_notification_settings_form = EditNotificationSettingsForm(
- prefix='edit_notification_settings_form'
+ prefix='edit-notification-settings-form'
)
if change_password_form.submit.data and change_password_form.validate():
@@ -34,58 +34,38 @@ def index():
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.index'))
- if (
- edit_general_settings_form.submit.data
- and edit_general_settings_form.validate()
- ):
+ if (edit_general_settings_form.submit.data
+ and edit_general_settings_form.validate()):
current_user.email = edit_general_settings_form.email.data
current_user.username = edit_general_settings_form.username.data
db.session.commit()
flash('Your changes have been saved')
- return redirect(url_for('.index'))
- if (
- edit_interface_settings_form.submit.data
- and edit_interface_settings_form.validate()
- ):
- current_user.setting_dark_mode = \
- edit_interface_settings_form.dark_mode.data
+ return redirect(url_for('.settings'))
+ if (edit_interface_settings_form.submit.data
+ and edit_interface_settings_form.validate()):
+ current_user.setting_dark_mode = (
+ edit_interface_settings_form.dark_mode.data)
db.session.commit()
flash('Your changes have been saved')
- return redirect(url_for('.index'))
- if (
- edit_notification_settings_form.submit.data
- and edit_notification_settings_form.validate()
- ):
- current_user.setting_job_status_mail_notification_level = \
+ return redirect(url_for('.settings'))
+ if (edit_notification_settings_form.submit.data
+ and edit_notification_settings_form.validate()):
+ current_user.setting_job_status_mail_notification_level = (
UserSettingJobStatusMailNotificationLevel[
edit_notification_settings_form.job_status_mail_notification_level.data # noqa
]
+ )
db.session.commit()
flash('Your changes have been saved')
- return redirect(url_for('.index'))
- edit_general_settings_form.email.data = current_user.email
- edit_general_settings_form.username.data = current_user.username
- edit_interface_settings_form.dark_mode.data = \
- current_user.setting_dark_mode
- edit_notification_settings_form.job_status_mail_notification_level.data = \
- current_user.setting_job_status_mail_notification_level.name
+ return redirect(url_for('.settings'))
+ edit_general_settings_form.prefill(current_user)
+ edit_interface_settings_form.prefill(current_user)
+ edit_notification_settings_form.prefill(current_user)
return render_template(
- 'settings/index.html.j2',
+ 'settings/settings.html.j2',
change_password_form=change_password_form,
edit_general_settings_form=edit_general_settings_form,
edit_interface_settings_form=edit_interface_settings_form,
edit_notification_settings_form=edit_notification_settings_form,
title='Settings'
)
-
-
-@bp.route('/delete')
-@login_required
-def delete():
- """
- View to delete current_user and all associated data.
- """
- tasks.delete_user(current_user.id)
- logout_user()
- flash('Your account has been marked for deletion')
- return redirect(url_for('main.index'))
diff --git a/app/settings/tasks.py b/app/settings/tasks.py
deleted file mode 100644
index 2bd82ca92c36f9030fa1a3e9ef7c572ef6d27231..0000000000000000000000000000000000000000
--- a/app/settings/tasks.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from app import db
-from app.decorators import background
-from app.models import User
-
-
-@background
-def delete_user(user_id, *args, **kwargs):
- with kwargs['app'].app_context():
- user = User.query.get(user_id)
- if user is None:
- raise Exception(f'User {user_id} not found')
- user.delete()
- db.session.commit()
diff --git a/app/socketio_event_listeners.py b/app/socketio_event_listeners.py
deleted file mode 100644
index 92cfc1fa7c137b24aded9cdba827ea7fbc47ecaa..0000000000000000000000000000000000000000
--- a/app/socketio_event_listeners.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from app import hashids, socketio
-from app.decorators import socketio_login_required
-from app.models import TesseractOCRModel, TranskribusHTRModel, User
-from flask_login import current_user
-from flask_socketio import join_room
-
-
-@socketio.on('users.user.get')
-@socketio_login_required
-def users_user_get(user_hashid):
- user_id = hashids.decode(user_hashid)
- user = User.query.get(user_id)
- if user is None:
- return {'code': 404, 'msg': 'Not found'}
- if not (user == current_user or current_user.is_administrator):
- return {'code': 403, 'msg': 'Forbidden'}
- # corpora = [x.to_dict() for x in user.corpora]
- # jobs = [x.to_dict() for x in user.jobs]
- # transkribus_htr_models = TranskribusHTRModel.query.filter(
- # (TranskribusHTRModel.shared == True) | (TranskribusHTRModel.user == user)
- # ).all()
- # tesseract_ocr_models = TesseractOCRModel.query.filter(
- # (TesseractOCRModel.shared == True) | (TesseractOCRModel.user == user)
- # ).all()
- # response = {
- # 'code': 200,
- # 'msg': 'OK',
- # 'payload': {
- # 'user': user.to_dict(),
- # 'corpora': corpora,
- # 'jobs': jobs,
- # 'transkribus_htr_models': transkribus_htr_models,
- # 'tesseract_ocr_models': tesseract_ocr_models
- # }
- # }
- join_room(f'users.{user.hashid}')
- return {
- 'code': 200,
- 'msg': 'OK',
- 'payload': user.to_dict(backrefs=True, relationships=True)
- }
diff --git a/app/sqlalchemy_event_listeners.py b/app/sqlalchemy_event_listeners.py
deleted file mode 100644
index 94470591ca1a05f3583e3abc0eaf84b6f80be81b..0000000000000000000000000000000000000000
--- a/app/sqlalchemy_event_listeners.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from app import db, mail, socketio
-from app.email import create_message
-from app.models import (
- Corpus,
- CorpusFile,
- Job,
- JobInput,
- JobResult,
- JobStatus,
- UserSettingJobStatusMailNotificationLevel
-)
-from datetime import datetime
-from enum import Enum
-
-
-@db.event.listens_for(Corpus, 'after_delete')
-@db.event.listens_for(CorpusFile, 'after_delete')
-@db.event.listens_for(Job, 'after_delete')
-@db.event.listens_for(JobInput, 'after_delete')
-@db.event.listens_for(JobResult, 'after_delete')
-def ressource_after_delete(mapper, connection, ressource):
- jsonpatch = [{'op': 'remove', 'path': ressource.jsonpatch_path}]
- room = f'users.{ressource.user_hashid}'
- socketio.emit('users.patch', jsonpatch, room=room)
-
-
-@db.event.listens_for(Corpus, 'after_insert')
-@db.event.listens_for(CorpusFile, 'after_insert')
-@db.event.listens_for(Job, 'after_insert')
-@db.event.listens_for(JobInput, 'after_insert')
-@db.event.listens_for(JobResult, 'after_insert')
-def ressource_after_insert_handler(mapper, connection, ressource):
- value = ressource.to_dict(backrefs=False, relationships=False)
- for attr in mapper.relationships:
- value[attr.key] = {}
- jsonpatch = [
- {'op': 'add', 'path': ressource.jsonpatch_path, 'value': value}
- ]
- room = f'users.{ressource.user_hashid}'
- socketio.emit('users.patch', jsonpatch, room=room)
-
-
-@db.event.listens_for(Corpus, 'after_update')
-@db.event.listens_for(CorpusFile, 'after_update')
-@db.event.listens_for(Job, 'after_update')
-@db.event.listens_for(JobInput, 'after_update')
-@db.event.listens_for(JobResult, 'after_update')
-def ressource_after_update_handler(mapper, connection, ressource):
- jsonpatch = []
- for attr in db.inspect(ressource).attrs:
- if attr.key in mapper.relationships:
- continue
- if not attr.load_history().has_changes():
- continue
- if isinstance(attr.value, datetime):
- value = attr.value.isoformat() + 'Z'
- elif isinstance(attr.value, Enum):
- value = attr.value.name
- else:
- value = attr.value
- jsonpatch.append(
- {
- 'op': 'replace',
- 'path': f'{ressource.jsonpatch_path}/{attr.key}',
- 'value': value
- }
- )
- if isinstance(ressource, Job) and attr.key == 'status':
- _job_status_email_handler(ressource)
- if jsonpatch:
- room = f'users.{ressource.user_hashid}'
- socketio.emit('users.patch', jsonpatch, room=room)
-
-
-def _job_status_email_handler(job):
- if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.NONE: # noqa
- return
- if job.user.setting_job_status_mail_notification_level == UserSettingJobStatusMailNotificationLevel.END: # noqa
- if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
- return
- msg = create_message(
- job.user.email,
- f'Status update for your Job "{job.title}"',
- 'tasks/email/notification',
- job=job
- )
- mail.send(msg)
diff --git a/app/sqlalchemy_type_decorators.py b/app/sqlalchemy_type_decorators.py
deleted file mode 100644
index ac97e308014c3d755598a1a62f0ad475db469f18..0000000000000000000000000000000000000000
--- a/app/sqlalchemy_type_decorators.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from app import db
-import json
-
-
-class IntEnumColumn(db.TypeDecorator):
- impl = db.Integer
-
- def __init__(self, enum_type, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.enum_type = enum_type
-
- def process_bind_param(self, value, dialect):
- if isinstance(value, self.enum_type) and isinstance(value.value, int):
- return value.value
- elif isinstance(value, int):
- return self.enum_type(value).value
- else:
- return TypeError()
-
- def process_result_value(self, value, dialect):
- return self.enum_type(value)
-
-
-class ContainerColumn(db.TypeDecorator):
- impl = db.String
-
- def __init__(self, container_type, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.container_type = container_type
-
- def process_bind_param(self, value, dialect):
- if isinstance(value, self.container_type):
- return json.dumps(value)
- elif (
- isinstance(value, str)
- and isinstance(json.loads(value), self.container_type)
- ):
- return value
- else:
- return TypeError()
-
- def process_result_value(self, value, dialect):
- return json.loads(value)
\ No newline at end of file
diff --git a/app/static/css/helpers.scss b/app/static/css/helpers.scss
new file mode 100644
index 0000000000000000000000000000000000000000..b7595c26fb16bf18a3757a2f4286ce5b91fd9989
--- /dev/null
+++ b/app/static/css/helpers.scss
@@ -0,0 +1,31 @@
+/*
+ * Spacing
+ */
+$spacing-shortcuts: ("margin": "mg", "padding": "pd");
+$spacing-directions: ("top": "t", "right": "r", "bottom": "b", "left": "l");
+$spacing-values: ("0": 0, "1": 0.25rem, "2": 0.5rem, "3": 0.75rem, "4": 1rem, "5": 1.5rem, "6": 3rem, "auto": auto);
+
+@each $spacing-shortcut-name, $spacing-shortcut-value in $spacing-shortcuts {
+ @each $spacing-name, $spacing-value in $spacing-values {
+ // All directions
+ .#{$spacing-shortcut-value}-#{$spacing-name} {
+ #{$spacing-shortcut-name}: $spacing-value !important;
+ }
+ // Horizontal axis
+ .#{$spacing-shortcut-value}x-#{$spacing-name} {
+ #{$spacing-shortcut-name}-left: $spacing-value !important;
+ #{$spacing-shortcut-name}-right: $spacing-value !important;
+ }
+ // Vertical axis
+ .#{$spacing-shortcut-value}y-#{$spacing-name} {
+ #{$spacing-shortcut-name}-top: $spacing-value !important;
+ #{$spacing-shortcut-name}-bottom: $spacing-value !important;
+ }
+ // Cardinal directions
+ @each $spacing-direction-name, $spacing-direction-value in $spacing-directions {
+ .#{$spacing-shortcut-value}#{$spacing-direction-value}-#{$spacing-name} {
+ #{$spacing-shortcut-name}-#{$spacing-direction-name}: $spacing-value !important;
+ }
+ }
+ }
+}
diff --git a/app/static/css/style.css b/app/static/css/style.css
index 0a6c575a867b58f4fa54c235fb9a7aa998a4510f..e0c288b5da551193dd6ea8cae8c2f98b851aa767 100644
--- a/app/static/css/style.css
+++ b/app/static/css/style.css
@@ -49,5 +49,14 @@ h1 .nopaque-icons, h2 .nopaque-icons, h3 .nopaque-icons, h4 .nopaque-icons, .tab
.nopaque-icons.service-icon[data-service="spacy-nlp-pipeline"]:empty:before {content: "G";}
.nopaque-icons.service-icon[data-service="corpus-analysis"]:empty:before {content: "H";}
-.hoverable {cursor: pointer;}
+.clickable {
+ cursor: pointer !important;
+ pointer-events: all !important;
+}
.chip.s-attr .chip.p-attr {background-color: inherit;}
+
+
+.width-25 {width: 25%;}
+.width-50 {width: 50%;}
+.width-75 {width: 75%;}
+.width-100 {width: 100%;}
diff --git a/app/static/js/App.js b/app/static/js/App.js
index 3761be8cf7a5503b5d5500649cf67b0d24fef7f1..e20b30f2ff779d3d0453c140d3c6a63cb8f4056e 100644
--- a/app/static/js/App.js
+++ b/app/static/js/App.js
@@ -1,78 +1,111 @@
class App {
constructor() {
- this.data = {users: {}};
- this.eventListeners = {'users.patch': []};
- this.promises = {users: {}};
+ this.data = {
+ promises: {getUser: {}, subscribeUser: {}},
+ users: {},
+ };
this.socket = io({transports: ['websocket'], upgrade: false});
- this.socket.on('users.patch', patch => this.usersPatchHandler(patch));
+ this.socket.on('PATCH', (patch) => {this.onPatch(patch);});
}
- get users() {
- return this.data.users;
+ getUser(userId) {
+ if (userId in this.data.promises.getUser) {
+ return this.data.promises.getUser[userId];
+ }
+
+ this.data.promises.getUser[userId] = new Promise((resolve, reject) => {
+ fetch(`/users/${userId}?backrefs=true&relationships=true`, {headers: {Accept: 'application/json'}})
+ .then(
+ (response) => {return response.json();},
+ (response) => {
+ if (response.status === 403) {this.flash('Forbidden', 'error');}
+ if (response.status === 404) {this.flash('Not Found', 'error');}
+ reject(response);
+ }
+ )
+ .then(
+ (user) => {
+ this.data.users[userId] = user;
+ resolve(this.data.users[userId]);
+ },
+ (error) => {
+ console.error(error, 'error');
+ reject(error);
+ }
+ );
+ });
+
+ return this.data.promises.getUser[userId];
}
- addEventListener(type, listener) {
- if (!(type in this.eventListeners)) {
- throw `Unknown event type: ${type}`;
+ subscribeUser(userId) {
+ if (userId in this.data.promises.subscribeUser) {
+ return this.data.promises.subscribeUser[userId];
}
- this.eventListeners[type].push(listener);
+
+ this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => {
+ this.socket.emit('SUBSCRIBE /users/<user_id>', userId, (response) => {
+ if (response.code === 200) {
+ resolve(response);
+ } else {
+ reject(response);
+ }
+ });
+ });
+
+ return this.data.promises.subscribeUser[userId];
}
flash(message, category) {
- let iconPrefix;
- let toast;
- let toastCloseActionElement;
-
+ let iconPrefix = '';
switch (category) {
- case 'corpus':
+ case 'corpus': {
iconPrefix = '<i class="left material-icons">book</i>';
break;
- case 'error':
+ }
+ case 'error': {
iconPrefix = '<i class="error-color-text left material-icons">error</i>';
break;
- case 'job':
+ }
+ case 'job': {
iconPrefix = '<i class="left nopaque-icons">J</i>';
break;
- default:
+ }
+ default: {
iconPrefix = '<i class="left material-icons">notifications</i>';
break;
+ }
}
- toast = M.toast(
+ let toast = M.toast(
{
html: `
<span>${iconPrefix}${message}</span>
- <button class="btn-flat toast-action white-text" data-action="close">
+ <button class="action-button btn-flat toast-action white-text" data-action="close">
<i class="material-icons">close</i>
</button>
`.trim()
}
);
- toastCloseActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
+ let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]');
toastCloseActionElement.addEventListener('click', () => {toast.dismiss();});
}
+
+ onPatch(patch) {
+ // Filter Patch to only include operations on users that are initialized
+ let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`);
+ let filteredPatch = patch.filter(operation => regExp.test(operation.path));
- getUserById(userId) {
- if (userId in this.promises.users) {
- return this.promises.users[userId];
+ // Handle job status updates
+ let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`);
+ let subFilteredPatch = filteredPatch
+ .filter((operation) => {return operation.op === 'replace';})
+ .filter((operation) => {return subRegExp.test(operation.path);});
+ for (let operation of subFilteredPatch) {
+ let [match, userId, jobId] = operation.path.match(subRegExp);
+ this.flash(`[<a href="/jobs/${jobId}">${this.data.users[userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
}
- this.promises.users[userId] = new Promise((resolve, reject) => {
- this.socket.emit('users.user.get', userId, response => {
- if (response.code === 200) {
- this.data.users[userId] = response.payload;
- resolve(this.data.users[userId]);
- } else {
- reject(response);
- }
- });
- });
- return this.promises.users[userId];
- }
-
- usersPatchHandler(patch) {
- let listener;
- this.data = jsonpatch.applyPatch(this.data, patch).newDocument;
- //this.data = jsonpatch.apply_patch(this.data, patch);
- for (listener of this.eventListeners['users.patch']) {listener(patch);}
+ // Apply Patch
+ jsonpatch.applyPatch(this.data, filteredPatch);
}
}
diff --git a/app/static/js/Forms/CreateCorpusFileForm.js b/app/static/js/Forms/CreateCorpusFileForm.js
new file mode 100644
index 0000000000000000000000000000000000000000..ae8dba3b1a317720408a09e12366d270cae54420
--- /dev/null
+++ b/app/static/js/Forms/CreateCorpusFileForm.js
@@ -0,0 +1,18 @@
+class CreateCorpusFileForm extends Form {
+ static autoInit() {
+ let createCorpusFileFormElements = document.querySelectorAll('.create-corpus-file-form');
+ for (let createCorpusFileFormElement of createCorpusFileFormElements) {
+ new CreateCorpusFileForm(createCorpusFileFormElement);
+ }
+ }
+
+ constructor(formElement) {
+ super(formElement);
+
+ this.addEventListener('requestLoad', (event) => {
+ if (event.target.status === 201) {
+ window.location.href = event.target.getResponseHeader('Location');
+ }
+ });
+ }
+}
diff --git a/app/static/js/Forms/CreateJobForm.js b/app/static/js/Forms/CreateJobForm.js
new file mode 100644
index 0000000000000000000000000000000000000000..6aa2d1b6eb2165d2306bf75d2206a1975dfa9c57
--- /dev/null
+++ b/app/static/js/Forms/CreateJobForm.js
@@ -0,0 +1,25 @@
+class CreateJobForm extends Form {
+ static autoInit() {
+ let createJobFormElements = document.querySelectorAll('.create-job-form');
+ for (let createJobFormElement of createJobFormElements) {
+ new CreateJobForm(createJobFormElement);
+ }
+ }
+
+ constructor(formElement) {
+ super(formElement);
+
+ let versionField = this.formElement.querySelector('#create-job-form-version');
+ versionField.addEventListener('change', (event) => {
+ let url = new URL(window.location.href);
+ url.search = `?version=${event.target.value}`;
+ window.location.href = url.toString();
+ });
+
+ this.addEventListener('requestLoad', (event) => {
+ if (event.target.status === 201) {
+ window.location.href = event.target.getResponseHeader('Location');
+ }
+ });
+ }
+}
diff --git a/app/static/js/Forms/Form.js b/app/static/js/Forms/Form.js
new file mode 100644
index 0000000000000000000000000000000000000000..9a21e98661a2e2ba8c64a7d9c846af4af25fd5a3
--- /dev/null
+++ b/app/static/js/Forms/Form.js
@@ -0,0 +1,141 @@
+class Form {
+ static autoInit() {
+ CreateCorpusFileForm.autoInit();
+ CreateJobForm.autoInit();
+ }
+
+ constructor(formElement) {
+ this.formElement = formElement;
+ this.eventListeners = {
+ 'requestLoad': []
+ };
+ this.afterRequestListeners = [];
+
+ for (let selectElement of this.formElement.querySelectorAll('select')) {
+ selectElement.removeAttribute('required');
+ }
+
+ this.formElement.addEventListener('submit', (event) => {
+ event.preventDefault();
+ this.submit(event);
+ });
+ }
+
+ addEventListener(eventType, listener) {
+ if (eventType in this.eventListeners) {
+ this.eventListeners[eventType].push(listener);
+ } else {
+ throw `Unknown event type ${eventType}`;
+ }
+ }
+
+ submit(event) {
+ let request = new XMLHttpRequest();
+ let modalElement = Utils.elementFromString(
+ `
+ <div class="modal">
+ <div class="modal-content">
+ <h4><i class="material-icons left">file_upload</i>Submitting...</h4>
+ <div class="progress">
+ <div class="determinate" style="width: 0%"></div>
+ </div>
+ </div>
+ <div class="modal-footer">
+ <a class="action-button btn red waves-effect waves-light modal-close" data-action="cancel">Cancel</a>
+ </div>
+ </div>
+ `
+ );
+ document.querySelector('#modals').appendChild(modalElement);
+ let modal = M.Modal.init(
+ modalElement,
+ {
+ dismissible: false,
+ onCloseEnd: () => {
+ modal.destroy();
+ modalElement.remove();
+ }
+ }
+ );
+ modal.open();
+
+ // Remove all previous helper text elements that indicate errors
+ let errorHelperTextElements = this.formElement
+ .querySelectorAll('.helper-text[data-helper-text-type="error"]');
+ for (let errorHelperTextElement of errorHelperTextElements) {
+ errorHelperTextElement.remove();
+ }
+
+ // Check if select elements are filled out properly
+ for (let selectElement of this.formElement.querySelectorAll('select')) {
+ if (selectElement.value === '') {
+ let inputFieldElement = selectElement.closest('.input-field');
+ let errorHelperTextElement = Utils.elementFromString(
+ '<span class="helper-text error-color-text" data-helper-text-type="error">Please select an option.</span>'
+ );
+ inputFieldElement.appendChild(errorHelperTextElement);
+ inputFieldElement.querySelector('.select-dropdown').classList.add('invalid');
+ modal.close();
+ return;
+ }
+ }
+
+ // Setup abort handling
+ let cancelElement = modalElement.querySelector('.action-button[data-action="cancel"]');
+ cancelElement.addEventListener('click', (event) => {request.abort();});
+
+ // Setup load handling (after the request completed)
+ request.addEventListener('load', (event) => {
+ for (let listener of this.eventListeners['requestLoad']) {
+ listener(event);
+ }
+ if (request.status === 400) {
+ let responseJson = JSON.parse(request.responseText);
+ console.log(responseJson);
+ for (let [inputName, inputErrors] of Object.entries(responseJson.errors)) {
+ let inputFieldElement = this.formElement
+ .querySelector(`input[name$="${inputName}"], select[name$="${inputName}"]`)
+ .closest('.input-field');
+ for (let inputError of inputErrors) {
+ let errorHelperTextElement = Utils.elementFromString(
+ `<span class="helper-text error-color-text" data-helper-type="error">${inputError}</span>`
+ );
+ inputFieldElement.appendChild(errorHelperTextElement);
+ }
+ }
+ }
+ if (request.status === 500) {
+ app.flash('Internal Server Error', 'error');
+ }
+ modal.close();
+ });
+
+ // Setup progress handling
+ let progressBarElement = modalElement.querySelector('.progress > .determinate');
+ request.upload.addEventListener('progress', (event) => {
+ let progress = Math.floor(100 * event.loaded / event.total);
+ progressBarElement.style.width = `${progress}%`;
+ });
+
+ request.open(this.formElement.method, this.formElement.action);
+ request.setRequestHeader('Accept', 'application/json');
+ let formData = new FormData(this.formElement);
+ switch (this.formElement.enctype) {
+ case 'application/x-www-form-urlencoded':
+ let urlSearchParams = new URLSearchParams(formData);
+ request.send(urlSearchParams);
+ break;
+ case 'multipart/form-data': {
+ request.send(formData);
+ break;
+ }
+ case 'text/plain': {
+ throw 'enctype "text/plain" is not supported';
+ break;
+ }
+ default: {
+ break;
+ }
+ }
+ }
+}
diff --git a/app/static/js/JobStatusNotifier.js b/app/static/js/JobStatusNotifier.js
deleted file mode 100644
index bb0ca44d76ae92ef2c3a14252decca17baee1b83..0000000000000000000000000000000000000000
--- a/app/static/js/JobStatusNotifier.js
+++ /dev/null
@@ -1,22 +0,0 @@
-class JobStatusNotifier {
- constructor(userId) {
- this.userId = userId;
- }
-
- usersPatchHandler(patch) {
- let filteredPatch;
- let jobId;
- let match;
- let operation;
- let re;
-
- re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)/status$`)
- filteredPatch = patch
- .filter(operation => operation.op === 'replace')
- .filter(operation => re.test(operation.path));
- for (operation of filteredPatch) {
- [match, jobId] = operation.path.match(re);
- app.flash(`[<a href="/jobs/${jobId}">${app.users[this.userId].jobs[jobId].title}</a>] New status: <span class="job-status-text" data-job-status="${operation.value}"></span>`, 'job');
- }
- }
-}
diff --git a/app/static/js/RessourceDisplays/CorpusDisplay.js b/app/static/js/RessourceDisplays/CorpusDisplay.js
index 6920b3bcab621edb9e688383bbf8cd8259f2cced..9bdc4800b6a9583f27ae5e05d1d51a1450402e79 100644
--- a/app/static/js/RessourceDisplays/CorpusDisplay.js
+++ b/app/static/js/RessourceDisplays/CorpusDisplay.js
@@ -2,12 +2,20 @@ class CorpusDisplay extends RessourceDisplay {
constructor(displayElement) {
super(displayElement);
this.corpusId = displayElement.dataset.corpusId;
+ this.displayElement
+ .querySelector('.action-button[data-action="build-request"]')
+ .addEventListener('click', (event) => {
+ Utils.buildCorpusRequest(this.userId, this.corpusId);
+ });
+ this.displayElement
+ .querySelector('.action-button[data-action="delete-request"]')
+ .addEventListener('click', (event) => {
+ Utils.deleteCorpusRequest(this.userId, this.corpusId);
+ });
}
init(user) {
- let corpus;
-
- corpus = user.corpora[this.corpusId];
+ let corpus = user.corpora[this.corpusId];
this.setCreationDate(corpus.creation_date);
this.setDescription(corpus.description);
this.setLastEditedDate(corpus.last_edited_date);
@@ -16,17 +24,20 @@ class CorpusDisplay extends RessourceDisplay {
this.setNumTokens(corpus.num_tokens);
}
- usersPatchHandler(patch) {
- let filteredPatch;
- let operation;
- let re;
-
- re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}`);
- filteredPatch = patch.filter(operation => re.test(operation.path));
- for (operation of filteredPatch) {
+ onPatch(patch) {
+ let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}`);
+ let filteredPatch = patch.filter(operation => re.test(operation.path));
+ for (let operation of filteredPatch) {
switch(operation.op) {
- case 'replace':
- re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/last_edited_date$`);
+ case 'remove': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}$`);
+ if (re.test(operation.path)) {
+ window.location.href = '/dashboard#corpora';
+ }
+ break;
+ }
+ case 'replace': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/last_edited_date$`);
if (re.test(operation.path)) {
this.setLastEditedDate(operation.value);
break;
@@ -42,8 +53,10 @@ class CorpusDisplay extends RessourceDisplay {
break;
}
break;
- default:
+ }
+ default: {
break;
+ }
}
}
}
@@ -55,7 +68,7 @@ class CorpusDisplay extends RessourceDisplay {
setNumTokens(numTokens) {
this.setElements(
this.displayElement.querySelectorAll('.corpus-token-ratio'),
- `${numTokens}/${app.users[this.userId].corpora[this.corpusId].max_num_tokens}`
+ `${numTokens}/${app.data.users[this.userId].corpora[this.corpusId].max_num_tokens}`
);
}
@@ -64,31 +77,28 @@ class CorpusDisplay extends RessourceDisplay {
}
setStatus(status) {
- let element;
- let elements;
-
- elements = this.displayElement.querySelectorAll('.corpus-analyse-trigger')
- for (element of elements) {
+ let elements = this.displayElement.querySelectorAll('.corpus-analyse-trigger')
+ for (let element of elements) {
if (['BUILT', 'STARTING_ANALYSIS_SESSION', 'RUNNING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) {
element.classList.remove('disabled');
} else {
element.classList.add('disabled');
}
}
- elements = this.displayElement.querySelectorAll('.corpus-build-trigger');
- for (element of elements) {
- if (['UNPREPARED', 'FAILED'].includes(status) && Object.values(app.users[this.userId].corpora[this.corpusId].files).length > 0) {
+ elements = this.displayElement.querySelectorAll('.action-button[data-action="build-request"]');
+ for (let element of elements) {
+ if (['UNPREPARED', 'FAILED'].includes(status) && Object.values(app.data.users[this.userId].corpora[this.corpusId].files).length > 0) {
element.classList.remove('disabled');
} else {
element.classList.add('disabled');
}
}
elements = this.displayElement.querySelectorAll('.corpus-status');
- for (element of elements) {
+ for (let element of elements) {
element.dataset.corpusStatus = status;
}
elements = this.displayElement.querySelectorAll('.corpus-status-spinner');
- for (element of elements) {
+ for (let element of elements) {
if (['SUBMITTED', 'QUEUED', 'BUILDING', 'STARTING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'].includes(status)) {
element.classList.remove('hide');
} else {
diff --git a/app/static/js/RessourceDisplays/JobDisplay.js b/app/static/js/RessourceDisplays/JobDisplay.js
index f00242a503d0fe60636eca37e3c4c80a3cc25160..d6669450215034b5d7b78e77043f7337ec18179e 100644
--- a/app/static/js/RessourceDisplays/JobDisplay.js
+++ b/app/static/js/RessourceDisplays/JobDisplay.js
@@ -2,12 +2,25 @@ class JobDisplay extends RessourceDisplay {
constructor(displayElement) {
super(displayElement);
this.jobId = this.displayElement.dataset.jobId;
+ this.displayElement
+ .querySelector('.action-button[data-action="delete-request"]')
+ .addEventListener('click', (event) => {
+ Utils.deleteJobRequest(this.userId, this.jobId);
+ });
+ this.displayElement
+ .querySelector('.action-button[data-action="get-log-request"]')
+ .addEventListener('click', (event) => {
+ Utils.getJobLogRequest(this.userId, this.jobId);
+ });
+ this.displayElement
+ .querySelector('.action-button[data-action="restart-request"]')
+ .addEventListener('click', (event) => {
+ Utils.restartJobRequest(this.userId, this.jobId);
+ });
}
init(user) {
- let job;
-
- job = user.jobs[this.jobId];
+ let job = user.jobs[this.jobId];
this.setCreationDate(job.creation_date);
this.setEndDate(job.creation_date);
this.setDescription(job.description);
@@ -18,17 +31,20 @@ class JobDisplay extends RessourceDisplay {
this.setTitle(job.title);
}
- usersPatchHandler(patch) {
- let filteredPatch;
- let operation;
- let re;
-
- re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}`);
- filteredPatch = patch.filter(operation => re.test(operation.path));
- for (operation of filteredPatch) {
+ onPatch(patch) {
+ let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}`);
+ let filteredPatch = patch.filter(operation => re.test(operation.path));
+ for (let operation of filteredPatch) {
switch(operation.op) {
- case 'replace':
- re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/end_date$`);
+ case 'remove': {
+ let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}$`);
+ if (re.test(operation.path)) {
+ window.location.href = '/dashboard#jobs';
+ }
+ break;
+ }
+ case 'replace': {
+ let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/end_date$`);
if (re.test(operation.path)) {
this.setEndDate(operation.value);
break;
@@ -39,8 +55,10 @@ class JobDisplay extends RessourceDisplay {
break;
}
break;
- default:
+ }
+ default: {
break;
+ }
}
}
}
@@ -54,29 +72,42 @@ class JobDisplay extends RessourceDisplay {
}
setStatus(status) {
- let element;
- let elements;
-
- elements = this.displayElement.querySelectorAll('.job-status');
- for (element of elements) {
+ let elements = this.displayElement.querySelectorAll('.job-status');
+ for (let element of elements) {
element.dataset.jobStatus = status;
}
elements = this.displayElement.querySelectorAll('.job-status-spinner');
- for (element of elements) {
+ for (let element of elements) {
if (['COMPLETED', 'FAILED'].includes(status)) {
element.classList.add('hide');
} else {
element.classList.remove('hide');
}
}
- elements = this.displayElement.querySelectorAll('.job-restart-trigger');
- for (element of elements) {
+ elements = this.displayElement.querySelectorAll('.job-log-trigger');
+ for (let element of elements) {
if (['COMPLETED', 'FAILED'].includes(status)) {
element.classList.remove('hide');
} else {
element.classList.add('hide');
}
}
+ elements = this.displayElement.querySelectorAll('.action-button[data-action="get-log-request"]');
+ for (let element of elements) {
+ if (['COMPLETED', 'FAILED'].includes(status)) {
+ element.classList.remove('disabled');
+ } else {
+ element.classList.add('disabled');
+ }
+ }
+ elements = this.displayElement.querySelectorAll('.action-button[data-action="restart-request"]');
+ for (let element of elements) {
+ if (status === 'FAILED') {
+ element.classList.remove('disabled');
+ } else {
+ element.classList.add('disabled');
+ }
+ }
}
setCreationDate(creationDate) {
diff --git a/app/static/js/RessourceDisplays/RessourceDisplay.js b/app/static/js/RessourceDisplays/RessourceDisplay.js
index c8c16be1367ea35d7a8ee4d6a3b7ef9268336479..a07c2163be0692e8022e7019e09a98af11b52efe 100644
--- a/app/static/js/RessourceDisplays/RessourceDisplay.js
+++ b/app/static/js/RessourceDisplays/RessourceDisplay.js
@@ -2,30 +2,42 @@ class RessourceDisplay {
constructor(displayElement) {
this.displayElement = displayElement;
this.userId = this.displayElement.dataset.userId;
- app.addEventListener('users.patch', patch => this.usersPatchHandler(patch));
- app.getUserById(this.userId).then(user => this.init(user));
+ this.isInitialized = false;
+ if (this.userId) {
+ app.subscribeUser(this.userId)
+ .then((response) => {
+ app.socket.on('PATCH', (patch) => {
+ if (this.isInitialized) {this.onPatch(patch);}
+ });
+ });
+ app.getUser(this.userId)
+ .then((user) => {
+ this.init(user);
+ this.isInitialized = true;
+ });
+ }
}
init(user) {throw 'Not implemented';}
- usersPatchHandler(patch) {throw 'Not implemented';}
+ onPatch(patch) {throw 'Not implemented';}
setElement(element, value) {
switch (element.tagName) {
- case 'INPUT':
+ case 'INPUT': {
element.value = value;
M.updateTextFields();
break;
- default:
+ }
+ default: {
element.innerText = value;
break;
+ }
}
}
setElements(elements, value) {
- let element;
-
- for (element of elements) {
+ for (let element of elements) {
this.setElement(element, value);
}
}
diff --git a/app/static/js/RessourceLists/CorpusFileList.js b/app/static/js/RessourceLists/CorpusFileList.js
index a4d76dadf973c84c5a0ac7a5fec50fb27a1669b7..a24fcf7e8661a4c2adc00b75e98041b75a381236 100644
--- a/app/static/js/RessourceLists/CorpusFileList.js
+++ b/app/static/js/RessourceLists/CorpusFileList.js
@@ -1,19 +1,47 @@
class CorpusFileList extends RessourceList {
+ static autoInit() {
+ for (let corpusFileListElement of document.querySelectorAll('.corpus-file-list:not(.no-autoinit)')) {
+ new CorpusFileList(corpusFileListElement);
+ }
+ }
+
static options = {
+ initialHtmlGenerator: (id) => {
+ return `
+ <div class="input-field">
+ <i class="material-icons prefix">search</i>
+ <input id="${id}-search" class="search" type="search"></input>
+ <label for="${id}-search">Search corpus file</label>
+ </div>
+ <table>
+ <thead>
+ <tr>
+ <th>Filename</th>
+ <th>Author</th>
+ <th>Title</th>
+ <th>Publishing year</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody class="list"></tbody>
+ </table>
+ <ul class="pagination"></ul>
+ `.trim();
+ },
item: `
- <tr class="hoverable">
+ <tr class="clickable hoverable">
<td><span class="filename"></span></td>
<td><span class="author"></span></td>
<td><span class="title"></span></td>
<td><span class="publishing-year"></span></td>
<td class="right-align">
- <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
- <a class="action-button btn-floating tooltipped nopaque-service-color darken waves-effect waves-light" data-action="download" data-position="top" data-service="corpus-analysis" data-tooltip="View"><i class="material-icons">file_download</i></a>
- <a class="action-button btn-floating tooltipped nopaque-service-color darken waves-effect waves-light" data-action="view" data-position="top" data-service="corpus-analysis" data-tooltip="View"><i class="material-icons">send</i></a>
+ <a class="action-button btn-floating red waves-effect waves-light" data-action="delete"><i class="material-icons">delete</i></a>
+ <a class="action-button btn-floating service-color darken waves-effect waves-light" data-action="download" data-service="corpus-analysis"><i class="material-icons">file_download</i></a>
+ <a class="action-button btn-floating service-color darken waves-effect waves-light" data-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
- ressourceMapper: corpusFile => {
+ ressourceMapper: (corpusFile) => {
return {
'id': corpusFile.id,
'author': corpusFile.author,
@@ -23,7 +51,7 @@ class CorpusFileList extends RessourceList {
'title': corpusFile.title
};
},
- sortValueName: 'creation-date',
+ sortArgs: ['creation-date', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
@@ -34,7 +62,6 @@ class CorpusFileList extends RessourceList {
]
};
-
constructor(listElement, options = {}) {
super(listElement, {...CorpusFileList.options, ...options});
this.corpusId = listElement.dataset.corpusId;
@@ -44,92 +71,59 @@ class CorpusFileList extends RessourceList {
this._init(user.corpora[this.corpusId].files);
}
- onclick(event) {
- let action;
- let actionButtonElement;
- let corpusFileElement;
- let corpusFileId;
- let deleteModal;
- let deleteModalElement;
- let tmp;
-
- corpusFileElement = event.target.closest('tr[data-id]');
- if (corpusFileElement === null) {return;}
- corpusFileId = corpusFileElement.dataset.id;
- actionButtonElement = event.target.closest('.action-button[data-action]');
- action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+ onClick(event) {
+ let actionButtonElement = event.target.closest('.action-button');
+ let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+ let corpusFileElement = event.target.closest('tr');
+ let corpusFileId = corpusFileElement.dataset.id;
switch (action) {
- case 'delete':
- tmp = document.createElement('div');
- tmp.innerHTML = `
- <div class="modal">
- <div class="modal-content">
- <h4>Confirm corpus deletion</h4>
- <p>Do you really want to delete the corpus file <b>${app.users[this.userId].corpora[this.corpusId].files[corpusFileId].filename}</b>? It will be permanently deleted!</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
- <a class="btn modal-close red waves-effect waves-light" href="/corpora/${this.corpusId}/files/${corpusFileId}/delete"><i class="material-icons left">delete</i>Delete</a>
- </div>
- </div>
- `.trim();
- deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
- deleteModal = M.Modal.init(
- deleteModalElement,
- {
- onCloseEnd: () => {
- deleteModal.destroy();
- deleteModalElement.remove();
- }
- }
- );
- deleteModal.open();
+ case 'delete': {
+ Utils.deleteCorpusFileRequest(this.userId, this.corpusId, corpusFileId);
break;
- case 'download':
+ }
+ case 'download': {
window.location.href = `/corpora/${this.corpusId}/files/${corpusFileId}/download`;
break;
- case 'view':
+ }
+ case 'view': {
window.location.href = `/corpora/${this.corpusId}/files/${corpusFileId}`;
break;
- default:
+ }
+ default: {
break;
+ }
}
}
- usersPatchHandler(patch) {
- let corpusFileId;
- let filteredPatch;
- let match;
- let operation;
- let re;
- let valueName;
-
- re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)`);
- filteredPatch = patch.filter(operation => re.test(operation.path));
- for (operation of filteredPatch) {
+ onPatch(patch) {
+ let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)`);
+ let filteredPatch = patch.filter(operation => re.test(operation.path));
+ for (let operation of filteredPatch) {
switch(operation.op) {
- case 'add':
- re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
- if (re.test(operation.path)) {
- this.add(operation.value);
- }
+ case 'add': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
+ if (re.test(operation.path)) {this.add(operation.value);}
break;
- case 'remove':
- re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
+ }
+ case 'remove': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)$`);
if (re.test(operation.path)) {
- [match, corpusFileId] = operation.path.match(re);
+ let [match, corpusFileId] = operation.path.match(re);
this.remove(corpusFileId);
}
break;
- case 'replace':
- re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)/(author|filename|publishing_year|title)$`);
+ }
+ case 'replace': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/${this.corpusId}/files/([A-Za-z0-9]*)/(author|filename|publishing_year|title)$`);
if (re.test(operation.path)) {
- [match, corpusFileId, valueName] = operation.path.match(re);
+ let [match, corpusFileId, valueName] = operation.path.match(re);
this.replace(corpusFileId, valueName.replace('_', '-'), operation.value);
}
break;
- default:
+ }
+ default: {
break;
+ }
}
}
}
diff --git a/app/static/js/RessourceLists/CorpusList.js b/app/static/js/RessourceLists/CorpusList.js
index 068f446ad0ee790c3165ec6d0c4a7a7cd5a7eed3..0721a8074c361138da3035f89c14c77c9757368a 100644
--- a/app/static/js/RessourceLists/CorpusList.js
+++ b/app/static/js/RessourceLists/CorpusList.js
@@ -1,17 +1,44 @@
class CorpusList extends RessourceList {
+ static autoInit() {
+ for (let corpusListElement of document.querySelectorAll('.corpus-list:not(.no-autoinit)')) {
+ new CorpusList(corpusListElement);
+ }
+ }
+
static options = {
+ initialHtmlGenerator: (id) => {
+ return `
+ <div class="input-field">
+ <i class="material-icons prefix">search</i>
+ <input id="${id}-search" class="search" type="search"></input>
+ <label for="${id}-search">Search corpus</label>
+ </div>
+ <table>
+ <thead>
+ <tr>
+ <th></th>
+ <th>Title and Description</th>
+ <th>Status</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody class="list"></tbody>
+ </table>
+ <ul class="pagination"></ul>
+ `.trim();
+ },
item: `
- <tr class="hoverable">
+ <tr class="clickable hoverable">
<td><a class="btn-floating disabled"><i class="material-icons service-color darken" data-service="corpus-analysis">book</i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="status badge new corpus-status-color corpus-status-text" data-badge-caption=""></span></td>
<td class="right-align">
- <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
- <a class="action-button btn-floating nopaque-service-color darken tooltipped waves-effect waves-light" data-action="view" data-position="top" data-service="corpus-analysis" data-tooltip="View"><i class="material-icons">send</i></a>
+ <a class="action-button btn-floating red waves-effect waves-light" data-action="delete-request"><i class="material-icons">delete</i></a>
+ <a class="action-button btn-floating service-color darken waves-effect waves-light" data-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
- ressourceMapper: corpus => {
+ ressourceMapper: (corpus) => {
return {
'id': corpus.id,
'creation-date': corpus.creation_date,
@@ -20,7 +47,7 @@ class CorpusList extends RessourceList {
'title': corpus.title
};
},
- sortValueName: 'creation-date',
+ sortArgs: ['creation-date', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
@@ -30,96 +57,63 @@ class CorpusList extends RessourceList {
]
};
-
constructor(listElement, options = {}) {
super(listElement, {...CorpusList.options, ...options});
}
init(user) {
- super._init(user.corpora);
+ this._init(user.corpora);
}
- onclick(event) {
- let action;
- let actionButtonElement;
- let corpusElement;
- let corpusId;
- let deleteModal;
- let deleteModalElement;
- let tmp;
-
- corpusElement = event.target.closest('tr[data-id]');
- if (corpusElement === null) {return;}
- corpusId = corpusElement.dataset.id;
- actionButtonElement = event.target.closest('.action-button[data-action]');
- action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+ onClick(event) {
+ let actionButtonElement = event.target.closest('.action-button');
+ let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+ let corpusElement = event.target.closest('tr');
+ let corpusId = corpusElement.dataset.id;
switch (action) {
- case 'delete':
- tmp = document.createElement('div');
- tmp.innerHTML = `
- <div class="modal">
- <div class="modal-content">
- <h4>Confirm corpus deletion</h4>
- <p>Do you really want to delete the corpus <b>${app.users[this.userId].corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
- <a class="btn modal-close red waves-effect waves-light" href="/corpora/${corpusId}/delete"><i class="material-icons left">delete</i>Delete</a>
- </div>
- </div>
- `.trim();
- deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
- deleteModal = M.Modal.init(
- deleteModalElement,
- {
- onCloseEnd: () => {
- deleteModal.destroy();
- deleteModalElement.remove();
- }
- }
- );
- deleteModal.open();
+ case 'delete-request': {
+ Utils.deleteCorpusRequest(this.userId, corpusId);
break;
- case 'view':
+ }
+ case 'view': {
window.location.href = `/corpora/${corpusId}`;
break;
- default:
+ }
+ default: {
break;
+ }
}
}
- usersPatchHandler(patch) {
- let corpusId;
- let filteredPatch;
- let match;
- let operation;
- let re;
- let valueName;
-
- re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)`);
- filteredPatch = patch.filter(operation => re.test(operation.path));
- for (operation of filteredPatch) {
+ onPatch(patch) {
+ let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)`);
+ let filteredPatch = patch.filter(operation => re.test(operation.path));
+ for (let operation of filteredPatch) {
switch(operation.op) {
- case 'add':
- re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
+ case 'add': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
if (re.test(operation.path)) {this.add(operation.value);}
break;
- case 'remove':
- re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
+ }
+ case 'remove': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)$`);
if (re.test(operation.path)) {
- [match, corpusId] = operation.path.match(re);
+ let [match, corpusId] = operation.path.match(re);
this.remove(corpusId);
}
break;
- case 'replace':
- re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)/(status|description|title)$`);
+ }
+ case 'replace': {
+ let re = new RegExp(`^/users/${this.userId}/corpora/([A-Za-z0-9]*)/(status|description|title)$`);
if (re.test(operation.path)) {
- [match, corpusId, valueName] = operation.path.match(re);
+ let [match, corpusId, valueName] = operation.path.match(re);
this.replace(corpusId, valueName, operation.value);
}
break;
- default:
+ }
+ default: {
break;
+ }
}
}
}
diff --git a/app/static/js/RessourceLists/JobInputList.js b/app/static/js/RessourceLists/JobInputList.js
index c1f313122903af6efd6ab7a5c5cf6eda6615319a..2cd14aa9822bb08834d8e8849ff95ddfbc6c822b 100644
--- a/app/static/js/RessourceLists/JobInputList.js
+++ b/app/static/js/RessourceLists/JobInputList.js
@@ -1,21 +1,46 @@
class JobInputList extends RessourceList {
+ static autoInit() {
+ for (let jobInputListElement of document.querySelectorAll('.job-input-list:not(.no-autoinit)')) {
+ new JobInputList(jobInputListElement);
+ }
+ }
+
static options = {
+ initialHtmlGenerator: (id) => {
+ return `
+ <div class="input-field">
+ <i class="material-icons prefix">search</i>
+ <input id="${id}-search" class="search" type="search"></input>
+ <label for="${id}-search">Search job input</label>
+ </div>
+ <table>
+ <thead>
+ <tr>
+ <th>Filename</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody class="list"></tbody>
+ </table>
+ <ul class="pagination"></ul>
+ `.trim();
+ },
item: `
- <tr class="hoverable">
+ <tr class="clickable hoverable">
<td><span class="filename"></span></td>
<td class="right-align">
- <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
+ <a class="action-button btn-floating waves-effect waves-light" data-action="download"><i class="material-icons">file_download</i></a>
</td>
</tr>
`.trim(),
- ressourceMapper: jobInput => {
+ ressourceMapper: (jobInput) => {
return {
'id': jobInput.id,
'creation-date': jobInput.creation_date,
'filename': jobInput.filename
};
},
- sortValueName: 'creation-date',
+ sortArgs: ['filename', {order: 'asc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
@@ -23,7 +48,6 @@ class JobInputList extends RessourceList {
]
};
-
constructor(listElement, options = {}) {
super(listElement, {...JobInputList.options, ...options});
this.jobId = listElement.dataset.jobId;
@@ -33,26 +57,21 @@ class JobInputList extends RessourceList {
this._init(user.jobs[this.jobId].inputs);
}
- onclick(event) {
- let jobInputElement;
- let jobInputId;
- let action;
- let actionButtonElement;
-
- jobInputElement = event.target.closest('tr[data-id]');
- if (jobInputElement === null) {return;}
- jobInputId = jobInputElement.dataset.id;
- actionButtonElement = event.target.closest('.action-button[data-action]');
- if (actionButtonElement === null) {return;}
- action = actionButtonElement.dataset.action;
+ onClick(event) {
+ let actionButtonElement = event.target.closest('.action-button');
+ let action = actionButtonElement === null ? 'download' : actionButtonElement.dataset.action;
+ let jobInputElement = event.target.closest('tr');
+ let jobInputId = jobInputElement.dataset.id;
switch (action) {
- case 'download':
+ case 'download': {
window.location.href = `/jobs/${this.jobId}/inputs/${jobInputId}/download`;
break;
- default:
+ }
+ default: {
break;
+ }
}
}
- usersPatchHandler(patch) {return;}
+ onPatch(patch) {return;}
}
diff --git a/app/static/js/RessourceLists/JobList.js b/app/static/js/RessourceLists/JobList.js
index a487c557725ba4ad710821a35573274fc4fe4cb5..d6fa789416f9292217cc0d32264f5782d06eedb5 100644
--- a/app/static/js/RessourceLists/JobList.js
+++ b/app/static/js/RessourceLists/JobList.js
@@ -1,17 +1,44 @@
class JobList extends RessourceList {
+ static autoInit() {
+ for (let jobListElement of document.querySelectorAll('.job-list:not(.no-autoinit)')) {
+ new JobList(jobListElement);
+ }
+ }
+
static options = {
+ initialHtmlGenerator: (id) => {
+ return `
+ <div class="input-field">
+ <i class="material-icons prefix">search</i>
+ <input id="${id}-search" class="search" type="search"></input>
+ <label for="${id}-search">Search job</label>
+ </div>
+ <table>
+ <thead>
+ <tr>
+ <th>Service</th>
+ <th>Title and Description</th>
+ <th>Status</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody class="list"></tbody>
+ </table>
+ <ul class="pagination"></ul>
+ `.trim();
+ },
item: `
- <tr class="hoverable service-color lighten">
+ <tr class="clickable hoverable service-color lighten">
<td><a class="btn-floating disabled"><i class="service-1 nopaque-icons service-color darken service-icon"></i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="status badge new job-status-color job-status-text" data-badge-caption=""></span></td>
<td class="right-align">
- <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
- <a class="service-2 action-button btn-floating nopaque-service-color darken tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+ <a class="action-button btn-floating red waves-effect waves-light" data-action="delete-request"><i class="material-icons">delete</i></a>
+ <a class="action-button btn-floating service-color darken waves-effect waves-light service-2" data-action="view"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
- ressourceMapper: job => {
+ ressourceMapper: (job) => {
return {
'id': job.id,
'creation-date': job.creation_date,
@@ -23,7 +50,7 @@ class JobList extends RessourceList {
'title': job.title
};
},
- sortValueName: 'creation-date',
+ sortArgs: ['creation-date', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
@@ -36,7 +63,6 @@ class JobList extends RessourceList {
]
};
-
constructor(listElement, options = {}) {
super(listElement, {...JobList.options, ...options});
}
@@ -45,89 +71,55 @@ class JobList extends RessourceList {
this._init(user.jobs);
}
- onclick(event) {
- let action;
- let actionButtonElement;
- let deleteModal;
- let deleteModalElement;
- let jobElement;
- let jobId;
- let tmp;
-
- jobElement = event.target.closest('tr[data-id]');
- if (jobElement === null) {return;}
- jobId = jobElement.dataset.id;
- actionButtonElement = event.target.closest('.action-button[data-action]');
- action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+ onClick(event) {
+ let actionButtonElement = event.target.closest('.action-button');
+ let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+ let jobElement = event.target.closest('tr');
+ let jobId = jobElement.dataset.id;
switch (action) {
- case 'delete':
- tmp = document.createElement('div');
- tmp.innerHTML = `
- <div class="modal">
- <div class="modal-content">
- <h4>Confirm job deletion</h4>
- <p>Do you really want to delete the job <b>${app.users[this.userId].jobs[jobId].title}</b>? All files will be permanently deleted!</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
- <a class="btn modal-close red waves-effect waves-light" href="/jobs/${jobId}/delete"><i class="material-icons left">delete</i>Delete</a>
- </div>
- </div>
- `.trim();
- deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
- deleteModal = M.Modal.init(
- deleteModalElement,
- {
- onCloseEnd: () => {
- deleteModal.destroy();
- deleteModalElement.remove();
- }
- }
- );
- deleteModal.open();
+ case 'delete-request': {
+ Utils.deleteJobRequest(this.userId, jobId);
break;
- case 'view':
+ }
+ case 'view': {
window.location.href = `/jobs/${jobId}`;
break;
- default:
+ }
+ default: {
break;
+ }
}
}
- usersPatchHandler(patch) {
- let filteredPatch;
- let jobId;
- let match;
- let operation;
- let re;
- let valueName;
-
- re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)`);
- filteredPatch = patch.filter(operation => re.test(operation.path));
- for (operation of filteredPatch) {
+ onPatch(patch) {
+ let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)`);
+ let filteredPatch = patch.filter(operation => re.test(operation.path));
+ for (let operation of filteredPatch) {
switch(operation.op) {
- case 'add':
- re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
- if (re.test(operation.path)) {
- this.add(operation.value);
- }
+ case 'add': {
+ let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
+ if (re.test(operation.path)) {this.add(operation.value);}
break;
- case 'remove':
- re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
+ }
+ case 'remove': {
+ let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)$`);
if (re.test(operation.path)) {
- [match, jobId] = operation.path.match(re);
+ let [match, jobId] = operation.path.match(re);
this.remove(jobId);
}
break;
- case 'replace':
- re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)/(service|status|description|title)$`);
+ }
+ case 'replace': {
+ let re = new RegExp(`^/users/${this.userId}/jobs/([A-Za-z0-9]*)/(service|status|description|title)$`);
if (re.test(operation.path)) {
- [match, jobId, valueName] = operation.path.match(re);
+ let [match, jobId, valueName] = operation.path.match(re);
this.replace(jobId, valueName, operation.value);
}
break;
- default:
+ }
+ default: {
break;
+ }
}
}
}
diff --git a/app/static/js/RessourceLists/JobResultList.js b/app/static/js/RessourceLists/JobResultList.js
index 5f9da3b59bef588bdee2d6375710058686fa3196..3623363a080948aca9e88a34446550eb42c10e77 100644
--- a/app/static/js/RessourceLists/JobResultList.js
+++ b/app/static/js/RessourceLists/JobResultList.js
@@ -1,15 +1,41 @@
class JobResultList extends RessourceList {
+ static autoInit() {
+ for (let jobResultListElement of document.querySelectorAll('.job-result-list:not(.no-autoinit)')) {
+ new JobResultList(jobResultListElement);
+ }
+ }
+
static options = {
+ initialHtmlGenerator: (id) => {
+ return `
+ <div class="input-field">
+ <i class="material-icons prefix">search</i>
+ <input id="${id}-search" class="search" type="search"></input>
+ <label for="${id}-search">Search job result</label>
+ </div>
+ <table>
+ <thead>
+ <tr>
+ <th>Description</th>
+ <th>Filename</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody class="list"></tbody>
+ </table>
+ <ul class="pagination"></ul>
+ `.trim();
+ },
item: `
- <tr class="hoverable">
+ <tr class="clickable hoverable">
<td><span class="description"></span></td>
<td><span class="filename"></span></td>
<td class="right-align">
- <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
+ <a class="action-button btn-floating waves-effect waves-light" data-action="download"><i class="material-icons">file_download</i></a>
</td>
</tr>
`.trim(),
- ressourceMapper: jobResult => {
+ ressourceMapper: (jobResult) => {
return {
'id': jobResult.id,
'creation-date': jobResult.creation_date,
@@ -17,7 +43,7 @@ class JobResultList extends RessourceList {
'filename': jobResult.filename
};
},
- sortValueName: 'creation-date',
+ sortArgs: ['filename', {order: 'asc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
@@ -26,7 +52,6 @@ class JobResultList extends RessourceList {
]
};
-
constructor(listElement, options = {}) {
super(listElement, {...JobResultList.options, ...options});
this.jobId = listElement.dataset.jobId;
@@ -36,44 +61,35 @@ class JobResultList extends RessourceList {
super._init(user.jobs[this.jobId].results);
}
- onclick(event) {
- let action;
- let actionButtonElement;
- let jobResultElement;
- let jobResultId;
-
- jobResultElement = event.target.closest('tr[data-id]');
- if (jobResultElement === null) {return;}
- jobResultId = jobResultElement.dataset.id;
- actionButtonElement = event.target.closest('.action-button[data-action]');
- if (actionButtonElement === null) {return;}
- action = actionButtonElement.dataset.action;
+ onClick(event) {
+ let actionButtonElement = event.target.closest('.action-button');
+ let action = actionButtonElement === null ? 'download' : actionButtonElement.dataset.action;
+ let jobResultElement = event.target.closest('tr');
+ let jobResultId = jobResultElement.dataset.id;
switch (action) {
- case 'download':
+ case 'download': {
window.location.href = `/jobs/${this.jobId}/results/${jobResultId}/download`;
break;
- default:
+ }
+ default: {
break;
+ }
}
}
- usersPatchHandler(patch) {
- let filteredPatch;
- let operation;
- let re;
-
- re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)`);
- filteredPatch = patch.filter(operation => re.test(operation.path));
- for (operation of filteredPatch) {
+ onPatch(patch) {
+ let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)`);
+ let filteredPatch = patch.filter(operation => re.test(operation.path));
+ for (let operation of filteredPatch) {
switch(operation.op) {
- case 'add':
- re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)$`);
- if (re.test(operation.path)) {
- this.add(operation.value);
- }
+ case 'add': {
+ let re = new RegExp(`^/users/${this.userId}/jobs/${this.jobId}/results/([A-Za-z0-9]*)$`);
+ if (re.test(operation.path)) {this.add(operation.value);}
break;
- default:
+ }
+ default: {
break;
+ }
}
}
}
diff --git a/app/static/js/RessourceLists/QueryResultList.js b/app/static/js/RessourceLists/QueryResultList.js
index c78ea3cbf142b55894f8a02cb7b73269390c544e..cffc43181ed04f9ddcdefcb10ec2ca3128a73fcf 100644
--- a/app/static/js/RessourceLists/QueryResultList.js
+++ b/app/static/js/RessourceLists/QueryResultList.js
@@ -1,12 +1,18 @@
class QueryResultList extends RessourceList {
+ static autoInit() {
+ for (let queryResultListElement of document.querySelectorAll('.query-result-list:not(.no-autoinit)')) {
+ new QueryResultList(queryResultListElement);
+ }
+ }
+
static options = {
item: `
<tr class="hoverable">
<td><b class="title"></b><br><i class="description"></i><br></td>
<td><span class="corpus-title"></span><br><span class="query"></span></td>
<td class="right-align">
- <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
- <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+ <a class="action-button btn-floating red waves-effect waves-light" data-action="delete"><i class="material-icons">delete</i></a>
+ <a class="action-button btn-floating waves-effect waves-light" data-action="view"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
@@ -20,7 +26,7 @@ class QueryResultList extends RessourceList {
'title': queryResult.title
};
},
- sortValueName: 'creation-date',
+ sortArgs: ['creation-date', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
@@ -31,7 +37,6 @@ class QueryResultList extends RessourceList {
]
};
-
constructor(listElement, options = {}) {
super(listElement, {...QueryResultList.options, ...options});
}
@@ -89,7 +94,7 @@ class QueryResultList extends RessourceList {
}
}
- usersPatchHandler(patch) {
+ onPATCH(patch) {
let filteredPatch;
let match;
let operation;
diff --git a/app/static/js/RessourceLists/RessourceList.js b/app/static/js/RessourceLists/RessourceList.js
index ca23d8200ff74998dba098a39a63c9996be3d4f7..824db3d180a46724293bcf7fe6669ce7a14c0f31 100644
--- a/app/static/js/RessourceLists/RessourceList.js
+++ b/app/static/js/RessourceLists/RessourceList.js
@@ -3,45 +3,22 @@ class RessourceList {
* This class is not meant to be used directly, instead it should be used as
* a base class for concrete ressource list implementations.
*/
- static autoInit() {
- const nopaqueRessourceListElements = document.querySelectorAll('.nopaque-ressource-list[data-ressource-type]:not(.no-autoinit)');
- let nopaqueRessourceListElement;
- for (nopaqueRessourceListElement of nopaqueRessourceListElements) {
- switch (nopaqueRessourceListElement.dataset.ressourceType) {
- case 'Corpus':
- new CorpusList(nopaqueRessourceListElement);
- break;
- case 'CorpusFile':
- new CorpusFileList(nopaqueRessourceListElement);
- break;
- case 'Job':
- new JobList(nopaqueRessourceListElement);
- break;
- case 'JobInput':
- new JobInputList(nopaqueRessourceListElement);
- break;
- case 'JobResult':
- new JobResultList(nopaqueRessourceListElement);
- break;
- case 'QueryResult':
- new QueryResultList(nopaqueRessourceListElement);
- break;
- case 'User':
- new UserList(nopaqueRessourceListElement);
- break;
- default:
- break;
- }
- }
+ static autoInit() {
+ CorpusList.autoInit();
+ CorpusFileList.autoInit();
+ JobList.autoInit();
+ JobInputList.autoInit();
+ JobResultList.autoInit();
+ QueryResultList.autoInit();
+ UserList.autoInit();
}
- static options = {page: 5, pagination: {innerWindow: 4, outerWindow: 1}};
+ static options = {page: 5, pagination: {innerWindow: 4, outerWindow: 1}};
constructor(listElement, options = {}) {
- let i;
-
if (!(listElement.hasAttribute('id'))) {
+ let i;
for (i = 0; true; i++) {
if (document.querySelector(`#ressource-list-${i}`)) {continue;}
listElement.id = `ressource-list-${i}`;
@@ -56,9 +33,14 @@ class RessourceList {
this.ressourceMapper = options.ressourceMapper;
delete options.ressourceMapper;
}
- if ('sortValueName' in options) {
- this.sortValueName = options.sortValueName;
- delete options.sortValueName;
+ if ('initialHtmlGenerator' in options) {
+ this.initialHtmlGenerator = options.initialHtmlGenerator;
+ listElement.innerHTML = this.initialHtmlGenerator(listElement.id);
+ delete options.initialHtmlGenerator;
+ }
+ if ('sortArgs' in options) {
+ this.sortArgs = options.sortArgs;
+ delete options.sortArgs;
}
this.listjs = new List(listElement, {...RessourceList.options, ...options});
this.listjs.list.innerHTML = `
@@ -87,47 +69,54 @@ class RessourceList {
</td>
</tr>
`.trim();
- this.listjs.list.style.cursor = 'pointer';
this.userId = this.listjs.listContainer.dataset.userId;
- this.listjs.list.addEventListener('click', event => this.onclick(event));
+ this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
+ this.isInitialized = false;
if (this.userId) {
- app.addEventListener('users.patch', patch => this.usersPatchHandler(patch));
- app.getUserById(this.userId).then(
- user => this.init(user),
- error => {throw JSON.stringify(error);}
- );
+ app.subscribeUser(this.userId)
+ .then((response) => {
+ app.socket.on('PATCH', (patch) => {
+ if (this.isInitialized) {this.onPatch(patch);}
+ });
+ });
+ app.getUser(this.userId)
+ .then((user) => {
+ this.init(user);
+ this.isInitialized = true;
+ });
}
}
_init(ressources) {
this.listjs.clear();
this.add(Object.values(ressources));
- let emptyListElementHTML = `
- <tr class="show-if-only-child">
- <td colspan="100%">
- <span class="card-title"><i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...</span>
- <p>No ressource available.</p>
- </td>
- </tr>
- `.trim();
- this.listjs.list.insertAdjacentHTML('afterbegin', emptyListElementHTML);
+ this.listjs.list.insertAdjacentHTML(
+ 'afterbegin',
+ `
+ <tr class="show-if-only-child">
+ <td colspan="100%">
+ <span class="card-title"><i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...</span>
+ <p>No ressource available.</p>
+ </td>
+ </tr>
+ `.trim()
+ );
}
init(user) {throw 'Not implemented';}
- onclick(event) {throw 'Not implemented';}
+ onClick(event) {throw 'Not implemented';}
- usersPatchHandler(patch) {throw 'Not implemented';}
+ onPatch(patch) {throw 'Not implemented';}
add(ressources) {
let values = Array.isArray(ressources) ? ressources : [ressources];
-
if ('ressourceMapper' in this) {
- values = values.map(value => this.ressourceMapper(value));
+ values = values.map((value) => {return this.ressourceMapper(value);});
}
this.listjs.add(values, () => {
- if ('sortValueName' in this) {
- this.listjs.sort(this.sortValueName, {order: 'desc'});
+ if ('sortArgs' in this) {
+ this.listjs.sort(...this.sortArgs);
}
});
}
@@ -137,6 +126,6 @@ class RessourceList {
}
replace(id, valueName, newValue) {
- this.listjs.get('id', id)[0].values({[valueName]: newValue});
+ this.listjs.get('id', id)[0].values({[valueName]: newValue});
}
}
diff --git a/app/static/js/RessourceLists/UserList.js b/app/static/js/RessourceLists/UserList.js
index 5f488f47ef3e5ef897b812e7d5efcf00640cee60..986685bafcd9b7a74c9295e498e15ba96833717a 100644
--- a/app/static/js/RessourceLists/UserList.js
+++ b/app/static/js/RessourceLists/UserList.js
@@ -1,31 +1,60 @@
class UserList extends RessourceList {
+ static autoInit() {
+ for (let userListElement of document.querySelectorAll('.user-list:not(.no-autoinit)')) {
+ new UserList(userListElement);
+ }
+ }
+
static options = {
+ initialHtmlGenerator: (id) => {
+ return `
+ <div class="input-field">
+ <i class="material-icons prefix">search</i>
+ <input id="${id}-search" class="search" type="search"></input>
+ <label for="${id}-search">Search user</label>
+ </div>
+ <table>
+ <thead>
+ <tr>
+ <th>Id</th>
+ <th>Username</th>
+ <th>Email</th>
+ <th>Last seen</th>
+ <th>Role</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody class="list"></tbody>
+ </table>
+ <ul class="pagination"></ul>
+ `.trim();
+ },
item: `
- <tr class="hoverable">
+ <tr class="clickable hoverable">
<td><span class="id-1"></span></td>
<td><span class="username"></span></td>
<td><span class="email"></span></td>
<td><span class="last-seen"></span></td>
<td><span class="role"></span></td>
<td class="right-align">
- <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
- <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="edit" data-position="top" data-tooltip="Edit"><i class="material-icons">edit</i></a>
- <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
+ <a class="action-button btn-floating red waves-effect waves-light" data-action="delete"><i class="material-icons">delete</i></a>
+ <a class="action-button btn-floating waves-effect waves-light" data-action="edit"><i class="material-icons">edit</i></a>
+ <a class="action-button btn-floating waves-effect waves-light" data-action="view"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
- ressourceMapper: user => {
+ ressourceMapper: (user) => {
return {
'id': user.id,
'id-1': user.id,
'username': user.username,
'email': user.email,
- 'last-seen': new Date(user.last_seen).toLocaleString("en-US"),
+ 'last-seen': new Date(user.last_seen).toLocaleString('en-US'),
'member-since': user.member_since,
'role': user.role.name
};
},
- sortValueName: 'member-since',
+ sortArgs: ['member-since', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['member-since']},
@@ -37,8 +66,6 @@ class UserList extends RessourceList {
]
};
-
-
constructor(listElement, options = {}) {
super(listElement, {...UserList.options, ...options});
}
@@ -47,55 +74,28 @@ class UserList extends RessourceList {
super._init(Object.values(users));
}
- onclick(event) {
- let action;
- let actionButtonElement;
- let deleteModal;
- let deleteModalElement;
- let tmp;
- let userElement;
- let userId;
-
- userElement = event.target.closest('tr[data-id]');
- if (userElement === null) {return;}
- userId = userElement.dataset.id;
- actionButtonElement = event.target.closest('.action-button[data-action]');
- action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
+ onClick(event) {
+ let actionButtonElement = event.target.closest('.action-button');
+ let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
+ let userElement = event.target.closest('tr');
+ let userId = userElement.dataset.id;
switch (action) {
- case 'delete':
- tmp = document.createElement('div');
- tmp.innerHTML = `
- <div class="modal">
- <div class="modal-content">
- <h4>Confirm user deletion</h4>
- <p>Do you really want to delete user <b>${userId}</b>? All files will be permanently deleted!</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
- <a class="btn modal-close red waves-effect waves-light" href="/admin/users/${userId}/delete"><i class="material-icons left">delete</i>Delete</a>
- </div>
- </div>
- `.trim();
- deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
- deleteModal = M.Modal.init(
- deleteModalElement,
- {
- onCloseEnd: () => {
- deleteModal.destroy();
- deleteModalElement.remove();
- }
- }
- );
- deleteModal.open();
+ case 'delete': {
+ Utils.deleteUserRequest(userId);
+ if (userId === currentUserId) {window.location.href = '/';}
break;
- case 'edit':
+ }
+ case 'edit': {
window.location.href = `/admin/users/${userId}/edit`;
break;
- case 'view':
+ }
+ case 'view': {
window.location.href = `/admin/users/${userId}`;
break;
- default:
+ }
+ default: {
break;
+ }
}
}
}
diff --git a/app/static/js/UploadForm.js b/app/static/js/UploadForm.js
deleted file mode 100644
index 2a1df808cacffc5c1cf24adf0d4e05770c1b1620..0000000000000000000000000000000000000000
--- a/app/static/js/UploadForm.js
+++ /dev/null
@@ -1,125 +0,0 @@
-class UploadForm {
- static autoInit() {
- const nopaqueSubmitForms = document.querySelectorAll('.nopaque-upload-form');
- let nopaqueSubmitForm;
-
- for (nopaqueSubmitForm of nopaqueSubmitForms) {
- new UploadForm(nopaqueSubmitForm);
- }
- }
-
-
- constructor(formElement) {
- this.formElement = formElement;
- this.request = new XMLHttpRequest();
-
- this.formElement.addEventListener('submit', (event) => {
- event.preventDefault();
- this.submit();
- });
- }
-
- submit() {
- const selectElements = this.formElement.querySelectorAll('select');
- let abortElement;
- let helperTextElement;
- let helperTextElements;
- let inputFieldElement;
- let modal;
- let modalElement;
- let progressElement;
- let selectElement;
- let tmp;
-
- // Check if select elements are filled out properly
- for (selectElement of selectElements) {
- if (selectElement.value === '') {
- inputFieldElement = selectElement.closest('.input-field');
- inputFieldElement.querySelector('.select-dropdown').classList.add('invalid');
- helperTextElements = inputFieldElement.querySelectorAll('.helper-text');
- for (helperTextElement of helperTextElements) {
- helperTextElement.remove();
- }
- inputFieldElement.insertAdjacentHTML(
- 'beforeend',
- '<span class="helper-text error-color-text">Please select an option.</span>'
- );
- return;
- }
- }
-
- // Setup modal
- tmp = document.createElement('div');
- tmp.innerHTML = `
- <div class="modal">
- <div class="modal-content">
- <h4><i class="material-icons left">file_upload</i>Uploading files...</h4>
- <div class="progress">
- <div class="determinate" style="width: 0%"></div>
- </div>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn red waves-effect waves-light abort">Cancel</a>
- </div>
- </div>
- `.trim();
- modalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
- modal = M.Modal.init(
- modalElement,
- {
- dismissible: false,
- onCloseEnd: () => {
- modal.destroy();
- modalElement.remove();
- }
- }
- );
- modal.open();
-
- // Setup abort handling
- abortElement = modalElement.querySelector('.abort');
- abortElement.addEventListener('click', event => {this.request.abort();});
- this.request.addEventListener('abort', event => {
- this.request.abort();
- modal.close();
- });
-
- // Setup load handling (after the request completed)
- this.request.addEventListener('load', event => {
- const response = JSON.parse(this.request.responseText);
- let inputError;
- let inputErrors;
- let inputFieldElement;
- let inputName;
-
- if (this.request.status === 201) {
- window.location.href = response.redirect_url;
- }
- if (this.request.status === 400) {
- for ([inputName, inputErrors] of Object.entries(response)) {
- inputFieldElement = this.formElement.querySelector(`input[name="${inputName}"], select[name="${inputName}"]`).closest('.input-field');
- for (inputError of inputErrors) {
- inputFieldElement.insertAdjacentHTML(
- 'beforeend',
- `<span class="helper-text red-text">${inputError}</span>`
- );
- }
- }
- }
- if (this.request.status === 500) {
- location.reload();
- }
- modal.close();
- });
-
- // Setup progress handling
- progressElement = modalElement.querySelector('.progress > .determinate');
- this.request.upload.addEventListener('progress', event => {
- const progress = Math.floor(100 * event.loaded / event.total);
- progressElement.style.width = `${progress}%`;
- });
-
- this.request.open('POST', window.location.href);
- this.request.send(new FormData(this.formElement));
- }
-}
diff --git a/app/static/js/Utils.js b/app/static/js/Utils.js
new file mode 100644
index 0000000000000000000000000000000000000000..2e7cbd4cde08c7bc26b0d4794ebcb33e1de16ceb
--- /dev/null
+++ b/app/static/js/Utils.js
@@ -0,0 +1,326 @@
+class Utils {
+ static elementFromString(string) {
+ let tmpElement = document.createElement('div');
+ tmpElement.innerHTML = string.trim();
+ return tmpElement.firstChild;
+ }
+
+ static buildCorpusRequest(userId, corpusId) {
+ return new Promise((resolve, reject) => {
+ let corpus = app.data.users[userId].corpora[corpusId];
+
+ fetch(`/corpora/${corpus.id}/build`, {method: 'POST', headers: {Accept: 'application/json'}})
+ .then(
+ (response) => {
+ app.flash(`Corpus "${corpus.title}" marked for building`, 'corpus');
+ resolve(response);
+ },
+ (response) => {
+ if (response.status === 403) {app.flash('Forbidden', 'error');}
+ if (response.status === 404) {app.flash('Not Found', 'error');}
+ if (response.status === 409) {app.flash('Conflict', 'error');}
+ reject(response);
+ }
+ );
+ });
+ }
+
+ static deleteCorpusRequest(userId, corpusId) {
+ return new Promise((resolve, reject) => {
+ let corpus = app.data.users[userId].corpora[corpusId];
+
+ let modalElement = Utils.elementFromString(
+ `
+ <div class="modal">
+ <div class="modal-content">
+ <h4>Confirm job deletion</h4>
+ <p>Do you really want to delete the job <b>${corpus.title}</b>? All files will be permanently deleted!</p>
+ </div>
+ <div class="modal-footer">
+ <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+ <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+ </div>
+ </div>
+ `
+ );
+ document.querySelector('#modals').appendChild(modalElement);
+ let modal = M.Modal.init(
+ modalElement,
+ {
+ dismissible: false,
+ onCloseEnd: () => {
+ modal.destroy();
+ modalElement.remove();
+ }
+ }
+ );
+
+ let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+ confirmElement.addEventListener('click', (event) => {
+ let corpusTitle = corpus.title;
+ fetch(`/corpora/${corpus.id}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+ .then(
+ (response) => {
+ app.flash(`Corpus "${corpusTitle}" marked for deletion`, 'corpus');
+ resolve(response);
+ },
+ (response) => {
+ if (response.status === 403) {app.flash('Forbidden', 'error');}
+ if (response.status === 404) {app.flash('Not Found', 'error');}
+ reject(response);
+ }
+ );
+ });
+ modal.open();
+ });
+ }
+
+ static deleteCorpusFileRequest(userId, corpusId, corpusFileId) {
+ return new Promise((resolve, reject) => {
+ let corpus = app.data.users[userId].corpora[corpusId];
+ let corpusFile = corpus.files[corpusFileId];
+
+ let modalElement = Utils.elementFromString(
+ `
+ <div class="modal">
+ <div class="modal-content">
+ <h4>Confirm job deletion</h4>
+ <p>Do you really want to delete the job <b>${corpusFile.title}</b>? All files will be permanently deleted!</p>
+ </div>
+ <div class="modal-footer">
+ <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+ <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+ </div>
+ </div>
+ `
+ );
+ document.querySelector('#modals').appendChild(modalElement);
+ let modal = M.Modal.init(
+ modalElement,
+ {
+ dismissible: false,
+ onCloseEnd: () => {
+ modal.destroy();
+ modalElement.remove();
+ }
+ }
+ );
+
+ let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+ confirmElement.addEventListener('click', (event) => {
+ let corpusFileTitle = corpusFile.title;
+ fetch(`/corpora/${corpusId}/files/${corpusFileId}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+ .then(
+ (response) => {
+ app.flash(`Corpus file "${corpusFileTitle}" marked for deletion`, 'corpus');
+ resolve(response);
+ },
+ (response) => {
+ if (response.status === 403) {app.flash('Forbidden', 'error');}
+ if (response.status === 404) {app.flash('Not Found', 'error');}
+ reject(response);
+ }
+ );
+ });
+ modal.open();
+ });
+ }
+
+ static deleteJobRequest(userId, jobId) {
+ return new Promise((resolve, reject) => {
+ let job = app.data.users[userId].jobs[jobId];
+
+ let modalElement = Utils.elementFromString(
+ `
+ <div class="modal">
+ <div class="modal-content">
+ <h4>Confirm job deletion</h4>
+ <p>Do you really want to delete the job <b>${job.title}</b>? All files will be permanently deleted!</p>
+ </div>
+ <div class="modal-footer">
+ <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+ <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+ </div>
+ </div>
+ `
+ );
+ document.querySelector('#modals').appendChild(modalElement);
+ let modal = M.Modal.init(
+ modalElement,
+ {
+ dismissible: false,
+ onCloseEnd: () => {
+ modal.destroy();
+ modalElement.remove();
+ }
+ }
+ );
+
+ let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+ confirmElement.addEventListener('click', (event) => {
+ let jobTitle = job.title;
+ fetch(`/jobs/${job.id}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+ .then(
+ (response) => {
+ app.flash(`Job "${jobTitle}" marked for deletion`, 'job');
+ resolve(response);
+ },
+ (response) => {
+ if (response.status === 403) {app.flash('Forbidden', 'error');}
+ if (response.status === 404) {app.flash('Not Found', 'error');}
+ reject(response);
+ }
+ );
+ });
+ modal.open();
+ });
+ }
+
+ static getJobLogRequest(userId, jobId) {
+ return new Promise((resolve, reject) => {
+ let job = app.data.users[userId].jobs[jobId];
+
+ fetch(`/jobs/${job.id}/log`, {method: 'GET', headers: {Accept: 'application/json, text/plain'}})
+ .then(
+ (response) => {
+ resolve(response);
+ return response.text();
+ },
+ (response) => {
+ if (response.status === 403) {app.flash('Forbidden', 'error');}
+ if (response.status === 404) {app.flash('Not Found', 'error');}
+ reject(response);
+ }
+ )
+ .then(
+ (text) => {
+ let modalElement = Utils.elementFromString(
+ `
+ <div class="modal">
+ <div class="modal-content">
+ <h4>Job logs</h4>
+ <pre><code>${text}</code></pre>
+ </div>
+ <div class="modal-footer">
+ <a class="btn modal-close waves-effect waves-light">Close</a>
+ </div>
+ </div>
+ `
+ );
+ document.querySelector('#modals').appendChild(modalElement);
+ let modal = M.Modal.init(
+ modalElement,
+ {
+ onCloseEnd: () => {
+ modal.destroy();
+ modalElement.remove();
+ }
+ }
+ );
+ modal.open();
+ }
+ );
+ });
+ }
+
+ static restartJobRequest(userId, jobId) {
+ return new Promise((resolve, reject) => {
+ let job = app.data.users[userId].jobs[jobId];
+
+ let modalElement = Utils.elementFromString(
+ `
+ <div class="modal">
+ <div class="modal-content">
+ <h4>Confirm job restart</h4>
+ <p>Do you really want to restart the job <b>${job.title}</b>? All log and result files will be permanently deleted.</p>
+ </div>
+ <div class="modal-footer">
+ <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+ <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Restart</a>
+ </div>
+ </div>
+ `
+ );
+ document.querySelector('#modals').appendChild(modalElement);
+ let modal = M.Modal.init(
+ modalElement,
+ {
+ dismissible: false,
+ onCloseEnd: () => {
+ modal.destroy();
+ modalElement.remove();
+ }
+ }
+ );
+
+ let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+ confirmElement.addEventListener('click', (event) => {
+ let jobTitle = job.title;
+ fetch(`/jobs/${job.id}/restart`, {method: 'POST', headers: {Accept: 'application/json'}})
+ .then(
+ (response) => {
+ app.flash(`Job "${jobTitle}" restarted.`, 'job');
+ resolve(response);
+ },
+ (response) => {
+ if (response.status === 403) {app.flash('Forbidden', 'error');}
+ if (response.status === 404) {app.flash('Not Found', 'error');}
+ if (response.status === 409) {app.flash('Conflict', 'error');}
+ reject(response);
+ }
+ );
+ });
+ modal.open();
+ });
+ }
+
+ static deleteUserRequest(userId) {
+ return new Promise((resolve, reject) => {
+ let user = app.data.users[userId];
+
+ let modalElement = Utils.elementFromString(
+ `
+ <div class="modal">
+ <div class="modal-content">
+ <h4>Confirm job deletion</h4>
+ <p>Do you really want to delete the user <b>${user.username}</b>? All files will be permanently deleted!</p>
+ </div>
+ <div class="modal-footer">
+ <a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
+ <a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
+ </div>
+ </div>
+ `
+ );
+ document.querySelector('#modals').appendChild(modalElement);
+ let modal = M.Modal.init(
+ modalElement,
+ {
+ dismissible: false,
+ onCloseEnd: () => {
+ modal.destroy();
+ modalElement.remove();
+ }
+ }
+ );
+
+ let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
+ confirmElement.addEventListener('click', (event) => {
+ let userName = user.username;
+ fetch(`/users/${user.id}`, {method: 'DELETE', headers: {Accept: 'application/json'}})
+ .then(
+ (response) => {
+ app.flash(`User "${userName}" marked for deletion`);
+ resolve(response);
+ },
+ (response) => {
+ if (response.status === 403) {app.flash('Forbidden', 'error');}
+ if (response.status === 404) {app.flash('Not Found', 'error');}
+ reject(response);
+ }
+ );
+ });
+ modal.open();
+ });
+ }
+}
diff --git a/app/templates/_navbar.html.j2 b/app/templates/_navbar.html.j2
index 63631a011ce7b9bf9bf44fe4869b644010b3262e..e2d4db6452116377e4e007a86e0674ea63373b7a 100644
--- a/app/templates/_navbar.html.j2
+++ b/app/templates/_navbar.html.j2
@@ -29,7 +29,7 @@
<ul class="dropdown-content" id="nav-more-dropdown">
<li><a href="{{ url_for('main.user_manual') }}"><i class="material-icons left">help</i>Manual</a></li>
{% if current_user.is_authenticated %}
- <li><a href="{{ url_for('settings.index') }}"><i class="material-icons left">settings</i>Settings</a></li>
+ <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons left">settings</i>Settings</a></li>
<li class="divider" tabindex="-1"></li>
<li><a href="{{ url_for('auth.logout') }}">Log out</a></li>
{% else %}
diff --git a/app/templates/_roadmap.html.j2 b/app/templates/_roadmap.html.j2
index 3e87d6079d413dbc80598e9bc575685b37b209ad..50cc18cd7387a4992b66591a6334103a727dced2 100644
--- a/app/templates/_roadmap.html.j2
+++ b/app/templates/_roadmap.html.j2
@@ -11,19 +11,19 @@
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a{%if request.path == url_for('services.spacy_nlp_pipeline') %} class="active"{% endif %} href="{{ url_for('services.spacy_nlp_pipeline') }}" target="_self">NLP</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
- <li class="tab"><a{%if request.path == url_for('corpora.add_corpus') %} class="active"{% endif %} href="{{ url_for('corpora.add_corpus') }}" target="_self">Add corpus</a></li>
+ <li class="tab"><a{%if request.path == url_for('corpora.create_corpus') %} class="active"{% endif %} href="{{ url_for('corpora.create_corpus') }}" target="_self">Create corpus</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
{% if corpus %}
- <li class="tab"><a{%if request.path == url_for('corpora.add_corpus_file', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" target="_self">Add corpus file(s)</a></li>
+ <li class="tab"><a{%if request.path == url_for('corpora.create_corpus_file', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.create_corpus_file', corpus_id=corpus.id) }}" target="_self">Create corpus file(s)</a></li>
{% else %}
- <li class="tab disabled tooltipped" data-tooltip="Select a corpus first" target="_self"><a>Add corpus file(s)</a></li>
+ <li class="tab disabled tooltipped" data-tooltip="Select a corpus first" target="_self"><a>Create corpus file(s)</a></li>
{% endif %}
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
{% if corpus %}
{% if corpus.files.all() %}
<li class="tab"><a{%if request.path == url_for('corpora.analyse_corpus', corpus_id=corpus.id) %} class="active"{% endif %} href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}" target="_self">Corpus analysis</a></li>
{% else %}
- <li class="tab disabled tooltipped" data-tooltip="Add at least one corpus file first"><a>Corpus analysis</a></li>
+ <li class="tab disabled tooltipped" data-tooltip="Create at least one corpus file first"><a>Corpus analysis</a></li>
{% endif %}
{% else %}
<li class="tab disabled tooltipped" data-tooltip="Select a corpus first"><a>Corpus analysis</a></li>
diff --git a/app/templates/_scripts.html.j2 b/app/templates/_scripts.html.j2
index 54095e0018c4416b6d8eb92c73602954a54eb850..47b7543fdc57ae38821734166bbbc4d2a5edd056 100644
--- a/app/templates/_scripts.html.j2
+++ b/app/templates/_scripts.html.j2
@@ -1,16 +1,18 @@
-<script src="https://cdnjs.cloudflare.com/ajax/libs/fast-json-patch/3.1.0/fast-json-patch.min.js" integrity="sha512-KrvLlmKBiDoTa0Fke92aFoEv4xS0+cuYGP27nt39w0yLZWvVOhArmZ29uuOe3uOOBcbnkpvnLhkvYcYjahSOwg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/fast-json-patch/3.1.1/fast-json-patch.min.js" integrity="sha512-5uDdefwnzyq4N+SkmMBmekZLZNmc6dLixvVxCdlHBfqpyz0N3bzLdrJ55OLm7QrZmgZuhLGgHLDtJwU6RZoFCA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/2.3.1/list.min.js" integrity="sha512-93wYgwrIFL+b+P3RvYxi/WUFRXXUDSLCT2JQk9zhVGXuS2mHl2axj6d+R6pP+gcU5isMHRj1u0oYE/mWyt/RjA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.0/socket.io.min.js" integrity="sha512-pxLMWs4E33rW9tdIhovcCp2dCo9k4Q8eHw7CETjyjdXf4aX6wvsEBq+KdOJJRFALr6FxNoXx+jksgbE74TZjEw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.1/socket.io.min.js" integrity="sha512-mHO4BJ0ELk7Pb1AzhTi3zvUeRgq3RXVOu9tTRfnA6qOxGK4pG2u57DJYolI4KrEnnLTcH9/J5wNOozRTDaybXg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
{%- assets
filters='rjsmin',
output='gen/app.%(version)s.js',
'js/App.js',
+ 'js/Utils.js',
+ 'js/Forms/Form.js',
+ 'js/Forms/CreateCorpusFileForm.js',
+ 'js/Forms/CreateJobForm.js',
'js/CorpusAnalysis/CQiClient.js',
'js/CorpusAnalysis/CorpusAnalysisApp.js',
'js/CorpusAnalysis/CorpusAnalysisConcordance.js',
'js/CorpusAnalysis/CorpusAnalysisReader.js',
- 'js/CorpusAnalysis/QueryBuilder.js',
- 'js/JobStatusNotifier.js',
'js/RessourceDisplays/RessourceDisplay.js',
'js/RessourceDisplays/CorpusDisplay.js',
'js/RessourceDisplays/JobDisplay.js',
@@ -21,8 +23,7 @@
'js/RessourceLists/JobInputList.js',
'js/RessourceLists/JobResultList.js',
'js/RessourceLists/QueryResultList.js',
- 'js/RessourceLists/UserList.js',
- 'js/UploadForm.js'
+ 'js/RessourceLists/UserList.js'
%}
<script src="{{ ASSET_URL }}"></script>
{%- endassets %}
@@ -33,12 +34,8 @@
const jobStatusNotifier = new JobStatusNotifier(currentUserId);
// Initialize components for current user
- app.addEventListener('users.patch', patch => jobStatusNotifier.usersPatchHandler(patch));
- app.getUserById(currentUserId)
- .then(
- user => {return;},
- error => {throw JSON.stringify(error);}
- );
+ app.subscribeUser(currentUserId).catch((error) => {throw JSON.stringify(error);});
+ app.getUser(currentUserId, true, true);
{%- endif %}
// Disable all option elements with no value
@@ -59,7 +56,7 @@
{alignment: 'right', constrainWidth: false, coverTrigger: false}
);
RessourceList.autoInit();
- UploadForm.autoInit();
+ Form.autoInit();
// Display flashed messages
for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {
diff --git a/app/templates/_sidenav.html.j2 b/app/templates/_sidenav.html.j2
index 08d8f163d28c7ef24e35f129eb6e48a482d24fd5..553eb9db1be39bf60e5ed28119e63388d95084d5 100644
--- a/app/templates/_sidenav.html.j2
+++ b/app/templates/_sidenav.html.j2
@@ -23,7 +23,7 @@
<li class="service-color service-color-border border-darken" data-service="corpus-analysis" style="border-left: 10px solid; margin-top: 5px;"><a href="{{ url_for('services.corpus_analysis') }}"><i class="nopaque-icons service-icon" data-service="corpus-analysis"></i>Corpus analysis</a></li>
<li><div class="divider"></div></li>
<li><a class="subheader">Account</a></li>
- <li><a href="{{ url_for('settings.index') }}"><i class="material-icons">settings</i>Settings</a></li>
+ <li><a href="{{ url_for('settings.settings') }}"><i class="material-icons">settings</i>Settings</a></li>
<li><a href="{{ url_for('auth.logout') }}">Log out</a></li>
{% if current_user.can(Permission.ADMINISTRATE) or current_user.can(Permission.USE_API) %}
<li><div class="divider"></div></li>
@@ -31,10 +31,10 @@
{% if current_user.can(Permission.ADMINISTRATE) %}
<li><a href="{{ url_for('admin.index') }}"><i class="material-icons">admin_panel_settings</i>Administration</a></li>
{% endif %}
- {% if current_user.can(Permission.CONTRIBUTE) %}
- <li><a href="{{ url_for('contribute.index') }}"><i class="material-icons">new_label</i>Contribute</a></li>
- {% endif %}
{% if current_user.can(Permission.USE_API) %}
- <li><a href="{{ url_for('api.doc') }}"><i class="material-icons">api</i>API</a></li>
+ <li><a href="{{ url_for('apifairy.docs') }}"><i class="material-icons">api</i>API</a></li>
+ {% endif %}
+ {% if current_user.can(Permission.CONTRIBUTE) %}
+ <li><a href="{{ url_for('contributions.contributions') }}"><i class="material-icons">new_label</i>Contribute</a></li>
{% endif %}
</ul>
diff --git a/app/templates/_styles.html.j2 b/app/templates/_styles.html.j2
index de2bff7aebe1678276b7165f6f004e31b1801d30..2c1ea8f83d48196b3c60bb8fd20a4d1e4a7b284b 100644
--- a/app/templates/_styles.html.j2
+++ b/app/templates/_styles.html.j2
@@ -8,6 +8,7 @@
filters='pyscss',
output='gen/app.%(version)s.css',
'css/colors.scss',
+ 'css/helpers.scss',
'css/style.css'
%}
<link href="{{ ASSET_URL }}" media="screen,projection" rel="stylesheet">
diff --git a/app/templates/admin/edit_user.html.j2 b/app/templates/admin/edit_user.html.j2
index f44ccb7c18236f46ba0582c956eea5c93906a036..45c27c6a8b8c32f492791e6bf6046b3a722096b3 100644
--- a/app/templates/admin/edit_user.html.j2
+++ b/app/templates/admin/edit_user.html.j2
@@ -15,8 +15,8 @@
<div class="card">
<div class="card-content">
<span class="card-title">General settings</span>
- {{ wtf.render_field(edit_general_settings_form.username, data_length='64', material_icon='person') }}
- {{ wtf.render_field(edit_general_settings_form.email, data_length='254', material_icon='email') }}
+ {{ wtf.render_field(edit_general_settings_form.username, material_icon='person') }}
+ {{ wtf.render_field(edit_general_settings_form.email, material_icon='email') }}
</div>
<div class="card-action">
<div class="right-align">
diff --git a/app/templates/admin/user.html.j2 b/app/templates/admin/user.html.j2
index 9af7e4e02a04f8351e8beea72b3960f0110fbfc9..b4b0e30322c9dcdaecd98be307085f4b74d56130 100644
--- a/app/templates/admin/user.html.j2
+++ b/app/templates/admin/user.html.j2
@@ -37,52 +37,20 @@
</div>
</div>
- <div class="col s12 l6 nopaque-ressource-list" data-ressource-type="Corpus" data-user-id="{{ user.hashid }}">
+ <div class="col s12 l6">
<h3>Corpora</h3>
<div class="card">
<div class="card-content">
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input id="search-corpus" class="search" type="search"></input>
- <label for="search-corpus">Search corpus</label>
- </div>
- <table>
- <thead>
- <tr>
- <th></th>
- <th>Title and Description</th>
- <th>Status</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
+ <div class="corpus-list" data-user-id="{{ user.hashid }}"></div>
</div>
</div>
</div>
- <div class="col s12 l6 nopaque-ressource-list" data-ressource-type="Job" data-user-id="{{ user.hashid }}">
+ <div class="col s12 l6">
<h3>Jobs</h3>
<div class="card">
<div class="card-content">
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input id="search-job" class="search" type="search"></input>
- <label for="search-job">Search job</label>
- </div>
- <table>
- <thead>
- <tr>
- <th>Service</th>
- <th>Title and Description</th>
- <th>Status</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
+ <div class="job-list" data-user-id="{{ user.hashid }}"></div>
</div>
</div>
</div>
diff --git a/app/templates/admin/users.html.j2 b/app/templates/admin/users.html.j2
index 378998667bfbf69da59441c9b02b8282446ec6d0..75254b0e45a7eac5e6a65264ce4a51b5c36b2a10 100644
--- a/app/templates/admin/users.html.j2
+++ b/app/templates/admin/users.html.j2
@@ -8,28 +8,10 @@
<h1 id="title">{{ title }}</h1>
</div>
- <div class="col s12 nopaque-ressource-list no-autoinit" data-ressource-type="User" id="users">
+ <div class="col s12">
<div class="card">
<div class="card-content">
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input id="search-user" class="search" type="text"></input>
- <label for="search-user">Search user</label>
- </div>
- <table>
- <thead>
- <tr>
- <th>Id</th>
- <th>Username</th>
- <th>Email</th>
- <th>Last seen</th>
- <th>Role</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
+ <div class="user-list no-autoinit"></div>
</div>
</div>
</div>
@@ -40,7 +22,11 @@
{% block scripts %}
{{ super() }}
<script>
- let userList = new UserList(document.querySelector('#users'));
- userList.init({{ dict_users|tojson }});
+ for (let user of {{ json_users|tojson }}) {
+ if (user.id in app.data.users) {continue;}
+ app.data.users[user.id] = user;
+ }
+ let userList = new UserList(document.querySelector('.user-list'));
+ userList.init(app.data.users);
</script>
{% endblock scripts %}
diff --git a/app/templates/auth/login.html.j2 b/app/templates/auth/login.html.j2
index aea7138246dbe863a48dd98e247be561106caf29..213c0a5f19bc24183951b8c050f3fe2a862b7184 100644
--- a/app/templates/auth/login.html.j2
+++ b/app/templates/auth/login.html.j2
@@ -2,53 +2,30 @@
{% from "auth/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% import "materialize/wtf.html.j2" as wtf %}
-{% block styles %}
-{{ super() }}
-<style>
- main {
- background-image: url("{{ url_for('static', filename='images/parallax_lq/04_german_text_book_paper.jpg') }}");
- background-repeat: no-repeat;
- background-size: cover;
- }
-</style>
-{% endblock styles %}
{% block page_content %}
<div class="container">
<div class="row">
- <div class="col s12 m4">
- <div class="card medium">
- <div class="card-content">
- <h1 id="title">{{ title }}</h1>
- <p>Want to boost your research and get going? nopaque is free and no download is needed. Register now!</p>
- </div>
- <div class="card-action right-align">
- <a class="btn" href="{{ url_for('.register') }}"><i class="material-icons left">person_add</i>Register</a>
- </div>
- </div>
- </div>
+ <div class="col s12 m8 offset-m2">
+ <h1 id="title">{{ title }}</h1>
+ <p>Want to boost your research and get going? Nopaque is free and no download is needed. <a href="{{ url_for('.register') }}">Register now</a>!</p>
- <div class="col s12 m8">
- <div class="card medium">
- <form method="POST">
- <div class="card-content">
- {{ form.hidden_tag() }}
- {{ wtf.render_field(form.user, material_icon='person') }}
- {{ wtf.render_field(form.password, material_icon='vpn_key') }}
- <div class="row" style="margin-bottom: 0;">
- <div class="col s6 left-align">
- <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
- </div>
- <div class="col s6 right-align">
- {{ wtf.render_field(form.remember_me) }}
- </div>
+ <form method="POST">
+ <div class="card-panel">
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.user, material_icon='person') }}
+ {{ wtf.render_field(form.password, material_icon='vpn_key') }}
+ <div class="row">
+ <div class="col s6 left-align">
+ <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
+ </div>
+ <div class="col s6 right-align">
+ {{ wtf.render_field(form.remember_me) }}
</div>
</div>
- <div class="card-action right-align">
- {{ wtf.render_field(form.submit, material_icon='send') }}
- </div>
- </form>
- </div>
+ {{ wtf.render_field(form.submit, material_icon='send', class_='width-100') }}
+ </div>
+ </form>
</div>
</div>
</div>
diff --git a/app/templates/auth/register.html.j2 b/app/templates/auth/register.html.j2
index 5c195e568fca651278c5d8bc465b17edef5c28da..69a019126dd1eb314bfd628da3741c5cc3667e93 100644
--- a/app/templates/auth/register.html.j2
+++ b/app/templates/auth/register.html.j2
@@ -2,47 +2,31 @@
{% from "auth/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% import "materialize/wtf.html.j2" as wtf %}
-{% block styles %}
-{{ super() }}
-<style>
- main {
- background-image: url("{{ url_for('static', filename='images/parallax_lq/02_concept_document_focus_letter.jpg') }}");
- background-repeat: no-repeat;
- background-size: cover;
- }
-</style>
-{% endblock styles %}
{% block page_content %}
<div class="container">
<div class="row">
- <div class="col s12 m4">
- <div class="card medium">
- <div class="card-content">
- <h1 id="title">{{ title }}</h1>
- <p>Simply enter a username and password to receive your registration email. After that you can start right away.</p>
- <p>It goes without saying that the <a href="{{ url_for('main.privacy_policy') }}">General Data Protection Regulation</a> applies, only necessary data is stored.</p>
- <p>Please also read our <a href="{{ url_for('main.terms_of_use') }}">terms of use</a> before signing up for nopaque!</p>
- </div>
- </div>
- </div>
+ <div class="col s12 m8 offset-m2">
+ <h1 id="title">{{ title }}</h1>
+ <p>
+ Simply enter a username and password to receive your registration email.
+ After that you can start right away. It goes without saying that the
+ <a href="{{ url_for('main.privacy_policy') }}">General Data Protection Regulation</a>
+ applies, only necessary data is stored. Please also read our
+ <a href="{{ url_for('main.terms_of_use') }}">terms of use</a> before
+ signing up for nopaque!
+ </p>
- <div class="col s12 m8">
- <div class="card medium">
- <form method="POST">
- <div class="card-content">
- {{ form.hidden_tag() }}
- {{ wtf.render_field(form.username, material_icon='person') }}
- {{ wtf.render_field(form.password, material_icon='vpn_key') }}
- {{ wtf.render_field(form.password_confirmation, material_icon='vpn_key') }}
- {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
- </div>
- <div class="card-action right-align">
- {{ wtf.render_field(form.submit, material_icon='send') }}
- </div>
- </form>
- </div>
- </div>
+ <form method="POST">
+ <div class="card-panel">
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.username, material_icon='person') }}
+ {{ wtf.render_field(form.password, material_icon='vpn_key') }}
+ {{ wtf.render_field(form.password_2, material_icon='vpn_key') }}
+ {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
+ {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
+ </div>
+ </form>
</div>
</div>
{% endblock page_content %}
diff --git a/app/templates/auth/reset_password.html.j2 b/app/templates/auth/reset_password.html.j2
index a0f4c32bf8d9a7b270813a33a31aafbe9cffbfd2..06f11059b759db639572621195fefac4b2dc6f63 100644
--- a/app/templates/auth/reset_password.html.j2
+++ b/app/templates/auth/reset_password.html.j2
@@ -5,27 +5,18 @@
{% block page_content %}
<div class="container">
<div class="row">
- <div class="col s12">
+ <div class="col s12 m8 offset-m2">
<h1 id="title">{{ title }}</h1>
- </div>
-
- <div class="col s12 m4">
<p>Enter a new password and confirm it! After that, the entered password is your new one!</p>
- </div>
- <div class="col s12 m8">
- <div class="card">
- <form method="POST">
- <div class="card-content">
- {{ form.hidden_tag() }}
- {{ wtf.render_field(form.password) }}
- {{ wtf.render_field(form.password_confirmation) }}
- </div>
- <div class="card-action right-align">
- {{ wtf.render_field(form.submit, material_icon='send') }}
- </div>
- </form>
- </div>
+ <form method="POST">
+ <div class="card-panel">
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.password) }}
+ {{ wtf.render_field(form.password_2) }}
+ {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
+ </div>
+ </form>
</div>
</div>
</div>
diff --git a/app/templates/auth/reset_password_request.html.j2 b/app/templates/auth/reset_password_request.html.j2
index b91cd59b4bf3f5216eee4e004061a724b4a4395c..a94d18da91521dbd6b38ec67e04be35b1366dc3c 100644
--- a/app/templates/auth/reset_password_request.html.j2
+++ b/app/templates/auth/reset_password_request.html.j2
@@ -5,26 +5,17 @@
{% block page_content %}
<div class="container">
<div class="row">
- <div class="col s12">
+ <div class="col s12 m8 offset-m2">
<h1 id="title">{{ title }}</h1>
- </div>
-
- <div class="col s12 m4">
<p>After entering your email address you will receive instructions on how to reset your password.</p>
- </div>
- <div class="col s12 m8">
- <div class="card">
- <form method="POST">
- <div class="card-content">
- {{ form.hidden_tag() }}
- {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
- </div>
- <div class="card-action right-align">
- {{ wtf.render_field(form.submit, material_icon='send') }}
- </div>
- </form>
- </div>
+ <form method="POST">
+ <div class="card-panel">
+ {{ form.hidden_tag() }}
+ {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
+ {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
+ </div>
+ </form>
</div>
</div>
</div>
diff --git a/app/templates/auth/unconfirmed.html.j2 b/app/templates/auth/unconfirmed.html.j2
index db9bf8c4940611ab25e798c58d13a70e219f307e..26384eccbd71a207ee690c6486c87f5b5dca11d5 100644
--- a/app/templates/auth/unconfirmed.html.j2
+++ b/app/templates/auth/unconfirmed.html.j2
@@ -6,20 +6,13 @@
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
- </div>
-
- <div class="col s12">
- <div class="card">
- <div class="card-content">
- <span class="card-title">Hello, {{ current_user.username }}!</span>
- <p><b>You have not confirmed your account yet.</b></p>
- <p>Before you can access this site you need to confirm your account. Check your inbox, you should have received an email with a confirmation link.</p>
- <p>Need another confirmation email? Click the button below!</p>
- </div>
- <div class="card-action right-align">
- <a class="btn" href="{{ url_for('.resend_confirmation') }}">Resend confirmation mail</a>
- </div>
- </div>
+ <p>Hello, <b>{{ current_user.username }}</b>.</p>
+ <p>
+ You have not confirmed your account yet. Before you can access this
+ site you need to confirm your account. Check your inbox, you should
+ have received an email with a confirmation link.
+ </p>
+ <p>Need another confirmation email? <a href="{{ url_for('.confirm_request') }}">Get a new one</a>.</p>
</div>
</div>
</div>
diff --git a/app/templates/corpora/_breadcrumbs.html.j2 b/app/templates/corpora/_breadcrumbs.html.j2
index d91bc8c33cebd10f8fbf7e97f65aab1edaa74b6c..af6d2b78eec36b366fe81854fc64c3ae89945888 100644
--- a/app/templates/corpora/_breadcrumbs.html.j2
+++ b/app/templates/corpora/_breadcrumbs.html.j2
@@ -2,8 +2,8 @@
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('main.dashboard', _anchor='jobs') }}" target="_self">My corpora</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
-{% if request.path == url_for('.add_corpus') %}
-<li class="tab"><a class="active" href="{{ url_for('.add_corpus') }}" target="_self">{{ title }}</a></li>
+{% if request.path == url_for('.create_corpus') %}
+<li class="tab"><a class="active" href="{{ url_for('.create_corpus') }}" target="_self">{{ title }}</a></li>
{% elif request.path == url_for('.import_corpus') %}
<li class="tab"><a class="active" href="{{ url_for('.import_corpus') }}" target="_self">{{ title }}</a></li>
{% elif request.path == url_for('.corpus', corpus_id=corpus.id) %}
@@ -12,12 +12,12 @@
<li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id) }}" target="_self">{{ corpus.title }}</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.analyse_corpus', corpus_id=corpus.id) }}" target="_self">{{ title }}</a></li>
-{% elif request.path == url_for('.add_corpus_file', corpus_id=corpus.id) %}
+{% elif request.path == url_for('.create_corpus_file', corpus_id=corpus.id) %}
<li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id) }}" target="_self">{{ corpus.title }}</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id, _anchor='files') }}" target="_self">Corpus files</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
-<li class="tab"><a class="active" href="{{ url_for('.add_corpus_file', corpus_id=corpus.id) }}" target="_self">{{ title }}</a></li>
+<li class="tab"><a class="active" href="{{ url_for('.create_corpus_file', corpus_id=corpus.id) }}" target="_self">{{ title }}</a></li>
{% elif request.path == url_for('.corpus_file', corpus_file_id=corpus_file.id, corpus_id=corpus.id) %}
<li class="tab"><a href="{{ url_for('.corpus', corpus_id=corpus.id) }}" target="_self">{{ corpus.title }}</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
diff --git a/app/templates/corpora/corpus.html.j2 b/app/templates/corpora/corpus.html.j2
index 177d83b4bd662ddb22afaf3245a4fa87bd0314fb..d64b708497ff70c23b7f6f22be13578aa01bfddf 100644
--- a/app/templates/corpora/corpus.html.j2
+++ b/app/templates/corpora/corpus.html.j2
@@ -65,38 +65,21 @@
</div>
<div class="card-action right-align">
<a class="btn corpus-analyse-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">search</i>Analyze</a>
- <a class="btn corpus-build-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.build_corpus', corpus_id=corpus.id) }}"><i class="nopaque-icons left">K</i>Build</a>
+ <a class="action-button btn disabled waves-effect waves-light" data-action="build-request"><i class="nopaque-icons left">K</i>Build</a>
<a class="btn disabled export-corpus-trigger waves-effect waves-light" href="{{ url_for('corpora.export_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">import_export</i>Export</a>
- <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-corpus-modal"><i class="material-icons left">delete</i>Delete</a>
+ <a class="action-button btn red waves-effect waves-light" data-action="delete-request"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</div>
- <div class="col s12 nopaque-ressource-list" data-corpus-id="{{ corpus.hashid }}" data-ressource-type="CorpusFile" data-user-id="{{ corpus.user.hashid }}">
+ <div class="col s12">
<div class="card">
<div class="card-content">
<span class="card-title" id="files">Corpus files</span>
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input class="search" id="search-corpus-files" type="search"></input>
- <label for="search-corpus-files">Search corpus files</label>
- </div>
- <table>
- <thead>
- <tr>
- <th>Filename</th>
- <th>Author</th>
- <th>Title</th>
- <th>Publishing year</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
+ <div class="corpus-file-list" data-user-id="{{ corpus.user.hashid }}" data-corpus-id="{{ corpus.hashid }}"></div>
</div>
<div class="card-action right-align">
- <a href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a>
+ <a href="{{ url_for('corpora.create_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a>
</div>
</div>
</div>
@@ -104,20 +87,6 @@
</div>
{% endblock page_content %}
-{% block modals %}
-{{ super() }}
-<div id="delete-corpus-modal" class="modal">
- <div class="modal-content">
- <h4>Confirm corpus deletion</h4>
- <p>Do you really want to delete the corpus <span class="corpus-title"></span>? All files will be permanently deleted!</p>
- </div>
- <div class="modal-footer">
- <a class="btn modal-close waves-effect waves-light" href="#!">Cancel</a>
- <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">delete</i>Delete</a>
- </div>
-</div>
-{% endblock modals %}
-
{% block scripts %}
{{ super() }}
<script>
diff --git a/app/templates/corpora/add_corpus.html.j2 b/app/templates/corpora/create_corpus.html.j2
similarity index 100%
rename from app/templates/corpora/add_corpus.html.j2
rename to app/templates/corpora/create_corpus.html.j2
diff --git a/app/templates/corpora/add_corpus_file.html.j2 b/app/templates/corpora/create_corpus_file.html.j2
similarity index 66%
rename from app/templates/corpora/add_corpus_file.html.j2
rename to app/templates/corpora/create_corpus_file.html.j2
index 725ee0cbd949d72956a313b61524c81684a59211..55b078e40cfb5d44f2696524cbaf76da22d5a0a7 100644
--- a/app/templates/corpora/add_corpus_file.html.j2
+++ b/app/templates/corpora/create_corpus_file.html.j2
@@ -13,11 +13,10 @@
<div class="col s12 m4">
<p>Fill out the following form to add a corpus file in verticalized text format (.vrt).</p>
- <p><b>Do not use the .stand-off.vrt file!</b></p>
</div>
<div class="col s12 m8">
- <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+ <form class="create-corpus-file-form" enctype="multipart/form-data" method="POST">
<div class="card">
<div class="card-content">
{{ form.hidden_tag() }}
@@ -52,39 +51,8 @@
</div>
</li>
</ul>
- <br>
- <ul class="collapsible hoverable">
- <li>
- <div class="collapsible-header"><i class="material-icons">add</i>Add metadata with BibTex</div>
- <div class="collapsible-body">
- <span>
- <div class="row">
- <div class="col s12">
-
- </div>
- </div>
- </span>
- </div>
- </li>
- </ul>
</form>
</div>
</div>
</div>
{% endblock page_content %}
-
-
-{% block modals %}
-{{ super() }}
-<div id="progress-modal" class="modal">
- <div class="modal-content">
- <h4><i class="material-icons left">file_upload</i>Uploading files...</h4>
- <div class="progress">
- <div class="determinate" style="width: 0%"></div>
- </div>
- </div>
- <div class="modal-footer">
- <a href="#!" class="modal-close waves-effect waves-light btn red abort-request">Cancel</a>
- </div>
-</div>
-{% endblock modals %}
diff --git a/app/templates/errors/403.html.j2 b/app/templates/errors/403.html.j2
deleted file mode 100644
index db286454c46a8084963e9b3869635b7ddb07ef3d..0000000000000000000000000000000000000000
--- a/app/templates/errors/403.html.j2
+++ /dev/null
@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
- <h1 id="title">{{ title }}</h1>
- <p class="light">{{ request.path }}</p>
- <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
- <div class="modal-content">
- <h2>About the "{{ title }}" error</h2>
- <p>The request contained valid data and was understood by the server, but the server is refusing action. This may be due to the user not having the necessary permissions for a resource or needing an account of some sort, or attempting a prohibited action (e.g. creating a duplicate record where only one is allowed). This code is also typically used if the request provided authentication by answering the WWW-Authenticate header field challenge, but the server did not accept that authentication. The request should not be repeated.</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
- </div>
-</div>
-{% endblock page_content %}
diff --git a/app/templates/errors/404.html.j2 b/app/templates/errors/404.html.j2
deleted file mode 100644
index 62006da96207fcb23d52a46461adf6cbc101558d..0000000000000000000000000000000000000000
--- a/app/templates/errors/404.html.j2
+++ /dev/null
@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
- <h1 id="title">{{ title }}</h1>
- <p class="light">{{ request.path }}</p>
- <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
- <div class="modal-content">
- <h2>About the "{{ title }}" error</h2>
- <p>The requested resource could not be found but may be available in the future. Subsequent requests by the client are permissible.</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
- </div>
-</div>
-{% endblock page_content %}
diff --git a/app/templates/errors/413.html.j2 b/app/templates/errors/413.html.j2
deleted file mode 100644
index 6e4e3e7e92ad6a2181ebcbffc91a7962e4e4383d..0000000000000000000000000000000000000000
--- a/app/templates/errors/413.html.j2
+++ /dev/null
@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
- <h1 id="title">{{ title }}</h1>
- <p class="light">{{ request.path }}</p>
- <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
- <div class="modal-content">
- <h2>About the "{{ title }}" error</h2>
- <p>The request is larger than the server is willing or able to process. Previously called "Request Entity Too Large".</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
- </div>
-</div>
-{% endblock page_content %}
diff --git a/app/templates/errors/500.html.j2 b/app/templates/errors/500.html.j2
deleted file mode 100644
index 5e2d3e87d28f4bdd5c451b05559194330fd48016..0000000000000000000000000000000000000000
--- a/app/templates/errors/500.html.j2
+++ /dev/null
@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
- <h1 id="title">{{ title }}</h1>
- <p class="light">{{ request.path }}</p>
- <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
- <div class="modal-content">
- <h2>About the "{{ title }}" error</h2>
- <p>A generic error message, given when an unexpected condition was encountered and no more specific message is suitable.</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
- </div>
-</div>
-{% endblock page_content %}
diff --git a/app/templates/errors/503.html.j2 b/app/templates/errors/503.html.j2
deleted file mode 100644
index eb10cfb0e1c7e157076957af87f55c1ad04ad61a..0000000000000000000000000000000000000000
--- a/app/templates/errors/503.html.j2
+++ /dev/null
@@ -1,19 +0,0 @@
-{% extends "base.html.j2" %}
-
-{% block page_content %}
-<div class="container">
- <h1 id="title">{{ title }}</h1>
- <p class="light">{{ request.path }}</p>
- <p>Alternatively, you can visit the <a href="{{ url_for('main.index') }}">Main Page</a> or read <a class="modal-trigger" href="#more-information-modal">more information</a> about this type of error.</p>
-</div>
-
-<div class="modal" id="more-information-modal">
- <div class="modal-content">
- <h2>About the "{{ title }}" error</h2>
- <p>The server cannot handle the request (because it is overloaded or down for maintenance). Generally, this is a temporary state.</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn-flat modal-close waves-effect waves-green">Close</a>
- </div>
-</div>
-{% endblock page_content %}
diff --git a/app/templates/errors/error.html.j2 b/app/templates/errors/error.html.j2
new file mode 100644
index 0000000000000000000000000000000000000000..ef19de5fbee7dc9d918284cf332cee1149a9a809
--- /dev/null
+++ b/app/templates/errors/error.html.j2
@@ -0,0 +1,10 @@
+{% extends "base.html.j2" %}
+
+{% set title = error.name %}
+
+{% block page_content %}
+<div class="container">
+ <h1 id="title">{{ error.name }}</h1>
+ <p>{{ error.description }}</p>
+</div>
+{% endblock page_content %}
diff --git a/app/templates/jobs/_breadcrumbs.html.j2 b/app/templates/jobs/_breadcrumbs.html.j2
index 5bad7de317888574269990d0fb1e94b5a701c471..e3de43f313d7d460f3196d2889f10136803b0a82 100644
--- a/app/templates/jobs/_breadcrumbs.html.j2
+++ b/app/templates/jobs/_breadcrumbs.html.j2
@@ -1,8 +1,8 @@
{% set breadcrumbs %}
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('main.dashboard', _anchor='jobs') }}" target="_self">My jobs</a></li>
-<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
{% if request.path == url_for('.job', job_id=job.id) %}
+<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.job', job_id=job.id) }}" target="_self">{{ job.title }}</a></li>
{% endif %}
{% endset %}
diff --git a/app/templates/jobs/job.html.j2 b/app/templates/jobs/job.html.j2
index 7a2caa1036b0261ffedcf20973a2cba927e688e0..5b78d0555268dede68007a9a563f7bad140ad587 100644
--- a/app/templates/jobs/job.html.j2
+++ b/app/templates/jobs/job.html.j2
@@ -79,60 +79,31 @@
</div>
<div class="card-action right-align">
{% if current_user.is_administrator() %}
- <a class="btn hide modal-trigger restart-job-trigger waves-effect waves-light" data-target="restart-job-modal"><i class="material-icons left">repeat</i>Restart</a>
+ <a class="action-button btn disabled waves-effect waves-light" data-action="get-log-request"><i class="material-icons left">text_snippet</i>Log</a>
{% endif %}
- <!-- <a href="#" class="btn disabled waves-effect waves-light"><i class="material-icons left">settings</i>Export Parameters</a> -->
- <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-job-modal"><i class="material-icons left">delete</i>Delete</a>
+ <a class="action-button btn disabled waves-effect waves-light" data-action="restart-request"><i class="material-icons left">repeat</i>Restart</a>
+ <a class="action-button btn red waves-effect waves-light" data-action="delete-request"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</div>
- <div class="col s12 nopaque-ressource-list" data-job-id="{{ job.hashid }}" data-ressource-type="JobInput" data-user-id="{{ job.user.hashid }}">
+ <div class="col s12">
<div class="card">
<div class="card-content">
<div class="row">
- <div class="col s12 m2">
- <span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span>
- <p>Original input files.</p>
- </div>
- <div class="col s12 m10">
- <table>
- <thead>
- <tr>
- <th>Filename</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
- </div>
+ <span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span>
+ <div class="job-input-list" data-user-id="{{ job.user.hashid }}" data-job-id="{{ job.hashid }}"></div>
</div>
</div>
</div>
</div>
- <div class="col s12 nopaque-ressource-list" data-job-id="{{ job.hashid }}" data-ressource-type="JobResult" data-user-id="{{ job.user.hashid }}">
+ <div class="col s12">
<div class="card">
<div class="card-content">
<div class="row">
- <div class="col s12 m2">
- <span class="card-title"><i class="left material-icons" style="font-size: inherit;">done</i>Results</span>
- <p>Processed result files.</p>
- </div>
- <div class="col s12 m10">
- <table>
- <thead>
- <tr>
- <th>Description</th>
- <th>Filename</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
- </div>
+ <span class="card-title"><i class="left material-icons" style="font-size: inherit;">done</i>Results</span>
+ <div class="job-result-list" data-user-id="{{ job.user.hashid }}" data-job-id="{{ job.hashid }}"></div>
</div>
</div>
</div>
@@ -141,32 +112,6 @@
</div>
{% endblock page_content %}
-{% block modals %}
-{{ super() }}
-<div id="delete-job-modal" class="modal">
- <div class="modal-content">
- <h4>Confirm deletion</h4>
- <p>Do you really want to delete the job <span class="job-title"></span>? All associated files will be permanently deleted.</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
- <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
- </div>
-</div>
-
-{% if current_user.is_administrator() %}
-<div id="restart-job-modal" class="modal">
- <div class="modal-content">
- <h4>Confirm restart</h4>
- <p>Do you really want to restart the job <span class="job-title"></span>? All log and result files will be permanently deleted.</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
- <a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.restart', job_id=job.id) }}"><i class="material-icons left">restart</i>Restart</a>
- </div>
-</div>
-{% endif %}
-{% endblock modals %}
{% block scripts %}
{{ super() }}
diff --git a/app/templates/main/dashboard.html.j2 b/app/templates/main/dashboard.html.j2
index 6141e5d8b7906f32df249643ed7596a938e30707..0d15651763e1eb99d39cd50cf792a1c1adf9af13 100644
--- a/app/templates/main/dashboard.html.j2
+++ b/app/templates/main/dashboard.html.j2
@@ -6,100 +6,48 @@
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
- </div>
- <div class="col s12">
<h3>My Corpora and Query results</h3>
<p>Create a corpus to interactively perform linguistic analysis or import query results to save interesting passages.</p>
+
<div class="row">
<div class="col s12">
<ul class="tabs">
<li class="tab col s6"><a class="active" href="#corpora">Corpora</a></li>
- <li class="tab col s6"><a href="#query-results">Query results</a></li>
+ <li class="tab col s6 disabled"><a href="#query-results">Query results</a></li>
</ul>
</div>
- <div class="col s12 nopaque-ressource-list" data-ressource-type="Corpus" data-user-id="{{ current_user.hashid }}" id="corpora">
+ <div class="col s12" id="corpora">
<div class="card">
<div class="card-content">
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input id="search-corpus" class="search" type="search"></input>
- <label for="search-corpus">Search corpus</label>
- </div>
- <table>
- <thead>
- <tr>
- <th></th>
- <th>Title and Description</th>
- <th>Status</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
+ <div class="corpus-list" data-user-id="{{ current_user.hashid }}"></div>
</div>
<div class="card-action right-align">
- <a class="btn waves-effect waves-light" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
- <a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_corpus') }}">New corpus<i class="material-icons right">add</i></a>
- </div>
- </div>
- </div>
- {# <div class="col s12 nopaque-ressource-list" data-ressource-type="QueryResult" data-user-id="{{ current_user.hashid }}" id="query-results"> #}
- <div class="col s12" id="query-results">
- <div class="card">
- <div class="card-content">
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input id="search-query-results" class="search" type="search"></input>
- <label for="search-query-results">Search query result</label>
- </div>
- <table>
- <thead>
- <tr>
- <th>Title and Description</th>
- <th>Corpus and Query</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
- </div>
- <div class="card-action right-align">
- <a class="waves-effect waves-light btn disabled">Add query result<i class="material-icons right">file_upload</i></a>
+ <a class="btn disabled waves-effect waves-light" href="{{ url_for('corpora.import_corpus') }}">Import Corpus<i class="material-icons right">import_export</i></a>
+ <a class="btn waves-effect waves-light" href="{{ url_for('corpora.create_corpus') }}">Create corpus<i class="material-icons right">add</i></a>
</div>
</div>
</div>
</div>
</div>
- <div class="col s12 nopaque-ressource-list" data-ressource-type="Job" data-user-id="{{ current_user.hashid }}" id="jobs">
+ <div class="col s12" id="jobs">
<h3>My Jobs</h3>
- <p>A job is the execution of a service provided by nopaque. You can create any number of jobs and let them be processed simultaneously.</p>
+ <p>
+ A job is the execution of a service provided by nopaque. You can
+ create any number of jobs and let them be processed simultaneously. We
+ <b>strongly recommend</b> that you create a folder on your computer where you
+ save the various files that nopaque provides you with after each
+ pre-processing step. You will need the result of each step for the
+ next step.
+ </p>
+ <p><b>Where is my Job data?</b> Don't worry, please read <a href="{{ url_for('main.news', _anchor='april-2022-update') }}">this news</a> entry</p>
<div class="card">
<div class="card-content">
- <p><b>Where is my Job data?</b> Don't worry, please read <a href="{{ url_for('main.news', _anchor='april-2022-update') }}">this news</a> entry</p>
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input id="search-job" class="search" type="search"></input>
- <label for="search-job">Search job</label>
- </div>
- <table>
- <thead>
- <tr>
- <th>Service</th>
- <th>Title and Description</th>
- <th>Status</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
+ <div class="job-list" data-user-id="{{ current_user.hashid }}"></div>
</div>
<div class="card-action right-align">
- <p><a class="modal-trigger waves-effect waves-light btn" href="#" data-target="new-job-modal"><i class="material-icons left">add</i>New job</a></p>
+ <p><a class="btn modal-trigger waves-effect waves-light" data-target="create-job-modal"><i class="material-icons left">add</i>Create job</a></p>
</div>
</div>
</div>
@@ -109,7 +57,7 @@
{% block modals %}
{{ super() }}
-<div id="new-job-modal" class="modal">
+<div id="create-job-modal" class="modal">
<div class="modal-content">
<h4>Select a service</h4>
<p> </p>
@@ -153,7 +101,7 @@
</div>
</div>
<div class="modal-footer">
- <a href="#!" class="modal-close waves-effect waves-light btn-flat">Close</a>
+ <a class="btn-flat modal-close waves-effect waves-light">Close</a>
</div>
</div>
{% endblock modals %}
diff --git a/app/templates/services/corpus_analysis.html.j2 b/app/templates/services/corpus_analysis.html.j2
index f89b46fcb59cc581695bf91d89ee8ad36d7bfb84..e72d11b88b9d08d9fecd8dff94e3814b8bf628da 100644
--- a/app/templates/services/corpus_analysis.html.j2
+++ b/app/templates/services/corpus_analysis.html.j2
@@ -22,36 +22,20 @@
<p>Nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags. The results can either be displayed as text or abstract visualizations.</p>
</div>
- <div class="col s12 nopaque-ressource-list" data-ressource-type="Corpus" data-user-id="{{ current_user.hashid }}" id="corpora">
+ <div class="col s12" id="corpora">
<h2>My Corpora</h2>
<div class="card">
<div class="card-content">
- <div class="input-field">
- <i class="material-icons prefix">search</i>
- <input id="search-corpus" class="search" type="search"></input>
- <label for="search-corpus">Search corpus</label>
- </div>
- <table class="highlight">
- <thead>
- <tr>
- <th></th>
- <th>Title and Description</th>
- <th>Status</th>
- <th></th>
- </tr>
- </thead>
- <tbody class="list"></tbody>
- </table>
- <ul class="pagination"></ul>
+ <div class="corpus-list" data-user-id="{{ current_user.hashid }}"></div>
</div>
<div class="card-action right-align">
- <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
- <a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_corpus') }}">New corpus<i class="material-icons right">add</i></a>
+ <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}">Import Corpus<i class="material-icons right">import_export</i></a>
+ <a class="btn waves-effect waves-light" href="{{ url_for('corpora.create_corpus') }}">Create corpus<i class="material-icons right">add</i></a>
</div>
</div>
</div>
- <div class="col s12 nopaque-ressource-list" data-ressource-type="QueryResult" data-user-id="{{ current_user.hashid }}" id="query-results">
+ <div class="col s12 query-result-list" data-user-id="{{ current_user.hashid }}" id="query-results">
<h2>My query results</h2>
<div class="card">
<div class="card-content">
diff --git a/app/templates/services/file_setup_pipeline.html.j2 b/app/templates/services/file_setup_pipeline.html.j2
index dfb11fec88f7a77416fb11c5ff6cefe06290989a..eca9b8b7dc4b850ce152b99c209c033c6fd4a6b4 100644
--- a/app/templates/services/file_setup_pipeline.html.j2
+++ b/app/templates/services/file_setup_pipeline.html.j2
@@ -39,7 +39,7 @@
<div class="col s12">
<h2>Submit a job</h2>
<div class="card">
- <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+ <form class="create-job-form" enctype="multipart/form-data" method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
diff --git a/app/templates/services/spacy_nlp_pipeline.html.j2 b/app/templates/services/spacy_nlp_pipeline.html.j2
index 106e72e136e50367ba7260267c30ad851a304ff2..7552b5889e09f850041a2d9b40877ca8655e96e8 100644
--- a/app/templates/services/spacy_nlp_pipeline.html.j2
+++ b/app/templates/services/spacy_nlp_pipeline.html.j2
@@ -57,7 +57,7 @@
<div class="col s12">
<h2>Submit a job</h2>
<div class="card">
- <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+ <form class="create-job-form" enctype="multipart/form-data" method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
diff --git a/app/templates/services/tesseract_ocr_pipeline.html.j2 b/app/templates/services/tesseract_ocr_pipeline.html.j2
index 129b74aa93da9a91bff7acbc9337078f5808985e..c38c396593ecdd8267cccfe9717290f1263c9c72 100644
--- a/app/templates/services/tesseract_ocr_pipeline.html.j2
+++ b/app/templates/services/tesseract_ocr_pipeline.html.j2
@@ -39,7 +39,7 @@
<div class="col s12">
<h2>Submit a job</h2>
<div class="card">
- <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+ <form class="create-job-form" enctype="multipart/form-data" method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
@@ -178,28 +178,4 @@
<a href="#!" class="modal-close waves-effect waves-light btn">Close</a>
</div>
</div>
-
-<div id="progress-modal" class="modal">
- <div class="modal-content">
- <h4><i class="material-icons left">file_upload</i>Uploading files...</h4>
- <div class="progress">
- <div class="determinate" style="width: 0%"></div>
- </div>
- </div>
- <div class="modal-footer">
- <a href="#!" class="modal-close waves-effect waves-light btn red abort-request">Cancel</a>
- </div>
-</div>
{% endblock modals %}
-
-{% block scripts %}
-{{ super() }}
-<script>
- let versionField = document.querySelector('#add-job-form-version');
- versionField.addEventListener('change', (event) => {
- let url = new URL(window.location.href);
- url.search = `?version=${event.target.value}`;
- window.location.href = url.toString();
- });
-</script>
-{% endblock scripts %}
diff --git a/app/templates/services/transkribus_htr_pipeline.html.j2 b/app/templates/services/transkribus_htr_pipeline.html.j2
index 7708e8d8636ed7223c0b17730a1fc8c38d51f7af..a73839b5d389c67daf0425d578e30e2f42a047fb 100644
--- a/app/templates/services/transkribus_htr_pipeline.html.j2
+++ b/app/templates/services/transkribus_htr_pipeline.html.j2
@@ -44,7 +44,7 @@
<div class="col s12">
<h2>Submit a job</h2>
<div class="card">
- <form class="nopaque-upload-form" data-progress-modal="progress-modal">
+ <form class="create-job-form" enctype="multipart/form-data" method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
diff --git a/app/templates/settings/_breadcrumbs.html.j2 b/app/templates/settings/_breadcrumbs.html.j2
index 33b8984cef722fdd030696346afcbddf55c6ba89..3b5077bfd8e7dd07315128eef5e87fcf34b4721b 100644
--- a/app/templates/settings/_breadcrumbs.html.j2
+++ b/app/templates/settings/_breadcrumbs.html.j2
@@ -1,6 +1,6 @@
{% set breadcrumbs %}
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
-{% if request.path == url_for('settings.index') %}
-<li class="tab"><a{%if request.path == url_for('settings.index') %} class="active"{% endif %} href="{{ url_for('settings.index') }}" target="_self">Settings</a></li>
+{% if request.path == url_for('settings.settings') %}
+<li class="tab"><a{%if request.path == url_for('settings.settings') %} class="active"{% endif %} href="{{ url_for('settings.settings') }}" target="_self">Settings</a></li>
{% endif %}
{% endset %}
diff --git a/app/templates/settings/index.html.j2 b/app/templates/settings/settings.html.j2
similarity index 83%
rename from app/templates/settings/index.html.j2
rename to app/templates/settings/settings.html.j2
index 0a814ea2eea1ca7961029504f6dba9232299f6f8..441cd367834a78ee6692aa86799db87578d8328f 100644
--- a/app/templates/settings/index.html.j2
+++ b/app/templates/settings/settings.html.j2
@@ -81,7 +81,7 @@
<span class="card-title">Change Password</span>
{{ wtf.render_field(change_password_form.password, material_icon='vpn_key') }}
{{ wtf.render_field(change_password_form.new_password, material_icon='vpn_key') }}
- {{ wtf.render_field(change_password_form.new_password_confirmation, material_icon='vpn_key') }}
+ {{ wtf.render_field(change_password_form.new_password_2, material_icon='vpn_key') }}
</div>
<div class="card-action">
<div class="right-align">
@@ -101,23 +101,19 @@
</ul>
</div>
<div class="card-action right-align">
- <a href="#delete-account-modal" class="btn modal-trigger red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a>
+ <a class="btn red waves-effect waves-light" id="delete-user"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</div>
</div>
{% endblock page_content %}
-{% block modals %}
+{% block scripts %}
{{ super() }}
-<div class="modal" id="delete-account-modal">
- <div class="modal-content">
- <h4>Confirm deletion</h4>
- <p>Do you really want to delete your account and all associated data? All associated corpora, jobs and files will be permanently deleted!</p>
- </div>
- <div class="modal-footer">
- <a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
- <a href="{{ url_for('.delete') }}" class="btn red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a>
- </div>
-</div>
-{% endblock modals %}
+<script>
+document.querySelector('#delete-user').addEventListener('click', (event) => {
+ Utils.deleteUserRequest(currentUserId)
+ .then((response) => {window.location.href = '/';});
+});
+</script>
+{% endblock scripts %}
diff --git a/app/users/__init__.py b/app/users/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..885cdbe223e1f48f5e9708458f9c9e5be0a19795
--- /dev/null
+++ b/app/users/__init__.py
@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+
+bp = Blueprint('users', __name__)
+from . import events, routes
diff --git a/app/users/events.py b/app/users/events.py
new file mode 100644
index 0000000000000000000000000000000000000000..7cab2199ee3d6a802058a399ab2deabffcbcc9e7
--- /dev/null
+++ b/app/users/events.py
@@ -0,0 +1,31 @@
+from flask_login import current_user
+from flask_socketio import join_room, leave_room
+from app import hashids, socketio
+from app.decorators import socketio_login_required
+from app.models import User
+
+
+@socketio.on('SUBSCRIBE /users/<user_id>')
+@socketio_login_required
+def subscribe_user(user_hashid):
+ user_id = hashids.decode(user_hashid)
+ user = User.query.get(user_id)
+ if user is None:
+ return {'code': 404, 'msg': 'Not found'}
+ if not (user == current_user or current_user.is_administrator):
+ return {'code': 403, 'msg': 'Forbidden'}
+ join_room(f'/users/{user.hashid}')
+ return {'code': 200, 'msg': 'OK'}
+
+
+@socketio.on('UNSUBSCRIBE /users/<user_id>')
+@socketio_login_required
+def unsubscribe_user(user_hashid):
+ user_id = hashids.decode(user_hashid)
+ user = User.query.get(user_id)
+ if user is None:
+ return {'code': 404, 'msg': 'Not found'}
+ if not (user == current_user or current_user.is_administrator):
+ return {'code': 403, 'msg': 'Forbidden'}
+ leave_room(f'/users/{user.hashid}')
+ return {'code': 200, 'msg': 'OK'}
diff --git a/app/users/routes.py b/app/users/routes.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2f8abf27b40549512c71d3ee8b077bc99cb3672
--- /dev/null
+++ b/app/users/routes.py
@@ -0,0 +1,38 @@
+from flask import abort, current_app, request
+from flask_login import current_user, login_required
+from threading import Thread
+from app import db
+from app.models import User
+from . import bp
+
+
+@bp.route('/<hashid:user_id>')
+@login_required
+def user(user_id):
+ user = User.query.get_or_404(user_id)
+ if not (user == current_user or current_user.is_administrator()):
+ abort(403)
+ backrefs = request.args.get('backrefs', 'false').lower() == 'true'
+ relationships = (
+ request.args.get('relationships', 'false').lower() == 'true')
+ return user.to_json(backrefs=backrefs, relationships=relationships), 200
+
+
+@bp.route('/<hashid:user_id>', methods=['DELETE'])
+@login_required
+def delete_user(user_id):
+ def _delete_user(app, user_id):
+ with app.app_context():
+ user = User.query.get(user_id)
+ user.delete()
+ db.session.commit()
+
+ user = User.query.get_or_404(user_id)
+ if not (user == current_user or current_user.is_administrator()):
+ abort(403)
+ thread = Thread(
+ target=_delete_user,
+ args=(current_app._get_current_object(), user_id)
+ )
+ thread.start()
+ return {}, 202
diff --git a/boot.sh b/boot.sh
index 68765e62f39fef1675cbfbc03b0a11e6fc72303a..1d63652f74182a1406af14da858df7b279729b6f 100755
--- a/boot.sh
+++ b/boot.sh
@@ -1,28 +1,37 @@
#!/bin/bash
+
source venv/bin/activate
+
+display_help() {
+ local script_name=$(basename "${0}")
+ echo ""
+ echo "Usage: ${script_name} [COMMAND]"
+ echo ""
+ echo "Run wrapper for a nopaque instance"
+ echo ""
+ echo "Commands:"
+ echo " flask A general utility script for Flask applications."
+ echo ""
+ echo "Run '${script_name} COMMAND --help' for more information on a command."
+}
+
if [[ "${#}" -eq 0 ]]; then
if [[ "${NOPAQUE_IS_PRIMARY_INSTANCE:-True}" == "True" ]]; then
while true; do
- echo "INFO Run deployment tasks..."
flask deploy
if [[ "${?}" == "0" ]]; then
break
fi
- echo "WARNING ...Failed, retrying in 5 secs..."
+ echo "Deploy command failed, retrying in 5 secs..."
sleep 5
done
- echo "INFO Start nopaque daemon..."
- flask daemon run &
fi
- echo "INFO Start nopaque..."
python nopaque.py
elif [[ "${1}" == "flask" ]]; then
- exec ${@:1}
+ flask "${@:2}"
+elif [[ "${1}" == "--help" || "${1}" == "-h" ]]; then
+ display_help
else
- echo "${0} [COMMAND]"
- echo ""
- echo "nopaque startup script"
- echo ""
- echo "Commands:"
- echo " flask"
+ display_help
+ exit 1
fi
diff --git a/config.py b/config.py
index 773f808990c2441a1f8e234a5b0b3ce798568046..4eba99d2399285f4d05e5658554c0fd7940bd675 100644
--- a/config.py
+++ b/config.py
@@ -11,6 +11,13 @@ load_dotenv(os.path.join(basedir, '.env'))
class Config:
+ ''' APIFairy '''
+ APIFAIRY_TITLE = 'nopaque'
+ APIFAIRY_VERSION = '0.0.1'
+ APIFAIRY_UI = 'swagger_ui'
+ APIFAIRY_APISPEC_PATH = '/api/apispec.json'
+ APIFAIRY_UI_PATH = '/api'
+
''' # Flask # '''
PREFERRED_URL_SCHEME = os.environ.get('PREFERRED_URL_SCHEME', 'http')
SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')
@@ -18,6 +25,9 @@ class Config:
SESSION_COOKIE_SECURE = \
os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true'
+ ''' # Flask-APScheduler # '''
+ JOBS = []
+
''' # Flask-Assets '''
ASSETS_DEBUG = os.environ.get('ASSETS_DEBUG', 'false').lower() == 'true'
@@ -55,6 +65,9 @@ class Config:
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
+ NOPAQUE_JOB_EXPIRATION_ENABLED = os.environ.get('NOPAQUE_JOB_EXPIRATION_ENABLED', 'true').lower() == 'true'
+ NOPAQUE_JOB_EXPIRATION_TIME = int(os.environ.get('NOPAQUE_JOB_EXPIRATION_TIME', '120'))
+
NOPAQUE_DOCKER_REGISTRY = 'gitlab.ub.uni-bielefeld.de:4567'
NOPAQUE_DOCKER_IMAGE_PREFIX = f'{NOPAQUE_DOCKER_REGISTRY}/sfb1288inf/'
NOPAQUE_DOCKER_REGISTRY_USERNAME = \
@@ -112,6 +125,16 @@ class Config:
fmt=app.config['NOPAQUE_LOG_FORMAT'],
datefmt=app.config['NOPAQUE_LOG_DATE_FORMAT']
)
+ if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
+ app.config['JOBS'].append(
+ {
+ "id": "daemon",
+ "func": "app.daemon:daemon",
+ "args": (app,),
+ "trigger": "interval",
+ "seconds": 3,
+ }
+ )
if app.config['NOPAQUE_LOG_STDERR_ENABLED']:
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
diff --git a/docker-compose.development.yml b/docker-compose/docker-compose.development.yml
similarity index 100%
rename from docker-compose.development.yml
rename to docker-compose/docker-compose.development.yml
diff --git a/docker-compose.scale.yml b/docker-compose/docker-compose.scale.yml
similarity index 100%
rename from docker-compose.scale.yml
rename to docker-compose/docker-compose.scale.yml
diff --git a/docker-compose.traefik.yml b/docker-compose/docker-compose.traefik.yml
similarity index 100%
rename from docker-compose.traefik.yml
rename to docker-compose/docker-compose.traefik.yml
diff --git a/migrations/versions/116b4ab3ef9c_.py b/migrations/versions/116b4ab3ef9c_.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8a29e5769ea77453810e6fcfb2ab8a1ba9473c8
--- /dev/null
+++ b/migrations/versions/116b4ab3ef9c_.py
@@ -0,0 +1,41 @@
+"""Add API authentication token table
+
+Revision ID: 116b4ab3ef9c
+Revises: f9070ff1fa4a
+Create Date: 2022-09-02 11:12:01.995451
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '116b4ab3ef9c'
+down_revision = 'f9070ff1fa4a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('tokens',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('user_id', sa.Integer(), nullable=True),
+ sa.Column('access_token', sa.String(length=64), nullable=True),
+ sa.Column('access_expiration', sa.DateTime(), nullable=True),
+ sa.Column('refresh_token', sa.String(length=64), nullable=True),
+ sa.Column('refresh_expiration', sa.DateTime(), nullable=True),
+ sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.create_index(op.f('ix_tokens_access_token'), 'tokens', ['access_token'], unique=False)
+ op.create_index(op.f('ix_tokens_refresh_token'), 'tokens', ['refresh_token'], unique=False)
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_index(op.f('ix_tokens_refresh_token'), table_name='tokens')
+ op.drop_index(op.f('ix_tokens_access_token'), table_name='tokens')
+ op.drop_table('tokens')
+ # ### end Alembic commands ###
diff --git a/migrations/versions/9e8d7d15d950_.py b/migrations/versions/9e8d7d15d950_.py
index b76a490ea3ec0aedf252f49b78976a5cac4e263d..9d59da3994c38b32d2bfdb327ba275f7129258aa 100644
--- a/migrations/versions/9e8d7d15d950_.py
+++ b/migrations/versions/9e8d7d15d950_.py
@@ -1,4 +1,4 @@
-"""empty message
+"""Initial database setup
Revision ID: 9e8d7d15d950
Revises:
diff --git a/migrations/versions/f9070ff1fa4a_.py b/migrations/versions/f9070ff1fa4a_.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0cfb00f4373387afcdf4e10ccf2929e2d67d07d
--- /dev/null
+++ b/migrations/versions/f9070ff1fa4a_.py
@@ -0,0 +1,31 @@
+"""Remove token entries for rudimentary API authentication mechanism
+
+Revision ID: f9070ff1fa4a
+Revises: 9e8d7d15d950
+Create Date: 2022-09-01 13:46:47.425268
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'f9070ff1fa4a'
+down_revision = '9e8d7d15d950'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_index('ix_users_token', table_name='users')
+ op.drop_column('users', 'token')
+ op.drop_column('users', 'token_expiration')
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('users', sa.Column('token_expiration', sa.DateTime(), autoincrement=False, nullable=True))
+ op.add_column('users', sa.Column('token', sa.VARCHAR(length=32), autoincrement=False, nullable=True))
+ op.create_index('ix_users_token', 'users', ['token'], unique=False)
+ # ### end Alembic commands ###
diff --git a/nopaque.py b/nopaque.py
index 8eb4d3bba8fd4f10d22456e6ae588a7cdaf5c14b..96746d9750a00648055820c87b5b0321bb4c231f 100644
--- a/nopaque.py
+++ b/nopaque.py
@@ -3,7 +3,7 @@
import eventlet
eventlet.monkey_patch()
-from app import db, cli, create_app, socketio # noqa
+from app import cli, create_app, db, scheduler, socketio # noqa
from app.models import (
Corpus,
CorpusFile,
@@ -49,8 +49,14 @@ def make_shell_context() -> Dict[str, Any]:
def main():
+ with app.app_context():
+ if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
+ for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0).all():
+ corpus.num_analysis_sessions = 0
+ db.session.commit()
+ scheduler.start()
socketio.run(app, host='0.0.0.0')
if __name__ == '__main__':
- main()
\ No newline at end of file
+ main()
diff --git a/requirements.txt b/requirements.txt
index dd42f09bf4c4a8ee05dd58e078be2e25bdb0f7b2..40206acb939f8a2b11b1ced84d8e6d8eee3e1045 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,8 @@
+apifairy
cqi
docker
eventlet
-Flask==1.1.4
+Flask==2.1.3
Flask-APScheduler
Flask-Assets
Flask-Hashids
@@ -10,87 +11,17 @@ Flask-Login
Flask-Mail
Flask-Migrate
Flask-Paranoid
-Flask-RESTX
Flask-SocketIO
Flask-SQLAlchemy
Flask-WTF
hiredis
-jsonschema
MarkupSafe==2.0.1
+marshmallow-sqlalchemy
psycopg2
+PyJWT
pyScss
python-dotenv
pyyaml
redis
tqdm
wtforms[email]
-
-
-# Safe to use from 05.05.2022
-# alembic==1.7.7
-# aniso8601==9.0.1
-# APScheduler==3.9.1
-# async-timeout==4.0.2
-# attrs==21.4.0
-# backports.zoneinfo==0.2.1
-# bidict==0.22.0
-# blinker==1.4
-# certifi==2021.10.8
-# charset-normalizer==2.0.12
-# click==7.1.2
-# cqi==0.1.0
-# Deprecated==1.2.13
-# dnspython==2.2.1
-# docker==5.0.3
-# email-validator==1.2.1
-# eventlet==0.33.0
-# Flask==1.1.4
-# Flask-APScheduler==1.12.3
-# Flask-Assets==2.0
-# Flask-Hashids==0.2.0
-# Flask-HTTPAuth==4.6.0
-# Flask-Login==0.6.1
-# Flask-Mail==0.9.1
-# Flask-Migrate==3.1.0
-# Flask-Paranoid==0.3.0
-# flask-restx==0.5.1
-# Flask-SocketIO==5.1.2
-# Flask-SQLAlchemy==2.5.1
-# Flask-WTF==1.0.1
-# greenlet==1.1.2
-# hashids==1.3.1
-# hiredis==2.0.0
-# idna==3.3
-# importlib-metadata==4.11.3
-# importlib-resources==5.7.1
-# itsdangerous==1.1.0
-# Jinja2==2.11.3
-# jsonschema==4.4.0
-# Mako==1.2.0
-# MarkupSafe==2.0.1
-# packaging==21.3
-# psycopg2==2.9.3
-# pyparsing==3.0.8
-# pyrsistent==0.18.1
-# pyScss==1.4.0
-# python-dateutil==2.8.2
-# python-dotenv==0.20.0
-# python-engineio==4.3.2
-# python-socketio==5.6.0
-# pytz==2022.1
-# pytz-deprecation-shim==0.1.0.post0
-# PyYAML==6.0
-# redis==4.2.2
-# requests==2.27.1
-# six==1.16.0
-# SQLAlchemy==1.4.36
-# tqdm==4.64.0
-# tzdata==2022.1
-# tzlocal==4.2
-# urllib3==1.26.9
-# webassets==2.0
-# websocket-client==1.3.2
-# Werkzeug==1.0.1
-# wrapt==1.14.1
-# WTForms==3.0.1
-# zipp==3.8.0
\ No newline at end of file