diff --git a/web/app/models.py b/web/app/models.py
index fdade759826643d5a751ffcc105324954bf90e19..6cd94698e24f9da3e40a0a17e65b1b9894db920d 100644
--- a/web/app/models.py
+++ b/web/app/models.py
@@ -362,8 +362,6 @@ class Job(db.Model):
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
description = db.Column(db.String(255))
end_date = db.Column(db.DateTime())
- mem_mb = db.Column(db.Integer)
- n_cores = db.Column(db.Integer)
service = db.Column(db.String(64))
'''
' Service specific arguments as string list.
diff --git a/web/app/services/__init__.py b/web/app/services/__init__.py
index 03836993d52645f745fe5d5d4d9e19ba3f77a7d1..f374b18bd57b87786639c291ec03244e418c7502 100644
--- a/web/app/services/__init__.py
+++ b/web/app/services/__init__.py
@@ -25,7 +25,14 @@ SERVICES = {
'latest': '1.0.0',
'1.0.0': {
'check_encoding': True,
- 'models': {},
+ 'models': {
+ 'de': 'German',
+ 'en': 'English',
+ 'it': 'Italian',
+ 'nl': 'Dutch',
+ 'pl': 'Polish',
+ 'zh': 'Chinese'
+ },
'publishing_data': {
'date': None,
'title': 'nopaque NLP service',
@@ -42,14 +49,19 @@ SERVICES = {
'1.0.0': {
'binarization': True,
'models': {
+ 'ara': 'Arabic',
+ 'chi_tra': 'Chinese - Traditional',
+ 'dan': 'Danish',
'eng': 'English',
'enm': 'English, Middle 1100-1500',
'fra': 'French',
'frm': 'French, Middle ca. 1400-1600',
'deu': 'German',
'frk': 'German Fraktur',
+ 'ell': 'Greek, Modern (1453-)',
'ita': 'Italian',
'por': 'Portuguese',
+ 'rus': 'Russian',
'spa': 'Spanish; Castilian',
},
'publishing_data': {
diff --git a/web/app/services/forms.py b/web/app/services/forms.py
index cea741de756faea4c6275dd33e4ea8f0eb35b341..5d9f5ede30432dbb918026a06e6cc3bb40f20235 100644
--- a/web/app/services/forms.py
+++ b/web/app/services/forms.py
@@ -10,16 +10,18 @@ class AddJobForm(FlaskForm):
validators=[DataRequired(), Length(1, 255)])
submit = SubmitField()
title = StringField('Title', validators=[DataRequired(), Length(1, 32)])
+ version = SelectField('Version', validators=[DataRequired()])
class AddNLPJobForm(AddJobForm):
- files = MultipleFileField('Files', validators=[DataRequired()])
- model = SelectField('Model', validators=[DataRequired()])
- version = SelectField('Version',
- choices=[(x, x) for x in SERVICES['nlp']['versions'] if x != 'latest'], # noqa
- default=SERVICES['nlp']['versions']['latest'],
- validators=[DataRequired()])
check_encoding = BooleanField('Check encoding')
+ files = MultipleFileField('Files', validators=[DataRequired()])
+ language = SelectField('Language', choices=[('', 'Choose your option')],
+ default='', validators=[DataRequired()])
+
+ def validate_check_encoding(self, field):
+ if field.data and 'check_encoding' not in SERVICES['nlp']['versions'][self.version.data]: # noqa
+ raise ValidationError('Check encoding is not available in this version') # noqa
def validate_files(form, field):
for file in field.data:
@@ -27,37 +29,44 @@ class AddNLPJobForm(AddJobForm):
raise ValidationError('File does not have an approved '
'extension: .txt')
+ def __init__(self, *args, **kwargs):
+ version = kwargs.pop('version', SERVICES['nlp']['versions']['latest'])
+ super().__init__(*args, **kwargs)
+ if 'check_encoding' not in SERVICES['nlp']['versions'][version]:
+ self.check_encoding.render_kw = {'disabled': True}
+ self.language.choices += [(x, y) for x, y in SERVICES['nlp']['versions'][version]['models'].items()] # noqa
+ self.version.choices = [(x, x) for x in SERVICES['nlp']['versions'] if x != 'latest'] # noqa
+ self.version.default = version
+
class AddOCRJobForm(AddJobForm):
binarization = BooleanField('Binarazation')
files = MultipleFileField('Files', validators=[DataRequired()])
- model = SelectField('Model', validators=[DataRequired()])
- version = SelectField('Version',
- choices=[(x, x) for x in SERVICES['ocr']['versions'] if x != 'latest'], # noqa
- default=SERVICES['ocr']['versions']['latest'],
- validators=[DataRequired()])
-
- def validate_binarization(form, field):
- if field.data and 'binarization' not in SERVICES['ocr'][form.version.data]: # noqa
+ language = SelectField('Language', choices=[('', 'Choose your option')],
+ default='', validators=[DataRequired()])
+
+ def validate_binarization(self, field):
+ if field.data and 'binarization' not in SERVICES['ocr']['versions'][self.version.data]: # noqa
raise ValidationError('Binarization is not available in this version') # noqa
- def validate_files(form, field):
+ def validate_files(self, field):
for file in field.data:
if not file.filename.lower().endswith('.pdf'):
raise ValidationError('File does not have an approved '
'extension: .pdf')
- def validate_model(form, field):
- if field.data not in SERVICES['ocr'][form.versiondata]['models']:
- raise ValidationError('Model is not available in this version')
+ def __init__(self, *args, **kwargs):
+ version = kwargs.pop('version', SERVICES['ocr']['versions']['latest'])
+ super().__init__(*args, **kwargs)
+ if 'binarization' not in SERVICES['ocr']['versions'][version]:
+ self.binarization.render_kw = {'disabled': True}
+ self.language.choices += [(x, y) for x, y in SERVICES['ocr']['versions'][version]['models'].items()] # noqa
+ self.version.choices = [(x, x) for x in SERVICES['ocr']['versions'] if x != 'latest'] # noqa
+ self.version.default = version
class AddFileSetupJobForm(AddJobForm):
files = MultipleFileField('Files', validators=[DataRequired()])
- version = SelectField('Version',
- choices=[(x, x) for x in SERVICES['file-setup']['versions'] if x != 'latest'], # noqa
- default=SERVICES['file-setup']['versions']['latest'],
- validators=[DataRequired()])
def validate_files(form, field):
for file in field.data:
@@ -66,3 +75,9 @@ class AddFileSetupJobForm(AddJobForm):
raise ValidationError('File does not have an approved '
'extension: .jpeg | .jpg | .png | .tiff '
'| .tif')
+
+ def __init__(self, *args, **kwargs):
+ version = kwargs.pop('version', SERVICES['file-setup']['versions']['latest'])
+ super().__init__(*args, **kwargs)
+ self.version.choices = [(x, x) for x in SERVICES['file-setup']['versions'] if x != 'latest'] # noqa
+ self.version.default = version
diff --git a/web/app/services/views.py b/web/app/services/views.py
index 4436d1ff49b70ba0d2dff2aa28c1933ac8ff504d..4aaac006d041250fb7f46e3e293098afeea78b2b 100644
--- a/web/app/services/views.py
+++ b/web/app/services/views.py
@@ -23,19 +23,30 @@ def service(service):
elif service == 'file-setup':
form = AddFileSetupJobForm(prefix='add-file-setup-job-form')
elif service == 'nlp':
- form = AddNLPJobForm(prefix='add-nlp-job-form')
+ version = request.args.get('version')
+ if version is None or version not in SERVICES['nlp']['versions']:
+ form = AddNLPJobForm(prefix='add-nlp-job-form')
+ else:
+ form = AddNLPJobForm(prefix='add-nlp-job-form', version=version)
+ form.version.data = version
elif service == 'ocr':
- form = AddOCRJobForm(prefix='add-ocr-job-form')
+ version = request.args.get('version')
+ if version is None or version not in SERVICES['ocr']['versions']:
+ form = AddOCRJobForm(prefix='add-ocr-job-form')
+ else:
+ form = AddOCRJobForm(prefix='add-ocr-job-form', version=version)
+ form.version.data = version
if form.is_submitted():
if not form.validate():
+ logging.error(form.errors)
return make_response(form.errors, 400)
service_args = []
if service == 'nlp':
- service_args.append('-l {}'.format(form.model.data))
+ service_args.append('-l {}'.format(form.language.data))
if form.check_encoding.data:
service_args.append('--check-encoding')
if service == 'ocr':
- service_args.append('-l {}'.format(form.model.data))
+ service_args.append('-l {}'.format(form.language.data))
if form.binarization.data:
service_args.append('--binarize')
job = Job(creator=current_user,
@@ -70,4 +81,5 @@ def service(service):
return make_response(
{'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
return render_template('services/{}.html.j2'.format(service),
- form=form, title=SERVICES[service]['name'])
+ form=form, title=SERVICES[service]['name'],
+ versions=SERVICES[service]['versions'])
diff --git a/web/app/tasks/job_utils.py b/web/app/tasks/job_utils.py
index 9baadde79b15ff267756f5b7da676920dcec503a..cdd0160de09187d352c4fc9e94e07adff45a7b2f 100644
--- a/web/app/tasks/job_utils.py
+++ b/web/app/tasks/job_utils.py
@@ -9,13 +9,31 @@ import json
import os
+service_settings = {
+ 'file-setup': {
+ 'ressources': docker.types.Resources(mem_reservation=1024 * (10 ** 6),
+ cpu_reservation=1 * (10 ** 9))
+ },
+ 'nlp': {
+ 'default_args': ' --n-cores 2 --mem-mb 2048',
+ 'ressources': docker.types.Resources(mem_reservation=2048 * (10 ** 6),
+ cpu_reservation=2 * (10 ** 9))
+ },
+ 'ocr': {
+ 'default_args': ' --n-cores 4 --mem-mb 4096',
+ 'ressources': docker.types.Resources(mem_reservation=4096 * (10 ** 6),
+ cpu_reservation=4 * (10 ** 9))
+ }
+}
+
+
class CheckJobsMixin:
def check_jobs(self):
jobs = Job.query.all()
- canceling_jobs = list(filter(lambda job: job.status == 'canceling', jobs))
+ canceling_jobs = list(filter(lambda job: job.status == 'canceling', jobs)) # noqa
queued_jobs = list(filter(lambda job: job.status == 'queued', jobs))
running_jobs = list(filter(lambda job: job.status == 'running', jobs))
- submitted_jobs = list(filter(lambda job: job.status == 'submitted', jobs))
+ submitted_jobs = list(filter(lambda job: job.status == 'submitted', jobs)) # noqa
for job in submitted_jobs:
self.create_job_service(job)
for job in queued_jobs + running_jobs:
@@ -25,26 +43,24 @@ class CheckJobsMixin:
def create_job_service(self, job):
cmd = '{} -i /files -o /files/output'.format(job.service)
+ if 'default_args' in service_settings[job.service]:
+ cmd += service_settings[job.service]['default_args']
if job.service == 'file-setup':
cmd += ' -f {}'.format(secure_filename(job.title))
- ressources = docker.types.Resources(cpu_reservation=4 * (10 ** 9), mem_reservation=4096 * (10 ** 6)) # noqa
- elif job.service == 'nlp':
- ressources = docker.types.Resources(cpu_reservation=2 * (10 ** 9), mem_reservation=4096 * (10 ** 6)) # noqa
- elif job.service == 'ocr':
- ressources = docker.types.Resources(cpu_reservation=4 * (10 ** 9), mem_reservation=8192 * (10 ** 6)) # noqa
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title))
cmd += ' ' + ' '.join(json.loads(job.service_args))
+ ressources = service_settings[job.service]['ressources']
service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
- 'type': 'service.{}'.format(job.service),
+ 'type': 'job',
'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': ressources,
'restart_policy': docker.types.RestartPolicy()}
- service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(job.service, job.service_version) # noqa
+ service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:latest'.format(job.service) # noqa
try:
self.docker.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
diff --git a/web/app/templates/main/news.html.j2 b/web/app/templates/main/news.html.j2
index 7b90f497f71e18c943c8ce737d71df5d466ce22c..6e00ed09c52c2b305a4cc3c5f58b3fe03d677970 100644
--- a/web/app/templates/main/news.html.j2
+++ b/web/app/templates/main/news.html.j2
@@ -11,6 +11,17 @@
<h1 id="title">{{ title }}</h1>
</div>
+ <div class="col s12">
+ <div class="card" id="nlp-removed-language-support">
+ <div class="card-content">
+ <span class="card-title">Natural Language Processing removed language support</span>
+ <p>Dear users</p>
+ <br>
+ <p>Not all language models support all features we utizlize in our NLP service. Thats why we had to drop them, as soon as they meet our requirements we will add them back!</p>
+ </div>
+ </div>
+ </div>
+
<div class="col s12">
<div class="card" id="beta-launch">
<div class="card-content">
diff --git a/web/app/templates/services/file-setup.html.j2 b/web/app/templates/services/file-setup.html.j2
index b39fec385dc0842e36cfdfab19d587c93b0e2c38..ffb0efb015b15eb0aa3ca568403390397221d50d 100644
--- a/web/app/templates/services/file-setup.html.j2
+++ b/web/app/templates/services/file-setup.html.j2
@@ -56,10 +56,10 @@
<div class="col s12 l8">
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
- <div class="col s12">
+ <div class="col s12 l9">
{{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
</div>
- <div class="col s12 hide">
+ <div class="col s12 l3">
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
</div>
diff --git a/web/app/templates/services/nlp.html.j2 b/web/app/templates/services/nlp.html.j2
index 4c5018bcc59ee13f091234468b46b2c7a65a588b..6e805f3ee9bbcd9947e6faf79cfa77742402c919 100644
--- a/web/app/templates/services/nlp.html.j2
+++ b/web/app/templates/services/nlp.html.j2
@@ -23,7 +23,7 @@
<p class="hide-on-small-only"> </p>
<p class="hide-on-small-only"> </p>
<a class="btn-floating btn-large waves-effect waves-light" style="transform: scale(2);">
- <i class="material-icons service" data-service="nlp"></i>
+ <i class="nopaque-icons service" data-service="nlp"></i>
</a>
</div>
</div>
@@ -127,3 +127,16 @@
</div>
</div>
{% endblock %}
+
+
+{% block scripts %}
+{{ super() }}
+<script>
+ var versionField = document.querySelector('#add-nlp-job-form-version');
+ versionField.addEventListener('change', (event) => {
+ let url = new URL(window.location.href);
+ url.search = `?version=${event.target.value}`;
+ window.location.href = url.toString();
+ });
+</script>
+{% endblock %}
diff --git a/web/app/templates/services/ocr.html.j2 b/web/app/templates/services/ocr.html.j2
index 5cb520bf596766028b561f63cb9539bcae4973bc..af8205a045f646c6f85ec26d80d2822f7c52c158 100644
--- a/web/app/templates/services/ocr.html.j2
+++ b/web/app/templates/services/ocr.html.j2
@@ -23,7 +23,7 @@
<p class="hide-on-small-only"> </p>
<p class="hide-on-small-only"> </p>
<a class="btn-floating btn-large waves-effect waves-light" style="transform: scale(2);">
- <i class="material-icons service" data-service="ocr"></i>
+ <i class="nopaque-icons service" data-service="ocr"></i>
</a>
</div>
</div>
@@ -60,7 +60,7 @@
{{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
</div>
<div class="col s12 l4">
- {{ wtf.render_field(form.model, material_icon='language') }}
+ {{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
{{ wtf.render_field(form.version, material_icon='apps') }}
@@ -154,3 +154,16 @@
</div>
</div>
{% endblock %}
+
+
+{% block scripts %}
+{{ super() }}
+<script>
+ var versionField = document.querySelector('#add-ocr-job-form-version');
+ versionField.addEventListener('change', (event) => {
+ let url = new URL(window.location.href);
+ url.search = `?version=${event.target.value}`;
+ window.location.href = url.toString();
+ });
+</script>
+{% endblock %}