Add Docker Swarm interface.

app/__init__.py

@@ -3,6 +3,7 @@ from flask import Flask
 from flask_login import LoginManager
 from flask_mail import Mail
 from flask_sqlalchemy import SQLAlchemy
+from .swarm import Swarm
 
 
 db = SQLAlchemy()
@@ -11,6 +12,7 @@ login_manager = LoginManager()
 login_manager.login_view = 'auth.login'
 
 mail = Mail()
+swarm = Swarm()
 
 
 def create_app(config_name):
@@ -21,6 +23,8 @@ def create_app(config_name):
     db.init_app(app)
     login_manager.init_app(app)
     mail.init_app(app)
+    if not hasattr(app, 'extensions'):
+        app.extensions = {}
 
     from .auth import auth as auth_blueprint
     app.register_blueprint(auth_blueprint, url_prefix='/auth')

app/main/forms.py

+from flask_wtf import FlaskForm
+from wtforms import SubmitField
+
+
+class SwarmForm(FlaskForm):
+    submit = SubmitField('Submit')

app/main/views.py

-from flask import render_template
+from flask import redirect, render_template, url_for
 from ..models import User
 from ..tables import AdminUserTable, AdminUserItem
 from . import main
 from ..decorators import admin_required
-from flask_login import login_required
+from flask_login import current_user, login_required
+from .forms import SwarmForm
+from ..import swarm
+from threading import Thread
 
 
 @main.route('/')
@@ -16,7 +19,7 @@ def about():
     return render_template('main/about.html.j2', title='About')
 
 
-@main.route('/admin')
+@main.route('/admin', methods=['GET', 'POST'])
 @login_required
 @admin_required
 def for_admins_only():
@@ -26,5 +29,32 @@ def for_admins_only():
     users = User.query.order_by(User.username).all()
     items = [AdminUserItem(u.username, u.email, u.role_id, u.confirmed) for u in users]
     table = AdminUserTable(items)
+
+    swarm_form = SwarmForm()
+    if swarm_form.validate_on_submit():
+        '''
+        ' TODO: Implement a Job class. For now a dictionary representation is
+        '       enough.
+        '''
+        job = {
+            'creator': current_user.id,
+            'id': '5fd40cb0cadef3ab5676c4968fc3d748',
+            'requested_cpus': 2,
+            'requested_memory': 2048,
+            'service': 'ocr',
+            'service_args': {
+                'lang': 'eng'
+            },
+            'status': 'queued'
+        }
+        '''
+        ' TODO: Let the scheduler run this job in the background. Using self
+        '       created threads is just for testing purpose as there is no
+        '       scheduler available.
+        '''
+        thread = Thread(target=swarm.run, args=(job,))
+        thread.start()
+        return redirect(url_for('main.for_admins_only'))
+
     return render_template('main/admin.html.j2', title='Administration tools',
-                           table=table.__html__())
+                           swarm_form=swarm_form, table=table.__html__())

app/swarm.py

+import docker
+import subprocess
+
+
+class Swarm:
+    def __init__(self):
+        self.docker = docker.from_env()
+        self.checkout()
+
+    def checkout(self):
+        cpus = 0
+        memory = 0
+        for node in self.docker.nodes.list(filters={'role': 'worker'}):
+            if node.attrs.get('Status').get('State') == 'ready':
+                cpus += 0 or node.attrs \
+                    .get('Description') \
+                    .get('Resources') \
+                    .get('NanoCPUs')
+                memory += 0 or node.attrs \
+                    .get('Description') \
+                    .get('Resources') \
+                    .get('MemoryBytes')
+        '''
+        ' For whatever reason the Python Docker SDK provides a CPU count in
+        ' nano (10^-6), whilst this is not that handy, it gets converted.
+        '''
+        cpus *= 10 ** -9
+        '''
+        ' For a more natural handling the memory information
+        ' gets converted from bytes to megabytes.
+        '''
+        memory *= 10 ** -6
+        self.cpus = int(cpus)
+        self.memory = int(memory)
+        self.available_cpus = self.cpus
+        self.available_memory = self.memory
+
+    def run(self, job):
+        if self.available_cpus < job['requested_cpus'] or \
+           self.available_memory < job['requested_memory']:
+            print('Not enough ressources available.')
+            '''
+            ' TODO: At this point the scheduler thinks that the job gets
+            '       processed, which apparently is not the case. So the job
+            '       needs to get rescheduled and gain a new chance to get
+            '       processed (next).
+            '
+            ' Note: Maybe it is a good idea to create a method that checks if
+            '       enough ressources are available before the run method gets
+            '       executed. This would replace the need of the TODO mentioned
+            '       above.
+            '''
+            return
+
+        job['status'] = 'running'
+        # TODO: Push job changes to the database
+        self.available_cpus -= job['requested_cpus']
+        self.available_memory -= job['requested_memory']
+
+        container_command = 'ocr' \
+                            + ' -i /input/{}'.format(job['id']) \
+                            + ' -l {}'.format(job['service_args']['lang']) \
+                            + ' -o /output' \
+                            + ' --keep-intermediates' \
+                            + ' --nCores {}'.format(job['requested_cpus'])
+        container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr'
+        container_mount = '/media/sf_files/=/input/'
+        '''
+        ' Swarm mode is intendet to run containers which are meant to serve a
+        ' non terminating service like a webserver. In order to process the
+        ' occuring jobs it is necessary to use one-shot (terminating)
+        ' containers. These one-shot containers are spawned with a programm
+        ' called JaaS¹ (Jobs as a Service), which is described in Alex Ellis'
+        ' short article "One-shot containers on Docker Swarm"².
+        '
+        ' ¹ https://github.com/alexellis/jaas
+        ' ² https://blog.alexellis.io/containers-on-swarm/
+        '''
+        cmd = ['jaas', 'run'] \
+            + ['--command', container_command] \
+            + ['--image', container_image] \
+            + ['--mount', container_mount] \
+            + ['--timeout', '86400s']
+        completed_process = subprocess.run(
+            cmd,
+            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL
+        )
+
+        self.available_cpus += job['requested_cpus']
+        self.available_memory += job['requested_memory']
+        if (completed_process.returncode == 0):
+            job['status'] = 'finished'
+        else:
+            job['status'] = 'failed'
+        # TODO: Push job changes to the database
+
+        return

app/templates/main/admin.html.j2

@@ -9,4 +9,16 @@
     </div>
   </div>
 </div>
+
+<div class="col s12">
+  <div class="card large">
+    <div class="card-content">
+      <span class="card-title">Swarm</span>
+      <form method="POST">
+        {{ swarm_form.hidden_tag() }}
+        {{ swarm_form.submit(class='btn') }}
+      </form>
+    </div>
+  </div>
+</div>
 {% endblock %}

requirements.txt

+docker==4.0.2
 Flask==1.0.3
 Flask-APScheduler==1.11.0
 Flask-Login==0.4.1