Skip to content
Snippets Groups Projects
Commit e1004a01 authored by Patrick Jentsch's avatar Patrick Jentsch
Browse files

First attempts to use type hinting

parent 9d30dda7
No related branches found
No related tags found
No related merge requests found
...@@ -16,22 +16,14 @@ The generated computational workload is handled by a [Docker](https://docs.docke ...@@ -16,22 +16,14 @@ The generated computational workload is handled by a [Docker](https://docs.docke
### **Create network storage** ### **Create network storage**
A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [samba](https://www.samba.org/) share is used. A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [samba](https://www.samba.org/) share can be used.
``` bash
# Example: Create a Samba share via Docker You can create a samba share by using [this](https://hub.docker.com/r/dperson/samba/) Docker image.
# More details can be found under https://hub.docker.com/r/dperson/samba/
username@hostname:~$ sudo mkdir -p /srv/samba/nopaque
username@hostname:~$ docker run \
--name opaque_storage \
-v /srv/samba/nopaque:/srv/samba/nopaque \
-p 139:139 \
-p 445:445 \
dperson/samba \
-p -r -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
``` bash
# Mount the Samba share on all swarm nodes (managers and workers) # Mount the Samba share on all swarm nodes (managers and workers)
username@hostname:~$ sudo mkdir /mnt/nopaque username@hostname:~$ sudo mkdir /mnt/nopaque
username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 //<SAMBA-SERVER-IP>/nopaque /mnt/nopaque username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 //<SAMBA-SERVER-IP>/<SAMBA-SHARE-NAME> /mnt/nopaque
``` ```
### **Download, configure and build nopaque** ### **Download, configure and build nopaque**
...@@ -39,8 +31,8 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa ...@@ -39,8 +31,8 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
``` bash ``` bash
# Clone the nopaque repository # Clone the nopaque repository
username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
# Create data directories for the database and message queue # Create data directories
username@hostname:~$ mkdir data/{db,mq} username@hostname:~$ mkdir data/{db,logs,mq}
username@hostname:~$ cp db.env.tpl db.env username@hostname:~$ cp db.env.tpl db.env
username@hostname:~$ cp .env.tpl .env username@hostname:~$ cp .env.tpl .env
# Fill out the variables within these files. # Fill out the variables within these files.
...@@ -56,8 +48,6 @@ username@hostname:~$ docker-compose build ...@@ -56,8 +48,6 @@ username@hostname:~$ docker-compose build
### Start your instance ### Start your instance
``` bash ``` bash
# Create log files
touch nopaque.log nopaqued.log
# For background execution add the -d flag # For background execution add the -d flag
username@hostname:~$ docker-compose up username@hostname:~$ docker-compose up
# To scale your app use the following command after starting it normally # To scale your app use the following command after starting it normally
...@@ -65,5 +55,7 @@ username@hostname:~$ docker-compose -f docker-compose.yml \ ...@@ -65,5 +55,7 @@ username@hostname:~$ docker-compose -f docker-compose.yml \
-f docker-compose.override.yml -f docker-compose.override.yml
-f docker-compose.scale.yml -f docker-compose.scale.yml
up up
-d --no-recreate --scale nopaque=<NUM_INSTANCES> -d
--no-recreate
--scale nopaque=<NUM_INSTANCES>
``` ```
...@@ -10,21 +10,23 @@ from hashids import Hashids ...@@ -10,21 +10,23 @@ from hashids import Hashids
import flask_assets import flask_assets
assets = flask_assets.Environment() assets: flask_assets.Environment = flask_assets.Environment()
db = SQLAlchemy() db: SQLAlchemy = SQLAlchemy()
hashids = Hashids(min_length=32) # , salt=current_app.config.get('SECRET_KEY') # TODO: Add 'SECRET_KEY' from as 'salt' kwarg
login = LoginManager() hashids: Hashids = Hashids(min_length=32)
login.login_view = 'auth.login' login: LoginManager = LoginManager()
login.login_message = 'Please log in to access this page.' login.login_view: str = 'auth.login'
mail = Mail() login.login_message: str = 'Please log in to access this page.'
migrate = Migrate() mail: Mail = Mail()
paranoid = Paranoid() migrate: Migrate = Migrate()
paranoid.redirect_view = '/' paranoid: Paranoid = Paranoid()
socketio = SocketIO() paranoid.redirect_view: str = '/'
socketio: SocketIO = SocketIO()
def create_app(config_class=Config):
app = Flask(__name__) def create_app(config_class: Config = Config) -> Flask:
''' Creates an initialized Flask (WSGI Application) object. '''
app: Flask = Flask(__name__)
app.config.from_object(config_class) app.config.from_object(config_class)
assets.init_app(app) assets.init_app(app)
...@@ -35,13 +37,10 @@ def create_app(config_class=Config): ...@@ -35,13 +37,10 @@ def create_app(config_class=Config):
migrate.init_app(app, db) migrate.init_app(app, db)
paranoid.init_app(app) paranoid.init_app(app)
socketio.init_app( socketio.init_app(
app, app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
message_queue=app.config.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
)
from .utils import HashidConverter, permission_context_processor from .utils import HashidConverter
app.url_map.converters['hashid'] = HashidConverter app.url_map.converters['hashid'] = HashidConverter
app.context_processor(permission_context_processor)
from .events import socketio as socketio_events from .events import socketio as socketio_events
from .events import sqlalchemy as sqlalchemy_events from .events import sqlalchemy as sqlalchemy_events
......
...@@ -6,7 +6,7 @@ from flask_migrate import upgrade ...@@ -6,7 +6,7 @@ from flask_migrate import upgrade
def register(app): def register(app):
@app.cli.command() @app.cli.command()
def deploy(): def deploy():
"""Run deployment tasks.""" ''' Run deployment tasks. '''
# migrate database to latest revision # migrate database to latest revision
upgrade() upgrade()
# create or update user roles # create or update user roles
...@@ -14,27 +14,29 @@ def register(app): ...@@ -14,27 +14,29 @@ def register(app):
@app.cli.group() @app.cli.group()
def daemon(): def daemon():
"""Daemon commands.""" ''' Daemon commands. '''
pass pass
@daemon.command('run') @daemon.command('run')
def run_daemon(): def run_daemon():
"""Run daemon""" ''' Run daemon '''
corpus: Corpus
for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0): for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0):
corpus.num_analysis_sessions = 0 corpus.num_analysis_sessions = 0
db.session.commit() db.session.commit()
from app.daemon import Daemon from app.daemon import Daemon
daemon = Daemon() daemon: Daemon = Daemon()
daemon.run() daemon.run()
@app.cli.group() @app.cli.group()
def test(): def test():
"""Test commands.""" ''' Test commands. '''
pass pass
@test.command('run') @test.command('run')
def run_test(): def run_test():
"""Run unit tests.""" ''' Run unit tests. '''
import unittest from unittest import TestLoader, TextTestRunner
tests = unittest.TestLoader().discover('tests') from unittest.suite import TestSuite
unittest.TextTestRunner(verbosity=2).run(tests) tests: TestSuite = TestLoader().discover('tests')
TextTestRunner(verbosity=2).run(tests)
from flask import current_app, render_template from flask import current_app, render_template
from flask_mail import Message from flask_mail import Message
from typing import Any, Text
from . import mail from . import mail
from .decorators import background from .decorators import background
def create_message(recipient, subject, template, **kwargs): def create_message(
msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa recipient: str,
msg.body = render_template('{}.txt.j2'.format(template), **kwargs) subject: str,
msg.html = render_template('{}.html.j2'.format(template), **kwargs) template: str,
**kwargs: Any
) -> Message:
subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
msg: Message = Message(
f'{subject_prefix} {subject}', recipients=[recipient])
msg.body: Text = render_template(f'{template}.txt.j2', **kwargs)
msg.html: Text = render_template(f'{template}.html.j2', **kwargs)
return msg return msg
@background @background
def send(msg, *args, **kwargs): def send(msg: Message, *args, **kwargs):
with kwargs['app'].app_context(): with kwargs['app'].app_context():
mail.send(msg) mail.send(msg)
from app import hashids from app import hashids
from werkzeug.routing import BaseConverter from werkzeug.routing import BaseConverter
from .models import Permission
class HashidConverter(BaseConverter): class HashidConverter(BaseConverter):
def to_python(self, value): def to_python(self, value: str) -> int:
return hashids.decode(value)[0] return hashids.decode(value)[0]
def to_url(self, value): def to_url(self, value: int) -> str:
return hashids.encode(value) return hashids.encode(value)
def permission_context_processor():
return {'Permission': Permission}
...@@ -6,21 +6,33 @@ eventlet.monkey_patch() ...@@ -6,21 +6,33 @@ eventlet.monkey_patch()
from app import db, cli, create_app # noqa from app import db, cli, create_app # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult, from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
QueryResult, Role, User) # noqa Permission, QueryResult, Role, User) # noqa
from flask import Flask # noqa
from typing import Any, Dict # noqa
app = create_app() app: Flask = create_app()
cli.register(app) cli.register(app)
@app.context_processor
def make_context() -> Dict[str, Any]:
''' Adds variables to the template context. '''
return {'Permission': Permission}
@app.shell_context_processor @app.shell_context_processor
def make_shell_context(): def make_shell_context() -> Dict[str, Any]:
return {'Corpus': Corpus, ''' Adds variables to the shell context. '''
'CorpusFile': CorpusFile, return {
'db': db, 'Corpus': Corpus,
'Job': Job, 'CorpusFile': CorpusFile,
'JobInput': JobInput, 'db': db,
'JobResult': JobResult, 'Job': Job,
'QueryResult': QueryResult, 'JobInput': JobInput,
'Role': Role, 'JobResult': JobResult,
'User': User} 'Permission': Permission,
'QueryResult': QueryResult,
'Role': Role,
'User': User
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment