Add compression of result dictionary and size infos for other compression or no compression methods

app/corpora/events.py

@@ -7,6 +7,7 @@ from .CQiWrapper.CQiWrapper import CQiWrapper
 import sys
 import gzip
 import zlib
+import json
 
 '''
 ' A dictionary containing lists of, with corpus ids associated, Socket.IO
@@ -64,14 +65,18 @@ def recv_query(message):
     analysis_client.query_subcorpus(query)
     results = analysis_client.show_query_results(result_len=int(message['hits_per_page']),
                                                  context_len=int(message['context']))
+    # logger.warning('RESULTS: {}'.format(results))
     size_internal_dict = sys.getsizeof(results) / 1000000
     size_dict_to_str = sys.getsizeof(str(results)) / 1000000
     compressed_str = gzip.compress(str(results).encode())
     size_dict_to_str_compressed = sys.getsizeof(compressed_str) / 1000000
-    logger.warning('Size of dict for {} hits per page: {} MB'.format(message['hits_per_page'], size_internal_dict))
-    logger.warning('Size of dict to string for {} hits per page: {} MB'.format(message['hits_per_page'], size_dict_to_str))
-    logger.warning('Size of compressed dict to string for {} hits per page: {} MB'.format(message['hits_per_page'], size_dict_to_str_compressed))
-    socketio.emit('query', compressed_str, room=request.sid)
+    zlib_compressed = zlib.compress(json.dumps(results).encode('utf-8'))
+    size_zlib_compressed = sys.getsizeof(zlib_compressed) / 1000000
+    logger.warning('Internal size of dict for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_internal_dict))
+    logger.warning('Size of dict as raw string for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_dict_to_str))
+    logger.warning('Size of gzip compressed dict to string for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_dict_to_str_compressed))
+    logger.warning('Size of zlib compressed and utf-8 encoded string for {} hits per page and context len {}: {} MB'.format(message['hits_per_page'], message['context'], size_zlib_compressed))
+    socketio.emit('query', zlib_compressed, room=request.sid)
 
 
 def observe_corpus_analysis_connection(app, corpus_id, session_id):

app/templates/base.html.j2

@@ -19,6 +19,7 @@
     <script src="{{ url_for('static', filename='js/CorpusList.js') }}"></script>
     <script src="{{ url_for('static', filename='js/JobList.js') }}"></script>
     <script src="{{ url_for('static', filename='js/darkreader.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/pako.js') }}"></script>
     <script>
       {% if current_user.is_dark == True %}
       DarkReader.enable({