Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
nopaque
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Monitor
Service Desk
Analyze
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Admin message
Looking for advice? Join the
Matrix channel for GitLab users in Bielefeld
!
Show more breadcrumbs
SFB 1288 - INF
nopaque
Commits
08e7df4a
Commit
08e7df4a
authored
5 years ago
by
Stephan Porada
Browse files
Options
Downloads
Patches
Plain Diff
Add function to resend queries during running query
parent
6b241842
No related branches found
No related tags found
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
app/corpora/events.py
+65
-16
65 additions, 16 deletions
app/corpora/events.py
app/static/js/nopaque.analyse_corpus.js
+59
-8
59 additions, 8 deletions
app/static/js/nopaque.analyse_corpus.js
app/templates/corpora/analyse_corpus.html.j2
+17
-1
17 additions, 1 deletion
app/templates/corpora/analyse_corpus.html.j2
with
141 additions
and
25 deletions
app/corpora/events.py
+
65
−
16
View file @
08e7df4a
...
@@ -15,22 +15,36 @@ corpus_analysis_sessions = {}
...
@@ -15,22 +15,36 @@ corpus_analysis_sessions = {}
'''
'''
'
A dictionary containing Socket.IO session id - CQi client pairs.
'
A dictionary containing Socket.IO session id - CQi client pairs.
'
{<sid>: CQiClient, ...}
'
{<sid>: CQiClient, ...}
'
A dictionary containing Socket.IO session id - and its status.
'
{<sid>:
'
running
'
|
'
idle
'
|
'
abort
'
}
'''
'''
corpus_analysis_clients
=
{}
corpus_analysis_clients
=
{}
corpus_analysis_clients_status
=
{}
@socketio.on
(
'
send_analysis_status_cli
'
)
@login_required
def
update_status
(
response
):
update_analysis_status
(
response
[
'
status
'
])
@socketio.on
(
'
corpus_analysis_init
'
)
@socketio.on
(
'
corpus_analysis_init
'
)
@login_required
@login_required
def
init_corpus_analysis
(
corpus_id
):
def
init_corpus_analysis
(
corpus_id
):
logger
.
warning
(
"
Initiating corpus analysis.
"
)
logger
.
warning
(
'
Initiating corpus analysis.
'
)
socketio
.
start_background_task
(
corpus_analysis_session_handler
,
socketio
.
start_background_task
(
corpus_analysis_session_handler
,
current_app
.
_get_current_object
(),
current_app
.
_get_current_object
(),
corpus_id
,
current_user
.
id
,
request
.
sid
)
corpus_id
,
current_user
.
id
,
request
.
sid
)
update_analysis_status
(
'
idle
'
)
@socketio.on
(
'
corpus_analysis_query
'
)
@socketio.on
(
'
corpus_analysis_query
'
)
@login_required
@login_required
def
corpus_analysis_query
(
query
):
def
corpus_analysis_query
(
query
):
update_analysis_status
(
'
running
'
)
logger
.
warning
(
'
Recieved a query.
'
)
logger
.
warning
(
'
Clients current session ID is: {}
'
.
format
(
request
.
sid
))
current_client_status
=
get_analysis_status
()
client
=
corpus_analysis_clients
.
get
(
request
.
sid
)
client
=
corpus_analysis_clients
.
get
(
request
.
sid
)
if
client
is
None
:
if
client
is
None
:
socketio
.
emit
(
'
query
'
,
'
[424]: Failed Dependency
'
,
room
=
request
.
sid
)
socketio
.
emit
(
'
query
'
,
'
[424]: Failed Dependency
'
,
room
=
request
.
sid
)
...
@@ -41,7 +55,7 @@ def corpus_analysis_query(query):
...
@@ -41,7 +55,7 @@ def corpus_analysis_query(query):
except
Exception
as
e
:
except
Exception
as
e
:
logger
.
warning
(
e
)
logger
.
warning
(
e
)
response
=
str
(
e
)
response
=
str
(
e
)
if
response
==
"
CQI_CQP_ERROR_GENERAL
"
:
if
response
==
'
CQI_CQP_ERROR_GENERAL
'
:
response
=
{
'
code
'
:
1
}
response
=
{
'
code
'
:
1
}
socketio
.
emit
(
'
corpus_analysis_query
'
,
response
,
room
=
request
.
sid
)
socketio
.
emit
(
'
corpus_analysis_query
'
,
response
,
room
=
request
.
sid
)
else
:
else
:
...
@@ -50,21 +64,52 @@ def corpus_analysis_query(query):
...
@@ -50,21 +64,52 @@ def corpus_analysis_query(query):
context
=
100
context
=
100
progress
=
0
progress
=
0
while
chunk_start
<=
results
.
size
:
while
chunk_start
<=
results
.
size
:
chunk
=
results
.
dump_values
(
context
=
context
,
current_client_status
=
get_analysis_status
()
first_result
=
chunk_start
,
logger
.
warning
(
'
Status at while start: {}
'
.
format
(
current_client_status
))
num_results
=
chunk_size
)
if
current_client_status
==
'
abort
'
:
if
(
results
.
size
==
0
):
update_analysis_status
(
'
idle
'
)
progress
=
100
logger
.
warning
(
'
Aborting due to status being
"
abort
"
.
'
)
break
else
:
else
:
progress
=
((
chunk_start
+
chunk_size
)
/
results
.
size
)
*
100
chunk
=
results
.
dump_values
(
context
=
context
,
progress
=
min
(
100
,
int
(
math
.
ceil
(
progress
)))
first_result
=
chunk_start
,
socketio
.
emit
(
'
corpus_analysis_query
'
,
num_results
=
chunk_size
)
{
'
chunk
'
:
chunk
,
if
(
results
.
size
==
0
):
'
progress
'
:
progress
,
progress
=
100
'
num_matches_total
'
:
results
.
size
,
else
:
'
code
'
:
0
},
progress
=
((
chunk_start
+
chunk_size
)
/
results
.
size
)
*
100
room
=
request
.
sid
)
progress
=
min
(
100
,
int
(
math
.
ceil
(
progress
)))
chunk_start
+=
chunk_size
socketio
.
emit
(
'
corpus_analysis_query
'
,
{
'
chunk
'
:
chunk
,
'
progress
'
:
progress
,
'
num_matches_total
'
:
results
.
size
,
'
code
'
:
0
},
room
=
request
.
sid
)
chunk_start
+=
chunk_size
current_client_status
=
get_analysis_status
()
if
(
progress
==
100
):
update_analysis_status
(
'
idle
'
)
elif
(
current_client_status
!=
'
abort
'
):
update_analysis_status
(
'
running
'
)
logger
.
warning
(
'
Status at while end: {}
'
.
format
(
get_analysis_status
()))
def
update_analysis_status
(
status_str
):
# first: sets new status on server side
session_id
=
request
.
sid
corpus_analysis_clients_status
[
session_id
]
=
status_str
logger
.
warning
(
'
Set new status to
"
{}
"
on server side for {}.
'
.
format
(
status_str
,
session_id
))
# second: send new status to client side
if
status_str
!=
'
abort
'
:
message
=
{
'
status
'
:
status_str
}
socketio
.
emit
(
'
send_analysis_status_server
'
,
message
,
room
=
session_id
)
logger
.
warning
(
'
Sending status
"
{}
"
to client.
'
.
format
(
status_str
))
def
get_analysis_status
():
status
=
corpus_analysis_clients_status
[
request
.
sid
]
logger
.
warning
(
'
Current status on server side is: {}
'
.
format
(
status
))
return
status
@socketio.on
(
'
inspect_match
'
)
@socketio.on
(
'
inspect_match
'
)
...
@@ -88,10 +133,12 @@ def corpus_analysis_session_handler(app, corpus_id, user_id, session_id):
...
@@ -88,10 +133,12 @@ def corpus_analysis_session_handler(app, corpus_id, user_id, session_id):
corpus
=
Corpus
.
query
.
get
(
corpus_id
)
corpus
=
Corpus
.
query
.
get
(
corpus_id
)
user
=
User
.
query
.
get
(
user_id
)
user
=
User
.
query
.
get
(
user_id
)
if
corpus
is
None
:
if
corpus
is
None
:
logger
.
warning
(
"
ERROR 1
"
)
response
=
{
'
code
'
:
404
,
'
msg
'
:
'
Not Found
'
}
response
=
{
'
code
'
:
404
,
'
msg
'
:
'
Not Found
'
}
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
return
return
elif
not
(
corpus
.
creator
==
user
or
user
.
is_administrator
()):
elif
not
(
corpus
.
creator
==
user
or
user
.
is_administrator
()):
logger
.
warning
(
"
ERROR 2
"
)
response
=
{
'
code
'
:
403
,
'
msg
'
:
'
Forbidden
'
}
response
=
{
'
code
'
:
403
,
'
msg
'
:
'
Forbidden
'
}
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
return
return
...
@@ -102,6 +149,7 @@ def corpus_analysis_session_handler(app, corpus_id, user_id, session_id):
...
@@ -102,6 +149,7 @@ def corpus_analysis_session_handler(app, corpus_id, user_id, session_id):
try
:
try
:
client
.
connect
()
client
.
connect
()
except
Exception
:
except
Exception
:
logger
.
warning
(
"
ERROR 3
"
)
response
=
{
'
code
'
:
500
,
'
msg
'
:
'
Internal Server Error
'
}
response
=
{
'
code
'
:
500
,
'
msg
'
:
'
Internal Server Error
'
}
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
return
return
...
@@ -111,6 +159,7 @@ def corpus_analysis_session_handler(app, corpus_id, user_id, session_id):
...
@@ -111,6 +159,7 @@ def corpus_analysis_session_handler(app, corpus_id, user_id, session_id):
else
:
else
:
corpus_analysis_sessions
[
corpus_id
].
append
(
session_id
)
corpus_analysis_sessions
[
corpus_id
].
append
(
session_id
)
response
=
{
'
code
'
:
201
,
'
msg
'
:
'
Created
'
}
response
=
{
'
code
'
:
201
,
'
msg
'
:
'
Created
'
}
logger
.
warning
(
"
ERROR 4
"
)
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
socketio
.
emit
(
'
corpus_analysis_init
'
,
response
,
room
=
session_id
)
'''
Observe analysis session
'''
'''
Observe analysis session
'''
while
session_id
in
connected_sessions
:
while
session_id
in
connected_sessions
:
...
...
This diff is collapsed.
Click to expand it.
app/static/js/nopaque.analyse_corpus.js
+
59
−
8
View file @
08e7df4a
// ### corpus analysis client status functions
function
sendAbortSignal
()
{
console
.
log
(
`Sending status "abort signal" from client side.`
);
message
=
{
'
status
'
:
"
abort
"
};
nopaque
.
socket
.
emit
(
"
send_analysis_status_cli
"
,
message
);
setAnalysisStatus
(
"
abort
"
);
}
function
setAnalysisStatus
(
statusStr
)
{
analysisClientStatus
[
sessionId
]
=
statusStr
;
console
.
log
(
`Set new status "
${
statusStr
}
" for "
${
sessionId
}
" on client side.`
);
console
.
log
(
"
Object with status is:
"
,
analysisClientStatus
);
}
function
checkAnalysisStatus
(
sessionId
)
{
let
status
=
analysisClientStatus
[
sessionId
];
console
.
log
(
`Check status on client side for
${
sessionId
}
.`
,
analysisClientStatus
);
return
status
;
}
// ### Send query functions ###
// ### Send query functions ###
//function to get current queryData from a given queryFormElement
//function to get current queryData from a given queryFormElement
...
@@ -12,11 +33,13 @@ function getQueryData(queryFormElement) {
...
@@ -12,11 +33,13 @@ function getQueryData(queryFormElement) {
}
}
// Function to send a query to the CQP server using queryData gatherd with getQueryData
// Function to send a query to the CQP server using queryData gatherd with getQueryData
function
sendQuery
(
event
)
{
async
function
sendQuery
(
event
)
{
const
delay
=
ms
=>
new
Promise
(
res
=>
setTimeout
(
res
,
ms
));
result
;
// global declaration, holds query results as JSON
result
;
// global declaration, holds query results as JSON
resultList
;
// global declaration, list.js object displaying the results
resultList
;
// global declaration, list.js object displaying the results
let
queryData
;
// holds data of query form
let
queryData
;
// holds data of query form
let
resultListOptions
;
// holding the options for the ResultList object
let
resultListOptions
;
// holding the options for the ResultList object
let
analysisStatus
;
// checks if one query has been finished before
// checks if one query has been finished before
// if true, result download will be disabled again until query is finished
// if true, result download will be disabled again until query is finished
...
@@ -26,9 +49,32 @@ function sendQuery(event) {
...
@@ -26,9 +49,32 @@ function sendQuery(event) {
queryResultsDeterminateElement
.
parentNode
.
parentNode
.
classList
.
remove
(
"
hide
"
);
queryResultsDeterminateElement
.
parentNode
.
parentNode
.
classList
.
remove
(
"
hide
"
);
}
}
console
.
log
(
"
Sending query.
"
);
console
.
log
(
"
Session status are:
"
,
analysisClientStatus
);
event
.
preventDefault
();
event
.
preventDefault
();
analysisStatus
=
checkAnalysisStatus
(
sessionId
);
queryData
=
getQueryData
(
queryFormElement
);
queryData
=
getQueryData
(
queryFormElement
);
nopaque
.
socket
.
emit
(
"
corpus_analysis_query
"
,
queryData
.
query
);
if
(
analysisStatus
===
"
idle
"
)
{
nopaque
.
toast
(
"
Query has been sent!
"
);
nopaque
.
socket
.
emit
(
"
corpus_analysis_query
"
,
queryData
.
query
);
helperSendQuery
(
queryData
);
analysisStatus
=
checkAnalysisStatus
(
sessionId
);
}
else
if
(
analysisStatus
===
"
running
"
)
{
sendAbortSignal
();
analysisStatus
=
checkAnalysisStatus
(
sessionId
);
while
(
analysisStatus
===
"
abort
"
)
{
queryResultsTableElement
.
classList
.
add
(
"
hide
"
);
queryLoadingElement
.
classList
.
remove
(
"
hide
"
);
analysisStatus
=
checkAnalysisStatus
(
sessionId
);
await
delay
(
100
);
}
sendQuery
(
event
)
}
}
function
helperSendQuery
(
queryData
)
{
// full results object declaration, global declaration!
// full results object declaration, global declaration!
// will always be reset if a query is sent, so that only the chunks of the
// will always be reset if a query is sent, so that only the chunks of the
// current query will be saved in it
// current query will be saved in it
...
@@ -42,7 +88,6 @@ function sendQuery(event) {
...
@@ -42,7 +88,6 @@ function sendQuery(event) {
// some hiding/showing for loading animation
// some hiding/showing for loading animation
queryLoadingElement
.
classList
.
remove
(
"
hide
"
);
queryLoadingElement
.
classList
.
remove
(
"
hide
"
);
queryResultsTableElement
.
classList
.
add
(
"
hide
"
);
queryResultsTableElement
.
classList
.
add
(
"
hide
"
);
nopaque
.
toast
(
"
Query has been sent!
"
);
resultListOptions
=
{
page
:
queryData
[
"
hits_per_page
"
],
resultListOptions
=
{
page
:
queryData
[
"
hits_per_page
"
],
pagination
:
[{
pagination
:
[{
...
@@ -114,10 +159,16 @@ function recieveResults(response) {
...
@@ -114,10 +159,16 @@ function recieveResults(response) {
result
[
"
query
"
]
=
queryData
[
"
query
"
];
result
[
"
query
"
]
=
queryData
[
"
query
"
];
console
.
log
(
result
);
console
.
log
(
result
);
// Some hiding and showing of loading animations
// Some hiding and showing of loading animations
queryLoadingElement
.
classList
.
add
(
"
hide
"
);
analysisStatus
=
checkAnalysisStatus
();
queryResultsTableElement
.
classList
.
remove
(
"
hide
"
);
if
(
analysisStatus
===
"
abort
"
)
{
queryResultsElement
=
document
.
getElementById
(
"
query-results
"
);
queryResultsTableElement
.
classList
.
add
(
"
hide
"
);
queryResultsElement
.
innerHTML
=
""
;
queryLoadingElement
.
classList
.
remove
(
"
hide
"
);
}
else
{
queryResultsTableElement
.
classList
.
remove
(
"
hide
"
);
queryLoadingElement
.
classList
.
add
(
"
hide
"
);
queryResultsElement
=
document
.
getElementById
(
"
query-results
"
);
queryResultsElement
.
innerHTML
=
""
;
}
// check if query has any results
// check if query has any results
if
(
chunk
[
"
matches
"
].
length
===
0
)
{
if
(
chunk
[
"
matches
"
].
length
===
0
)
{
...
@@ -252,7 +303,7 @@ function changeContext(event) {
...
@@ -252,7 +303,7 @@ function changeContext(event) {
console
.
log
(
"
This error is expected.
"
);
console
.
log
(
"
This error is expected.
"
);
}
finally
{
}
finally
{
newContextValue
=
document
.
getElementById
(
"
context
"
).
value
;
newContextValue
=
document
.
getElementById
(
"
context
"
).
value
;
console
.
log
(
newContextValue
);
console
.
log
(
"
Context value is:
"
,
newContextValue
);
lc
=
document
.
getElementsByClassName
(
"
left-context
"
);
lc
=
document
.
getElementsByClassName
(
"
left-context
"
);
rc
=
document
.
getElementsByClassName
(
"
right-context
"
);
rc
=
document
.
getElementsByClassName
(
"
right-context
"
);
for
(
let
element
of
lc
)
{
for
(
let
element
of
lc
)
{
...
...
This diff is collapsed.
Click to expand it.
app/templates/corpora/analyse_corpus.html.j2
+
17
−
1
View file @
08e7df4a
...
@@ -239,7 +239,9 @@ server side -->
...
@@ -239,7 +239,9 @@ server side -->
// create some global variables
// create some global variables
var result; // will gradually be filled with JSON result chunks of a query
var result; // will gradually be filled with JSON result chunks of a query
var resultList; // the list.js new ResultList object displaying the results
var resultList; // the list.js new ResultList object displaying the results
var queryFinished; // bool flag that shows if query is finished or nor
var queryFinished; // bool flag that shows if query is finished or not
var analysisClientStatus; // Object holding: {<sid>: 'running'|'idle'}
var sessionId; // the clients current session id
// create some global elements
// create some global elements
var exportQueryResultsElement; // export button opens onclick download modal
var exportQueryResultsElement; // export button opens onclick download modal
var queryResultsDeterminateElement; // progress bar for recieved query status
var queryResultsDeterminateElement; // progress bar for recieved query status
...
@@ -254,6 +256,13 @@ server side -->
...
@@ -254,6 +256,13 @@ server side -->
// set queryFinished to false initially
// set queryFinished to false initially
queryFinished = false;
queryFinished = false;
analysisClientStatus = {};
nopaque.socket.on("connect", function() {
sessionId = nopaque.socket.id;
console.log("Clients current session id:", sessionId);
setAnalysisStatus("idle", sessionId);
});
// creates some models on DOMContentLoaded
// creates some models on DOMContentLoaded
document.addEventListener("DOMContentLoaded", function() {
document.addEventListener("DOMContentLoaded", function() {
...
@@ -295,6 +304,13 @@ server side -->
...
@@ -295,6 +304,13 @@ server side -->
queryFormElement = document.getElementById("query-form");
queryFormElement = document.getElementById("query-form");
queryFormElement.addEventListener("submit", sendQuery);
queryFormElement.addEventListener("submit", sendQuery);
// recieves server side analysis status updates and sets those accordingly in
// analysisClientStatus
nopaque.socket.on("send_analysis_status_server", (response) => {
console.log("Recieved new analysis status from server:", response.status);
setAnalysisStatus(response.status);
});
// recieves results on "corpus_analysis_query" via socket.io
// recieves results on "corpus_analysis_query" via socket.io
nopaque.socket.on("corpus_analysis_query", recieveResults);
nopaque.socket.on("corpus_analysis_query", recieveResults);
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment