ma
/
project
1
0
Fork 0
Code Issues 5 Pull Requests Releases Wiki Activity

first analysis running detached with status updates through redis

activity_mapper
Clemens Klug 2018-03-20 18:01:28 +01:00
parent 7d93b5f6fd
commit 01e2433b8b
10 changed files with 299 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
analysis/analyzers/render/default.py
View File

@ -49,6 +49,7 @@ class KMLRender(Render):
result_types = [LocationAnalyzer]
def render(self, results: List[Result], name=None):
files = []
for result in self.filter(results):
times = ["<when>{time}</when>".format(time=format_time(entry["timestamp"])) for entry in result.get()]
coords = [
@ -62,6 +63,8 @@ class KMLRender(Render):
print(filename)
with open(filename, "w") as out:
out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times)))
files.append(filename)
return files

2
analysis/analyzers/settings.py
View File

@ -35,6 +35,8 @@ class LogSettings:
self.custom = json_dict['custom']
if "source" in json_dict:
self.source = load_source(json_dict['source'])
if "render" in json_dict:
self.render = json_dict['render']
def __repr__(self):
return str({

7
clients/webclients.py
View File

@ -75,7 +75,9 @@ class BiogamesClient(Client):
def login(self) -> bool:
csrf_request = self.get(self.list_url)
if not csrf_request.ok:
raise ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")")
log.exception(ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")"))
return False
if not 'csrftoken' in self.cookies:
self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
login_payload = {
'username': self.config['username'],
@ -85,7 +87,8 @@ class BiogamesClient(Client):
}
login = self.post(self.login_url, json.dumps(login_payload))
if not login.ok:
raise ConnectionError("Unable to authenticate", login, login.text)
log.exception(ConnectionError("Unable to authenticate", login, login.text))
return False
self.cookies['sessionid'] = login.cookies['sessionid']
print(self.cookies)
return True

29
docker-compose.yml
View File

@ -19,12 +19,6 @@ services:
- "traefik.docker.network=traefik_net"
- "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
redis:
image: redis:4-alpine
celery:
image: docker.clkl.de/ma/celery:0.1
build: .
@ -32,8 +26,27 @@ services:
- PYTHONPATH=/app
volumes:
- ./:/app
working_dir: /app/tasks
command: celery -A tasks worker --loglevel=info
- ./data/results:/data/results
working_dir: /app
command: celery -A tasks.tasks worker --loglevel=info
redis:
image: redis:4-alpine
volumes:
- ./data/redis:/data
command: redis-server --appendonly yes
nginx:
image: nginx:1.13-alpine
volumes:
- ./data/results:/usr/share/nginx/html:ro
labels:
- "traefik.enable=true"
- "traefik.port=80"
- "traefik.docker.network=traefik_net"
- "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de"
networks:
traefik_net:

3
selector/static/style.css
View File

@ -4,3 +4,6 @@ body {
#data{
display: none;
}
li{
list-style-type: none;
}

63
selector/temp_config.py Normal file
View File

@ -0,0 +1,63 @@
KML = """{
"logFormat": "zip",
"entryType": "@class",
"spatials": [
"de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
],
"actions": [],
"boards": [
"de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
],
"analyzers": {
"analysis.analyzers": [
"BiogamesCategorizer",
"LocationAnalyzer"
]
},
"sequences": {
"start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
"end": {
"@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
"action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
}
},
"custom": {
"simulation_rounds": [
"de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
],
"simu_data": [
"de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
],
"instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
"instance_id": "instance_id",
"instance_config_id": "config.@id",
"sequences2": {
"id_field": "sequence_id",
"start": {
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
"action": "START"
},
"end": {
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
"action": "PAUSE"
}
},
"coordinates": "location.coordinates",
"metadata": {
"timestamp": "timestamp",
"gamefield": "instance_id",
"user": "player_group_name"
}
},
"source": {
"type": "Biogames",
"username": "ba",
"password": "853451",
"host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
},
"render": [
"KMLRender"
]
}"""
CONFIGS = {"KML": KML}#TODO

26
selector/templates/games.html
View File

@ -1,7 +1,7 @@
{% extends "base.html" %}
{% block body %}
<form action="/start" method="post">
<div id="data"> {{logs}} </div>
<div id="data"> {{logs}}</div>
<ul>
{% for log in logs %}
<li>
@ -11,9 +11,31 @@
<!--{{log}}-->
{% endfor %}
</ul>
<input type="checkbox" id="safety"><label for="safety">Confirm selection</label>
<!--input type="checkbox" id="safety"><label for="safety">Confirm selection</label-->
<input type="text" id="name" maxlength="128" placeholder="name" name="name"/><br>
<select name="config">
{% for config in configs %}
<option>{{config}}</option>
{% endfor %}
</select>
<input type="submit">
</form>
<div id="results">
<ul>
{% for job in jobs %}
<li> {{jobs[job].status}}: "{{job}}":
<ul>
{% for r in jobs[job].results %}
<li>{{r}}</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
</div>
{% endblock %}

40
selector/webserver.py
View File

@ -1,14 +1,23 @@
import json
import logging
import typing
import uuid
import time
from clients.webclients import Client, CLIENTS
from flask import Flask, render_template, request, redirect, session
from tasks import tasks
from selector.temp_config import CONFIGS
BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
app = Flask(__name__)
clients: typing.Dict[str, Client] = {}
log: logging.Logger = logging.getLogger(__name__)
@app.route("/")
def index():
@ -24,6 +33,7 @@ def login():
if client.login():
session['logged_in'] = True
session['uid'] = str(uuid.uuid4())
session['username'] = request.form['username']
session['cookies'] = client.cookies
session['game'] = game
session['host'] = BIOGAMES_HOST
@ -38,16 +48,38 @@ def games():
return redirect("/")
if session['logged_in'] and not session['uid'] in clients:
clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
return render_template("games.html", logs=clients[session['uid']].list())
job_status = json.loads(tasks.redis.get(session['username']))
return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
@app.route("/start", methods=['POST'])
def start():
pass
print(str(request.form['logs']))
status = {
"status": "PENDING",
"submit": time.strftime("%c"),
"log_ids": request.form.getlist('logs'),
"config": request.form['config'],
}
params = {
"log_ids": request.form.getlist('logs'),
"config": CONFIGS[request.form['config']],
"username": session['username'],
"cookies": session['cookies'],
"host": session['host'],
"clientName": session['game'],
"name": request.form['name'],
}
tasks.status_update(session['username'], request.form['name'], status)
tasks.analyze.delay(**params)
return redirect("/games")
@app.route("/status")
def status():
pass
return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
if __name__ == '__main__':
app.config.update({"SECRET_KEY":"59765798-2784-11e8-8d05-db4d6f6606c9"})
app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
app.run(host="0.0.0.0", debug=True)

65
tasks/__init__.py
View File

@ -0,0 +1,65 @@
from .tasks import add, analyze
__log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
__config__ = """{
"logFormat": "zip",
"entryType": "@class",
"spatials": [
"de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
],
"actions": [
"...QuestionAnswerEvent",
"...SimuAnswerEvent"
],
"boards": [
"de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
],
"analyzers": {
"analysis.analyzers": [
"BiogamesCategorizer",
"LocationAnalyzer"
]
},
"sequences": {
"start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
"end": {
"@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
"action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
}
},
"custom": {
"simulation_rounds": [
"de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
],
"simu_data": [
"de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
],
"instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
"instance_id": "instance_id",
"instance_config_id": "config.@id",
"sequences2": {
"id_field": "sequence_id",
"start": {
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
"action": "START"
},
"end": {
"@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
"action": "PAUSE"
}
},
"coordinates": "location.coordinates",
"metadata": {
"timestamp": "timestamp",
"gamefield": "instance_id",
"user": "player_group_name"
}
},
"source": {
"type": "Biogames",
"username": "ba",
"password": "853451",
"host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
}
}"""

77
tasks/tasks.py
View File

@ -1,14 +1,81 @@
import json
import logging
import shutil
import uuid
import os.path
import os
import redis as redis_lib
import time
from celery import Celery
from analysis import log_analyzer as la
from analysis.analyzers import KMLRender
from clients.webclients import CLIENTS
FLASK_DB = 1
REDIS_HOST = "redis"
DATA_PATH = "/app/data/results_test/"
RENDERERS = { # TODO
"KMLRender": KMLRender
}
app = Celery('tasks', backend='redis://redis', broker='redis://redis')
redis = redis_lib.StrictRedis(host=REDIS_HOST, db=FLASK_DB)
log: logging.Logger = logging.getLogger(__name__)
def update_status(username, name, state, **kwargs):
status = json.loads(redis.get(username))
status[name][state[0]] = time.strftime("%c")
status[name]['status'] = state[1]
for i in kwargs:
status[name][i] = kwargs[i]
redis.set(username, json.dumps(status))
@app.task
def add(x, y):
return x + y
def analyze(config, log_ids, **kwargs):
update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
log.info("start analysis")
client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
logs = client.list()
id_urls = {str(x['@id']): x['file_url'] for x in logs}
urls = [id_urls[i] for i in log_ids]
tmpdir = client.download_files(urls)
log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
@app.task
def analyze(config, log_ids):
settings = la.parse_settings(config)
store = la.run_analysis(log_ids, settings, la.LOADERS)
store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
render = RENDERERS[settings.render[0]]() # TODO
files = render.render(store.get_all())
uid = str(uuid.uuid4())
results = []
log.error(files)
os.mkdir(os.path.join(DATA_PATH, uid))
for file in files:
try:
head, tail = os.path.split(file)
target = os.path.join(DATA_PATH, uid, tail)
shutil.move(file, target)
results.append(target)
except FileNotFoundError as e:
log.exception(e)
tmpdir.cleanup()
update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
def status_update(key, status_key, status):
record = redis.get(key)
if not record:
redis.set(key, json.dumps({status_key: status}))
else:
data = json.loads(record)
data[status_key] = status
redis.set(key, json.dumps(data))
redis.save()