diff --git a/analysis/analyzers/render/default.py b/analysis/analyzers/render/default.py index d0d94c1..6f74600 100644 --- a/analysis/analyzers/render/default.py +++ b/analysis/analyzers/render/default.py @@ -49,6 +49,7 @@ class KMLRender(Render): result_types = [LocationAnalyzer] def render(self, results: List[Result], name=None): + files = [] for result in self.filter(results): times = ["{time}".format(time=format_time(entry["timestamp"])) for entry in result.get()] coords = [ @@ -62,6 +63,8 @@ class KMLRender(Render): print(filename) with open(filename, "w") as out: out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times))) + files.append(filename) + return files diff --git a/analysis/analyzers/settings.py b/analysis/analyzers/settings.py index 17631a5..269eb00 100644 --- a/analysis/analyzers/settings.py +++ b/analysis/analyzers/settings.py @@ -35,6 +35,8 @@ class LogSettings: self.custom = json_dict['custom'] if "source" in json_dict: self.source = load_source(json_dict['source']) + if "render" in json_dict: + self.render = json_dict['render'] def __repr__(self): return str({ diff --git a/clients/webclients.py b/clients/webclients.py index 025dbf4..13e13e9 100644 --- a/clients/webclients.py +++ b/clients/webclients.py @@ -75,8 +75,10 @@ class BiogamesClient(Client): def login(self) -> bool: csrf_request = self.get(self.list_url) if not csrf_request.ok: - raise ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")") - self.cookies['csrftoken'] = csrf_request.cookies['csrftoken'] + log.exception(ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")")) + return False + if not 'csrftoken' in self.cookies: + self.cookies['csrftoken'] = csrf_request.cookies['csrftoken'] login_payload = { 'username': self.config['username'], 'password': self.config['password'], @@ -85,7 +87,8 @@ class BiogamesClient(Client): } login = self.post(self.login_url, json.dumps(login_payload)) if not login.ok: - raise ConnectionError("Unable to authenticate", login, login.text) + log.exception(ConnectionError("Unable to authenticate", login, login.text)) + return False self.cookies['sessionid'] = login.cookies['sessionid'] print(self.cookies) return True diff --git a/docker-compose.yml b/docker-compose.yml index fc01d0d..4943cb5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,12 +19,6 @@ services: - "traefik.docker.network=traefik_net" - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de" - - redis: - image: redis:4-alpine - - - celery: image: docker.clkl.de/ma/celery:0.1 build: . @@ -32,8 +26,27 @@ services: - PYTHONPATH=/app volumes: - ./:/app - working_dir: /app/tasks - command: celery -A tasks worker --loglevel=info + - ./data/results:/data/results + working_dir: /app + command: celery -A tasks.tasks worker --loglevel=info + + + redis: + image: redis:4-alpine + volumes: + - ./data/redis:/data + command: redis-server --appendonly yes + + nginx: + image: nginx:1.13-alpine + volumes: + - ./data/results:/usr/share/nginx/html:ro + labels: + - "traefik.enable=true" + - "traefik.port=80" + - "traefik.docker.network=traefik_net" + - "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de" + networks: traefik_net: diff --git a/selector/static/style.css b/selector/static/style.css index 9eb023c..4ddd162 100644 --- a/selector/static/style.css +++ b/selector/static/style.css @@ -3,4 +3,7 @@ body { } #data{ display: none; +} +li{ + list-style-type: none; } \ No newline at end of file diff --git a/selector/temp_config.py b/selector/temp_config.py new file mode 100644 index 0000000..9f73f7f --- /dev/null +++ b/selector/temp_config.py @@ -0,0 +1,63 @@ +KML = """{ + "logFormat": "zip", + "entryType": "@class", + "spatials": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation" + ], + "actions": [], + "boards": [ + "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry" + ], + "analyzers": { + "analysis.analyzers": [ + "BiogamesCategorizer", + "LocationAnalyzer" + ] + }, + "sequences": { + "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache", + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction", + "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction" + } + }, + "custom": { + "simulation_rounds": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion" + ], + "simu_data": [ + "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData" + ], + "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance", + "instance_id": "instance_id", + "instance_config_id": "config.@id", + "sequences2": { + "id_field": "sequence_id", + "start": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "START" + }, + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "PAUSE" + } + }, + "coordinates": "location.coordinates", + "metadata": { + "timestamp": "timestamp", + "gamefield": "instance_id", + "user": "player_group_name" + } + }, + "source": { + "type": "Biogames", + "username": "ba", + "password": "853451", + "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de" + }, + "render": [ + "KMLRender" + ] +}""" + +CONFIGS = {"KML": KML}#TODO diff --git a/selector/templates/games.html b/selector/templates/games.html index 2692bb7..3ef5a0a 100644 --- a/selector/templates/games.html +++ b/selector/templates/games.html @@ -1,19 +1,41 @@ {% extends "base.html" %} {% block body %}
-
{{logs}}
+
{{logs}}
- + +
+
+
+ +
+ {% endblock %} \ No newline at end of file diff --git a/selector/webserver.py b/selector/webserver.py index de901c2..3c369d7 100644 --- a/selector/webserver.py +++ b/selector/webserver.py @@ -1,14 +1,23 @@ +import json +import logging import typing import uuid +import time + from clients.webclients import Client, CLIENTS from flask import Flask, render_template, request, redirect, session +from tasks import tasks +from selector.temp_config import CONFIGS + BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de" app = Flask(__name__) clients: typing.Dict[str, Client] = {} +log: logging.Logger = logging.getLogger(__name__) + @app.route("/") def index(): @@ -24,6 +33,7 @@ def login(): if client.login(): session['logged_in'] = True session['uid'] = str(uuid.uuid4()) + session['username'] = request.form['username'] session['cookies'] = client.cookies session['game'] = game session['host'] = BIOGAMES_HOST @@ -38,16 +48,38 @@ def games(): return redirect("/") if session['logged_in'] and not session['uid'] in clients: clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies']) - return render_template("games.html", logs=clients[session['uid']].list()) + job_status = json.loads(tasks.redis.get(session['username'])) + return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status) + @app.route("/start", methods=['POST']) def start(): - pass + print(str(request.form['logs'])) + status = { + "status": "PENDING", + "submit": time.strftime("%c"), + "log_ids": request.form.getlist('logs'), + "config": request.form['config'], + } + params = { + "log_ids": request.form.getlist('logs'), + "config": CONFIGS[request.form['config']], + "username": session['username'], + "cookies": session['cookies'], + "host": session['host'], + "clientName": session['game'], + "name": request.form['name'], + } + tasks.status_update(session['username'], request.form['name'], status) + tasks.analyze.delay(**params) + return redirect("/games") + @app.route("/status") def status(): - pass + return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2) + if __name__ == '__main__': - app.config.update({"SECRET_KEY":"59765798-2784-11e8-8d05-db4d6f6606c9"}) + app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"}) app.run(host="0.0.0.0", debug=True) diff --git a/tasks/__init__.py b/tasks/__init__.py index e69de29..37e3696 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -0,0 +1,65 @@ +from .tasks import add, analyze + +__log__ = ["/app/data/008cad400ab848f729913d034a.zip"] + +__config__ = """{ + "logFormat": "zip", + "entryType": "@class", + "spatials": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation" + ], + "actions": [ + "...QuestionAnswerEvent", + "...SimuAnswerEvent" + ], + "boards": [ + "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry" + ], + "analyzers": { + "analysis.analyzers": [ + "BiogamesCategorizer", + "LocationAnalyzer" + ] + }, + "sequences": { + "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache", + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction", + "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction" + } + }, + "custom": { + "simulation_rounds": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion" + ], + "simu_data": [ + "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData" + ], + "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance", + "instance_id": "instance_id", + "instance_config_id": "config.@id", + "sequences2": { + "id_field": "sequence_id", + "start": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "START" + }, + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "PAUSE" + } + }, + "coordinates": "location.coordinates", + "metadata": { + "timestamp": "timestamp", + "gamefield": "instance_id", + "user": "player_group_name" + } + }, + "source": { + "type": "Biogames", + "username": "ba", + "password": "853451", + "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de" + } +}""" diff --git a/tasks/tasks.py b/tasks/tasks.py index b173f73..54051c6 100644 --- a/tasks/tasks.py +++ b/tasks/tasks.py @@ -1,14 +1,81 @@ +import json +import logging +import shutil +import uuid +import os.path +import os + +import redis as redis_lib +import time from celery import Celery from analysis import log_analyzer as la +from analysis.analyzers import KMLRender +from clients.webclients import CLIENTS + +FLASK_DB = 1 +REDIS_HOST = "redis" +DATA_PATH = "/app/data/results_test/" + +RENDERERS = { # TODO + "KMLRender": KMLRender +} app = Celery('tasks', backend='redis://redis', broker='redis://redis') +redis = redis_lib.StrictRedis(host=REDIS_HOST, db=FLASK_DB) +log: logging.Logger = logging.getLogger(__name__) + + +def update_status(username, name, state, **kwargs): + status = json.loads(redis.get(username)) + status[name][state[0]] = time.strftime("%c") + status[name]['status'] = state[1] + for i in kwargs: + status[name][i] = kwargs[i] + redis.set(username, json.dumps(status)) @app.task -def add(x, y): - return x + y +def analyze(config, log_ids, **kwargs): + update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING')) + + log.info("start analysis") + client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies']) + logs = client.list() + id_urls = {str(x['@id']): x['file_url'] for x in logs} + urls = [id_urls[i] for i in log_ids] + tmpdir = client.download_files(urls) + log.info(tmpdir.name, list(os.scandir(tmpdir.name))) + + update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING')) -@app.task -def analyze(config, log_ids): settings = la.parse_settings(config) - store = la.run_analysis(log_ids, settings, la.LOADERS) \ No newline at end of file + store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS) + render = RENDERERS[settings.render[0]]() # TODO + files = render.render(store.get_all()) + + uid = str(uuid.uuid4()) + results = [] + log.error(files) + os.mkdir(os.path.join(DATA_PATH, uid)) + for file in files: + try: + head, tail = os.path.split(file) + target = os.path.join(DATA_PATH, uid, tail) + shutil.move(file, target) + results.append(target) + except FileNotFoundError as e: + log.exception(e) + tmpdir.cleanup() + + update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results) + + +def status_update(key, status_key, status): + record = redis.get(key) + if not record: + redis.set(key, json.dumps({status_key: status})) + else: + data = json.loads(record) + data[status_key] = status + redis.set(key, json.dumps(data)) + redis.save()