first analysis running detached with status updates through redis

analysis/analyzers/render/default.py

@@ -49,6 +49,7 @@ class KMLRender(Render):
 	result_types = [LocationAnalyzer]
 
 	def render(self, results: List[Result], name=None):
+		files = []
 		for result in self.filter(results):
 			times = ["<when>{time}</when>".format(time=format_time(entry["timestamp"])) for entry in result.get()]
 			coords = [
@@ -62,6 +63,8 @@ class KMLRender(Render):
 			print(filename)
 			with open(filename, "w") as out:
 				out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times)))
+			files.append(filename)
+		return files
 
 
 

analysis/analyzers/settings.py

@@ -35,6 +35,8 @@ class LogSettings:
 			self.custom = json_dict['custom']
 		if "source" in json_dict:
 			self.source = load_source(json_dict['source'])
+		if "render" in json_dict:
+			self.render = json_dict['render']
 
 	def __repr__(self):
 		return str({

clients/webclients.py

@@ -75,8 +75,10 @@ class BiogamesClient(Client):
 	def login(self) -> bool:
 		csrf_request = self.get(self.list_url)
 		if not csrf_request.ok:
-			raise ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")")
+			log.exception(ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")"))
-		self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
+			return False
+		if not 'csrftoken' in self.cookies:
+			self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
 		login_payload = {
 			'username': self.config['username'],
 			'password': self.config['password'],
@@ -85,7 +87,8 @@ class BiogamesClient(Client):
 		}
 		login = self.post(self.login_url, json.dumps(login_payload))
 		if not login.ok:
-			raise ConnectionError("Unable to authenticate", login, login.text)
+			log.exception(ConnectionError("Unable to authenticate", login, login.text))
+			return False
 		self.cookies['sessionid'] = login.cookies['sessionid']
 		print(self.cookies)
 		return True

docker-compose.yml

@@ -19,12 +19,6 @@ services:
    - "traefik.docker.network=traefik_net"
    - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
 
-
- redis:
-  image: redis:4-alpine
-
-
-
  celery:
   image: docker.clkl.de/ma/celery:0.1
   build: .
@@ -32,8 +26,27 @@ services:
     - PYTHONPATH=/app
   volumes:
     - ./:/app
-  working_dir: /app/tasks
+    - ./data/results:/data/results
-  command: celery -A tasks worker --loglevel=info
+  working_dir: /app
+  command: celery -A tasks.tasks worker --loglevel=info
+
+
+ redis:
+  image: redis:4-alpine
+  volumes:
+    - ./data/redis:/data
+  command: redis-server --appendonly yes
+
+ nginx:
+  image: nginx:1.13-alpine
+  volumes:
+   - ./data/results:/usr/share/nginx/html:ro
+  labels:
+   - "traefik.enable=true"
+   - "traefik.port=80"
+   - "traefik.docker.network=traefik_net"
+   - "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de"
+
 
 networks:
  traefik_net:

selector/static/style.css

@@ -3,4 +3,7 @@ body {
 }
 #data{
     display: none;
+}
+li{
+    list-style-type: none;
 }

selector/temp_config.py

@@ -0,0 +1,63 @@
+KML = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  },
+  "render": [
+    "KMLRender"
+  ]
+}"""
+
+CONFIGS = {"KML": KML}#TODO

selector/templates/games.html

@@ -1,19 +1,41 @@
 {% extends "base.html" %}
 {% block body %}
 <form action="/start" method="post">
-    <div id="data"> {{logs}} </div>
+    <div id="data"> {{logs}}</div>
     <ul>
         {% for log in logs %}
-            <li>
+        <li>
-                <input type="checkbox" name="logs" value="{{log['@id']}}">
+            <input type="checkbox" name="logs" value="{{log['@id']}}">
-                {{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
+            {{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
-            </li>
+        </li>
         <!--{{log}}-->
         {% endfor %}
     </ul>
-    <input type="checkbox" id="safety"><label for="safety">Confirm selection</label>
+    <!--input type="checkbox" id="safety"><label for="safety">Confirm selection</label-->
+    <input type="text" id="name" maxlength="128" placeholder="name" name="name"/><br>
+    <select name="config">
+        {% for config in configs %}
+        <option>{{config}}</option>
+        {% endfor %}
+    </select>
     <input type="submit">
 
 </form>
 
+<div id="results">
+    <ul>
+    {% for job in jobs %}
+        <li> {{jobs[job].status}}: "{{job}}":
+            <ul>
+                {% for r in jobs[job].results %}
+                <li>{{r}}</li>
+                {% endfor %}
+            </ul>
+
+
+        </li>
+    {% endfor %}
+    </ul>
+</div>
+
 {% endblock %}

selector/webserver.py

@@ -1,14 +1,23 @@
+import json
+import logging
 import typing
 import uuid
 
+import time
+
 from clients.webclients import Client, CLIENTS
 
 from flask import Flask, render_template, request, redirect, session
 
+from tasks import tasks
+from selector.temp_config import CONFIGS
+
 BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
 
 app = Flask(__name__)
 clients: typing.Dict[str, Client] = {}
+log: logging.Logger = logging.getLogger(__name__)
+
 
 @app.route("/")
 def index():
@@ -24,6 +33,7 @@ def login():
 	if client.login():
 		session['logged_in'] = True
 		session['uid'] = str(uuid.uuid4())
+		session['username'] = request.form['username']
 		session['cookies'] = client.cookies
 		session['game'] = game
 		session['host'] = BIOGAMES_HOST
@@ -38,16 +48,38 @@ def games():
 		return redirect("/")
 	if session['logged_in'] and not session['uid'] in clients:
 		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
-	return render_template("games.html", logs=clients[session['uid']].list())
+	job_status = json.loads(tasks.redis.get(session['username']))
+	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
+
 
 @app.route("/start", methods=['POST'])
 def start():
-	pass
+	print(str(request.form['logs']))
+	status = {
+		"status": "PENDING",
+		"submit": time.strftime("%c"),
+		"log_ids": request.form.getlist('logs'),
+		"config": request.form['config'],
+	}
+	params = {
+		"log_ids": request.form.getlist('logs'),
+		"config": CONFIGS[request.form['config']],
+		"username": session['username'],
+		"cookies": session['cookies'],
+		"host": session['host'],
+		"clientName": session['game'],
+		"name": request.form['name'],
+	}
+	tasks.status_update(session['username'], request.form['name'], status)
+	tasks.analyze.delay(**params)
+	return redirect("/games")
+
 
 @app.route("/status")
 def status():
-	pass
+	return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
+
 
 if __name__ == '__main__':
-	app.config.update({"SECRET_KEY":"59765798-2784-11e8-8d05-db4d6f6606c9"})
+	app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
 	app.run(host="0.0.0.0", debug=True)

tasks/__init__.py

@@ -0,0 +1,65 @@
+from .tasks import add, analyze
+
+__log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
+
+__config__ = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [
+    "...QuestionAnswerEvent",
+    "...SimuAnswerEvent"
+  ],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  }
+}"""

tasks/tasks.py

@@ -1,14 +1,81 @@
+import json
+import logging
+import shutil
+import uuid
+import os.path
+import os
+
+import redis as redis_lib
+import time
 from celery import Celery
 from analysis import log_analyzer as la
+from analysis.analyzers import KMLRender
+from clients.webclients import CLIENTS
+
+FLASK_DB = 1
+REDIS_HOST = "redis"
+DATA_PATH = "/app/data/results_test/"
+
+RENDERERS = {  # TODO
+	"KMLRender": KMLRender
+}
 
 app = Celery('tasks', backend='redis://redis', broker='redis://redis')
+redis = redis_lib.StrictRedis(host=REDIS_HOST, db=FLASK_DB)
+log: logging.Logger = logging.getLogger(__name__)
+
+
+def update_status(username, name, state, **kwargs):
+	status = json.loads(redis.get(username))
+	status[name][state[0]] = time.strftime("%c")
+	status[name]['status'] = state[1]
+	for i in kwargs:
+		status[name][i] = kwargs[i]
+	redis.set(username, json.dumps(status))
 
 
 @app.task
-def add(x, y):
+def analyze(config, log_ids, **kwargs):
-	return x + y
+	update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
+
+	log.info("start analysis")
+	client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
+	logs = client.list()
+	id_urls = {str(x['@id']): x['file_url'] for x in logs}
+	urls = [id_urls[i] for i in log_ids]
+	tmpdir = client.download_files(urls)
+	log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
+
+	update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
 
-@app.task
-def analyze(config, log_ids):
 	settings = la.parse_settings(config)
-	store = la.run_analysis(log_ids, settings, la.LOADERS)
+	store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
+	render = RENDERERS[settings.render[0]]()  # TODO
+	files = render.render(store.get_all())
+
+	uid = str(uuid.uuid4())
+	results = []
+	log.error(files)
+	os.mkdir(os.path.join(DATA_PATH, uid))
+	for file in files:
+		try:
+			head, tail = os.path.split(file)
+			target = os.path.join(DATA_PATH, uid, tail)
+			shutil.move(file, target)
+			results.append(target)
+		except FileNotFoundError as e:
+			log.exception(e)
+	tmpdir.cleanup()
+
+	update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
+
+
+def status_update(key, status_key, status):
+	record = redis.get(key)
+	if not record:
+		redis.set(key, json.dumps({status_key: status}))
+	else:
+		data = json.loads(record)
+		data[status_key] = status
+		redis.set(key, json.dumps(data))
+	redis.save()