activityMapper working with celery

analysis/analyzers/render/biogames.py

@@ -152,12 +152,14 @@ class ActivityMapperRender(Render):
 
 	def render(self, results: List[Result], name=None):
 		print(os.getcwd())
+		files = []
 		for result in self.filter(results):
 			data = result.get()
-			with open(os.path.join("static", "progress", "data", data['instance'] + "_" + str(name) + ".json"),
-			          "w") as out:
+			path = os.path.join("/tmp", data['instance'] + "_" + str(name) + ".json")
+			with open(path, "w") as out:
 				json.dump(data, out, indent=1)
-		return "ok"
+			files.append(path)
+		return files
 
 
 class StoreRender(Render):
@@ -186,7 +188,7 @@ class SimulationOrderRender(Render):
 
 class SimulationGroupRender(Render):
 	def render(self, results: List[Result], name=None):
-		#data = [r.get() for r in self.filter(results)]
+		# data = [r.get() for r in self.filter(results)]
 		data = []
 		for r in self.filter(results):
 			raw = r.get()

analysis/util/download.py

@@ -6,12 +6,10 @@ from analysis.util import json_path
 logger = logging.getLogger(__name__)
 
 
-def download_board(board_id, instance_config_id, sequence_id, source):
-	local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
-		config_id=instance_config_id,
-		sequence_id=sequence_id,
-		board_id=board_id)
-	if os.path.exists(local_file):
+def download_board(board_id, instance_config_id, sequence_id, source, path="/data/results/"):
+	local_file = os.path.join("static", instance_config_id, sequence_id, board_id)
+	abs_path = os.path.join(path, local_file)
+	if os.path.exists(abs_path):
 		return local_file
 	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
 		config_id=instance_config_id,
@@ -20,12 +18,12 @@ def download_board(board_id, instance_config_id, sequence_id, source):
 	)
 	board = source.get(url)
 	if not board.ok:
-		raise ConnectionError()
+		raise ConnectionError(url, board, board.status_code)
 	data = board.json()
 	preview_url = json_path(data, "preview_url.medium")
 	logger.debug(preview_url)
-	os.makedirs(local_file[:-len(board_id)], exist_ok=True)
-	source.download_file(preview_url, local_file)
+	os.makedirs(abs_path[:-len(board_id)], exist_ok=True)
+	source.download_file(preview_url, abs_path)
 	return local_file
 
 

clients/webclients.py

@@ -22,9 +22,11 @@ class Client:
 		return path
 
 	def get(self, url, **kwargs) -> requests.models.Response:
+		log.info("GET " + str(url))
 		return requests.get(self.url(url), cookies=self.cookies, headers=self.headers, **kwargs)
 
 	def post(self, url, data, **kwargs) -> requests.models.Response:
+		log.info("POST " + str(url))
 		return requests.post(self.url(url), data, cookies=self.cookies, headers=self.headers, **kwargs)
 
 	def download_file(self, url, target, **kwargs) -> bool:

docker-compose.yml

@@ -41,6 +41,8 @@ services:
   image: nginx:1.13-alpine
   volumes:
    - ./data/results:/usr/share/nginx/html:ro
+  networks:
+   - traefik_net
   labels:
    - "traefik.enable=true"
    - "traefik.port=80"

selector/temp_config.py

@@ -60,4 +60,69 @@ KML = """{
   ]
 }"""
 
-CONFIGS = {"KML": KML}#TODO
+ACTIVITY = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "ActivityMapper"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  },
+  "render": [
+    "ActivityMapper"
+  ]
+}"""
+
+CONFIGS = {  # TODO
+	"KML": KML,
+	"ActivityMapper": ACTIVITY,
+}

selector/templates/games.html

@@ -28,7 +28,7 @@
         <li> {{jobs[job].status}}: "{{job}}":
             <ul>
                 {% for r in jobs[job].results %}
-                <li>{{r}}</li>
+                <li><a href="{{r | get_url}}">{{r|get_name}}</a></li>
                 {% endfor %}
             </ul>
 

selector/webserver.py

@@ -13,6 +13,7 @@ from tasks import tasks
 from selector.temp_config import CONFIGS
 
 BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/"
 
 app = Flask(__name__)
 clients: typing.Dict[str, Client] = {}
@@ -44,11 +45,15 @@ def login():
 
 @app.route("/games")
 def games():
-	if not session['logged_in']:
+	if not ('logged_in' in session and session['logged_in']):
 		return redirect("/")
 	if session['logged_in'] and not session['uid'] in clients:
 		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
-	job_status = json.loads(tasks.redis.get(session['username']))
+	status = tasks.redis.get(session['username'])
+	if status:
+		job_status = json.loads(status)
+	else:
+		job_status = {}
 	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
 
 
@@ -80,6 +85,15 @@ def status():
 	return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
 
 
+@app.template_filter('get_url')
+def get_url(path: str):
+	return path.replace(tasks.DATA_PATH, RESULT_HOST)
+
+@app.template_filter('get_name')
+def get_url(path: str):
+	return path.replace(tasks.DATA_PATH, "")
+
+
 if __name__ == '__main__':
 	app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
 	app.run(host="0.0.0.0", debug=True)

tasks/__init__.py

@@ -1,4 +1,4 @@
-from .tasks import add, analyze
+from .tasks import analyze
 
 __log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
 

tasks/tasks.py

@@ -9,15 +9,16 @@ import redis as redis_lib
 import time
 from celery import Celery
 from analysis import log_analyzer as la
-from analysis.analyzers import KMLRender
+from analysis.analyzers import KMLRender, ActivityMapperRender
 from clients.webclients import CLIENTS
 
 FLASK_DB = 1
 REDIS_HOST = "redis"
-DATA_PATH = "/app/data/results_test/"
+DATA_PATH = "/app/data/results/"
 
 RENDERERS = {  # TODO
-	"KMLRender": KMLRender
+	"KMLRender": KMLRender,
+	"ActivityMapper": ActivityMapperRender,
 }
 
 app = Celery('tasks', backend='redis://redis', broker='redis://redis')
@@ -38,6 +39,7 @@ def update_status(username, name, state, **kwargs):
 def analyze(config, log_ids, **kwargs):
 	update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
 
+	try:
 		log.info("start analysis")
 		client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
 		logs = client.list()
@@ -68,6 +70,9 @@ def analyze(config, log_ids, **kwargs):
 		tmpdir.cleanup()
 
 		update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
+	except Exception as e:
+		log.exception(e)
+		update_status(kwargs['username'], kwargs['name'], ('abort', 'ERROR'), exception=str(e))
 
 
 def status_update(key, status_key, status):