Dockerfile

@@ -2,7 +2,9 @@ FROM alpine:edge
 
 ADD ["requirements.txt", "/"]
 RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing/" >> /etc/apk/repositories && \
-    apk add --update --no-cache libpng libpng-dev freetype freetype-dev g++ python3 python3-dev libstdc++ openblas-dev && \
+    sed -i 's/numpy/#numpy/' requirements.txt && \
-    pip3 --no-cache-dir install -r requirements.txt && \
+    sed -i 's/scipy/#scipy/' requirements.txt && \
-    apk del libpng-dev freetype-dev g++ python3-dev openblas-dev && \
+    apk add --update libpng libpng-dev freetype freetype-dev g++ python3 py3-numpy python3-dev py-numpy-dev py3-scipy&& \
+    pip3 install -r requirements.txt && \
+    apk del libpng-dev freetype-dev g++ python3-dev py-numpy-dev && \
     rm requirements.txt

analysis/analyzers/render/biogames.py

@@ -40,7 +40,7 @@ def __plot_or_show(name=None):
 
 
 def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
-		 rotation='vertical', name=None):
+         rotation='vertical', name=None):
 	names, datas = list(zip(*src_data))
 	# plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True)
 	rand = np.random.rand(len(datas), len(datas[0]))
@@ -53,7 +53,7 @@ def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", titl
 
 
 def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="sequential simulation retries",
-			   rotation='vertical', name=None):
+               rotation='vertical', name=None):
 	config_name = CONFIG_NAMES[name] if name in CONFIG_NAMES else "---"
 	counts_per_group = [sum(i) for i in src_data]
 	label = "{}: n={n}; # of sim runs: ⌀={avg:.2f}, median={median}".format(
@@ -152,14 +152,12 @@ class ActivityMapperRender(Render):
 
 	def render(self, results: List[Result], name=None):
 		print(os.getcwd())
-		files = []
 		for result in self.filter(results):
 			data = result.get()
-			path = os.path.join("/tmp", data['instance'] + "_" + str(name) + ".json")
+			with open(os.path.join("static", "progress", "data", data['instance'] + "_" + str(name) + ".json"),
-			with open(path, "w") as out:
+			          "w") as out:
 				json.dump(data, out, indent=1)
-			files.append(path)
+		return "ok"
-		return files
 
 
 class StoreRender(Render):
@@ -188,7 +186,7 @@ class SimulationOrderRender(Render):
 
 class SimulationGroupRender(Render):
 	def render(self, results: List[Result], name=None):
-		# data = [r.get() for r in self.filter(results)]
+		#data = [r.get() for r in self.filter(results)]
 		data = []
 		for r in self.filter(results):
 			raw = r.get()
@@ -198,6 +196,6 @@ class SimulationGroupRender(Render):
 		print(name, len(data))
 		# graph_fit(list(data), name=name)
 		graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None,
-				   name=name)
+		           name=name)
 
 	result_types = [SimulationOrderAnalyzer]

analysis/util/download.py

@@ -6,10 +6,12 @@ from analysis.util import json_path
 logger = logging.getLogger(__name__)
 
 
-def download_board(board_id, instance_config_id, sequence_id, source, path="/data/results/"):
+def download_board(board_id, instance_config_id, sequence_id, source):
-	local_file = os.path.join("static", instance_config_id, sequence_id, board_id)
+	local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
-	abs_path = os.path.join(path, local_file)
+		config_id=instance_config_id,
-	if os.path.exists(abs_path):
+		sequence_id=sequence_id,
+		board_id=board_id)
+	if os.path.exists(local_file):
 		return local_file
 	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
 		config_id=instance_config_id,
@@ -18,12 +20,12 @@ def download_board(board_id, instance_config_id, sequence_id, source, path="/dat
 	)
 	board = source.get(url)
 	if not board.ok:
-		raise ConnectionError(url, board, board.status_code)
+		raise ConnectionError()
 	data = board.json()
 	preview_url = json_path(data, "preview_url.medium")
 	logger.debug(preview_url)
-	os.makedirs(abs_path[:-len(board_id)], exist_ok=True)
+	os.makedirs(local_file[:-len(board_id)], exist_ok=True)
-	source.download_file(preview_url, abs_path)
+	source.download_file(preview_url, local_file)
 	return local_file
 
 
@@ -69,8 +71,7 @@ def get_json(source, url):
 		data = source.get(url).json()
 
 	except Exception as e:
-		print("exception", e, e.args)
+		print("exception", e, e.args)  # TODO: logging
-		logger.exception(e)
 		data = None
 	cache[url] = data
 	return data

clients/webclients.py

@@ -22,11 +22,9 @@ class Client:
 		return path
 
 	def get(self, url, **kwargs) -> requests.models.Response:
-		log.info("GET " + str(url))
 		return requests.get(self.url(url), cookies=self.cookies, headers=self.headers, **kwargs)
 
 	def post(self, url, data, **kwargs) -> requests.models.Response:
-		log.info("POST " + str(url))
 		return requests.post(self.url(url), data, cookies=self.cookies, headers=self.headers, **kwargs)
 
 	def download_file(self, url, target, **kwargs) -> bool:

docker-compose.yml

@@ -1,10 +1,9 @@
-version: "2.2"
+version: "3"
 
 services:
  app:
-  image: docker.clkl.de/ma/celery:0.3.3
+  image: docker.clkl.de/ma/celery:0.1
-  build: ./selector
+  build: .
-  cpu_count: 4
   volumes:
     - ./:/app
   working_dir: /app/selector
@@ -21,7 +20,8 @@ services:
    - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
 
  celery:
-  image: docker.clkl.de/ma/celery:0.3.3
+  image: docker.clkl.de/ma/celery:0.1
+  build: .
   environment:
     - PYTHONPATH=/app
   volumes:
@@ -41,8 +41,6 @@ services:
   image: nginx:1.13-alpine
   volumes:
    - ./data/results:/usr/share/nginx/html:ro
-  networks:
-   - traefik_net
   labels:
    - "traefik.enable=true"
    - "traefik.port=80"

requirements.txt

@@ -1,10 +1,10 @@
 requests==2.18.4
-numpy==1.14.2
+numpy==1.13.1
 matplotlib==2.1.0
 #osmnx==0.6
 networkx==2.0
 #pydot==1.2.3
-scipy==1.0.1
+scipy==1.0.0
 #ipython==6.2.1
 
 flask==0.12.2

selector/temp_config.py

@@ -60,74 +60,4 @@ KML = """{
   ]
 }"""
 
-ACTIVITY = """{
+CONFIGS = {"KML": KML}#TODO
-  "logFormat": "zip",
-  "entryType": "@class",
-  "spatials": [
-    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
-  ],
-  "actions": [],
-  "boards": [
-    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
-  ],
-  "analyzers": {
-    "analysis.analyzers": [
-      "BiogamesCategorizer",
-      "ActivityMapper"
-    ]
-  },
-  "sequences": {
-    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
-    "end": {
-      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
-      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
-    }
-  },
-  "custom": {
-    "simulation_rounds": [
-      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
-    ],
-    "simu_data": [
-      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
-    ],
-    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
-    "instance_id": "instance_id",
-    "instance_config_id": "config.@id",
-    "sequences2": {
-      "id_field": "sequence_id",
-      "start": {
-        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
-        "action": "START"
-      },
-      "end": {
-        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
-        "action": "PAUSE"
-      }
-    },
-    "coordinates": "location.coordinates",
-    "metadata": {
-      "timestamp": "timestamp",
-      "gamefield": "instance_id",
-      "user": "player_group_name"
-    }
-  },
-  "source": {
-    "type": "Biogames",
-   "username": "ba",
-    "password": "853451",
-    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
-  },
-  "render": [
-    "ActivityMapper"
-  ]
-}"""
-
-CONFIGS = {  # TODO
-	"KML": KML,
-	"ActivityMapper": ACTIVITY,
-}
-
-URLS = {
-	"KML": "/",
-	"ActivityMapper": "#",
-}

selector/templates/games.html

@@ -21,5 +21,21 @@
     <input type="submit">
 
 </form>
-<a href="/results">show analysis progress/results</a>
+
+<div id="results">
+    <ul>
+    {% for job in jobs %}
+        <li> {{jobs[job].status}}: "{{job}}":
+            <ul>
+                {% for r in jobs[job].results %}
+                <li>{{r}}</li>
+                {% endfor %}
+            </ul>
+
+
+        </li>
+    {% endfor %}
+    </ul>
+</div>
+
 {% endblock %}

selector/templates/results.html

@@ -1,22 +0,0 @@
-{% extends "base.html" %}
-{% block body %}
-
-<a href="/games">create new analysis</a>
-
-<div id="results">
-    <ul>
-        {% for job in jobs %}
-        <li> {{jobs[job].status}}: "{{job}}":
-            <ul>
-                {% for r in jobs[job].results %}
-                <li><a href="{{jobs[job] | get_prefix}}{{r | get_name}}">{{r|get_name}} {{jobs[job].start}}</a></li>
-                {% endfor %}
-            </ul>
-
-
-        </li>
-        {% endfor %}
-    </ul>
-</div>
-
-{% endblock %}

selector/webserver.py

@@ -10,11 +10,9 @@ from clients.webclients import Client, CLIENTS
 from flask import Flask, render_template, request, redirect, session
 
 from tasks import tasks
-from selector.temp_config import CONFIGS, URLS
+from selector.temp_config import CONFIGS
 
 BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
-#BIOGAMES_HOST = "http://www.biodiv2go.de"
-RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/"
 
 app = Flask(__name__)
 clients: typing.Dict[str, Client] = {}
@@ -30,7 +28,7 @@ def index():
 def login():
 	game = request.form["game"]
 	if not game in CLIENTS:
-		return redirect("/?invalid_game")
+		return redirect("/")
 	client = CLIENTS[game](host=BIOGAMES_HOST, username=request.form['username'], password=request.form['password'])
 	if client.login():
 		session['logged_in'] = True
@@ -40,36 +38,18 @@ def login():
 		session['game'] = game
 		session['host'] = BIOGAMES_HOST
 		clients[session['uid']] = client
-		return redirect("/results")
+		return redirect("/games")
-	return redirect("/?fail")
+	return redirect("/")
-
-
-@app.route("/results")
-def results():
-	if not ('logged_in' in session and session['logged_in']):
-		return redirect("/")
-	if session['logged_in'] and not session['uid'] in clients:
-		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
-	status = tasks.redis.get(session['username'])
-	if status:
-		job_status = json.loads(status)
-	else:
-		job_status = {}
-	#for job in job_status:
-	#	results = []
-	#	for path in job_status[job]['results']:
-	#		results.append(path.replace(tasks.DATA_PATH, RESULT_HOST))
-	#	print(results) #TODO???
-	return render_template("results.html",  jobs=job_status)
 
 
 @app.route("/games")
 def games():
-	if not ('logged_in' in session and session['logged_in']):
+	if not session['logged_in']:
 		return redirect("/")
 	if session['logged_in'] and not session['uid'] in clients:
 		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
-	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS)
+	job_status = json.loads(tasks.redis.get(session['username']))
+	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
 
 
 @app.route("/start", methods=['POST'])
@@ -92,7 +72,7 @@ def start():
 	}
 	tasks.status_update(session['username'], request.form['name'], status)
 	tasks.analyze.delay(**params)
-	return redirect("/results")
+	return redirect("/games")
 
 
 @app.route("/status")
@@ -100,25 +80,6 @@ def status():
 	return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
 
 
-@app.template_filter('get_url')
-def get_url(path: str):
-	return path.replace(tasks.DATA_PATH, RESULT_HOST)
-
-@app.template_filter('get_name')
-def get_url(path: str):
-	return path.replace(tasks.DATA_PATH, "")
-
-
-@app.template_filter('get_prefix')
-def get_prefix(job):
-	print(job)
-	try:
-		return RESULT_HOST + URLS[job['config']]
-	except:
-		return RESULT_HOST + "#"
-
-
-
 if __name__ == '__main__':
 	app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
 	app.run(host="0.0.0.0", debug=True)

tasks/__init__.py

@@ -1,4 +1,4 @@
-from .tasks import analyze
+from .tasks import add, analyze
 
 __log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
 

tasks/tasks.py

@@ -9,16 +9,15 @@ import redis as redis_lib
 import time
 from celery import Celery
 from analysis import log_analyzer as la
-from analysis.analyzers import KMLRender, ActivityMapperRender
+from analysis.analyzers import KMLRender
 from clients.webclients import CLIENTS
 
 FLASK_DB = 1
 REDIS_HOST = "redis"
-DATA_PATH = "/app/data/results/"
+DATA_PATH = "/app/data/results_test/"
 
 RENDERERS = {  # TODO
-	"KMLRender": KMLRender,
+	"KMLRender": KMLRender
-	"ActivityMapper": ActivityMapperRender,
 }
 
 app = Celery('tasks', backend='redis://redis', broker='redis://redis')
@@ -39,40 +38,36 @@ def update_status(username, name, state, **kwargs):
 def analyze(config, log_ids, **kwargs):
 	update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
 
-	try:
+	log.info("start analysis")
-		log.info("start analysis")
+	client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
-		client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
+	logs = client.list()
-		logs = client.list()
+	id_urls = {str(x['@id']): x['file_url'] for x in logs}
-		id_urls = {str(x['@id']): x['file_url'] for x in logs}
+	urls = [id_urls[i] for i in log_ids]
-		urls = [id_urls[i] for i in log_ids]
+	tmpdir = client.download_files(urls)
-		tmpdir = client.download_files(urls)
+	log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
-		log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
 
-		update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
+	update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
 
-		settings = la.parse_settings(config)
+	settings = la.parse_settings(config)
-		store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
+	store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
-		render = RENDERERS[settings.render[0]]()  # TODO
+	render = RENDERERS[settings.render[0]]()  # TODO
-		files = render.render(store.get_all())
+	files = render.render(store.get_all())
 
-		uid = str(uuid.uuid4())
+	uid = str(uuid.uuid4())
-		results = []
+	results = []
-		log.error(files)
+	log.error(files)
-		os.mkdir(os.path.join(DATA_PATH, uid))
+	os.mkdir(os.path.join(DATA_PATH, uid))
-		for file in files:
+	for file in files:
-			try:
+		try:
-				head, tail = os.path.split(file)
+			head, tail = os.path.split(file)
-				target = os.path.join(DATA_PATH, uid, tail)
+			target = os.path.join(DATA_PATH, uid, tail)
-				shutil.move(file, target)
+			shutil.move(file, target)
-				results.append(target)
+			results.append(target)
-			except FileNotFoundError as e:
+		except FileNotFoundError as e:
-				log.exception(e)
+			log.exception(e)
-		tmpdir.cleanup()
+	tmpdir.cleanup()
 
-		update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
+	update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
-	except Exception as e:
-		log.exception(e)
-		update_status(kwargs['username'], kwargs['name'], ('abort', 'ERROR'), exception=str(e))
 
 
 def status_update(key, status_key, status):