From 1748ce1f67535bccd3269957d8855b58b2ee87ad Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Tue, 7 Nov 2017 12:50:45 +0100
Subject: [PATCH 01/24] add graph-based traversal density

---
 analyzers/__init__.py        |  5 +--
 analyzers/render/biogames.py | 62 +++++++++++++++++++++++++++++++++++-
 biogames2.json               |  4 +--
 log_analyzer.py              |  6 ++++
 static/progress/my.js        |  6 ++--
 5 files changed, 75 insertions(+), 8 deletions(-)

diff --git a/analyzers/__init__.py b/analyzers/__init__.py
index 5a95e21..e535085 100644
--- a/analyzers/__init__.py
+++ b/analyzers/__init__.py
@@ -9,7 +9,7 @@ from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
 from .analyzer.mask import MaskSpatials
 from .render import Render
 from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
-	ActivityMapperRender, StoreRender, SimulationOrderRender
+	ActivityMapperRender, StoreRender, SimulationOrderRender, SimulationGroupRender
 from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender
 from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
 	LocomotionActionRatioRender
@@ -49,7 +49,8 @@ __MAPPING__ = {
 	],
 	SimulationOrderAnalyzer: [
 		JSONRender,
-SimulationOrderRender
+#SimulationOrderRender,
+SimulationGroupRender
 	]
 }
 
diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py
index 65e8fe7..e2de09e 100644
--- a/analyzers/render/biogames.py
+++ b/analyzers/render/biogames.py
@@ -4,22 +4,73 @@ from typing import List, Tuple
 
 import matplotlib.pyplot as plt
 import os
+import numpy as np
+from scipy.interpolate import interp1d
+import networkx as nx
+import itertools
 
 from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
 from . import Render
 from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
 
+def add_edge(graph, src, dest):
+	if graph.has_edge(src, dest):
+		weight = graph.get_edge_data(src, dest)['weight'] + 1
+	else:
+		weight = 1
+	graph.add_edge(tuple(src),tuple(dest),weight=weight)
+
+def pairs(iterable):
+	a,b = itertools.tee(iterable)
+	next(b,None)
+	return zip(a,b)
 
 def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
          rotation='vertical'):
 	names, datas = list(zip(*src_data))
-	plt.boxplot(datas, labels=names)
+	#plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True)
+	rand = np.random.rand(len(datas),len(datas[0]))
+	plt.plot(datas+rand, linewidth=.2)
 	plt.xticks(rotation=rotation)
 	# plt.margins()
 	plt.ylabel(ylabel)
 	plt.title(title)
 	plt.show()
 
+def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="sequential simulation retries",
+         rotation='vertical'):
+	g = nx.Graph()
+	for group in src_data:
+		for i in pairs(enumerate(group)):
+			add_edge(g, i[0], i[1])
+	positions = {}
+	for node in g.nodes():
+		positions[node] = node
+	widths = [x[2]/10.0 for x in g.edges.data('weight')]
+	print(max(widths))
+	nx.draw_networkx_edges(g, positions, width=widths)
+	#rand = np.random.rand(len(datas),len(datas[0]))
+	#plt.plot(datas+rand, linewidth=.2)
+	plt.xticks(rotation=rotation)
+	# plt.margins()
+	plt.ylabel(ylabel)
+	plt.title(title)
+	plt.show()
+
+def graph_fit(src_data, deg=5):
+	plt.title("polyfit(x,y,deg="+str(deg)+")")
+	for i in src_data:
+		#plt.plot(i)
+		count = len(i)
+		xp = np.linspace(0, count-1, num=count, endpoint=True)
+		#fit = np.poly1d(np.polyfit(range(len(i)), i, deg=deg))
+		#plt.plot(xp, fit(xp), linewidth=0.1)
+		xnew = np.linspace(0, count-1, num=count*20, endpoint=True)
+		f = interp1d(xp, i, kind='quadratic')
+		
+		plt.plot(range(count), i, '.', markersize=1)
+		plt.plot(xnew, f(xnew), linewidth=0.2)
+	plt.show()
 
 class SimulationRoundsRender(Render):
 	def render(self, results: List[Result]):
@@ -101,4 +152,13 @@ class SimulationOrderRender(Render):
 		#plot(enumerate([r.get() for r in self.filter(results)]))
 		plot(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
 
+	result_types = [SimulationOrderAnalyzer]
+
+
+class SimulationGroupRender(Render):
+	def render(self, results: List[Result]):
+		data = [r.get() for r in self.filter(results)]
+		#graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
+		graph_fit(list(data))
+
 	result_types = [SimulationOrderAnalyzer]
\ No newline at end of file
diff --git a/biogames2.json b/biogames2.json
index 2dfb70f..9071e5a 100644
--- a/biogames2.json
+++ b/biogames2.json
@@ -68,8 +68,8 @@
     "type": "Biogames",
     "url": "http://0.0.0.0:5000/game2/instance/log/list/",
     "login_url": "http://localhost:5000/game2/auth/json-login",
-    "username": "dev",
-    "password": "dev",
+    "username": "ba",
+    "password": "853451",
     "host":"http://0.0.0.0:5000"
   }
 }
\ No newline at end of file
diff --git a/log_analyzer.py b/log_analyzer.py
index cc0f71c..f292a04 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -17,6 +17,7 @@ requests_log.setLevel(logging.WARN)
 
 def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]:
 	logfile: str = "data/inst_{id}.{format}".format(id=log_id, format=settings.log_format)
+	logfile = log_id
 	loader = LOADERS[settings.log_format]()
 	try:
 		loader.load(logfile)
@@ -54,6 +55,11 @@ if __name__ == '__main__':
 		#	"fe1331481f85560681f86827ec",
 			"fe1331481f85560681f86827ec"]
 		#"fec57041458e6cef98652df625", ]
+	log_ids = []
+	with open("/home/clemens/git/ma/test/filtered") as src:
+		for line in src:
+			line = line.strip()
+			log_ids.append(line)
 	store: ResultStore = ResultStore()
 	for log_id in log_ids:
 		for analysis in process_log(log_id, settings):
diff --git a/static/progress/my.js b/static/progress/my.js
index fec1c8c..e094e0e 100644
--- a/static/progress/my.js
+++ b/static/progress/my.js
@@ -1,11 +1,11 @@
 $.getJSON("tmp3.json", function (data) {
     var list = $("<ul />");
     var maps = {};
-    $.each(data, function (key, value) {
+    $.each(data, function (index, entry) {
         //key: instance_id, value: AnlysisResult
         //value.result.instance: InstanceConfig_id
         // console.log(key, value[0].result.store[0].timestamp);
-        $.each(value[0].result.store, function (index, entry) {
+        //$.each(value[0].result.store, function (index, entry) {
             //console.log(entry);
             var time = new Date(entry.timestamp);
             var item = $("<li>", {html: entry.sequence + " @ " + time.toLocaleDateString() + " "+ time.toLocaleTimeString()});
@@ -44,7 +44,7 @@ $.getJSON("tmp3.json", function (data) {
             });
             container.appendTo(item);
             item.appendTo(list);
-        });
+        //});
     });
     list.appendTo("body");
     var slider = $("<input />", {type: "range" })

From 076a12f72824190f0af83009c12b57d9d39ce4d2 Mon Sep 17 00:00:00 2001
From: agp8x <git@agp8x.org>
Date: Wed, 8 Nov 2017 00:31:33 +0100
Subject: [PATCH 02/24] 	neue Datei:     static/progress/docker-compose.yml

---
 static/progress/docker-compose.yml | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 static/progress/docker-compose.yml

diff --git a/static/progress/docker-compose.yml b/static/progress/docker-compose.yml
new file mode 100644
index 0000000..6e4a7f6
--- /dev/null
+++ b/static/progress/docker-compose.yml
@@ -0,0 +1,8 @@
+version: "2"
+services:
+ http:
+  image: httpd:alpine
+  volumes:
+   - ./:/usr/local/apache2/htdocs
+  ports:
+   - 5001:80

From e1105244f469aa5ed485d2f36efeb57cb112791c Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Wed, 8 Nov 2017 21:43:14 +0100
Subject: [PATCH 03/24] WIP snapshot

---
 .gitignore                     |   2 +-
 analyzers/__init__.py          |  17 ++--
 analyzers/analyzer/__init__.py |   3 +-
 analyzers/analyzer/biogames.py | 139 +++++++++++++++++++++++++++-----
 analyzers/analyzer/default.py  |   1 +
 analyzers/render/__init__.py   |   2 +-
 analyzers/render/biogames.py   | 107 ++++++++++++++++---------
 analyzers/render/default.py    |  47 +++++++++--
 analyzers/render/locomotion.py |   6 +-
 biogames2.json                 |   8 +-
 log_analyzer.py                | 142 ++++++++++++++++++++++++++++++---
 plots/.gitkeep                 |   0
 requirements.txt               |   3 +-
 sources/biogames.py            |   5 +-
 util/download.py               |  42 +++++++++-
 util/meta_temp.py              |  99 +++++++++++++++++++++++
 16 files changed, 526 insertions(+), 97 deletions(-)
 create mode 100644 plots/.gitkeep
 create mode 100644 util/meta_temp.py

diff --git a/.gitignore b/.gitignore
index 3b2357b..428a58b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,4 +4,4 @@ _*
 *.pyc
 logs/
 data/
-
+plots/*
diff --git a/analyzers/__init__.py b/analyzers/__init__.py
index e535085..4557f9d 100644
--- a/analyzers/__init__.py
+++ b/analyzers/__init__.py
@@ -2,7 +2,7 @@ from typing import List
 
 from .analyzer import Analyzer, Result
 from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
-	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer
+	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
 	CategorizerStub, Store, ProgressAnalyzer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
@@ -10,7 +10,8 @@ from .analyzer.mask import MaskSpatials
 from .render import Render
 from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
 	ActivityMapperRender, StoreRender, SimulationOrderRender, SimulationGroupRender
-from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender
+from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender, LogEntryCountAnalyzerPlot, \
+	LogEntryCountCSV
 from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
 	LocomotionActionRatioRender
 
@@ -21,7 +22,9 @@ __MAPPING__ = {
 		LocomotionActionRelativeRender,
 		LocomotionActionRatioRender, ],
 	LogEntryCountAnalyzer: [
-		JSONRender,
+		# JSONRender,
+		LogEntryCountAnalyzerPlot,
+		LogEntryCountCSV,
 	],
 	SimulationRoundsAnalyzer: [
 		JSONRender,
@@ -49,8 +52,8 @@ __MAPPING__ = {
 	],
 	SimulationOrderAnalyzer: [
 		JSONRender,
-#SimulationOrderRender,
-SimulationGroupRender
+		# SimulationOrderRender,
+		SimulationGroupRender
 	]
 }
 
@@ -61,10 +64,10 @@ def get_renderer(cls: type) -> [type]:
 	return __MAPPING__[cls]
 
 
-def render(cls: type, results: List[Result]):
+def render(cls: type, results: List[Result], name=None):
 	for r in get_renderer(cls):
 		p = r()
 		p.result_types.append(cls)
-		rendered = p.render(results)
+		rendered = p.render(results, name=name)
 		if rendered:
 			print(str(r))
diff --git a/analyzers/analyzer/__init__.py b/analyzers/analyzer/__init__.py
index 3b8e54f..e9f4e30 100644
--- a/analyzers/analyzer/__init__.py
+++ b/analyzers/analyzer/__init__.py
@@ -59,7 +59,8 @@ class ResultStore:
 
 	def get_category(self, key):
 		if key not in self.store:
-			return self.entry
+			return self.entry()
+		log.error("get_category %s %s", key, len(self.store[key]))
 		return self.store[key]
 
 	def serializable(self):
diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py
index 79b21a9..96215e3 100644
--- a/analyzers/analyzer/biogames.py
+++ b/analyzers/analyzer/biogames.py
@@ -4,7 +4,7 @@ from types import SimpleNamespace
 from typing import List, NamedTuple
 
 from util import json_path, combinate
-from util.download import download_board
+from util.download import download_board, get_board_data
 from . import Result, LogSettings, Analyzer, ResultStore
 from .default import CategorizerStub, Store
 
@@ -52,6 +52,36 @@ class BoardDurationAnalyzer(Analyzer):
 		self.last = {}
 
 
+class TypedBoardDuration(Analyzer):
+	__name__ = "BoardDuration"
+
+	def result(self, store: ResultStore) -> None:
+		pass
+
+	def process(self, entry: dict) -> bool:
+		entry_type = entry[self.settings.type_field]
+		if entry_type in self.settings.boards:
+			pass
+
+	def add_board(self, entry):
+		board_data = get_board_data(self.settings.source, )
+
+	def add_location(self, entry):
+		self.track['coordinates'].append(json_path(entry, self.settings.custom['coordinates']))
+
+	def add_track(self, **props):
+		self.track['properties'] = props
+		self.tracks.append(self.track)
+		self.track = dict(self.template)
+
+	def __init__(self, settings: LogSettings):
+		super().__init__(settings)
+		self.last_board = {}
+		self.tracks = []
+		self.template = {"type": "LineString", "coordinates": [], "properties": {}}
+		self.track = dict(self.template)
+
+
 class SimulationRoundsAnalyzer(Analyzer):
 	__name__ = "SimuRounds"
 
@@ -90,7 +120,7 @@ class ActivationSequenceAnalyzer(Analyzer):
 		return False
 
 
-class BiogamesCategorizer(CategorizerStub):
+class BiogamesCategorizer(CategorizerStub):  # TODO: refactor
 	__name__ = "BiogamesCategorizer"
 
 	def __init__(self, settings: LogSettings):
@@ -105,10 +135,21 @@ class BiogamesCategorizer(CategorizerStub):
 
 class ActivityMapper(Analyzer):
 	__name__ = "ActivityMapper"
+	classes = {
+		"sequence.simulation.": "simu",
+		"sequence.question.": "question",
+		"error": "error"
+	}
 
 	def __init__(self, settings: LogSettings) -> None:
 		super().__init__(settings)
 		self.store: List[self.State] = []
+		self.timeline = []
+		self.last_board = {}
+		self.last_board_type = "other"
+		self.last_coordinate = None
+		self.tracks = []
+		self.track = None
 		self.instance_config_id: str = None
 		self.filters = SimpleNamespace()
 		self.filters.start = lambda entry: combinate(self.settings.custom["sequences2"]["start"], entry)
@@ -116,7 +157,7 @@ class ActivityMapper(Analyzer):
 
 		self.State: NamedTuple = namedtuple("State", ["sequence", "events", "track", "timestamp"])
 
-	def result(self, store: ResultStore) -> None:
+	def result_old(self, store: ResultStore) -> None:
 		instance_config_id = self.instance_config_id
 		for active_segment in self.store:  # active_segment → sequence or None (None → map active)
 			seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
@@ -136,32 +177,69 @@ class ActivityMapper(Analyzer):
 						event["image"] = local_file[16:]
 		store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]}))
 
+	def result(self, store: ResultStore) -> None:
+
+		store.add(Result(type(self), {"instance": self.instance_config_id, "track": self.tracks, "boards": self.timeline}))
+
 	def process(self, entry: dict) -> bool:
+		if self.track is None:
+			self.track = self.new_track(entry['timestamp'])
 		if self.instance_config_id is None:
 			if entry[self.settings.type_field] in self.settings.custom['instance_start']:
 				self.instance_config_id = json_path(entry, self.settings.custom['instance_config_id'])
-		if self.filters.start(entry):
-			self.store.append(
-				self.State(
-					sequence=json_path(entry, json_path(self.settings.custom, "sequences2.id_field")),
-					events=[],
-					track=[],
-					timestamp=entry['timestamp']))
-		elif self.filters.end(entry) or not self.store:
-			self.store.append(self.State(sequence=None, events=[], track=[], timestamp=entry['timestamp']))
 
+		self.update_board_type(entry)
 		if entry[self.settings.type_field] in self.settings.spatials:
-			self.store[-1].track.append(
-				{
-					'timestamp': entry['timestamp'],
-					'coordinates': json_path(entry, "location.coordinates"),
-					'accuracy': entry['accuracy']
-				}
-			)
-		else:
-			self.store[-1].events.append(entry)
+			self.add_location(entry)
+		elif entry[self.settings.type_field] in self.settings.boards:
+			board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
+										entry["board_id"])
+			entry["extra_data"] = board_data
+			entry['coordinate'] = self.new_coordinate()
+			self.timeline.append(entry)
 		return False
 
+	def update_board_type(self, entry):
+		type = self.classify_entry(entry)
+		if not type == self.last_board_type:
+			self.add_track(activity_type=self.last_board_type,end_timestamp=entry['timestamp'])
+			self.last_board_type = type
+
+	def classify_entry(self, entry):
+		entry_type = entry[self.settings.type_field]
+		if self.filters.end(entry):
+			return "map"
+		if not entry_type in self.settings.boards:
+			return self.last_board_type
+		board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
+									entry["board_id"])
+		for pattern in self.classes:
+			if pattern in board_data['class']:
+				return self.classes[pattern]
+		if board_data['has_video']:
+			return "video"
+		elif  board_data['has_audio']:
+			return "audio"
+		elif  board_data['has_image']:
+			return "image"
+		return "other"
+
+	def new_coordinate(self):
+		return {"type": "Point", "coordinates": self.last_coordinate}
+
+	def add_location(self, entry):
+		coordinates = json_path(entry, self.settings.custom['coordinates'])
+		self.track['coordinates'].append(coordinates)
+		self.last_coordinate = coordinates
+
+	def add_track(self, **props):
+		self.track['properties'].update(props)
+		self.tracks.append(self.track)
+		self.track = self.new_track(props['end_timestamp'])
+
+	def new_track(self, timestamp):
+		return {"type": "LineString", "coordinates": [], "properties": {'start_timestamp': timestamp}}
+
 
 class BiogamesStore(Store):
 	__name__ = "BiogamesStore"
@@ -223,4 +301,21 @@ class SimulationOrderAnalyzer(Analyzer):
 				self.store[simu_id] += 1
 				if not simu_id in self.order:
 					self.order.append(simu_id)
-		return False
\ No newline at end of file
+		return False
+
+
+class SimulationCategorizer(CategorizerStub):  # TODO: refactor categorizer
+	__name__ = "SimulationCategorizer"
+
+	def __init__(self, settings: LogSettings):
+		super().__init__(settings)
+
+	def process(self, entry: dict) -> bool:
+		if self.key is "default":
+			if entry[self.settings.type_field] in self.settings.custom['instance_start']:
+				try:
+					self.key = json_path(entry, self.settings.custom['instance_config_id'])
+				except KeyError as e:
+					print(entry)
+					raise e
+		return False
diff --git a/analyzers/analyzer/default.py b/analyzers/analyzer/default.py
index 9841162..bcb9d2a 100644
--- a/analyzers/analyzer/default.py
+++ b/analyzers/analyzer/default.py
@@ -28,6 +28,7 @@ class LocationAnalyzer(Analyzer):
 
 
 class LogEntryCountAnalyzer(Analyzer):
+	#TODO: flexibler: z.b. min/max lat/long
 	"""
 	count occurrences of log entry types
 	"""
diff --git a/analyzers/render/__init__.py b/analyzers/render/__init__.py
index 25b87fc..aeb6a20 100644
--- a/analyzers/render/__init__.py
+++ b/analyzers/render/__init__.py
@@ -5,7 +5,7 @@ from .. import Result
 class Render:
 	result_types = []
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		raise NotImplementedError()
 
 	def filter(self, results: List[Result]):
diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py
index e2de09e..415b8a9 100644
--- a/analyzers/render/biogames.py
+++ b/analyzers/render/biogames.py
@@ -10,35 +10,60 @@ import networkx as nx
 import itertools
 
 from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
+from util.meta_temp import CONFIG_NAMES
 from . import Render
 from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
 
+
 def add_edge(graph, src, dest):
 	if graph.has_edge(src, dest):
 		weight = graph.get_edge_data(src, dest)['weight'] + 1
 	else:
 		weight = 1
-	graph.add_edge(tuple(src),tuple(dest),weight=weight)
+	graph.add_edge(tuple(src), tuple(dest), weight=weight)
+
 
 def pairs(iterable):
-	a,b = itertools.tee(iterable)
-	next(b,None)
-	return zip(a,b)
+	a, b = itertools.tee(iterable)
+	next(b, None)
+	return zip(a, b)
+
+
+def __plot_or_show(name=None):
+	if name:
+		plt.savefig(name)
+	else:
+		plt.show()
+	plt.cla()
+	plt.clf()
+	plt.close()
+
 
 def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
-         rotation='vertical'):
+		 rotation='vertical', name=None):
 	names, datas = list(zip(*src_data))
-	#plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True)
-	rand = np.random.rand(len(datas),len(datas[0]))
-	plt.plot(datas+rand, linewidth=.2)
+	# plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True)
+	rand = np.random.rand(len(datas), len(datas[0]))
+	plt.plot(datas + rand, linewidth=.2)
 	plt.xticks(rotation=rotation)
 	# plt.margins()
 	plt.ylabel(ylabel)
 	plt.title(title)
-	plt.show()
+	__plot_or_show(name)
+
 
 def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="sequential simulation retries",
-         rotation='vertical'):
+			   rotation='vertical', name=None):
+	config_name = CONFIG_NAMES[name] if name in CONFIG_NAMES else "---"
+	counts_per_group = [sum(i) for i in src_data]
+	label = "{}: n={n}; # of sim runs: ⌀={avg:.2f}, median={median}".format(
+		config_name,
+		n=len(src_data),
+		avg=np.mean(counts_per_group),
+		median=np.median(counts_per_group)
+	)
+	print(config_name)
+	name = "plots/{}.png".format(name)
 	g = nx.Graph()
 	for group in src_data:
 		for i in pairs(enumerate(group)):
@@ -46,34 +71,37 @@ def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds"
 	positions = {}
 	for node in g.nodes():
 		positions[node] = node
-	widths = [x[2]/10.0 for x in g.edges.data('weight')]
+	widths = [x[2] / 10.0 for x in g.edges.data('weight')]
 	print(max(widths))
 	nx.draw_networkx_edges(g, positions, width=widths)
-	#rand = np.random.rand(len(datas),len(datas[0]))
-	#plt.plot(datas+rand, linewidth=.2)
+	# rand = np.random.rand(len(datas),len(datas[0]))
+	# plt.plot(datas+rand, linewidth=.2)
 	plt.xticks(rotation=rotation)
 	# plt.margins()
 	plt.ylabel(ylabel)
 	plt.title(title)
-	plt.show()
+	plt.figtext(0.5, 0.13, label, ha="center")
+	__plot_or_show(name)
 
-def graph_fit(src_data, deg=5):
-	plt.title("polyfit(x,y,deg="+str(deg)+")")
+
+def graph_fit(src_data, deg=5, name=None):
+	plt.title("polyfit(x,y,deg=" + str(deg) + ")")
 	for i in src_data:
-		#plt.plot(i)
+		# plt.plot(i)
 		count = len(i)
-		xp = np.linspace(0, count-1, num=count, endpoint=True)
-		#fit = np.poly1d(np.polyfit(range(len(i)), i, deg=deg))
-		#plt.plot(xp, fit(xp), linewidth=0.1)
-		xnew = np.linspace(0, count-1, num=count*20, endpoint=True)
+		xp = np.linspace(0, count - 1, num=count, endpoint=True)
+		# fit = np.poly1d(np.polyfit(range(len(i)), i, deg=deg))
+		# plt.plot(xp, fit(xp), linewidth=0.1)
+		xnew = np.linspace(0, count - 1, num=count * 20, endpoint=True)
 		f = interp1d(xp, i, kind='quadratic')
-		
+
 		plt.plot(range(count), i, '.', markersize=1)
 		plt.plot(xnew, f(xnew), linewidth=0.2)
-	plt.show()
+		__plot_or_show(name)
+
 
 class SimulationRoundsRender(Render):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		data = defaultdict(list)
 		for result in self.filter(results):
 			get = result.get()
@@ -89,7 +117,7 @@ class SimulationRoundsRender(Render):
 class BoardDurationHistRender(Render):
 	result_types = [BoardDurationAnalyzer]
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		data = []
 		for result in self.filter(results):
 			session = result.get()
@@ -107,7 +135,7 @@ class BoardDurationHistRender(Render):
 class BoardDurationBoxRender(Render):
 	result_types = [BoardDurationAnalyzer]
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		data = defaultdict(list)
 		for result in self.filter(results):
 			get = result.get()
@@ -122,11 +150,11 @@ class BoardDurationBoxRender(Render):
 class ActivityMapperRender(Render):
 	result_types = [ActivityMapper]
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		print(os.getcwd())
 		for result in self.filter(results):
 			data = result.get()
-			with open(os.path.join("static", "progress", "data", data['instance']),"w") as out:
+			with open(os.path.join("static", "progress", "data", data['instance']), "w") as out:
 				json.dump(data["store"], out, indent=1)
 		return "ok"
 
@@ -134,31 +162,32 @@ class ActivityMapperRender(Render):
 class StoreRender(Render):
 	result_types = [Store, BiogamesStore]
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		for result in self.filter(results):
-			with open(os.path.join("static","progress","data","fooo"), "w") as out:
+			with open(os.path.join("static", "progress", "data", "fooo"), "w") as out:
 				json.dump(result.get(), out, indent=1)
 
 
 class SimulationOrderRender(Render):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		data = defaultdict(list)
 		for result in self.filter(results):
 			get = result.get()
-			for i,value in enumerate(get):
+			for i, value in enumerate(get):
 				data[i].append(value)
-		#data_tuples = [(key, data[key]) for key in sorted(data)]
-		#data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
-		#plot(enumerate([r.get() for r in self.filter(results)]))
+		# data_tuples = [(key, data[key]) for key in sorted(data)]
+		# data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
+		# plot(enumerate([r.get() for r in self.filter(results)]))
 		plot(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
 
 	result_types = [SimulationOrderAnalyzer]
 
 
 class SimulationGroupRender(Render):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		data = [r.get() for r in self.filter(results)]
-		#graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
-		graph_fit(list(data))
+		print(name, len(data))
+		graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None, name=name)
+		#graph_fit(list(data), name=name)
 
-	result_types = [SimulationOrderAnalyzer]
\ No newline at end of file
+	result_types = [SimulationOrderAnalyzer]
diff --git a/analyzers/render/default.py b/analyzers/render/default.py
index 044fdf3..bf57f61 100644
--- a/analyzers/render/default.py
+++ b/analyzers/render/default.py
@@ -2,6 +2,9 @@ import json
 import logging
 from typing import List
 
+import matplotlib.pyplot as plt
+
+from analyzers import LogEntryCountAnalyzer
 from . import Render, Result
 from .. import LocationAnalyzer
 
@@ -9,25 +12,26 @@ log = logging.getLogger(__name__)
 
 
 class PrintRender(Render):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		print("\t" + "\n\t".join([str(r) for r in results]))
 
 
 class JSONRender(Render):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		print(json.dumps([r.get() for r in self.filter(results)], indent=1))
 
 
 class TrackRender(Render):
 	result_types = [LocationAnalyzer]
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		data = []
 		log.debug(results)
 		for result in self.filter(results):
 			if len(result.get()) > 0:
 				data.append(
-					[[entry['location']['coordinates'][1], entry['location']['coordinates'][0]] for entry in  # TODO: configurable
+					[[entry['location']['coordinates'][1], entry['location']['coordinates'][0]] for entry in
+					 # TODO: configurable
 					 result.get()])
 		dumps = json.dumps(data)
 		with open("track_data.js", "w") as out:
@@ -38,7 +42,7 @@ class TrackRender(Render):
 class HeatMapRender(TrackRender):
 	weight = 0.01
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		raw = super(HeatMapRender, self).render(results)
 		data = []
 		for session in json.loads(raw):
@@ -47,3 +51,36 @@ class HeatMapRender(TrackRender):
 		with open('heat_data.js', 'w') as out:
 			out.write("coords = " + dumps + ";")
 		return dumps
+
+
+class LogEntryCountAnalyzerPlot(Render):
+	result_types = [LogEntryCountAnalyzer]
+
+	def render(self, results: List[Result], name=None):
+		raw_data = list(self.filter(results))[0].get()
+		print(raw_data)
+		labels = []
+		data = []
+		for x in sorted(raw_data.items()):
+			labels.append(str(x[0]).split(".")[-1])
+			data.append(x[1])
+		plt.bar(range(len(data)), list(data))
+		plt.xticks(range(len(data)), labels, rotation="vertical")
+		plt.tight_layout()
+
+		name = "plots/{}.png".format(name)
+		plt.savefig(name)
+		plt.cla()
+		plt.clf()
+		plt.close()
+
+class LogEntryCountCSV(Render):
+	result_types = [LogEntryCountAnalyzer]
+	summary = None
+
+	def render(self, results: List[Result], name=None):
+		if self.summary is None:
+			return
+		for result in self.filter(results):
+			raw_data = result.get()
+			self.summary[name] = raw_data
\ No newline at end of file
diff --git a/analyzers/render/locomotion.py b/analyzers/render/locomotion.py
index 7ec52f4..230a7d6 100644
--- a/analyzers/render/locomotion.py
+++ b/analyzers/render/locomotion.py
@@ -51,18 +51,18 @@ class LocomotionActionRender(Render):
 
 
 class LocomotionActionAbsoluteRender(LocomotionActionRender):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		results = filter_results(self.filter(results), ['locomotion_sum', 'action_sum'])
 		plot(results, "time", "abs loc/action")
 
 
 class LocomotionActionRelativeRender(LocomotionActionRender):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		results = filter_results(self.filter(results), ['locomotion_relative', 'action_relative'])
 		plot(results, "fraction of time", "rel loc/action")
 
 
 class LocomotionActionRatioRender(LocomotionActionRender):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		results = filter_results(self.filter(results), ['locomotion_action_ratio'])
 		plot_line(results, ylabel="Ratio", title="Locomotion/Action Ratio")
diff --git a/biogames2.json b/biogames2.json
index 9071e5a..83072ea 100644
--- a/biogames2.json
+++ b/biogames2.json
@@ -13,12 +13,14 @@
   ],
   "analyzers": {
     "analyzers": [
-      "BiogamesCategorizer",
-      "SimulationOrderAnalyzer"
+      "SimulationCategorizer",
+      "ActivityMapper"
     ]
   },
   "dis":[
-      "ActivityMapper",
+      "BiogamesCategorizer",
+      "LogEntryCountAnalyzer",
+      "SimulationOrderAnalyzer",
       "ProgressAnalyzer",
       "InstanceConfig"],
   "disabled_analyzers": [
diff --git a/log_analyzer.py b/log_analyzer.py
index f292a04..c0b0723 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -2,9 +2,12 @@ import json
 import logging
 from typing import List
 
+import numpy as np
+
 import analyzers
 from analyzers import get_renderer, Analyzer, render, Store
 from analyzers.analyzer import ResultStore
+from analyzers.render.default import LogEntryCountCSV
 from analyzers.settings import LogSettings, load_settings
 from loaders import LOADERS
 
@@ -30,8 +33,11 @@ def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]:
 	log.debug("process entries")
 	for entry in loader.get_entry():
 		for analyzer in analyzers:
-			if analyzer.process(entry):
-				break
+			try:
+				if analyzer.process(entry):
+					break
+			except KeyError as e:
+				log.exception(e)
 	return analyzers
 
 
@@ -44,18 +50,19 @@ if __name__ == '__main__':
 		"e32b16998440475b994ab46d481d3e0c",
 	]
 	log_ids: List[str] = [
-		#"34fecf49dbaca3401d745fb467",
+		# "34fecf49dbaca3401d745fb467",
 		#	"44ea194de594cd8d63ac0314be",
 		#	"57c444470dbf88605433ca935c",
 		#	"78e0c545b594e82edfad55bd7f",
 		#	"91abfd4b31a5562b1c66be37d9",
-			"597b704fe9ace475316c345903",
-			"e01a684aa29dff9ddd9705edf8",
-			"fbf9d64ae0bdad0de7efa3eec6",
+		"597b704fe9ace475316c345903",
+		"e01a684aa29dff9ddd9705edf8",
+		"fbf9d64ae0bdad0de7efa3eec6",
 		#	"fe1331481f85560681f86827ec",
-			"fe1331481f85560681f86827ec"]
-		#"fec57041458e6cef98652df625", ]
+		"fe1331481f85560681f86827ec"]
+	# "fec57041458e6cef98652df625", ]
 	log_ids = []
+	# with open("/home/clemens/git/ma/test/filtered") as src:
 	with open("/home/clemens/git/ma/test/filtered") as src:
 		for line in src:
 			line = line.strip()
@@ -72,17 +79,132 @@ if __name__ == '__main__':
 			r().render(store.get_all())
 	if False:
 		render(analyzers.LocationAnalyzer, store.get_all())
-	#print(json.dumps(store.serializable(), indent=1))
+	# print(json.dumps(store.serializable(), indent=1))
 	if False:
 		render(analyzers.ActivityMapper, store.get_all())
 		render(analyzers.ProgressAnalyzer, store.get_all())
 
 	if False:
 		from analyzers.postprocessing import graph
+
 		g = graph.Cache(settings)
 		g.run(store)
+	if False:
+		# render(analyzers.SimulationOrderAnalyzer, store.get_all())
+		for cat in store.get_categories():
+			data = store.get_category(cat)
+			render(analyzers.SimulationOrderAnalyzer, data, name=cat)
+	if False:
+		LogEntryCountCSV.summary = None
+		for cat in store.get_categories():
+			data = store.get_category(cat)
+			render(analyzers.LogEntryCountAnalyzer, data, name=cat)
+		if LogEntryCountCSV.summary:
+			headers = []
+			lines = []
+			for name in LogEntryCountCSV.summary:
+				data = LogEntryCountCSV.summary[name]
+				for head in data:
+					if not head in headers:
+						headers.append(head)
+				line = [name]
+				for head in headers:
+					line.append(data[head]) if head in data else line.append(0)
+				lines.append(line)
+			import csv
+
+			with open('logentrycount.csv', 'w', newline='') as csvfile:
+				writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE)
+				writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
+				for line in lines:
+					writer.writerow(line)
+
 	if True:
-		render(analyzers.SimulationOrderAnalyzer, store.get_all())
+		#json.dump(store.serializable(), open("new.json", "w"), indent=1)
+		from collections import defaultdict
+
+		keys = [
+			"simu",
+			"question",
+			"image",
+			"audio",
+			"video",
+			"other",
+			"map"
+		]
+		import matplotlib.pyplot as plt
+		#results = []
+
+		places = defaultdict(list)
+
+		for log in store.get_all():
+			result = defaultdict(lambda: 0)
+			for i in log.get()['track']:
+				duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
+				result[i['properties']['activity_type']] += duration
+			print(json.dumps(result, indent=4))
+			total = sum(result.values())
+			print(total)
+			percentage = defaultdict(lambda :0)
+			minutes = defaultdict(lambda:0)
+			for i in result:
+				percentage[i]= result[i]/total
+				minutes[i] = result[i]/60_000
+			print(json.dumps(percentage,indent=4))
+			if not 'error' in result:
+				#places[log.get()['instance']].append(percentage)
+				places[log.get()['instance']].append(minutes)
+
+		for place in places:
+			places[place] = sorted(places[place], key=lambda item:item['map'])
+
+		dummy = [0]*len(keys)
+		results = []
+		sites = []
+		from util.meta_temp import CONFIG_NAMES
+		for i in places:
+			for j in places[i]:
+				ordered = []
+				for k in keys:
+					ordered.append(j[k])
+				results.append(ordered)
+			results.append(dummy)
+			sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
+
+
+		size = len(results)
+		ind = np.arange(size)
+		width=0.9
+		print(results)
+		data = list(zip(*results))
+		print(data)
+		lines = []
+		bottom = [0]*len(results)
+		for i in range(0, len(data)):
+			lines.append(plt.bar(ind,data[i], bottom=bottom, width=width)[0])
+			for k,x in enumerate(data[i]):
+				bottom[k] += x
+		plt.legend(lines, keys)
+		plt.title(", ".join(sites))
+		plt.show()
+
+	#size = len(results)
+	#ind = np.arange(size)
+	#width = 0.9
+	#print(results)
+	#data = list(zip(*results))
+	#print(data)
+	#lines = []
+	#bottom = [0] * len(results)
+	#for i in range(0, len(data)):
+	#	lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+	#	for k, x in enumerate(data[i]):
+	#		bottom[k] += x
+	#plt.legend(lines, keys)
+	#plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)")
+	#plt.show()
+
+
 
 # for analyzers in analyzers:
 #	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:
diff --git a/plots/.gitkeep b/plots/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/requirements.txt b/requirements.txt
index dc88295..6168760 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,4 +3,5 @@ numpy==1.13.1
 matplotlib==2.1.0
 osmnx==0.6
 networkx==2.0
-pydot==1.2.3
\ No newline at end of file
+pydot==1.2.3
+scipy==1.0.0
\ No newline at end of file
diff --git a/sources/biogames.py b/sources/biogames.py
index e63a328..fa39b38 100644
--- a/sources/biogames.py
+++ b/sources/biogames.py
@@ -73,7 +73,10 @@ class Biogames(Source):
 				os.remove(filename)
 
 	def get_json(self, url):
-		return self._get(url, stream=False).json()
+		http = self._get(url, stream=False)
+		if not http.ok:
+			raise ConnectionError("HTTP status is not OK", http.url)
+		return http.json()
 
 	def close(self):
 		pass
diff --git a/util/download.py b/util/download.py
index 78dadc5..4e7dd84 100644
--- a/util/download.py
+++ b/util/download.py
@@ -31,12 +31,48 @@ def download_board(board_id, instance_config_id, sequence_id, source):
 
 def get_config(source, instance_id):
 	url = "/game2/editor/config/{config_id}/".format(config_id=instance_id)
-	instance_data = source.get_json(url)
+	instance_data = get_json(source, url)
 	caches = url + "cache/"
-	cache_data = source.get_json(caches)
+	cache_data = get_json(source, caches)
 
-	return {
+	result = {
 		"name": instance_data["name"],
 		"id": instance_data["@id"],
 		"caches": cache_data
 	}
+	return result
+
+
+def get_board_data(source, instance_id, sequence_id, board_id):
+	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
+		config_id=instance_id,
+		sequence_id=sequence_id,
+		board_id=board_id
+	)
+	instance_data = get_json(source, url)
+	if instance_data is None:
+		return {"class": "error"}
+	result = {
+		"class": instance_data["@class"],
+		"id": instance_data["@id"]
+	}
+	for i in ["image", "audio", "video"]:
+		key = i + "_file"
+		result["has_" + i] = bool(key in instance_data and instance_data[key])
+	return result
+
+
+cache = {}
+
+
+def get_json(source, url):
+	if url in cache:
+		return cache[url]
+	try:
+		data = source.get_json(url)
+
+	except Exception as e:
+		print("exception", e, e.args)  # TODO: logging
+		data = None
+	cache[url] = data
+	return data
diff --git a/util/meta_temp.py b/util/meta_temp.py
new file mode 100644
index 0000000..7efaf63
--- /dev/null
+++ b/util/meta_temp.py
@@ -0,0 +1,99 @@
+CONFIG_NAMES = {
+	'06c627ac-09fb-4f03-9a70-49261adefed9': u'Kopie 2 - Filderstadt',
+	'07c3566b-371b-4627-9fa6-96fdcf421ad8': u'Stadt\xf6kologieLindlarAWG-V.1',
+	'08369b6c-f699-41ba-9313-9d6ea2d22f78': u'Schierke',
+	'0bd1bba5-cde8-40e5-ad1c-326e55bf1247': u'Simulation: Wildkatzen',
+	'0d2e711b-be77-46e1-92f5-73199626b68c': u'Kopie 2 - Simulation: Rinder',
+	'11b9793e-7b4f-41ec-98fc-e46de557ae07': u'Kopie 11 - Bad Urach',
+	'13241906-cdae-441a-aed0-d57ebeb37cac': u'Stadt\xf6kologie',
+	'14dee52a-d040-4c70-9e1f-359c7faadfab': u'Kopie 5 - Bad Urach',
+	'14e8f4be-d27e-43a4-95e1-e033950a99bd': u'Kopie 13 - Bad Urach',
+	'16fc3117-61db-4f50-b84f-81de6310206f': u'Oberelsbach',
+	'17926099-4ed3-4ca0-996d-cc577c6fdaed': u'Kopie 6 - Bad Urach',
+	'17d401a9-de21-49a2-95bc-7dafa53dda64': u'Oberelsbach 2016',
+	'1cae4e4c-6d8b-43f0-b17d-1034b213cbaf': u'Test-Born',
+	'1f56b428-7c2c-4333-8fe1-c740ccbff40f': u'Bad Gateway',
+	'1f8b9d55-3b95-4739-914a-e2eff2dc52c3': u'Kopie 2 - Bad Gateway',
+	'2b3975be-242a-4c9d-80c7-8d9a370c9fe0': u'Simulation: Rinder',
+	'2bdc24f5-c51d-41a3-9cbd-adfc3a77a5ce': u'Simulation: Landnutzung',
+	'2c7cdb5d-7012-4a06-b4c8-980ad2470f10': u'Kopie 3 - Bad Gateway',
+	'30b743e6-144c-4fd7-a055-e87680f74c27': u'Nakundu2go',
+	'31da50e3-d59f-4166-91f2-7c84454c5769': u'Kopie 4 - Taltitz',
+	'3269d0d4-dc13-46f7-b666-1db94350fcd4': u'simu rindfleisch',
+	'32ed9be7-6fc2-4c50-afdb-8f32453d8409': u'Kopie 1 - Eichstaett - Stadt\xf6kologie2',
+	'33024899-7c10-4d44-b950-38fd0c2b0e16': u'Kopie 3 - Stadt\xf6kologie',
+	'3fe38d6e-04d8-49e7-a6d9-7a5f12635369': u'Kopie 8 - Bad Urach',
+	'50716b57-e494-46e0-9449-919cecb02a3d': u'Kopie 2 - Lindlar',
+	'5436357d-9644-4e3d-a181-bb4c6c0b3055': u'Kopie 4 - Bad Urach',
+	'543ac9b8-e990-4540-8277-994c3c756f47': u'Kopie 4 - Lindlar',
+	'5544ec80-5928-41e1-ba89-c13e570bda88': u'Test - Nakundu2Go',
+	'5637a078-c931-458d-865d-adc5d0653147': u'Kopie 2 - Bad Gateway',
+	'57e079b1-0a58-4150-a358-627fc9e896cc': u'Kopie 1 - Schierke',
+	'5cb3318a-cb5f-412f-bfd6-6467101ed505': u'Eichst\xe4tt-Schafe-1',
+	'5e64ce07-1c16-4d50-ac4e-b3117847ea43': u'Filderstadt',
+	'60e77829-2686-4022-84e6-b9e8875f7ca0': u'Kopie 10 - Bad Urach',
+	'6140a24e-32c6-4872-92b8-c463468f79a2': u'Taltitz neu',
+	'63610965-7a82-471b-a11a-0f696b4d6996': u'Kopie 3 - Lindlar',
+	'6479e339-f70a-4ed7-9b9e-9884a8037d81': u'Kopie 5 - Lindlar',
+	'658e1856-d04a-4284-9fb3-95c8e89843d9': u'Simulation: Streuobst',
+	'66fd2366-5985-4cac-8777-51a83e169d93': u'Kopie 1 - Test - Garnison Stadt\xf6kologie',
+	'681b9c2a-2547-4ffd-b510-ef28f5a2d355': u'Kopie 6 - Bad Gateway',
+	'74f0bd8c-c53c-4293-b583-1d7aec98fafa': u'Simulation: Luchse',
+	'78a00aac-422c-4772-9327-3241b32cea03': u'Kopie 2 - Stadt\xf6kologie',
+	'7a056d76-5636-45cc-a0bf-0555eff6101c': u'Test - Osnabr\xfcck Stadt\xf6kologie',
+	'7bf1de94-2627-489b-a310-cbad568d2230': u'Kopie 2 - Taltitz',
+	'7ea9ff83-c015-4ede-a561-8a16a1fb0833': u'Kopie 1 - Stadt\xf6kologieLindlarAWG-V.1',
+	'81ebf491-a556-43a8-b5d7-48ee193e2636': u'Test - Oberelsbach Wildkatze',
+	'877a8c70-fe0c-464b-98c1-73f8669cabd6': u'Mittenwald',
+	'890e99b0-eeed-4a20-ac21-ea027daf16f3': u'Kopie 3 - Bad Urach',
+	'8aa01b71-2609-4a47-a14c-8c3b51905fd2': u'? (lb)',
+	'8c002b38-606b-45cd-b046-bc4641188e18': u'Kopie 7 - Bad Urach',
+	'8cf124c1-3041-4e9b-a35a-1d4e09694917': u'Kopie 1 - AAAAA Bad Gateway',
+	'8e13e952-180c-4498-8730-9691dc837515': u'Test_Eichst\xe4tt_Schafe',
+	'90278021-4c57-464e-90b1-d603799d07eb': u'Eichst\xe4tt',
+	'92fc4eef-1489-4b31-b9e1-9d9436f7f43e': u'Kopie 5 - Taltitz',
+	'98c6aed7-3632-467e-9f20-5bdc3276f616': u'Kopie 8 - Bad Gateway',
+	'995e06bf-abc4-4103-a572-9f096d71d192': u'Eichstaett - Stadt\xf6kologie',
+	'9cb95913-8245-49e6-8416-ee6635e67aab': u'Kopie 2 - Simulation: Landnutzung',
+	'9e819058-f7f9-459e-9947-84349c7d849c': u'Kopie 9 - Bad Urach',
+	'9f99c761-4fb6-4636-92da-a9627977d8b3': u'Simulation: Schafe',
+	'a5fa36f5-7531-4821-ba0e-cf8f2a502ad4': u'Garmisch',
+	'a79bb488-5fea-4bf9-9b25-395691c8e7cd': u'Kopie 1 - A kopietest docker',
+	'abdf9bd0-9b7e-4286-a25e-2cb14742db30': u'Test - Bad Urach Streuobst',
+	'ac0eb831-0f47-4cf1-a1a5-2e6a535e70e9': u'Kopie 1 - Vorlagen',
+	'ae726f64-cfa5-4f86-9538-a1e63dd914cf': u'AAAAA Bad Gateway',
+	'b307e954-5f0e-43bb-855f-d39c1a8858bd': u'Kopie 7 - Bad Gateway',
+	'b58ea5b3-b864-42e3-aaf4-1a76633c037e': u'Kopie 4 - Bad Gateway',
+	'b5c27cc6-e2bc-4288-be97-1e5bc1f6f94f': u'Kopie 1 - Simulation: Landnutzung',
+	'b623a3c8-7ff8-47b5-853e-7b08e200dd27': u'Taltitz',
+	'be1b2167-293c-4a4b-beda-28d6aa8b47a7': u'Kopie 1 - Stadt\xf6kologie berichtigt',
+	'bf7146bd-c83b-4497-b948-dd6dfc8607aa': u'Kopie 1 - Rinder Lindlar gps',
+	'c2c87501-f180-40de-af7b-1a3288c82292': u'Eichstaett - Stadt\xf6kologie2',
+	'c3016b66-3c26-4ec4-bcf1-d36be07be037': u'?? (lb)',
+	'c3598b20-e8a5-45eb-953a-2b474fd2695a': u'Test - Eichst\xe4tt Schafe',
+	'c39fe95e-2cfd-461b-8103-cfd3e2b45e67': u'??? (lb)',
+	'c46994cc-5ca7-4f9f-b548-7bd6a6fff026': u'Kopie 1 - Bad Gateway',
+	'c528e93f-3469-4de9-b05d-e809575d2999': u'????',
+	'c6606915-3b7e-48f4-adfe-cbcf3130e76a': u'Kopie 12 - Bad Urach',
+	'c8ed936c-25d6-40ea-a731-2741c1d53b48': u'Born',
+	'c9df06e1-33a7-40fc-bd3d-0eba6abba245': u'Test - Schierke Luchse',
+	'ce093332-dc98-4cfa-9ff4-47903897e84f': u'Kopie 5 - Bad Gateway',
+	'ceb78b48-495d-4761-bb61-563fa4dd41fb': u'Kopie 2 - Eichstaett - Stadt\xf6kologie2',
+	'd5712976-59fa-452c-a5e3-8b4232b5cb44': u'Kopie 1 - Garmisch Test',
+	'd7d0be7e-a0ac-4f4f-910d-c55e13a01e88': u'Test - Garmisch Rinder',
+	'db1cd7aa-878b-4c30-9234-0739498996d6': u'Bad Urach - \xfcberarbeitet',
+	'df68db4d-3d51-45f3-96ed-08429a7de3c9': u'A kopietest docker',
+	'e3b0ffce-6135-400e-9796-d1aef173aaf5': u'Kopie 3 - Taltitz',
+	'e3e86e25-4d92-11e6-b176-00199963ac6e': u'Garmisch (geschrumpft)',
+	'e7b3094d-d3c6-41c7-92e3-28638365e018': u'Born II',
+	'e7f36db3-b919-4208-8d98-b5d400b5d972': u'Kopie 15 - Bad Urach',
+	'e9f35c27-6f4f-487c-b07e-9d0ab27a7b85': u'Kopie 1 - Filderstadt',
+	'ea94b249-439b-46dd-b621-e0fbe99aa4ee': u'Stadt\xf6kologie berichtigt',
+	'ec782ab1-eb9d-43b9-a2d1-4699f8432adb': u'Kopie 2 - Bad Urach',
+	'ecfdfd0b-28be-4df2-8994-8092a7fe87b5': u'Kopie 3 - Simulation: Landnutzung',
+	'f7bb56a3-fb15-413a-9a3e-f61e22d0a7d1': u'Kopie 2 - Schierke',
+	'f8f65e9d-de9e-4f8d-8bb6-d1e4e01593a0': u'Arbeitstitel',
+	'fca28f01-ea17-4c41-8e60-4726f96dfca8': u'Kopie 1 - Test-Born',
+	'fe43a0f0-3dea-11e6-a065-00199963ac6e': u'Vorlagen',
+	'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771': u'Bad Urach'
+}

From 4401757bef1321f5af882d52334c72b9fcbd3f40 Mon Sep 17 00:00:00 2001
From: agp8x <git@agp8x.org>
Date: Sat, 11 Nov 2017 16:27:48 +0100
Subject: [PATCH 04/24] add code to understand/explore simulationData flags

---
 analyzers/__init__.py          |  3 +-
 analyzers/analyzer/__init__.py | 14 ++++--
 analyzers/analyzer/biogames.py | 25 ++++++++--
 analyzers/analyzer/default.py  |  2 +-
 biogames2.json                 |  7 +--
 filter_todo/pre_filter.py      | 90 ++++++++++++++++++++++++++++++++++
 log_analyzer.py                | 23 +++++++--
 7 files changed, 148 insertions(+), 16 deletions(-)
 create mode 100644 filter_todo/pre_filter.py

diff --git a/analyzers/__init__.py b/analyzers/__init__.py
index 4557f9d..b928374 100644
--- a/analyzers/__init__.py
+++ b/analyzers/__init__.py
@@ -2,7 +2,8 @@ from typing import List
 
 from .analyzer import Analyzer, Result
 from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
-	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer
+	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer, \
+	SimulationFlagsAnalyzer
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
 	CategorizerStub, Store, ProgressAnalyzer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
diff --git a/analyzers/analyzer/__init__.py b/analyzers/analyzer/__init__.py
index e9f4e30..6b7d5e0 100644
--- a/analyzers/analyzer/__init__.py
+++ b/analyzers/analyzer/__init__.py
@@ -8,10 +8,11 @@ log: logging.Logger = logging.getLogger(__name__)
 
 
 class Result:
-	def __init__(self, analysis: Type, result: Sized):
+	def __init__(self, analysis: Type, result: Sized, name: str = None):
 		self.result = result
 		self.__analysis__ = analysis
 		log.debug("set" + str(len(self.result)))
+		self.name = name
 
 	def analysis(self):
 		return self.__analysis__
@@ -21,7 +22,8 @@ class Result:
 		return self.result
 
 	def __repr__(self):
-		return "<Result " + str(self.__analysis__) + ": " + str(type(self.result)) + " " + str(len(self.result)) + ">"
+		return "<Result " + str(self.__analysis__) + ": " + str(type(self.result)) + " " + str(
+			len(self.result)) + " for " + str(self.name) + ">"
 
 
 class ResultStore:
@@ -66,7 +68,11 @@ class ResultStore:
 	def serializable(self):
 		values = {}
 		for key in self.store:
-			values[key] = [{"analysis": str(result.analysis()), "result": result.get()} for result in self.store[key]]
+			values[key] = [{
+				"analysis": str(result.analysis()),
+				"result": result.get(),
+				"name": result.name
+			} for result in self.store[key]]
 		return values
 
 
@@ -84,7 +90,7 @@ class Analyzer:
 		"""
 		raise NotImplementedError()
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, name=None) -> None:
 		raise NotImplementedError()
 
 	def name(self) -> str:
diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py
index 96215e3..cbf9306 100644
--- a/analyzers/analyzer/biogames.py
+++ b/analyzers/analyzer/biogames.py
@@ -305,10 +305,7 @@ class SimulationOrderAnalyzer(Analyzer):
 
 
 class SimulationCategorizer(CategorizerStub):  # TODO: refactor categorizer
-	__name__ = "SimulationCategorizer"
-
-	def __init__(self, settings: LogSettings):
-		super().__init__(settings)
+	__name__ = "SimulationCategorizer"# TODO: rename -.- (InstanceConfigIDCategorizer)
 
 	def process(self, entry: dict) -> bool:
 		if self.key is "default":
@@ -319,3 +316,23 @@ class SimulationCategorizer(CategorizerStub):  # TODO: refactor categorizer
 					print(entry)
 					raise e
 		return False
+
+
+class SimulationFlagsAnalyzer(Analyzer):
+	__name__ = "SimuFlags"
+
+	def __init__(self, settings: LogSettings) -> None:
+		super().__init__(settings)
+		self.store = []
+
+	def process(self, entry: dict) -> bool:
+		entry_type = entry[self.settings.type_field]
+		if entry_type in self.settings.custom['simulation_rounds']:
+			if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]:
+				self.store.append(entry)
+		return False
+
+	def result(self, store: ResultStore, name=None) -> None:
+		store.add(Result(type(self), self.store, name=name))
+
+
diff --git a/analyzers/analyzer/default.py b/analyzers/analyzer/default.py
index bcb9d2a..9151313 100644
--- a/analyzers/analyzer/default.py
+++ b/analyzers/analyzer/default.py
@@ -89,7 +89,7 @@ class CategorizerStub(Analyzer):
 
 	__name__ = "Categorizer"
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, name=None) -> None:
 		store.new_category(self.key)
 
 	def __init__(self, settings: LogSettings):
diff --git a/biogames2.json b/biogames2.json
index 83072ea..7bedcb9 100644
--- a/biogames2.json
+++ b/biogames2.json
@@ -14,10 +14,11 @@
   "analyzers": {
     "analyzers": [
       "SimulationCategorizer",
-      "ActivityMapper"
+      "SimulationFlagsAnalyzer"
     ]
   },
   "dis":[
+      "ActivityMapper",
       "BiogamesCategorizer",
       "LogEntryCountAnalyzer",
       "SimulationOrderAnalyzer",
@@ -70,8 +71,8 @@
     "type": "Biogames",
     "url": "http://0.0.0.0:5000/game2/instance/log/list/",
     "login_url": "http://localhost:5000/game2/auth/json-login",
-    "username": "ba",
-    "password": "853451",
+    "username": "dev",
+    "password": "dev",
     "host":"http://0.0.0.0:5000"
   }
 }
\ No newline at end of file
diff --git a/filter_todo/pre_filter.py b/filter_todo/pre_filter.py
new file mode 100644
index 0000000..27480c4
--- /dev/null
+++ b/filter_todo/pre_filter.py
@@ -0,0 +1,90 @@
+import os
+from zipfile import ZipFile
+import sqlite3
+import json
+from collections import defaultdict
+
+
+def get_json(filename):
+	log = []
+	id = None
+	with ZipFile(filename) as zipf:
+		zipf.extract('instance_log.sqlite')
+		sql = sqlite3.connect('instance_log.sqlite')
+		cursor = sql.cursor()
+		for r in cursor.execute('SELECT json FROM log_entry;'):
+			entry = json.loads(r[0])
+			log.append(entry)
+			if id is None:
+				id = entry['instance_id']
+		sql.close()
+		os.remove('instance_log.sqlite')
+	return id, log
+
+
+def is_finished(log):
+	for entry in log:
+		if "action" in entry:
+			if "LogEntryInstanceAction" in entry["@class"] and entry["action"][
+				"@class"] == "de.findevielfalt.games.game2.instance.action.EndGameEnableAction" and entry['action'][
+				'enable']:
+				return True
+	return False
+
+
+def get_simus(log):
+	simus = defaultdict(lambda: 0)
+	order = []
+	actions = 0
+	for entry in log:
+		if "LogEntryQuestion" in entry["@class"]:
+			if "SimulationBoardData" in entry["answers"]["@class"]:
+				id = entry["answers"]["@id"]
+				simus[id] += 1
+				actions += 1 if entry['selected_actions'] else 0
+				if not id in order:
+					order.append(id)
+	return dict(simus), order, actions
+
+
+def simu_dist(simus):
+	dist = defaultdict(lambda: 0)
+	for instance in simus:
+		sim = simus[instance]
+		dist[len(sim)] += 1
+	return dist
+
+
+logs = {}
+finished = []
+simus = {}
+distribution = defaultdict(lambda: 0)
+finished_and_simu = defaultdict(list)
+files = {}
+actions_dist = defaultdict(list)
+with open('/home/agp8x/git/uni/ma/project/data/0000_ref') as src:
+	for line in src:
+		line = line.strip()
+		instance_id, log = get_json(line)
+		logs[instance_id] = log
+		files[instance_id] = line
+for id in logs:
+	simus[id] = get_simus(logs[id])
+	simu_count = len(simus[id][1])
+	distribution[simu_count] += 1
+	actions_dist[simus[id][2]].append(id)
+	if is_finished(logs[id]):
+		finished.append(id)
+		finished_and_simu[simu_count].append(id)
+print("total: ", len(logs))
+print("finished: ", len(finished))
+print("simu_dist: ", len(distribution), json.dumps(distribution, sort_keys=True))
+for i in sorted(finished_and_simu):
+	print("fin+sim" + str(i) + ": ", len(finished_and_simu[i]))
+for i in sorted(actions_dist):
+	print("actions: ", i, len(actions_dist[i]))
+print(json.dumps(actions_dist[4], sort_keys=True, indent=2))
+
+# print(finished_and_simu)
+# for instance in finished_and_simu:
+#	print(files[instance])
diff --git a/log_analyzer.py b/log_analyzer.py
index c0b0723..9b773e0 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -63,7 +63,7 @@ if __name__ == '__main__':
 	# "fec57041458e6cef98652df625", ]
 	log_ids = []
 	# with open("/home/clemens/git/ma/test/filtered") as src:
-	with open("/home/clemens/git/ma/test/filtered") as src:
+	with open("/home/agp8x/git/uni/ma/project/data/0000_ref") as src:
 		for line in src:
 			line = line.strip()
 			log_ids.append(line)
@@ -73,7 +73,7 @@ if __name__ == '__main__':
 			log.info("* Result for " + analysis.name())
 			# print(analysis.result())
 			# print(analysis.render())
-			analysis.result(store)
+			analysis.result(store, name=log_id)
 	if False:
 		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
 			r().render(store.get_all())
@@ -118,8 +118,25 @@ if __name__ == '__main__':
 				writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
 				for line in lines:
 					writer.writerow(line)
-
 	if True:
+		from datetime import datetime
+		json.dump(store.serializable(), open("simus.json", "w"), indent=2)
+		with open("simus.csv", "w") as csvfile:
+			csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n")
+			for key in store.get_store():
+				csvfile.write("{}\n".format(key))
+				for result in store.store[key]:
+					csvfile.write(",{}\n".format(result.name))
+					for i in result.get():
+						csvfile.write(",,{},{},{},{},{},{}\n".format(
+							i['answers']['@id'],
+							i['answers']['answered'],
+							len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0,
+							len(i['selected_actions']) if i['selected_actions'] else 0,
+							i['timestamp'],
+							str(datetime.fromtimestamp(i['timestamp']/1000))
+						))
+	if False:
 		#json.dump(store.serializable(), open("new.json", "w"), indent=1)
 		from collections import defaultdict
 

From b40efa4bbb60a35e19ecf91d175f14d403d846cf Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Mon, 13 Nov 2017 13:52:57 +0100
Subject: [PATCH 05/24] WIP snapshot

---
 analyzers/__init__.py          |   2 +-
 analyzers/analyzer/__init__.py |   2 +-
 analyzers/render/biogames.py   |   8 +-
 biogames2.json                 |   1 +
 log_analyzer.py                | 317 +++++++++++++++++++++++++--------
 requirements.txt               |   3 +-
 6 files changed, 253 insertions(+), 80 deletions(-)

diff --git a/analyzers/__init__.py b/analyzers/__init__.py
index 4557f9d..d09114a 100644
--- a/analyzers/__init__.py
+++ b/analyzers/__init__.py
@@ -51,7 +51,7 @@ __MAPPING__ = {
 		StoreRender
 	],
 	SimulationOrderAnalyzer: [
-		JSONRender,
+		#JSONRender,
 		# SimulationOrderRender,
 		SimulationGroupRender
 	]
diff --git a/analyzers/analyzer/__init__.py b/analyzers/analyzer/__init__.py
index e9f4e30..1740d08 100644
--- a/analyzers/analyzer/__init__.py
+++ b/analyzers/analyzer/__init__.py
@@ -50,7 +50,7 @@ class ResultStore:
 		:return:
 		"""
 		result = []
-		for key in self.store:
+		for key in sorted(self.store):
 			result += self.store[key]
 		return result
 
diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py
index 415b8a9..dd6fc01 100644
--- a/analyzers/render/biogames.py
+++ b/analyzers/render/biogames.py
@@ -185,7 +185,13 @@ class SimulationOrderRender(Render):
 
 class SimulationGroupRender(Render):
 	def render(self, results: List[Result], name=None):
-		data = [r.get() for r in self.filter(results)]
+		#data = [r.get() for r in self.filter(results)]
+		data = []
+		for r in self.filter(results):
+			raw = r.get()
+			if len(raw) < 6:
+				raw = [0] + raw
+			data.append(raw)
 		print(name, len(data))
 		graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None, name=name)
 		#graph_fit(list(data), name=name)
diff --git a/biogames2.json b/biogames2.json
index 83072ea..cc1633b 100644
--- a/biogames2.json
+++ b/biogames2.json
@@ -14,6 +14,7 @@
   "analyzers": {
     "analyzers": [
       "SimulationCategorizer",
+      "SimulationOrderAnalyzer",
       "ActivityMapper"
     ]
   },
diff --git a/log_analyzer.py b/log_analyzer.py
index c0b0723..bb542ea 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -63,17 +63,18 @@ if __name__ == '__main__':
 	# "fec57041458e6cef98652df625", ]
 	log_ids = []
 	# with open("/home/clemens/git/ma/test/filtered") as src:
-	with open("/home/clemens/git/ma/test/filtered") as src:
-		for line in src:
-			line = line.strip()
-			log_ids.append(line)
-	store: ResultStore = ResultStore()
-	for log_id in log_ids:
-		for analysis in process_log(log_id, settings):
-			log.info("* Result for " + analysis.name())
-			# print(analysis.result())
-			# print(analysis.render())
-			analysis.result(store)
+	if False:
+		with open("/home/clemens/git/ma/test/filtered_5_actions") as src:
+			for line in src:
+				line = line.strip()
+				log_ids.append(line)
+		store: ResultStore = ResultStore()
+		for log_id in log_ids:
+			for analysis in process_log(log_id, settings):
+				log.info("* Result for " + analysis.name())
+				# print(analysis.result())
+				# print(analysis.render())
+				analysis.result(store)
 	if False:
 		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
 			r().render(store.get_all())
@@ -120,8 +121,10 @@ if __name__ == '__main__':
 					writer.writerow(line)
 
 	if True:
-		#json.dump(store.serializable(), open("new.json", "w"), indent=1)
+		# json.dump(store.serializable(), open("new.json", "w"), indent=1)
 		from collections import defaultdict
+		import matplotlib.pyplot as plt
+		from util.meta_temp import CONFIG_NAMES
 
 		keys = [
 			"simu",
@@ -130,81 +133,243 @@ if __name__ == '__main__':
 			"audio",
 			"video",
 			"other",
-			"map"
+			"map",
+			# "error"
 		]
-		import matplotlib.pyplot as plt
-		#results = []
 
-		places = defaultdict(list)
 
-		for log in store.get_all():
-			result = defaultdict(lambda: 0)
-			for i in log.get()['track']:
-				duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
-				result[i['properties']['activity_type']] += duration
-			print(json.dumps(result, indent=4))
-			total = sum(result.values())
-			print(total)
-			percentage = defaultdict(lambda :0)
-			minutes = defaultdict(lambda:0)
-			for i in result:
-				percentage[i]= result[i]/total
-				minutes[i] = result[i]/60_000
-			print(json.dumps(percentage,indent=4))
-			if not 'error' in result:
-				#places[log.get()['instance']].append(percentage)
-				places[log.get()['instance']].append(minutes)
+		def get_data(store, relative_values=True, sort=True, show_errors=False):
+			places = defaultdict(list)
 
-		for place in places:
-			places[place] = sorted(places[place], key=lambda item:item['map'])
+			for log in store.get_all():
+				if not log.analysis() == analyzers.ActivityMapper:
+					continue
+				result = defaultdict(lambda: 0)
+				for i in log.get()['track']:
+					duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
+					result[i['properties']['activity_type']] += duration
+				print(json.dumps(result, indent=4))
+				total = sum(result.values())
+				print(total)
+				percentage = defaultdict(lambda: 0)
+				minutes = defaultdict(lambda: 0)
+				for i in result:
+					percentage[i] = result[i] / total
+					minutes[i] = result[i] / 60_000
+				print(json.dumps(percentage, indent=4))
+				if not 'error' in result or show_errors:
+					if relative_values:
+						places[log.get()['instance']].append(percentage)
+					else:
+						places[log.get()['instance']].append(minutes)
+			if sort:
+				for place in places:
+					places[place] = sorted(places[place], key=lambda item: item['map'])
+			return places
 
-		dummy = [0]*len(keys)
-		results = []
-		sites = []
-		from util.meta_temp import CONFIG_NAMES
-		for i in places:
-			for j in places[i]:
-				ordered = []
+
+		from shapely.geometry import LineString
+		from shapely.ops import transform
+		from functools import partial
+		import pyproj
+
+
+		def calc_distance(coordinates):
+			track = LineString(coordinates)
+			project = partial(
+				pyproj.transform,
+				pyproj.Proj(init='EPSG:4326'),
+				pyproj.Proj(init='EPSG:32633'))
+			return transform(project, track).length
+
+
+		def get_data_distance(store, relative_values=True, sort=True, show_errors=False):
+			places = defaultdict(list)
+
+			for log in store.get_all():
+				if not log.analysis() == analyzers.ActivityMapper:
+					continue
+				result = defaultdict(lambda: 0)
+				for i in log.get()['track']:
+					coords = i['coordinates']
+					if len(coords) > 1:
+						distance = calc_distance(coords)
+						result[i['properties']['activity_type']] += distance
+				total = sum(result.values())
+				percentage = defaultdict(lambda: 0)
+				for i in result:
+					if not total == 0:
+						percentage[i] = result[i] / total
+				if not 'error' in result or show_errors:
+					if relative_values:
+						places[log.get()['instance']].append(percentage)
+					else:
+						places[log.get()['instance']].append(result)
+			if sort:
+				for place in places:
+					places[place] = sorted(places[place], key=lambda item: item['map'])
+			return places
+
+
+		def get_all_data(store):
+			places = defaultdict(list)
+
+			for log in store.get_all():
+				if not log.analysis() == analyzers.ActivityMapper:
+					continue
+				result = defaultdict(lambda: defaultdict(lambda: 0))
+				for i in log.get()['track']:
+					coords = i['coordinates']
+					if len(coords) > 1:
+						distance = calc_distance(coords)
+					else:
+						distance = 0.1
+					result["space"][i['properties']['activity_type']] += distance
+					duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
+					result["time"][i['properties']['activity_type']] += duration
+				total_space = sum(result["space"].values())
+				total_time = sum(result["time"].values())
+				percentage = defaultdict(lambda: defaultdict(lambda: 0))
+				for i in result["space"]:
+					if not total_space == 0:
+						percentage[i]["space"] = result["space"][i] / total_space
+					else:
+						percentage[i]["space"] = 0
+					if not total_time == 0:
+						percentage[i]["time"] = result["time"][i] / total_time
+					else:
+						percentage[i]["time"] = 0
+				print(percentage)
+				if not 'error' in result:
+					places[log.get()['instance']].append(percentage)
+			return places
+
+
+		def stack_data(keys, places):
+			dummy = [0] * len(keys)
+			results = []
+			sites = []
+			for i in places:
+				for j in places[i]:
+					ordered = []
+					for k in keys:
+						ordered.append(j[k])
+					results.append(ordered)
+				results.append(dummy)
+				sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
+			return results, sites
+
+
+		def plot_data(places, keys):
+			results, sites = stack_data(keys, places)
+
+			size = len(results)
+			print("{} elements total".format(size))
+			ind = np.arange(size)
+			width = 1
+			# print(results)
+			data = list(zip(*results))
+			# print(data)
+			lines = []
+			bottom = [0] * size
+			for i in range(0, len(data)):
+				lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+				for k, x in enumerate(data[i]):
+					bottom[k] += x
+			plt.legend(lines, keys)
+			plt.title(", ".join(sites))
+			plt.show()
+
+
+		colors = {
+			"simu": "blue",
+			"question": "orange",
+			"image": "green",
+			"audio": "red",
+			"video": "purple",
+			"other": "brown",
+			"map": "violet",
+			# "error":"grey"
+		}
+		markers = [".", "o", "x", "s", "*", "D", "p", ",", "<", ">", "^", "v", "1", "2", "3", "4"]
+
+
+		def plot_time_space(time_data, space_data, keys):
+			# assuming time_data and space_data are in same order!
+			marker = 0
+			for id in time_data:
 				for k in keys:
-					ordered.append(j[k])
-				results.append(ordered)
-			results.append(dummy)
-			sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
+					for i in range(len(time_data[id])):
+						print(time_data[id][i][k], space_data[id][i][k])
+						plt.plot(time_data[id][i][k], space_data[id][i][k], color=colors[k], marker=markers[marker])
+				marker += 1
+			plt.show()
 
 
-		size = len(results)
-		ind = np.arange(size)
-		width=0.9
-		print(results)
-		data = list(zip(*results))
-		print(data)
-		lines = []
-		bottom = [0]*len(results)
-		for i in range(0, len(data)):
-			lines.append(plt.bar(ind,data[i], bottom=bottom, width=width)[0])
-			for k,x in enumerate(data[i]):
-				bottom[k] += x
-		plt.legend(lines, keys)
-		plt.title(", ".join(sites))
-		plt.show()
+		# plt.cla()
+		# plt.clf()
+		# plt.close()
 
-	#size = len(results)
-	#ind = np.arange(size)
-	#width = 0.9
-	#print(results)
-	#data = list(zip(*results))
-	#print(data)
-	#lines = []
-	#bottom = [0] * len(results)
-	#for i in range(0, len(data)):
-	#	lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
-	#	for k, x in enumerate(data[i]):
-	#		bottom[k] += x
-	#plt.legend(lines, keys)
-	#plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)")
-	#plt.show()
+		def plot_time_space_rel(combined, keys):
+			groups = defaultdict(list)
+			keys = list(keys)
+			keys.remove("other")
+			for k in keys:
+				for id in sorted(combined):
+					group = 0.0
+					count = 0
+					for item in combined[id]:
+						if k in item:
+							time = item[k]["time"]
+							distance = item[k]["space"]
+							if time > 0:
+								group += (distance / time)
+								count+=1
+							else:
+								print("div by zero", distance, time)
+					if count > 0:
+						groups[k].append(group/count)
+					else:
+						groups[k].append(0.0)
+			ind = np.arange(len(combined.keys()))
+			width = .7 / len(groups)
+			print(ind)
+			print(json.dumps(groups, indent=1))
+			bars = []
+			fig, ax = plt.subplots()
+			for k in groups:
+				print(groups[k])
+				if not len(groups[k]):
+					groups[k].append(0)
+				bars.append(ax.bar(ind, groups[k], width, color=colors[k]))
+				ind = ind + width
+			ax.set_xticks(ind + width / 2)
+			ax.set_xticklabels(list([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in sorted(combined.keys())]))
+			plt.legend(bars, keys)
+			print(combined.keys())
+			print([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in sorted(combined.keys())])
+			plt.show()
 
 
+		# spatial_data = get_data_distance(store,relative_values=False)
+		# temporal_data = get_data(store,relative_values=False)
+		# spatial_data_rel = get_data_distance(store,relative_values=True)
+		# temporal_data_rel = get_data(store,relative_values=True)
+		#temporal_data_rel = json.load(open("temporal_rel.json"))
+		#spatial_data_rel = json.load(open("spatial_rel.json"))
+		# import IPython
+		# IPython.embed()
+
+		#print(json.dumps(get_all_data(store)))
+#		json.dump(get_all_data(store), open("combined.json", "w"))
+		combined = json.load(open("combined.json"))
+		plot_time_space_rel(combined, keys)
+
+		#plot_time_space_rel(temporal_data_rel, spatial_data_rel, keys)
+
+	# plot_data(data, keys)
+	# plot_data(get_data_distance(store,relative_values=False), keys)
+
 
 # for analyzers in analyzers:
 #	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:
diff --git a/requirements.txt b/requirements.txt
index 6168760..5813878 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,4 +4,5 @@ matplotlib==2.1.0
 osmnx==0.6
 networkx==2.0
 pydot==1.2.3
-scipy==1.0.0
\ No newline at end of file
+scipy==1.0.0
+ipython==6.2.1
\ No newline at end of file

From e94fd665ada44213a46fdc984ae19be963e7f261 Mon Sep 17 00:00:00 2001
From: agp8x <git@agp8x.org>
Date: Tue, 14 Nov 2017 19:32:10 +0100
Subject: [PATCH 06/24] activity mapper track display basic function

---
 analyzers/analyzer/biogames.py |  75 ++++++-----
 analyzers/render/biogames.py   |  14 +-
 biogames2.json                 |   4 +-
 log_analyzer.py                | 234 +++++++++++++++++++--------------
 static/progress/index.html     | 109 +++++++++++++--
 static/progress/my.js          | 149 ++++++++++++---------
 util/download.py               |   1 -
 7 files changed, 367 insertions(+), 219 deletions(-)

diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py
index cbf9306..e7ff5ae 100644
--- a/analyzers/analyzer/biogames.py
+++ b/analyzers/analyzer/biogames.py
@@ -140,6 +140,16 @@ class ActivityMapper(Analyzer):
 		"sequence.question.": "question",
 		"error": "error"
 	}
+	colors = {
+		"simu": "blue",
+		"question": "orange",
+		"image": "green",
+		"audio": "red",
+		"video": "purple",
+		"other": "brown",
+		"map": "violet",
+		"error": "grey"
+	}
 
 	def __init__(self, settings: LogSettings) -> None:
 		super().__init__(settings)
@@ -157,29 +167,28 @@ class ActivityMapper(Analyzer):
 
 		self.State: NamedTuple = namedtuple("State", ["sequence", "events", "track", "timestamp"])
 
-	def result_old(self, store: ResultStore) -> None:
-		instance_config_id = self.instance_config_id
-		for active_segment in self.store:  # active_segment → sequence or None (None → map active)
-			seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
-				config_id=instance_config_id,
-				sequence_id=active_segment.sequence,
-			)
-			source = self.settings.source
-			seq_data = source._get(seq_data_url).json()
-			# TODO: use sequence names
-			logger.warning(seq_data)
-			for event in active_segment.events:
-				if event[self.settings.type_field] in self.settings.boards:
-					sequence_id = active_segment.sequence
-					board_id = event["board_id"]
-					local_file = download_board(board_id, instance_config_id, sequence_id, source)
-					if local_file is not None:
-						event["image"] = local_file[16:]
-		store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]}))
-
-	def result(self, store: ResultStore) -> None:
-
-		store.add(Result(type(self), {"instance": self.instance_config_id, "track": self.tracks, "boards": self.timeline}))
+	def result(self, store: ResultStore, **kwargs) -> None:
+		for board in self.timeline:
+			if board[self.settings.type_field] in self.settings.boards:
+				if board["extra_data"]["activity_type"] == "simu":
+					board["image"] = "simu.png"
+					continue
+				local_file = download_board(board["board_id"], self.instance_config_id, board["sequence_id"],
+				                            self.settings.source)
+				if local_file:
+					board['image'] = local_file
+				else:
+					board['image'] = "ERROR_FETCHING_FILE"
+					logger.error("error downloading board! %s %s %s", self.instance_config_id, board["sequence_id"],
+					             board["board_id"])
+			else:
+				board["image"] = "map.png"
+		store.add(Result(type(self), {
+			"instance": self.instance_config_id,
+			"track": self.tracks,
+			"boards": self.timeline,
+			"colors": self.colors,
+		}))
 
 	def process(self, entry: dict) -> bool:
 		if self.track is None:
@@ -193,8 +202,9 @@ class ActivityMapper(Analyzer):
 			self.add_location(entry)
 		elif entry[self.settings.type_field] in self.settings.boards:
 			board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
-										entry["board_id"])
+			                            entry["board_id"])
 			entry["extra_data"] = board_data
+			entry["extra_data"]["activity_type"] = self.classify_entry(entry)
 			entry['coordinate'] = self.new_coordinate()
 			self.timeline.append(entry)
 		return False
@@ -202,25 +212,28 @@ class ActivityMapper(Analyzer):
 	def update_board_type(self, entry):
 		type = self.classify_entry(entry)
 		if not type == self.last_board_type:
-			self.add_track(activity_type=self.last_board_type,end_timestamp=entry['timestamp'])
+			self.add_track(activity_type=self.last_board_type, end_timestamp=entry['timestamp'])
 			self.last_board_type = type
 
 	def classify_entry(self, entry):
 		entry_type = entry[self.settings.type_field]
 		if self.filters.end(entry):
+			data = {"extra_data": {"activity_type": "map"},"coordinate": self.new_coordinate()}
+			data.update(entry)
+			self.timeline.append(data)
 			return "map"
 		if not entry_type in self.settings.boards:
 			return self.last_board_type
 		board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
-									entry["board_id"])
+		                            entry["board_id"])
 		for pattern in self.classes:
 			if pattern in board_data['class']:
 				return self.classes[pattern]
 		if board_data['has_video']:
 			return "video"
-		elif  board_data['has_audio']:
+		elif board_data['has_audio']:
 			return "audio"
-		elif  board_data['has_image']:
+		elif board_data['has_image']:
 			return "image"
 		return "other"
 
@@ -236,6 +249,8 @@ class ActivityMapper(Analyzer):
 		self.track['properties'].update(props)
 		self.tracks.append(self.track)
 		self.track = self.new_track(props['end_timestamp'])
+		if self.last_coordinate:
+			self.track['coordinates'].append(self.last_coordinate)
 
 	def new_track(self, timestamp):
 		return {"type": "LineString", "coordinates": [], "properties": {'start_timestamp': timestamp}}
@@ -305,7 +320,7 @@ class SimulationOrderAnalyzer(Analyzer):
 
 
 class SimulationCategorizer(CategorizerStub):  # TODO: refactor categorizer
-	__name__ = "SimulationCategorizer"# TODO: rename -.- (InstanceConfigIDCategorizer)
+	__name__ = "SimulationCategorizer"  # TODO: rename -.- (InstanceConfigIDCategorizer)
 
 	def process(self, entry: dict) -> bool:
 		if self.key is "default":
@@ -334,5 +349,3 @@ class SimulationFlagsAnalyzer(Analyzer):
 
 	def result(self, store: ResultStore, name=None) -> None:
 		store.add(Result(type(self), self.store, name=name))
-
-
diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py
index 415b8a9..ebe38dd 100644
--- a/analyzers/render/biogames.py
+++ b/analyzers/render/biogames.py
@@ -40,7 +40,7 @@ def __plot_or_show(name=None):
 
 
 def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
-		 rotation='vertical', name=None):
+         rotation='vertical', name=None):
 	names, datas = list(zip(*src_data))
 	# plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True)
 	rand = np.random.rand(len(datas), len(datas[0]))
@@ -53,7 +53,7 @@ def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", titl
 
 
 def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="sequential simulation retries",
-			   rotation='vertical', name=None):
+               rotation='vertical', name=None):
 	config_name = CONFIG_NAMES[name] if name in CONFIG_NAMES else "---"
 	counts_per_group = [sum(i) for i in src_data]
 	label = "{}: n={n}; # of sim runs: ⌀={avg:.2f}, median={median}".format(
@@ -154,8 +154,9 @@ class ActivityMapperRender(Render):
 		print(os.getcwd())
 		for result in self.filter(results):
 			data = result.get()
-			with open(os.path.join("static", "progress", "data", data['instance']), "w") as out:
-				json.dump(data["store"], out, indent=1)
+			with open(os.path.join("static", "progress", "data", data['instance'] + "_" + str(name) + ".json"),
+			          "w") as out:
+				json.dump(data, out, indent=1)
 		return "ok"
 
 
@@ -187,7 +188,8 @@ class SimulationGroupRender(Render):
 	def render(self, results: List[Result], name=None):
 		data = [r.get() for r in self.filter(results)]
 		print(name, len(data))
-		graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None, name=name)
-		#graph_fit(list(data), name=name)
+		# graph_fit(list(data), name=name)
+		graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None,
+		           name=name)
 
 	result_types = [SimulationOrderAnalyzer]
diff --git a/biogames2.json b/biogames2.json
index 7bedcb9..589805d 100644
--- a/biogames2.json
+++ b/biogames2.json
@@ -13,7 +13,8 @@
   ],
   "analyzers": {
     "analyzers": [
-      "SimulationCategorizer",
+      "BiogamesCategorizer",
+      "ActivityMapper",
       "SimulationFlagsAnalyzer"
     ]
   },
@@ -23,6 +24,7 @@
       "LogEntryCountAnalyzer",
       "SimulationOrderAnalyzer",
       "ProgressAnalyzer",
+      "SimulationCategorizer",
       "InstanceConfig"],
   "disabled_analyzers": [
       "LocomotionActionAnalyzer",
diff --git a/log_analyzer.py b/log_analyzer.py
index 9b773e0..3a492c6 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -14,13 +14,11 @@ from loaders import LOADERS
 logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG)
 log: logging.Logger = logging.getLogger(__name__)
 
-requests_log = logging.getLogger('requests')
-requests_log.setLevel(logging.WARN)
+logging.getLogger('requests').setLevel(logging.WARN)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 
-def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]:
-	logfile: str = "data/inst_{id}.{format}".format(id=log_id, format=settings.log_format)
-	logfile = log_id
+def process_log(logfile: str, settings: LogSettings) -> List[Analyzer]:
 	loader = LOADERS[settings.log_format]()
 	try:
 		loader.load(logfile)
@@ -41,48 +39,102 @@ def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]:
 	return analyzers
 
 
+def run_analysis(log_ids: list, settings):
+	store: ResultStore = ResultStore()
+	for log_id in log_ids:
+		for analysis in process_log(log_id, settings):
+			log.info("* Result for " + analysis.name())
+			analysis.result(store, name=log_id)
+	return store
+
+
+def load_ids(name: str):
+	log_ids = []
+	with open(name) as src:
+		for line in src:
+			line = line.strip()
+			log_ids.append(line)
+	return log_ids
+
+
+def urach_logs(log_ids, settings):
+	return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
+
+
+def write_logentry_count_csv():
+	global cat, data, lines, csvfile
+	LogEntryCountCSV.summary = None
+	for cat in store.get_categories():
+		data = store.get_category(cat)
+		render(analyzers.LogEntryCountAnalyzer, data, name=cat)
+	if LogEntryCountCSV.summary:
+		headers = []
+		lines = []
+		for name in LogEntryCountCSV.summary:
+			data = LogEntryCountCSV.summary[name]
+			for head in data:
+				if not head in headers:
+					headers.append(head)
+			line = [name]
+			for head in headers:
+				line.append(data[head]) if head in data else line.append(0)
+			lines.append(line)
+		import csv
+
+		with open('logentrycount.csv', 'w', newline='') as csvfile:
+			writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE)
+			writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
+			for line in lines:
+				writer.writerow(line)
+
+
+def write_simulation_flag_csv():
+	global csvfile, result, i
+	from datetime import datetime
+	json.dump(store.serializable(), open("simus.json", "w"), indent=2)
+	with open("simus.csv", "w") as csvfile:
+		csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n")
+		for key in store.get_store():
+			csvfile.write("{}\n".format(key))
+			for result in store.store[key]:
+				csvfile.write(",{}\n".format(result.name))
+				for i in result.get():
+					csvfile.write(",,{},{},{},{},{},{}\n".format(
+						i['answers']['@id'],
+						i['answers']['answered'],
+						len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0,
+						len(i['selected_actions']) if i['selected_actions'] else 0,
+						i['timestamp'],
+						str(datetime.fromtimestamp(i['timestamp'] / 1000))
+					))
+
+
 if __name__ == '__main__':
 	settings: LogSettings = load_settings("biogames2.json")
-	log_ids: List[str] = [
-		"20d4244719404ffab0ca386c76e4b112",
-		"56d9b64144ab44e7b90bf766f3be32e3",
-		"dc2cdc28ca074715b905e4aa5badff10",
-		"e32b16998440475b994ab46d481d3e0c",
-	]
-	log_ids: List[str] = [
+	log_ids_urach: List[str] = urach_logs([
 		# "34fecf49dbaca3401d745fb467",
 		#	"44ea194de594cd8d63ac0314be",
 		#	"57c444470dbf88605433ca935c",
 		#	"78e0c545b594e82edfad55bd7f",
 		#	"91abfd4b31a5562b1c66be37d9",
-		"597b704fe9ace475316c345903",
-		"e01a684aa29dff9ddd9705edf8",
-		"fbf9d64ae0bdad0de7efa3eec6",
-		#	"fe1331481f85560681f86827ec",
-		"fe1331481f85560681f86827ec"]
-	# "fec57041458e6cef98652df625", ]
-	log_ids = []
-	# with open("/home/clemens/git/ma/test/filtered") as src:
-	with open("/home/agp8x/git/uni/ma/project/data/0000_ref") as src:
-		for line in src:
-			line = line.strip()
-			log_ids.append(line)
-	store: ResultStore = ResultStore()
-	for log_id in log_ids:
-		for analysis in process_log(log_id, settings):
-			log.info("* Result for " + analysis.name())
-			# print(analysis.result())
-			# print(analysis.render())
-			analysis.result(store, name=log_id)
+		# "597b704fe9ace475316c345903",
+		# "e01a684aa29dff9ddd9705edf8",
+		# "fbf9d64ae0bdad0de7efa3eec6",
+		"fe1331481f85560681f86827ec",
+		# "fe1331481f85560681f86827ec"]
+		"fec57041458e6cef98652df625", ]
+		,settings)
+	store: ResultStore = run_analysis(log_ids_urach, settings)
 	if False:
 		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
 			r().render(store.get_all())
 	if False:
 		render(analyzers.LocationAnalyzer, store.get_all())
 	# print(json.dumps(store.serializable(), indent=1))
-	if False:
-		render(analyzers.ActivityMapper, store.get_all())
-		render(analyzers.ProgressAnalyzer, store.get_all())
+	if True:
+		for cat in store.get_categories():
+			render(analyzers.ActivityMapper, store.get_category(cat), name=cat)
+		# render(analyzers.ProgressAnalyzer, store.get_all())
 
 	if False:
 		from analyzers.postprocessing import graph
@@ -95,49 +147,26 @@ if __name__ == '__main__':
 			data = store.get_category(cat)
 			render(analyzers.SimulationOrderAnalyzer, data, name=cat)
 	if False:
-		LogEntryCountCSV.summary = None
-		for cat in store.get_categories():
-			data = store.get_category(cat)
-			render(analyzers.LogEntryCountAnalyzer, data, name=cat)
-		if LogEntryCountCSV.summary:
-			headers = []
-			lines = []
-			for name in LogEntryCountCSV.summary:
-				data = LogEntryCountCSV.summary[name]
-				for head in data:
-					if not head in headers:
-						headers.append(head)
-				line = [name]
-				for head in headers:
-					line.append(data[head]) if head in data else line.append(0)
-				lines.append(line)
-			import csv
-
-			with open('logentrycount.csv', 'w', newline='') as csvfile:
-				writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE)
-				writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
-				for line in lines:
-					writer.writerow(line)
-	if True:
-		from datetime import datetime
-		json.dump(store.serializable(), open("simus.json", "w"), indent=2)
-		with open("simus.csv", "w") as csvfile:
-			csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n")
-			for key in store.get_store():
-				csvfile.write("{}\n".format(key))
-				for result in store.store[key]:
-					csvfile.write(",{}\n".format(result.name))
-					for i in result.get():
-						csvfile.write(",,{},{},{},{},{},{}\n".format(
-							i['answers']['@id'],
-							i['answers']['answered'],
-							len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0,
-							len(i['selected_actions']) if i['selected_actions'] else 0,
-							i['timestamp'],
-							str(datetime.fromtimestamp(i['timestamp']/1000))
-						))
+		write_logentry_count_csv()
 	if False:
-		#json.dump(store.serializable(), open("new.json", "w"), indent=1)
+		write_simulation_flag_csv()
+
+
+	def calc_distance(geojson: str):
+		from shapely.geometry import LineString
+		from shapely.ops import transform
+		from functools import partial
+		import pyproj
+		track = LineString(json.loads(geojson)['coordinates'])
+		project = partial(
+			pyproj.transform,
+			pyproj.Proj(init='EPSG:4326'),
+			pyproj.Proj(init='EPSG:32633'))
+		return transform(project, track).length
+
+
+	if False:
+		# json.dump(store.serializable(), open("new.json", "w"), indent=1)
 		from collections import defaultdict
 
 		keys = [
@@ -150,7 +179,8 @@ if __name__ == '__main__':
 			"map"
 		]
 		import matplotlib.pyplot as plt
-		#results = []
+
+		# results = []
 
 		places = defaultdict(list)
 
@@ -162,23 +192,24 @@ if __name__ == '__main__':
 			print(json.dumps(result, indent=4))
 			total = sum(result.values())
 			print(total)
-			percentage = defaultdict(lambda :0)
-			minutes = defaultdict(lambda:0)
+			percentage = defaultdict(lambda: 0)
+			minutes = defaultdict(lambda: 0)
 			for i in result:
-				percentage[i]= result[i]/total
-				minutes[i] = result[i]/60_000
-			print(json.dumps(percentage,indent=4))
+				percentage[i] = result[i] / total
+				minutes[i] = result[i] / 60_000
+			print(json.dumps(percentage, indent=4))
 			if not 'error' in result:
-				#places[log.get()['instance']].append(percentage)
+				# places[log.get()['instance']].append(percentage)
 				places[log.get()['instance']].append(minutes)
 
 		for place in places:
-			places[place] = sorted(places[place], key=lambda item:item['map'])
+			places[place] = sorted(places[place], key=lambda item: item['map'])
 
-		dummy = [0]*len(keys)
+		dummy = [0] * len(keys)
 		results = []
 		sites = []
 		from util.meta_temp import CONFIG_NAMES
+
 		for i in places:
 			for j in places[i]:
 				ordered = []
@@ -188,38 +219,37 @@ if __name__ == '__main__':
 			results.append(dummy)
 			sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
 
-
 		size = len(results)
 		ind = np.arange(size)
-		width=0.9
+		width = 0.9
 		print(results)
 		data = list(zip(*results))
 		print(data)
 		lines = []
-		bottom = [0]*len(results)
+		bottom = [0] * len(results)
 		for i in range(0, len(data)):
-			lines.append(plt.bar(ind,data[i], bottom=bottom, width=width)[0])
-			for k,x in enumerate(data[i]):
+			lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+			for k, x in enumerate(data[i]):
 				bottom[k] += x
 		plt.legend(lines, keys)
 		plt.title(", ".join(sites))
 		plt.show()
 
-	#size = len(results)
-	#ind = np.arange(size)
-	#width = 0.9
-	#print(results)
-	#data = list(zip(*results))
-	#print(data)
-	#lines = []
-	#bottom = [0] * len(results)
-	#for i in range(0, len(data)):
+	# size = len(results)
+	# ind = np.arange(size)
+	# width = 0.9
+	# print(results)
+	# data = list(zip(*results))
+	# print(data)
+	# lines = []
+	# bottom = [0] * len(results)
+	# for i in range(0, len(data)):
 	#	lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
 	#	for k, x in enumerate(data[i]):
 	#		bottom[k] += x
-	#plt.legend(lines, keys)
-	#plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)")
-	#plt.show()
+	# plt.legend(lines, keys)
+	# plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)")
+	# plt.show()
 
 
 
diff --git a/static/progress/index.html b/static/progress/index.html
index efade76..bc641e6 100644
--- a/static/progress/index.html
+++ b/static/progress/index.html
@@ -1,15 +1,98 @@
 <script
-  src="https://code.jquery.com/jquery-3.2.1.min.js"
-  integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
-  crossorigin="anonymous"></script>
-    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.2.0/dist/leaflet.css"
-   integrity="sha512-M2wvCLH6DSRazYeZRIm1JnYyh22purTM+FDB5CsyxtQJYeKq83arPe5wgbNmcFXGqiSH2XR8dT/fJISVA1r/zQ=="
-   crossorigin=""/>
-    <script src="https://unpkg.com/leaflet@1.2.0/dist/leaflet.js"
-   integrity="sha512-lInM/apFSqyy1o6s89K4iQUKg6ppXEgsVxT35HbzUupEVRh2Eu9Wdl4tHj7dZO0s1uvplcYGmt3498TtHq+log=="
-   crossorigin=""></script>
-  
+        src="https://code.jquery.com/jquery-3.2.1.min.js"
+        integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+        crossorigin="anonymous"></script>
+<link rel="stylesheet" href="https://unpkg.com/leaflet@1.2.0/dist/leaflet.css"
+      integrity="sha512-M2wvCLH6DSRazYeZRIm1JnYyh22purTM+FDB5CsyxtQJYeKq83arPe5wgbNmcFXGqiSH2XR8dT/fJISVA1r/zQ=="
+      crossorigin=""/>
+<script src="https://unpkg.com/leaflet@1.2.0/dist/leaflet.js"
+        integrity="sha512-lInM/apFSqyy1o6s89K4iQUKg6ppXEgsVxT35HbzUupEVRh2Eu9Wdl4tHj7dZO0s1uvplcYGmt3498TtHq+log=="
+        crossorigin=""></script>
+
+<script src="https://rawgit.com/Leaflet/Leaflet.heat/gh-pages/dist/leaflet-heat.js"></script>
+
 <script src="my.js"></script>
-    <style>
-       .map { width: 512px; height: 256px; }
-    </style>
\ No newline at end of file
+<style>
+    .mapDiv {
+        width: 1024px;
+        height: 768px;
+    }
+
+    .board {
+        width: 32px;
+        height: 32px;
+        display: inline-block;
+        position: relative;
+    }
+
+    .board img {
+        max-width: 100%;
+        max-height: 100%;
+        position: absolute;
+        /*bottom: 0px;*/
+    }
+
+    .board img:hover {
+        max-width: inherit;
+        max-height: inherit;
+        z-index: 99;
+        top: 20px;
+    }
+
+    .highlight {
+        /*what a nice way to highlight*/
+        display: none;
+    }
+
+    .simu {
+        background-color: blue;
+    }
+
+    .question {
+        background-color: orange;
+    }
+
+    .image {
+        background-color: green;
+    }
+
+    .audio {
+        background-color: red;
+    }
+
+    .video {
+        background-color: purple;
+    }
+
+    .other {
+        background-color: brown;
+    }
+
+    .map {
+        background-color: violet;
+    }
+
+    .error {
+        background-color: grey;
+    }
+
+    ul {
+        list-style-type: none;
+        overflow: auto;
+        overflow-y: hidden;
+        display: inline-block;
+        /*max-width:100%;
+        margin: 0 0 1em;
+        white-space: nowrap;
+        height:200px;*/
+    }
+
+    li {
+        display: inline-block;
+        vertical-align: top;
+    }
+
+</style>
+<div style="font-size:0.1px;position:absolute;bottom:0;">OSM Logo: CC-BY-SA
+    http://wiki.openstreetmap.org/wiki/File:Mag_map-120x120.png
+</div>
\ No newline at end of file
diff --git a/static/progress/my.js b/static/progress/my.js
index e094e0e..ee04a3d 100644
--- a/static/progress/my.js
+++ b/static/progress/my.js
@@ -1,71 +1,90 @@
-$.getJSON("tmp3.json", function (data) {
-    var list = $("<ul />");
-    var maps = {};
-    $.each(data, function (index, entry) {
-        //key: instance_id, value: AnlysisResult
-        //value.result.instance: InstanceConfig_id
-        // console.log(key, value[0].result.store[0].timestamp);
-        //$.each(value[0].result.store, function (index, entry) {
-            //console.log(entry);
-            var time = new Date(entry.timestamp);
-            var item = $("<li>", {html: entry.sequence + " @ " + time.toLocaleDateString() + " "+ time.toLocaleTimeString()});
-            var container = $("<p />");
-            if (entry.track.length > 0) {
-                var mapName = "map" + index;
-                //console.log(mapName, entry.track.length);
-                var mapContainer = $("<div />", {id: mapName, class: "map"});
-                var track = [];
-                $.each(entry.track, function (i, elem) {
-                    track.push([elem.coordinates[1], elem.coordinates[0]]);
-                });
-                maps[mapName] = track;
+//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) {
+$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-68854ffab9aa.json", function (data) {
+    var images = {};
+    var mapContainer = $("<div />", {id: "mainMap", class: "mapDiv"});
+    mapContainer.appendTo("body");
+    var tiles = {
+        "osm": L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
+            attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
+        }),
+        "openstreetmap": L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
+            attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a>'
+        }),
+        "esri sat": L.tileLayer('https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}', {
+            attribution: 'Tiles &copy; Esri &mdash; Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community'
+        }),
+        "google sat": L.tileLayer('http://{s}.google.com/vt/lyrs=s&x={x}&y={y}&z={z}', {
+            maxZoom: 20,
+            subdomains: ['mt0', 'mt1', 'mt2', 'mt3']
+        })
+    };
+    var map = L.map("mainMap", {layers: [tiles.osm], maxZoom: 22, maxNativeZoom: 19});
 
-                /* mapContainer.ready(function () {
-                     var map = L.map(mapName, {maxZoom: 22});
-                     L.control.scale().addTo(map);
-                     var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
-                         attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
-                     }).addTo(map);
-                     var track = [];
-                     $.each(entry.track, function (i, elem) {
-                         track.push([elem.coordinates[1], elem.coordinates[0]]);
-                     });
-                     var layer = L.polyline(track, {color: "green"});
-                     console.log(track);
-                     L.control.layers(null, [layer]).addTo(map);
-                 });*/
+    function styleTrack(feature) {
+        var styles = {};
+        styles.color = data.colors[feature.properties.activity_type];
+        return styles;
+    }
 
-                mapContainer.appendTo(container);
-            }
-            $.each(entry.events, function (i, event) {
-                if ("image" in event) {
-                    $("<img />", {src: event.image, height: 200}).appendTo(container);
+    var highlighted = null;
+
+    function onClick(e) {
+        var start = e.target.feature.geometry.properties.start_timestamp;
+        var end = e.target.feature.geometry.properties.end_timestamp;
+        var changed = highlighted !== e.target.feature;
+        $.each(images, function (timestamp, board) {
+                if ((timestamp >= start && timestamp < end) && changed) {
+                    board.image.first().addClass("highlight");
+                } else {
+                    board.image.removeClass("highlight");
+                    highlighted = null;
                 }
-            });
-            container.appendTo(item);
-            item.appendTo(list);
-        //});
+            }
+        );
+        if (changed) {
+            highlighted = e.target.feature;
+        }
+    }
+
+    var coords = [];
+
+    function onEachFeature(feature, layer) {
+        layer.setStyle(styleTrack(feature));
+        layer.on('click', onClick);
+        if (feature.coordinates.length > 1) {
+            coords = coords.concat(feature.coordinates.map(function (p) {
+                return [p[1], p[0], 0.1];
+            }));
+        }
+    }
+
+    var track = L.geoJSON(data['track'], {
+        //style: styleTrack,
+        onEachFeature: onEachFeature
+    }).addTo(map);
+
+    map.fitBounds(track.getBounds());
+
+    var heat = L.heatLayer(coords);
+    L.control.layers(tiles, {"heatmap": heat}).addTo(map);
+
+    var marker = null;
+    var list = $("<ul />");
+    $.each(data["boards"], function (index, entry) {
+        //console.log(index, entry);
+        var item = $("<li>", {class: entry.extra_data.activity_type});
+        var container = $("<div>", {class: "board"});
+        var image = $("<img>", {src: entry.image.replace("static/progress/", ""), height: 200});
+        image.attr("data-time", entry.timestamp);
+        image.hover(function () {
+            marker = L.geoJSON(entry.coordinate).addTo(map);
+        }, function () {
+            map.removeLayer(marker);
+        });
+        images[entry.timestamp] = {image: image, coordinate: entry.coordinate};
+        image.appendTo(container);
+        container.appendTo(item);
+        item.appendTo(list);
     });
     list.appendTo("body");
-    var slider = $("<input />", {type: "range" })
-    /*});
-
-    $(window).on("load", function () {*/
-    // setTimeout(function () {
-
-    //console.log(maps);
-    $.each(maps, function (mapName, track) {
-        //console.log("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa");
-        var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
-            attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
-        });
-        var map = L.map(mapName, {layers: [tiles]});
-        L.control.scale().addTo(map);
-        // console.log(mapName, track);
-        var layer = L.polyline(track, {color: "green"}).addTo(map);
-        map.fitBounds(layer.getBounds());
-        //console.log(layer)
-        //L.control.layers({"osm":tiles}, {layer]).addTo(map);
-    });
-    // }, 2000);
 });
\ No newline at end of file
diff --git a/util/download.py b/util/download.py
index 4e7dd84..e5dfb78 100644
--- a/util/download.py
+++ b/util/download.py
@@ -54,7 +54,6 @@ def get_board_data(source, instance_id, sequence_id, board_id):
 		return {"class": "error"}
 	result = {
 		"class": instance_data["@class"],
-		"id": instance_data["@id"]
 	}
 	for i in ["image", "audio", "video"]:
 		key = i + "_file"

From 33d63b120d6299ecb113e8cb51bc20cbc5b36c2c Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Wed, 15 Nov 2017 12:54:11 +0100
Subject: [PATCH 07/24] improve activity mapper

---
 biogames2.json                |   4 +-
 filter_todo/pre_filter.py     |   2 +-
 log_analyzer.py               |   3 +-
 static/progress/data/.gitkeep |   0
 static/progress/index.html    |  91 +++--------------------------
 static/progress/my.js         |  32 ++++++++---
 static/progress/style.css     | 105 ++++++++++++++++++++++++++++++++++
 7 files changed, 142 insertions(+), 95 deletions(-)
 create mode 100644 static/progress/data/.gitkeep
 create mode 100644 static/progress/style.css

diff --git a/biogames2.json b/biogames2.json
index 589805d..689a351 100644
--- a/biogames2.json
+++ b/biogames2.json
@@ -73,8 +73,8 @@
     "type": "Biogames",
     "url": "http://0.0.0.0:5000/game2/instance/log/list/",
     "login_url": "http://localhost:5000/game2/auth/json-login",
-    "username": "dev",
-    "password": "dev",
+    "username": "ba",
+    "password": "853451",
     "host":"http://0.0.0.0:5000"
   }
 }
\ No newline at end of file
diff --git a/filter_todo/pre_filter.py b/filter_todo/pre_filter.py
index 27480c4..8cc3c98 100644
--- a/filter_todo/pre_filter.py
+++ b/filter_todo/pre_filter.py
@@ -62,7 +62,7 @@ distribution = defaultdict(lambda: 0)
 finished_and_simu = defaultdict(list)
 files = {}
 actions_dist = defaultdict(list)
-with open('/home/agp8x/git/uni/ma/project/data/0000_ref') as src:
+with open('/home/clemens/git/ma/test/src') as src:
 	for line in src:
 		line = line.strip()
 		instance_id, log = get_json(line)
diff --git a/log_analyzer.py b/log_analyzer.py
index 3a492c6..c9c7a3b 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -58,7 +58,8 @@ def load_ids(name: str):
 
 
 def urach_logs(log_ids, settings):
-	return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
+	#return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
+	return ["data/{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
 
 
 def write_logentry_count_csv():
diff --git a/static/progress/data/.gitkeep b/static/progress/data/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/static/progress/index.html b/static/progress/index.html
index bc641e6..25464d6 100644
--- a/static/progress/index.html
+++ b/static/progress/index.html
@@ -12,87 +12,14 @@
 <script src="https://rawgit.com/Leaflet/Leaflet.heat/gh-pages/dist/leaflet-heat.js"></script>
 
 <script src="my.js"></script>
-<style>
-    .mapDiv {
-        width: 1024px;
-        height: 768px;
-    }
+<link href="style.css" rel="stylesheet"/>
+<main>
+    <div class="mapDiv" id="mainMap"></div>
+    <div class="sequenceContainer">
+        <div class="sequence"></div>
+    </div>
+</main>
 
-    .board {
-        width: 32px;
-        height: 32px;
-        display: inline-block;
-        position: relative;
-    }
-
-    .board img {
-        max-width: 100%;
-        max-height: 100%;
-        position: absolute;
-        /*bottom: 0px;*/
-    }
-
-    .board img:hover {
-        max-width: inherit;
-        max-height: inherit;
-        z-index: 99;
-        top: 20px;
-    }
-
-    .highlight {
-        /*what a nice way to highlight*/
-        display: none;
-    }
-
-    .simu {
-        background-color: blue;
-    }
-
-    .question {
-        background-color: orange;
-    }
-
-    .image {
-        background-color: green;
-    }
-
-    .audio {
-        background-color: red;
-    }
-
-    .video {
-        background-color: purple;
-    }
-
-    .other {
-        background-color: brown;
-    }
-
-    .map {
-        background-color: violet;
-    }
-
-    .error {
-        background-color: grey;
-    }
-
-    ul {
-        list-style-type: none;
-        overflow: auto;
-        overflow-y: hidden;
-        display: inline-block;
-        /*max-width:100%;
-        margin: 0 0 1em;
-        white-space: nowrap;
-        height:200px;*/
-    }
-
-    li {
-        display: inline-block;
-        vertical-align: top;
-    }
-
-</style>
-<div style="font-size:0.1px;position:absolute;bottom:0;">OSM Logo: CC-BY-SA
+<!--div style="font-size:0.1px;position:absolute;bottom:0;">OSM Logo: CC-BY-SA
     http://wiki.openstreetmap.org/wiki/File:Mag_map-120x120.png
-</div>
\ No newline at end of file
+</div-->
\ No newline at end of file
diff --git a/static/progress/my.js b/static/progress/my.js
index ee04a3d..55b22f4 100644
--- a/static/progress/my.js
+++ b/static/progress/my.js
@@ -1,24 +1,29 @@
 //$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) {
 $.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-68854ffab9aa.json", function (data) {
     var images = {};
-    var mapContainer = $("<div />", {id: "mainMap", class: "mapDiv"});
-    mapContainer.appendTo("body");
     var tiles = {
         "osm": L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
+            maxNativeZoom: 19,
+            maxZoom: 24,
             attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
         }),
         "openstreetmap": L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
+            maxNativeZoom: 19,
+            maxZoom: 24,
             attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a>'
         }),
         "esri sat": L.tileLayer('https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}', {
+            maxNativeZoom: 19,
+            maxZoom: 24,
             attribution: 'Tiles &copy; Esri &mdash; Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community'
         }),
         "google sat": L.tileLayer('http://{s}.google.com/vt/lyrs=s&x={x}&y={y}&z={z}', {
-            maxZoom: 20,
+            maxNativeZoom: 20,
+            maxZoom: 24,
             subdomains: ['mt0', 'mt1', 'mt2', 'mt3']
         })
     };
-    var map = L.map("mainMap", {layers: [tiles.osm], maxZoom: 22, maxNativeZoom: 19});
+    var map = L.map("mainMap", {layers: [tiles.osm]});
 
     function styleTrack(feature) {
         var styles = {};
@@ -68,23 +73,32 @@ $.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-688
     var heat = L.heatLayer(coords);
     L.control.layers(tiles, {"heatmap": heat}).addTo(map);
 
-    var marker = null;
     var list = $("<ul />");
+    var current = {
+        "pos":data["boards"][1].coordinate.coordinates
+    };
+    console.log(current);
+    var marker = L.marker([current.pos[1], current.pos[0]]).addTo(map);
     $.each(data["boards"], function (index, entry) {
         //console.log(index, entry);
         var item = $("<li>", {class: entry.extra_data.activity_type});
         var container = $("<div>", {class: "board"});
-        var image = $("<img>", {src: entry.image.replace("static/progress/", ""), height: 200});
+        var image = $("<img>", {src: entry.image.replace("static/progress/", "")});
         image.attr("data-time", entry.timestamp);
         image.hover(function () {
-            marker = L.geoJSON(entry.coordinate).addTo(map);
+            marker.setLatLng([entry.coordinate.coordinates[1], entry.coordinate.coordinates[0]]);
         }, function () {
-            map.removeLayer(marker);
+            marker.setLatLng(current.pos.coordinates[1], current.pos.coordinates[0]);
+        });
+        image.click(function (e) {
+            current.board = image;
+            current.pos = entry.coordinate;
         });
         images[entry.timestamp] = {image: image, coordinate: entry.coordinate};
         image.appendTo(container);
         container.appendTo(item);
         item.appendTo(list);
     });
-    list.appendTo("body");
+    current.board=images[data["boards"][1].timestamp];
+    list.appendTo(".sequence");
 });
\ No newline at end of file
diff --git a/static/progress/style.css b/static/progress/style.css
new file mode 100644
index 0000000..d33b09e
--- /dev/null
+++ b/static/progress/style.css
@@ -0,0 +1,105 @@
+/*.mapDiv {
+    width: 1024px;
+    height: 768px;
+}*/
+
+
+
+.highlight {
+    /*what a nice way to highlight*/
+    display: none;
+}
+
+.simu {
+    background-color: blue;
+}
+
+.question {
+    background-color: orange;
+}
+
+.image {
+    background-color: green;
+}
+
+.audio {
+    background-color: red;
+}
+
+.video {
+    background-color: purple;
+}
+
+.other {
+    background-color: brown;
+}
+
+.map {
+    background-color: violet;
+}
+
+.error {
+    background-color: grey;
+}
+
+
+.board {
+    width: 32px;
+    height: 32px;
+    display: inline-block;
+}
+
+.board img {
+    max-width: 32px;
+    max-height: 32px;
+    position: absolute;
+    /*bottom: 0px;*/
+}
+
+.board:hover img{
+    max-width: 205px;
+    max-height: 295px;
+    z-index: 99;
+    top: 5px;
+    right:0px;
+}
+ul {
+    list-style-type: none;
+    overflow: auto;
+    overflow-y: hidden;
+    display: inline-block;
+    /*max-width:100%;
+    margin: 0 0 1em;
+    white-space: nowrap;
+    height:200px;*/
+}
+
+li {
+    display: inline-block;
+    vertical-align: top;
+    padding: 2px;
+    margin-bottom: 2px;
+}
+
+body{
+    height: 100%;
+    padding:0;
+    margin:0;
+}
+
+main{
+    display: flex;
+    flex-direction: column;
+    height:100%;
+}
+
+.mapDiv {
+    flex-grow:1;
+}
+.sequenceContainer{
+    flex-grow: 0;
+    min-height:300px;
+    padding-right: 210px;
+    position: relative;
+
+}
\ No newline at end of file

From 75f41fb9f53bdc7e707ecb6ee5a50f74c32dba87 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Wed, 15 Nov 2017 18:39:42 +0100
Subject: [PATCH 08/24] WIP snapshot

---
 analyzers/analyzer/biogames.py |  3 ++
 log_analyzer.py                | 93 ++++++++++++++++++++++++++--------
 2 files changed, 75 insertions(+), 21 deletions(-)

diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py
index 96215e3..eab26e0 100644
--- a/analyzers/analyzer/biogames.py
+++ b/analyzers/analyzer/biogames.py
@@ -195,6 +195,7 @@ class ActivityMapper(Analyzer):
 			board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
 										entry["board_id"])
 			entry["extra_data"] = board_data
+			entry["extra_data"]["activity_type"] = self.last_board_type
 			entry['coordinate'] = self.new_coordinate()
 			self.timeline.append(entry)
 		return False
@@ -236,6 +237,8 @@ class ActivityMapper(Analyzer):
 		self.track['properties'].update(props)
 		self.tracks.append(self.track)
 		self.track = self.new_track(props['end_timestamp'])
+		if self.last_coordinate:
+			self.track['coordinates'].append(self.last_coordinate)
 
 	def new_track(self, timestamp):
 		return {"type": "LineString", "coordinates": [], "properties": {'start_timestamp': timestamp}}
diff --git a/log_analyzer.py b/log_analyzer.py
index bb542ea..9d9fc40 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -182,6 +182,9 @@ if __name__ == '__main__':
 				pyproj.Proj(init='EPSG:32633'))
 			return transform(project, track).length
 
+		whitelist = ['16fc3117-61db-4f50-b84f-81de6310206f', '5e64ce07-1c16-4d50-ac4e-b3117847ea43',
+					 '90278021-4c57-464e-90b1-d603799d07eb', 'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771']
+
 
 		def get_data_distance(store, relative_values=True, sort=True, show_errors=False):
 			places = defaultdict(list)
@@ -211,9 +214,10 @@ if __name__ == '__main__':
 			return places
 
 
-		def get_all_data(store):
+		def get_all_data(store, sort=False, relative=True):
 			places = defaultdict(list)
-
+			simu_distribution = defaultdict(lambda: 0)
+			#divisiors = {"time":60_000, "space":1000000}
 			for log in store.get_all():
 				if not log.analysis() == analyzers.ActivityMapper:
 					continue
@@ -223,13 +227,14 @@ if __name__ == '__main__':
 					if len(coords) > 1:
 						distance = calc_distance(coords)
 					else:
-						distance = 0.1
+						distance = 0.0
 					result["space"][i['properties']['activity_type']] += distance
 					duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
 					result["time"][i['properties']['activity_type']] += duration
 				total_space = sum(result["space"].values())
 				total_time = sum(result["time"].values())
 				percentage = defaultdict(lambda: defaultdict(lambda: 0))
+				total = defaultdict(lambda: defaultdict(lambda: 0))
 				for i in result["space"]:
 					if not total_space == 0:
 						percentage[i]["space"] = result["space"][i] / total_space
@@ -239,22 +244,51 @@ if __name__ == '__main__':
 						percentage[i]["time"] = result["time"][i] / total_time
 					else:
 						percentage[i]["time"] = 0
+					for t in ("space","time"):
+						#total[i][t] += (result[t][i] / divisiors[t])
+						total[i][t] += result[t][i]
 				print(percentage)
 				if not 'error' in result:
-					places[log.get()['instance']].append(percentage)
+					if relative:
+						value = percentage
+					else:
+						value = total
+					places[log.get()['instance']].append(value)
+					simus = defaultdict(lambda :0)
+					for item in log.get()['boards']:
+						if item["extra_data"]["activity_type"]=="simu":
+							simus[item["board_id"]] += 1
+					simu_distribution[len(simus)]+=1
+
+			if sort:
+				for place in places:
+					places[place] = sorted(places[place], key=lambda item: item['map']['time'])
+			print(simu_distribution)
 			return places
 
 
-		def stack_data(keys, places):
+		def stack_data(keys, places, type="time"):
+			divisiors = {"time": 60_000, "space": 1000}
+			divisiors = {"time": 1, "space": 1}
 			dummy = [0] * len(keys)
 			results = []
 			sites = []
-			for i in places:
+			for i in sorted(places):
+				if not i in whitelist:
+					continue
 				for j in places[i]:
 					ordered = []
 					for k in keys:
-						ordered.append(j[k])
-					results.append(ordered)
+						if k in j:
+							ordered.append(j[k][type]/divisiors[type])
+						else:
+							ordered.append(0)
+					print(sum(ordered))
+					if sum(ordered) > 0.9:
+						#print(sum(ordered), 1-sum(ordered))
+						#if sum(ordered)<1:
+						#	ordered[-2] = 1-sum(ordered[:-2], ordered[-1])
+						results.append(ordered)
 				results.append(dummy)
 				sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
 			return results, sites
@@ -262,7 +296,8 @@ if __name__ == '__main__':
 
 		def plot_data(places, keys):
 			results, sites = stack_data(keys, places)
-
+			dpi=86.1
+			plt.figure(figsize=(1280/dpi, 720/dpi))
 			size = len(results)
 			print("{} elements total".format(size))
 			ind = np.arange(size)
@@ -272,13 +307,16 @@ if __name__ == '__main__':
 			# print(data)
 			lines = []
 			bottom = [0] * size
+			plt.ticklabel_format(useMathText=False)
 			for i in range(0, len(data)):
 				lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
 				for k, x in enumerate(data[i]):
 					bottom[k] += x
 			plt.legend(lines, keys)
 			plt.title(", ".join(sites))
-			plt.show()
+			#plt.show()
+			dpi=86
+			plt.savefig("time_rel_{}.png".format(size), dpi=dpi,bbox_inches="tight")
 
 
 		colors = {
@@ -314,13 +352,18 @@ if __name__ == '__main__':
 			groups = defaultdict(list)
 			keys = list(keys)
 			keys.remove("other")
+			ids = []
 			for k in keys:
 				for id in sorted(combined):
+					if id not in whitelist:
+						continue
+					if not id in ids:
+						ids.append(id)
 					group = 0.0
 					count = 0
 					for item in combined[id]:
 						if k in item:
-							time = item[k]["time"]
+							time = item[k]["time"]/1000
 							distance = item[k]["space"]
 							if time > 0:
 								group += (distance / time)
@@ -331,24 +374,29 @@ if __name__ == '__main__':
 						groups[k].append(group/count)
 					else:
 						groups[k].append(0.0)
-			ind = np.arange(len(combined.keys()))
+			print(ids)
+			ind = np.arange(len(ids))
 			width = .7 / len(groups)
 			print(ind)
 			print(json.dumps(groups, indent=1))
 			bars = []
+			dpi=10
+			plt.figure(figsize=(1280/dpi, 720/dpi))
 			fig, ax = plt.subplots()
 			for k in groups:
 				print(groups[k])
 				if not len(groups[k]):
 					groups[k].append(0)
-				bars.append(ax.bar(ind, groups[k], width, color=colors[k]))
-				ind = ind + width
+				ind = ind + (width)
+				bars.append(ax.bar((ind + width*len(groups)/2), groups[k], width, color=colors[k]))
 			ax.set_xticks(ind + width / 2)
-			ax.set_xticklabels(list([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in sorted(combined.keys())]))
+			ax.set_xticklabels(list([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids]))
 			plt.legend(bars, keys)
-			print(combined.keys())
-			print([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in sorted(combined.keys())])
-			plt.show()
+			print(combined.keys(), ids)
+			print([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids])
+			#plt.show()
+			dpi=100
+			plt.savefig("speed.png", dpi=dpi)
 
 
 		# spatial_data = get_data_distance(store,relative_values=False)
@@ -361,13 +409,16 @@ if __name__ == '__main__':
 		# IPython.embed()
 
 		#print(json.dumps(get_all_data(store)))
-#		json.dump(get_all_data(store), open("combined.json", "w"))
-		combined = json.load(open("combined.json"))
+		#json.dump(get_all_data(store), open("combined.json", "w"))
+		#combined = get_all_data(store, sort=True, relative=True)
+		#json.dump(combined, open("combined_rel.json", "w"))
+		#combined = json.load(open("combined_rel.json"))
+		combined = json.load(open("combined_total.json"))
 		plot_time_space_rel(combined, keys)
 
 		#plot_time_space_rel(temporal_data_rel, spatial_data_rel, keys)
 
-	# plot_data(data, keys)
+		#plot_data(combined, keys)
 	# plot_data(get_data_distance(store,relative_values=False), keys)
 
 

From f0a6a1c8aa0a289386a370402f267d0165408633 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Mon, 20 Nov 2017 17:56:07 +0100
Subject: [PATCH 09/24] WIP snapshot

---
 log_analyzer.py | 108 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 69 insertions(+), 39 deletions(-)

diff --git a/log_analyzer.py b/log_analyzer.py
index 9d9fc40..ef04c38 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -137,6 +137,13 @@ if __name__ == '__main__':
 			# "error"
 		]
 
+		loc_keys=[
+			"question",
+			"image",
+			"audio",
+			"video"
+		]
+
 
 		def get_data(store, relative_values=True, sort=True, show_errors=False):
 			places = defaultdict(list)
@@ -182,6 +189,7 @@ if __name__ == '__main__':
 				pyproj.Proj(init='EPSG:32633'))
 			return transform(project, track).length
 
+
 		whitelist = ['16fc3117-61db-4f50-b84f-81de6310206f', '5e64ce07-1c16-4d50-ac4e-b3117847ea43',
 					 '90278021-4c57-464e-90b1-d603799d07eb', 'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771']
 
@@ -217,7 +225,7 @@ if __name__ == '__main__':
 		def get_all_data(store, sort=False, relative=True):
 			places = defaultdict(list)
 			simu_distribution = defaultdict(lambda: 0)
-			#divisiors = {"time":60_000, "space":1000000}
+			# divisiors = {"time":60_000, "space":1000000}
 			for log in store.get_all():
 				if not log.analysis() == analyzers.ActivityMapper:
 					continue
@@ -244,8 +252,8 @@ if __name__ == '__main__':
 						percentage[i]["time"] = result["time"][i] / total_time
 					else:
 						percentage[i]["time"] = 0
-					for t in ("space","time"):
-						#total[i][t] += (result[t][i] / divisiors[t])
+					for t in ("space", "time"):
+						# total[i][t] += (result[t][i] / divisiors[t])
 						total[i][t] += result[t][i]
 				print(percentage)
 				if not 'error' in result:
@@ -254,11 +262,11 @@ if __name__ == '__main__':
 					else:
 						value = total
 					places[log.get()['instance']].append(value)
-					simus = defaultdict(lambda :0)
+					simus = defaultdict(lambda: 0)
 					for item in log.get()['boards']:
-						if item["extra_data"]["activity_type"]=="simu":
+						if item["extra_data"]["activity_type"] == "simu":
 							simus[item["board_id"]] += 1
-					simu_distribution[len(simus)]+=1
+					simu_distribution[len(simus)] += 1
 
 			if sort:
 				for place in places:
@@ -267,26 +275,28 @@ if __name__ == '__main__':
 			return places
 
 
-		def stack_data(keys, places, type="time"):
+		def stack_data(keys, places, type="space"):
 			divisiors = {"time": 60_000, "space": 1000}
-			divisiors = {"time": 1, "space": 1}
+			#divisiors = {"time": 1, "space": 1}
 			dummy = [0] * len(keys)
 			results = []
 			sites = []
 			for i in sorted(places):
 				if not i in whitelist:
 					continue
-				for j in places[i]:
+				place = sorted(places[i], key=lambda item: item['map'][type])
+				for j in place:
 					ordered = []
 					for k in keys:
 						if k in j:
-							ordered.append(j[k][type]/divisiors[type])
+							ordered.append(j[k][type] / divisiors[type])
 						else:
 							ordered.append(0)
 					print(sum(ordered))
-					if sum(ordered) > 0.9:
-						#print(sum(ordered), 1-sum(ordered))
-						#if sum(ordered)<1:
+					#if sum(ordered) > 0.9 and sum(ordered) < 4000 and sum(ordered)>10:
+					if sum(ordered) > 0.9 and sum(ordered)<100:
+						# print(sum(ordered), 1-sum(ordered))
+						# if sum(ordered)<1:
 						#	ordered[-2] = 1-sum(ordered[:-2], ordered[-1])
 						results.append(ordered)
 				results.append(dummy)
@@ -296,8 +306,8 @@ if __name__ == '__main__':
 
 		def plot_data(places, keys):
 			results, sites = stack_data(keys, places)
-			dpi=86.1
-			plt.figure(figsize=(1280/dpi, 720/dpi))
+			dpi = 86.1
+			plt.figure(figsize=(1280 / dpi, 720 / dpi))
 			size = len(results)
 			print("{} elements total".format(size))
 			ind = np.arange(size)
@@ -314,9 +324,9 @@ if __name__ == '__main__':
 					bottom[k] += x
 			plt.legend(lines, keys)
 			plt.title(", ".join(sites))
-			#plt.show()
-			dpi=86
-			plt.savefig("time_rel_{}.png".format(size), dpi=dpi,bbox_inches="tight")
+			# plt.show()
+			dpi = 86
+			plt.savefig("space_abs_{}.png".format(size), dpi=dpi, bbox_inches="tight")
 
 
 		colors = {
@@ -327,7 +337,8 @@ if __name__ == '__main__':
 			"video": "purple",
 			"other": "brown",
 			"map": "violet",
-			# "error":"grey"
+			# "error":"grey",
+			"tasks": "olive",
 		}
 		markers = [".", "o", "x", "s", "*", "D", "p", ",", "<", ">", "^", "v", "1", "2", "3", "4"]
 
@@ -348,11 +359,27 @@ if __name__ == '__main__':
 		# plt.clf()
 		# plt.close()
 
+		def group_locationbased_tasks(data):
+			for id in data:
+				for log in data[id]:
+					loc = {"space":0,"time":0}
+					for k in log:
+						if k in loc_keys:
+							for i in ["space","time"]:
+								loc[i] += log[k][i]
+					log["tasks"] = loc
+
+
+
 		def plot_time_space_rel(combined, keys):
 			groups = defaultdict(list)
 			keys = list(keys)
 			keys.remove("other")
+			for i in loc_keys:
+				keys.remove(i)
+			keys.append("tasks")
 			ids = []
+			group_locationbased_tasks(combined)
 			for k in keys:
 				for id in sorted(combined):
 					if id not in whitelist:
@@ -363,15 +390,15 @@ if __name__ == '__main__':
 					count = 0
 					for item in combined[id]:
 						if k in item:
-							time = item[k]["time"]/1000
+							time = item[k]["time"] / 1000
 							distance = item[k]["space"]
 							if time > 0:
 								group += (distance / time)
-								count+=1
+								count += 1
 							else:
 								print("div by zero", distance, time)
 					if count > 0:
-						groups[k].append(group/count)
+						groups[k].append(group / count)
 					else:
 						groups[k].append(0.0)
 			print(ids)
@@ -380,45 +407,48 @@ if __name__ == '__main__':
 			print(ind)
 			print(json.dumps(groups, indent=1))
 			bars = []
-			dpi=10
-			plt.figure(figsize=(1280/dpi, 720/dpi))
+			dpi = 200
+			plt.figure(figsize=(1280 / dpi, 720 / dpi))
 			fig, ax = plt.subplots()
 			for k in groups:
 				print(groups[k])
 				if not len(groups[k]):
 					groups[k].append(0)
 				ind = ind + (width)
-				bars.append(ax.bar((ind + width*len(groups)/2), groups[k], width, color=colors[k]))
+				bars.append(ax.bar((ind + width * len(groups) / 2), groups[k], width, color=colors[k]))
 			ax.set_xticks(ind + width / 2)
 			ax.set_xticklabels(list([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids]))
-			plt.legend(bars, keys)
+			kmh = plt.hlines((1 / 3.6), 0.3, 4.2, linestyles="dashed", label="1 km/h", linewidths=1)
+			plt.legend(bars+[kmh], keys+[kmh.get_label()])
 			print(combined.keys(), ids)
 			print([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids])
-			#plt.show()
-			dpi=100
-			plt.savefig("speed.png", dpi=dpi)
+			# plt.show()
+			dpi = 200
+			plt.savefig("speed2.png", dpi=dpi)
 
 
 		# spatial_data = get_data_distance(store,relative_values=False)
 		# temporal_data = get_data(store,relative_values=False)
 		# spatial_data_rel = get_data_distance(store,relative_values=True)
 		# temporal_data_rel = get_data(store,relative_values=True)
-		#temporal_data_rel = json.load(open("temporal_rel.json"))
-		#spatial_data_rel = json.load(open("spatial_rel.json"))
+		# temporal_data_rel = json.load(open("temporal_rel.json"))
+		# spatial_data_rel = json.load(open("spatial_rel.json"))
 		# import IPython
 		# IPython.embed()
 
-		#print(json.dumps(get_all_data(store)))
-		#json.dump(get_all_data(store), open("combined.json", "w"))
-		#combined = get_all_data(store, sort=True, relative=True)
-		#json.dump(combined, open("combined_rel.json", "w"))
-		#combined = json.load(open("combined_rel.json"))
+		# print(json.dumps(get_all_data(store)))
+		# json.dump(get_all_data(store), open("combined.json", "w"))
+		# combined = get_all_data(store, sort=True, relative=True)
+		# json.dump(combined, open("combined_rel.json", "w"))
+		# combined = json.load(open("combined_rel.json"))
 		combined = json.load(open("combined_total.json"))
-		plot_time_space_rel(combined, keys)
+		#plot_time_space_rel(combined, keys)
+		plot_data(combined, keys)
 
-		#plot_time_space_rel(temporal_data_rel, spatial_data_rel, keys)
 
-		#plot_data(combined, keys)
+	# plot_time_space_rel(temporal_data_rel, spatial_data_rel, keys)
+
+	# plot_data(combined, keys)
 	# plot_data(get_data_distance(store,relative_values=False), keys)
 
 

From b21d0bf8ba832ed0059a5f8816b62b1e7dc52c3b Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Tue, 19 Dec 2017 13:12:07 +0100
Subject: [PATCH 10/24] cleanup, improve structure

---
 analyzers/analyzer/default.py |  69 +++++++++++
 analyzers/render/wip.py       |  87 ++++++++++++++
 biogames2.json                |   7 +-
 log_analyzer.py               | 214 ++++------------------------------
 static/progress/my.js         |  16 ++-
 util/geo.py                   |  12 ++
 util/processing.py            |  57 +++++++++
 7 files changed, 258 insertions(+), 204 deletions(-)
 create mode 100644 analyzers/render/wip.py
 create mode 100644 util/geo.py
 create mode 100644 util/processing.py

diff --git a/analyzers/analyzer/default.py b/analyzers/analyzer/default.py
index 9151313..8698a5c 100644
--- a/analyzers/analyzer/default.py
+++ b/analyzers/analyzer/default.py
@@ -137,3 +137,72 @@ class ProgressAnalyzer(Analyzer):
 		if entry[self.settings.type_field] in self.settings.boards:
 			self.board[entry["timestamp"]] = entry
 		return False
+
+
+class MetaDataAnalyzer(Analyzer):
+	"""collect metadata"""
+	__name__ = "MetaDataAnalyzer"
+
+	def result(self, store: ResultStore, name=None) -> None:
+		store.add(Result(type(self), dict(self.store)))
+
+	def process(self, entry: dict) -> bool:
+		if not "metadata" in self.settings.custom:
+			return False
+		for mdata in self.settings.custom["metadata"]:
+			key = self.settings.custom["metadata"]
+			if key in entry:
+				self.store[mdata] = json_path(entry, key)
+
+	def __init__(self, settings: LogSettings) -> None:
+		super().__init__(settings)
+		self.store = {}
+
+
+def write_logentry_count_csv(LogEntryCountCSV, store, render, analyzers):
+	global cat, data, lines, csvfile
+	LogEntryCountCSV.summary = None
+	for cat in store.get_categories():
+		data = store.get_category(cat)
+		render(analyzers.LogEntryCountAnalyzer, data, name=cat)
+	if LogEntryCountCSV.summary:
+		headers = []
+		lines = []
+		for name in LogEntryCountCSV.summary:
+			data = LogEntryCountCSV.summary[name]
+			for head in data:
+				if not head in headers:
+					headers.append(head)
+			line = [name]
+			for head in headers:
+				line.append(data[head]) if head in data else line.append(0)
+			lines.append(line)
+		import csv
+
+		with open('logentrycount.csv', 'w', newline='') as csvfile:
+			writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE)
+			writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
+			for line in lines:
+				writer.writerow(line)
+
+
+def write_simulation_flag_csv(store):
+	global csvfile, result, i
+	from datetime import datetime
+	import json
+	json.dump(store.serializable(), open("simus.json", "w"), indent=2)
+	with open("simus.csv", "w") as csvfile:
+		csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n")
+		for key in store.get_store():
+			csvfile.write("{}\n".format(key))
+			for result in store.store[key]:
+				csvfile.write(",{}\n".format(result.name))
+				for i in result.get():
+					csvfile.write(",,{},{},{},{},{},{}\n".format(
+						i['answers']['@id'],
+						i['answers']['answered'],
+						len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0,
+						len(i['selected_actions']) if i['selected_actions'] else 0,
+						i['timestamp'],
+						str(datetime.fromtimestamp(i['timestamp'] / 1000))
+					))
\ No newline at end of file
diff --git a/analyzers/render/wip.py b/analyzers/render/wip.py
new file mode 100644
index 0000000..3750529
--- /dev/null
+++ b/analyzers/render/wip.py
@@ -0,0 +1,87 @@
+def time_distribution(store):
+	# json.dump(store.serializable(), open("new.json", "w"), indent=1)
+	from collections import defaultdict
+	import json
+	import numpy as np
+
+	keys = [
+		"simu",
+		"question",
+		"image",
+		"audio",
+		"video",
+		"other",
+		"map"
+	]
+	import matplotlib.pyplot as plt
+
+	# results = []
+
+	places = defaultdict(list)
+
+	for log in store.get_all():
+		result = defaultdict(lambda: 0)
+		for i in log.get()['track']:
+			duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
+			result[i['properties']['activity_type']] += duration
+		print(json.dumps(result, indent=4))
+		total = sum(result.values())
+		print(total)
+		percentage = defaultdict(lambda: 0)
+		minutes = defaultdict(lambda: 0)
+		for i in result:
+			percentage[i] = result[i] / total
+			minutes[i] = result[i] / 60_000
+		print(json.dumps(percentage, indent=4))
+		if not 'error' in result:
+			# places[log.get()['instance']].append(percentage)
+			places[log.get()['instance']].append(minutes)
+
+	for place in places:
+		places[place] = sorted(places[place], key=lambda item: item['map'])
+
+	dummy = [0] * len(keys)
+	results = []
+	sites = []
+	from util.meta_temp import CONFIG_NAMES
+
+	for i in places:
+		for j in places[i]:
+			ordered = []
+			for k in keys:
+				ordered.append(j[k])
+			results.append(ordered)
+		results.append(dummy)
+		sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
+
+	size = len(results)
+	ind = np.arange(size)
+	width = 0.9
+	print(results)
+	data = list(zip(*results))
+	print(data)
+	lines = []
+	bottom = [0] * len(results)
+	for i in range(0, len(data)):
+		lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+		for k, x in enumerate(data[i]):
+			bottom[k] += x
+	plt.legend(lines, keys)
+	plt.title(", ".join(sites))
+	plt.show()
+
+	# size = len(results)
+	# ind = np.arange(size)
+	# width = 0.9
+	# print(results)
+	# data = list(zip(*results))
+	# print(data)
+	# lines = []
+	# bottom = [0] * len(results)
+	# for i in range(0, len(data)):
+	#	lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+	#	for k, x in enumerate(data[i]):
+	#		bottom[k] += x
+	# plt.legend(lines, keys)
+	# plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)")
+	# plt.show()
\ No newline at end of file
diff --git a/biogames2.json b/biogames2.json
index 689a351..04dfbe7 100644
--- a/biogames2.json
+++ b/biogames2.json
@@ -67,7 +67,12 @@
         "action":"PAUSE"
       }
     },
-    "coordinates": "location.coordinates"
+    "coordinates": "location.coordinates",
+    "metadata":{
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
   },
   "source":{
     "type": "Biogames",
diff --git a/log_analyzer.py b/log_analyzer.py
index c9c7a3b..6ab123c 100644
--- a/log_analyzer.py
+++ b/log_analyzer.py
@@ -7,9 +7,12 @@ import numpy as np
 import analyzers
 from analyzers import get_renderer, Analyzer, render, Store
 from analyzers.analyzer import ResultStore
+from analyzers.analyzer.default import write_logentry_count_csv, write_simulation_flag_csv
 from analyzers.render.default import LogEntryCountCSV
+from analyzers.render.wip import time_distribution
 from analyzers.settings import LogSettings, load_settings
 from loaders import LOADERS
+from util.processing import grep, run_analysis
 
 logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG)
 log: logging.Logger = logging.getLogger(__name__)
@@ -18,98 +21,11 @@ logging.getLogger('requests').setLevel(logging.WARN)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 
-def process_log(logfile: str, settings: LogSettings) -> List[Analyzer]:
-	loader = LOADERS[settings.log_format]()
-	try:
-		loader.load(logfile)
-	except BaseException as e:
-		raise RuntimeError(e)
-	analyzers: List[Analyzer] = []
-	log.debug("build analyzers")
-	for analyzer in settings.analyzers:
-		analyzers.append(analyzer(settings))
-	log.debug("process entries")
-	for entry in loader.get_entry():
-		for analyzer in analyzers:
-			try:
-				if analyzer.process(entry):
-					break
-			except KeyError as e:
-				log.exception(e)
-	return analyzers
-
-
-def run_analysis(log_ids: list, settings):
-	store: ResultStore = ResultStore()
-	for log_id in log_ids:
-		for analysis in process_log(log_id, settings):
-			log.info("* Result for " + analysis.name())
-			analysis.result(store, name=log_id)
-	return store
-
-
-def load_ids(name: str):
-	log_ids = []
-	with open(name) as src:
-		for line in src:
-			line = line.strip()
-			log_ids.append(line)
-	return log_ids
-
-
 def urach_logs(log_ids, settings):
-	#return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
+	# return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
 	return ["data/{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
 
 
-def write_logentry_count_csv():
-	global cat, data, lines, csvfile
-	LogEntryCountCSV.summary = None
-	for cat in store.get_categories():
-		data = store.get_category(cat)
-		render(analyzers.LogEntryCountAnalyzer, data, name=cat)
-	if LogEntryCountCSV.summary:
-		headers = []
-		lines = []
-		for name in LogEntryCountCSV.summary:
-			data = LogEntryCountCSV.summary[name]
-			for head in data:
-				if not head in headers:
-					headers.append(head)
-			line = [name]
-			for head in headers:
-				line.append(data[head]) if head in data else line.append(0)
-			lines.append(line)
-		import csv
-
-		with open('logentrycount.csv', 'w', newline='') as csvfile:
-			writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE)
-			writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
-			for line in lines:
-				writer.writerow(line)
-
-
-def write_simulation_flag_csv():
-	global csvfile, result, i
-	from datetime import datetime
-	json.dump(store.serializable(), open("simus.json", "w"), indent=2)
-	with open("simus.csv", "w") as csvfile:
-		csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n")
-		for key in store.get_store():
-			csvfile.write("{}\n".format(key))
-			for result in store.store[key]:
-				csvfile.write(",{}\n".format(result.name))
-				for i in result.get():
-					csvfile.write(",,{},{},{},{},{},{}\n".format(
-						i['answers']['@id'],
-						i['answers']['answered'],
-						len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0,
-						len(i['selected_actions']) if i['selected_actions'] else 0,
-						i['timestamp'],
-						str(datetime.fromtimestamp(i['timestamp'] / 1000))
-					))
-
-
 if __name__ == '__main__':
 	settings: LogSettings = load_settings("biogames2.json")
 	log_ids_urach: List[str] = urach_logs([
@@ -121,11 +37,19 @@ if __name__ == '__main__':
 		# "597b704fe9ace475316c345903",
 		# "e01a684aa29dff9ddd9705edf8",
 		# "fbf9d64ae0bdad0de7efa3eec6",
-		"fe1331481f85560681f86827ec",
+		"fe1331481f85560681f86827ec",  # urach
 		# "fe1331481f85560681f86827ec"]
-		"fec57041458e6cef98652df625", ]
-		,settings)
-	store: ResultStore = run_analysis(log_ids_urach, settings)
+		"fec57041458e6cef98652df625",
+	]
+		, settings)
+	log_ids_gf = grep(["9d11b749c78a57e786bf5c8d28",  # filderstadt
+					   "a192ff420b8bdd899fd28573e2",  # eichstätt
+					   "3a3d994c04b1b1d87168422309",  # stadtökologie
+					   "96f6d9cc556b42f3b2fec0a2cb7ed36e"  # oberelsbach
+					   ],
+					  "/home/clemens/git/ma/test/src",
+					  settings)
+	store: ResultStore = run_analysis(log_ids_gf, settings, LOADERS)
 	if False:
 		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
 			r().render(store.get_all())
@@ -148,110 +72,12 @@ if __name__ == '__main__':
 			data = store.get_category(cat)
 			render(analyzers.SimulationOrderAnalyzer, data, name=cat)
 	if False:
-		write_logentry_count_csv()
+		write_logentry_count_csv(LogEntryCountCSV, store, render, analyzers)
 	if False:
-		write_simulation_flag_csv()
-
-
-	def calc_distance(geojson: str):
-		from shapely.geometry import LineString
-		from shapely.ops import transform
-		from functools import partial
-		import pyproj
-		track = LineString(json.loads(geojson)['coordinates'])
-		project = partial(
-			pyproj.transform,
-			pyproj.Proj(init='EPSG:4326'),
-			pyproj.Proj(init='EPSG:32633'))
-		return transform(project, track).length
-
-
-	if False:
-		# json.dump(store.serializable(), open("new.json", "w"), indent=1)
-		from collections import defaultdict
-
-		keys = [
-			"simu",
-			"question",
-			"image",
-			"audio",
-			"video",
-			"other",
-			"map"
-		]
-		import matplotlib.pyplot as plt
-
-		# results = []
-
-		places = defaultdict(list)
-
-		for log in store.get_all():
-			result = defaultdict(lambda: 0)
-			for i in log.get()['track']:
-				duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
-				result[i['properties']['activity_type']] += duration
-			print(json.dumps(result, indent=4))
-			total = sum(result.values())
-			print(total)
-			percentage = defaultdict(lambda: 0)
-			minutes = defaultdict(lambda: 0)
-			for i in result:
-				percentage[i] = result[i] / total
-				minutes[i] = result[i] / 60_000
-			print(json.dumps(percentage, indent=4))
-			if not 'error' in result:
-				# places[log.get()['instance']].append(percentage)
-				places[log.get()['instance']].append(minutes)
-
-		for place in places:
-			places[place] = sorted(places[place], key=lambda item: item['map'])
-
-		dummy = [0] * len(keys)
-		results = []
-		sites = []
-		from util.meta_temp import CONFIG_NAMES
-
-		for i in places:
-			for j in places[i]:
-				ordered = []
-				for k in keys:
-					ordered.append(j[k])
-				results.append(ordered)
-			results.append(dummy)
-			sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
-
-		size = len(results)
-		ind = np.arange(size)
-		width = 0.9
-		print(results)
-		data = list(zip(*results))
-		print(data)
-		lines = []
-		bottom = [0] * len(results)
-		for i in range(0, len(data)):
-			lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
-			for k, x in enumerate(data[i]):
-				bottom[k] += x
-		plt.legend(lines, keys)
-		plt.title(", ".join(sites))
-		plt.show()
-
-	# size = len(results)
-	# ind = np.arange(size)
-	# width = 0.9
-	# print(results)
-	# data = list(zip(*results))
-	# print(data)
-	# lines = []
-	# bottom = [0] * len(results)
-	# for i in range(0, len(data)):
-	#	lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
-	#	for k, x in enumerate(data[i]):
-	#		bottom[k] += x
-	# plt.legend(lines, keys)
-	# plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)")
-	# plt.show()
+		write_simulation_flag_csv(store)
 
+	if True:
+		time_distribution(store)
 
 
 # for analyzers in analyzers:
diff --git a/static/progress/my.js b/static/progress/my.js
index 55b22f4..29c2dd8 100644
--- a/static/progress/my.js
+++ b/static/progress/my.js
@@ -1,12 +1,10 @@
-//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) {
-$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-68854ffab9aa.json", function (data) {
+$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) {    //urach
+//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-68854ffab9aa.json", function (data) {  //urach
+//$.getJSON("data/90278021-4c57-464e-90b1-d603799d07eb_07da99c9-398a-424f-99fc-2701763a63e9.json", function (data) {  //eichstätt
+//$.getJSON("data/13241906-cdae-441a-aed0-d57ebeb37cac_d33976a6-8a56-4a63-b492-fe5427dbf377.json", function (data) {  //stadtökologie
+//$.getJSON("data/5e64ce07-1c16-4d50-ac4e-b3117847ea43_2f664d7b-f0d8-42f5-8731-c034ef86703e.json", function (data) {  //filderstadt
     var images = {};
     var tiles = {
-        "osm": L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
-            maxNativeZoom: 19,
-            maxZoom: 24,
-            attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
-        }),
         "openstreetmap": L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
             maxNativeZoom: 19,
             maxZoom: 24,
@@ -17,13 +15,13 @@ $.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-688
             maxZoom: 24,
             attribution: 'Tiles &copy; Esri &mdash; Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community'
         }),
-        "google sat": L.tileLayer('http://{s}.google.com/vt/lyrs=s&x={x}&y={y}&z={z}', {
+        "google sat": L.tileLayer('https://{s}.google.com/vt/lyrs=s&x={x}&y={y}&z={z}', {
             maxNativeZoom: 20,
             maxZoom: 24,
             subdomains: ['mt0', 'mt1', 'mt2', 'mt3']
         })
     };
-    var map = L.map("mainMap", {layers: [tiles.osm]});
+    var map = L.map("mainMap", {layers: [tiles.openstreetmap]});
 
     function styleTrack(feature) {
         var styles = {};
diff --git a/util/geo.py b/util/geo.py
new file mode 100644
index 0000000..f50d6c5
--- /dev/null
+++ b/util/geo.py
@@ -0,0 +1,12 @@
+def calc_distance(geojson: str):
+	from shapely.geometry import LineString
+	from shapely.ops import transform
+	from functools import partial
+	import pyproj
+	import json
+	track = LineString(json.loads(geojson)['coordinates'])
+	project = partial(
+		pyproj.transform,
+		pyproj.Proj(init='EPSG:4326'),
+		pyproj.Proj(init='EPSG:32633'))
+	return transform(project, track).length
\ No newline at end of file
diff --git a/util/processing.py b/util/processing.py
new file mode 100644
index 0000000..67b8f6d
--- /dev/null
+++ b/util/processing.py
@@ -0,0 +1,57 @@
+import logging
+from typing import List
+
+from analyzers.analyzer import ResultStore, Analyzer
+from analyzers.settings import LogSettings
+
+log: logging.Logger = logging.getLogger(__name__)
+
+
+def process_log(logfile: str, settings: LogSettings, loaders) -> List[Analyzer]:
+	loader = loaders[settings.log_format]()
+	try:
+		loader.load(logfile)
+	except BaseException as e:
+		raise RuntimeError(e)
+	analyzers: List[Analyzer] = []
+	log.debug("build analyzers")
+	for analyzer in settings.analyzers:
+		analyzers.append(analyzer(settings))
+	log.debug("process entries")
+	for entry in loader.get_entry():
+		for analyzer in analyzers:
+			try:
+				if analyzer.process(entry):
+					break
+			except KeyError as e:
+				log.exception(e)
+	return analyzers
+
+
+def run_analysis(log_ids: list, settings, loaders):
+	store: ResultStore = ResultStore()
+	for log_id in log_ids:
+		for analysis in process_log(log_id, settings, loaders):
+			log.info("* Result for " + analysis.name())
+			analysis.result(store, name=log_id)
+	return store
+
+
+def load_ids(name: str):
+	log_ids = []
+	with open(name) as src:
+		for line in src:
+			line = line.strip()
+			log_ids.append(line)
+	return log_ids
+
+
+def grep(log_ids, source, settings):
+	logs = []
+	with open(source) as src:
+		lines = src.readlines()
+		for id in log_ids:
+			for line in lines:
+				if id in line:
+					logs.append(line.strip())
+	return logs

From 2c8eea0e6f7915acfd28b0dea7ddf535513a2f2b Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Wed, 14 Mar 2018 17:49:09 +0100
Subject: [PATCH 11/24] restructured

* add selector frontend
* basic celery backend
* client library
---
 Dockerfile                                    |  10 ++
 plots/.gitkeep => analysis/__init__.py        |   0
 {analyzers => analysis/analyzers}/__init__.py |   3 +-
 .../analyzers}/analyzer/__init__.py           |   2 +-
 .../analyzers}/analyzer/biogames.py           |  14 +--
 .../analyzers}/analyzer/default.py            |   6 +-
 .../analyzers}/analyzer/locomotion.py         |   0
 .../analyzers}/analyzer/mask.py               |   0
 .../analyzers}/render/__init__.py             |   0
 .../analyzers}/render/biogames.py             |   4 +-
 .../analyzers}/render/default.py              |  31 ++++-
 .../analyzers}/render/locomotion.py           |   0
 .../analyzers}/render/wip.py                  |   6 +-
 {analyzers => analysis/analyzers}/settings.py |   8 +-
 biogames.json => analysis/biogames.json       |   0
 biogames2.json => analysis/biogames2.json     |   0
 .../filter_todo}/pre_filter.py                |   0
 {loaders => analysis/loaders}/__init__.py     |   0
 {loaders => analysis/loaders}/biogames.py     |   0
 {loaders => analysis/loaders}/loader.py       |   0
 log_analyzer.py => analysis/log_analyzer.py   |  97 ++++++++-------
 {sources => analysis/sources}/__init__.py     |   0
 {sources => analysis/sources}/biogames.py     |   2 +-
 {sources => analysis/sources}/source.py       |   0
 {util => analysis/util}/__init__.py           |   0
 {util => analysis/util}/download.py           |   0
 {util => analysis/util}/geo.py                |   0
 {util => analysis/util}/iter.py               |   0
 {util => analysis/util}/meta_temp.py          |  17 +++
 {util => analysis/util}/processing.py         |   5 +-
 clients/__init__.py                           |   0
 clients/webclients.py                         | 110 ++++++++++++++++++
 docker-compose.yml                            |  41 +++++++
 requirements.txt                              |  11 +-
 selector/__init__.py                          |   0
 selector/static/script.js                     |   4 +
 selector/static/style.css                     |   6 +
 selector/templates/base.html                  |   5 +
 selector/templates/games.html                 |  19 +++
 selector/templates/index.html                 |  14 +++
 selector/webserver.py                         |  53 +++++++++
 static/progress/my.js                         |   5 +-
 tasks/__init__.py                             |   0
 tasks/tasks.py                                |  14 +++
 44 files changed, 417 insertions(+), 70 deletions(-)
 create mode 100644 Dockerfile
 rename plots/.gitkeep => analysis/__init__.py (100%)
 rename {analyzers => analysis/analyzers}/__init__.py (98%)
 rename {analyzers => analysis/analyzers}/analyzer/__init__.py (97%)
 rename {analyzers => analysis/analyzers}/analyzer/biogames.py (96%)
 rename {analyzers => analysis/analyzers}/analyzer/default.py (97%)
 rename {analyzers => analysis/analyzers}/analyzer/locomotion.py (100%)
 rename {analyzers => analysis/analyzers}/analyzer/mask.py (100%)
 rename {analyzers => analysis/analyzers}/render/__init__.py (100%)
 rename {analyzers => analysis/analyzers}/render/biogames.py (97%)
 rename {analyzers => analysis/analyzers}/render/default.py (68%)
 rename {analyzers => analysis/analyzers}/render/locomotion.py (100%)
 rename {analyzers => analysis/analyzers}/render/wip.py (98%)
 rename {analyzers => analysis/analyzers}/settings.py (87%)
 rename biogames.json => analysis/biogames.json (100%)
 rename biogames2.json => analysis/biogames2.json (100%)
 rename {filter_todo => analysis/filter_todo}/pre_filter.py (100%)
 rename {loaders => analysis/loaders}/__init__.py (100%)
 rename {loaders => analysis/loaders}/biogames.py (100%)
 rename {loaders => analysis/loaders}/loader.py (100%)
 rename log_analyzer.py => analysis/log_analyzer.py (53%)
 rename {sources => analysis/sources}/__init__.py (100%)
 rename {sources => analysis/sources}/biogames.py (98%)
 rename {sources => analysis/sources}/source.py (100%)
 rename {util => analysis/util}/__init__.py (100%)
 rename {util => analysis/util}/download.py (100%)
 rename {util => analysis/util}/geo.py (100%)
 rename {util => analysis/util}/iter.py (100%)
 rename {util => analysis/util}/meta_temp.py (94%)
 rename {util => analysis/util}/processing.py (90%)
 create mode 100644 clients/__init__.py
 create mode 100644 clients/webclients.py
 create mode 100644 docker-compose.yml
 create mode 100644 selector/__init__.py
 create mode 100644 selector/static/script.js
 create mode 100644 selector/static/style.css
 create mode 100644 selector/templates/base.html
 create mode 100644 selector/templates/games.html
 create mode 100644 selector/templates/index.html
 create mode 100644 selector/webserver.py
 create mode 100644 tasks/__init__.py
 create mode 100644 tasks/tasks.py

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..2de600e
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,10 @@
+FROM alpine:edge
+
+ADD ["requirements.txt", "/"]
+RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing/" >> /etc/apk/repositories && \
+    sed -i 's/numpy/#numpy/' requirements.txt && \
+    sed -i 's/scipy/#scipy/' requirements.txt && \
+    apk add --update libpng libpng-dev freetype freetype-dev g++ python3 py3-numpy python3-dev py-numpy-dev py3-scipy&& \
+    pip3 install -r requirements.txt && \
+    apk del libpng-dev freetype-dev g++ python3-dev py-numpy-dev && \
+    rm requirements.txt
\ No newline at end of file
diff --git a/plots/.gitkeep b/analysis/__init__.py
similarity index 100%
rename from plots/.gitkeep
rename to analysis/__init__.py
diff --git a/analyzers/__init__.py b/analysis/analyzers/__init__.py
similarity index 98%
rename from analyzers/__init__.py
rename to analysis/analyzers/__init__.py
index 41623e4..07a8186 100644
--- a/analyzers/__init__.py
+++ b/analysis/analyzers/__init__.py
@@ -12,7 +12,7 @@ from .render import Render
 from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
 	ActivityMapperRender, StoreRender, SimulationOrderRender, SimulationGroupRender
 from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender, LogEntryCountAnalyzerPlot, \
-	LogEntryCountCSV
+	LogEntryCountCSV, KMLRender
 from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
 	LocomotionActionRatioRender
 
@@ -41,6 +41,7 @@ __MAPPING__ = {
 	LocationAnalyzer: [
 		TrackRender,
 		HeatMapRender,
+		KMLRender,
 	],
 	ActivityMapper: [
 		ActivityMapperRender
diff --git a/analyzers/analyzer/__init__.py b/analysis/analyzers/analyzer/__init__.py
similarity index 97%
rename from analyzers/analyzer/__init__.py
rename to analysis/analyzers/analyzer/__init__.py
index 175ef92..422e30c 100644
--- a/analyzers/analyzer/__init__.py
+++ b/analysis/analyzers/analyzer/__init__.py
@@ -2,7 +2,7 @@ import logging
 from collections import KeysView
 from typing import Type, Sized, Collection
 
-from analyzers.settings import LogSettings
+from analysis.analyzers.settings import LogSettings
 
 log: logging.Logger = logging.getLogger(__name__)
 
diff --git a/analyzers/analyzer/biogames.py b/analysis/analyzers/analyzer/biogames.py
similarity index 96%
rename from analyzers/analyzer/biogames.py
rename to analysis/analyzers/analyzer/biogames.py
index fccc8d0..e87f21f 100644
--- a/analyzers/analyzer/biogames.py
+++ b/analysis/analyzers/analyzer/biogames.py
@@ -3,8 +3,8 @@ from collections import defaultdict, namedtuple, OrderedDict
 from types import SimpleNamespace
 from typing import List, NamedTuple
 
-from util import json_path, combinate
-from util.download import download_board, get_board_data
+from analysis.util import json_path, combinate
+from analysis.util.download import download_board, get_board_data
 from . import Result, LogSettings, Analyzer, ResultStore
 from .default import CategorizerStub, Store
 
@@ -174,13 +174,13 @@ class ActivityMapper(Analyzer):
 					board["image"] = "simu.png"
 					continue
 				local_file = download_board(board["board_id"], self.instance_config_id, board["sequence_id"],
-				                            self.settings.source)
+											self.settings.source)
 				if local_file:
 					board['image'] = local_file
 				else:
 					board['image'] = "ERROR_FETCHING_FILE"
 					logger.error("error downloading board! %s %s %s", self.instance_config_id, board["sequence_id"],
-					             board["board_id"])
+								 board["board_id"])
 			else:
 				board["image"] = "map.png"
 		store.add(Result(type(self), {
@@ -202,7 +202,7 @@ class ActivityMapper(Analyzer):
 			self.add_location(entry)
 		elif entry[self.settings.type_field] in self.settings.boards:
 			board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
-			                            entry["board_id"])
+										entry["board_id"])
 			entry["extra_data"] = board_data
 			entry["extra_data"]["activity_type"] = self.last_board_type
 			entry['coordinate'] = self.new_coordinate()
@@ -218,14 +218,14 @@ class ActivityMapper(Analyzer):
 	def classify_entry(self, entry):
 		entry_type = entry[self.settings.type_field]
 		if self.filters.end(entry):
-			data = {"extra_data": {"activity_type": "map"},"coordinate": self.new_coordinate()}
+			data = {"extra_data": {"activity_type": "map"}, "coordinate": self.new_coordinate()}
 			data.update(entry)
 			self.timeline.append(data)
 			return "map"
 		if not entry_type in self.settings.boards:
 			return self.last_board_type
 		board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
-		                            entry["board_id"])
+									entry["board_id"])
 		for pattern in self.classes:
 			if pattern in board_data['class']:
 				return self.classes[pattern]
diff --git a/analyzers/analyzer/default.py b/analysis/analyzers/analyzer/default.py
similarity index 97%
rename from analyzers/analyzer/default.py
rename to analysis/analyzers/analyzer/default.py
index 8698a5c..fce0e35 100644
--- a/analyzers/analyzer/default.py
+++ b/analysis/analyzers/analyzer/default.py
@@ -1,7 +1,7 @@
 import logging
 from collections import defaultdict, OrderedDict
 
-from util import json_path
+from analysis.util import json_path
 from . import Result, LogSettings, Analyzer, ResultStore
 
 
@@ -16,9 +16,9 @@ class LocationAnalyzer(Analyzer):
 		super().__init__(settings)
 		self.entries = []
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, **kwargs) -> None:
 		self.log.debug(len(self.entries))
-		store.add(Result(type(self), list(self.entries)))
+		store.add(Result(type(self), list(self.entries), name=kwargs['name']))
 
 	def process(self, entry: dict) -> bool:
 		if entry[self.settings.type_field] in self.settings.spatials:
diff --git a/analyzers/analyzer/locomotion.py b/analysis/analyzers/analyzer/locomotion.py
similarity index 100%
rename from analyzers/analyzer/locomotion.py
rename to analysis/analyzers/analyzer/locomotion.py
diff --git a/analyzers/analyzer/mask.py b/analysis/analyzers/analyzer/mask.py
similarity index 100%
rename from analyzers/analyzer/mask.py
rename to analysis/analyzers/analyzer/mask.py
diff --git a/analyzers/render/__init__.py b/analysis/analyzers/render/__init__.py
similarity index 100%
rename from analyzers/render/__init__.py
rename to analysis/analyzers/render/__init__.py
diff --git a/analyzers/render/biogames.py b/analysis/analyzers/render/biogames.py
similarity index 97%
rename from analyzers/render/biogames.py
rename to analysis/analyzers/render/biogames.py
index 5e2a2f1..608d045 100644
--- a/analyzers/render/biogames.py
+++ b/analysis/analyzers/render/biogames.py
@@ -9,8 +9,8 @@ from scipy.interpolate import interp1d
 import networkx as nx
 import itertools
 
-from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
-from util.meta_temp import CONFIG_NAMES
+from analysis.analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
+from analysis.util.meta_temp import CONFIG_NAMES
 from . import Render
 from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
 
diff --git a/analyzers/render/default.py b/analysis/analyzers/render/default.py
similarity index 68%
rename from analyzers/render/default.py
rename to analysis/analyzers/render/default.py
index bf57f61..d0d94c1 100644
--- a/analyzers/render/default.py
+++ b/analysis/analyzers/render/default.py
@@ -2,11 +2,13 @@ import json
 import logging
 from typing import List
 
+import datetime
 import matplotlib.pyplot as plt
 
-from analyzers import LogEntryCountAnalyzer
+from analysis.analyzers import LogEntryCountAnalyzer
+from analysis.util.meta_temp import KML_PATTERN
 from . import Render, Result
-from .. import LocationAnalyzer
+from analysis.analyzers import LocationAnalyzer
 
 log = logging.getLogger(__name__)
 
@@ -39,6 +41,31 @@ class TrackRender(Render):
 		return dumps
 
 
+def format_time(ts):
+	return datetime.datetime.fromtimestamp(ts/1000).strftime("%Y-%m-%dT%H:%M:%S.%f")
+
+
+class KMLRender(Render):
+	result_types = [LocationAnalyzer]
+
+	def render(self, results: List[Result], name=None):
+		for result in self.filter(results):
+			times = ["<when>{time}</when>".format(time=format_time(entry["timestamp"])) for entry in result.get()]
+			coords = [
+				"<gx:coord>{long} {lat} 0.0</gx:coord>"
+					.format(
+					lat=entry['location']['coordinates'][1],
+					long=entry['location']['coordinates'][0])
+				for entry in result.get()
+			]
+			filename = str(result.name)+".kml"
+			print(filename)
+			with open(filename, "w") as out:
+				out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times)))
+
+
+
+
 class HeatMapRender(TrackRender):
 	weight = 0.01
 
diff --git a/analyzers/render/locomotion.py b/analysis/analyzers/render/locomotion.py
similarity index 100%
rename from analyzers/render/locomotion.py
rename to analysis/analyzers/render/locomotion.py
diff --git a/analyzers/render/wip.py b/analysis/analyzers/render/wip.py
similarity index 98%
rename from analyzers/render/wip.py
rename to analysis/analyzers/render/wip.py
index 832c49f..19249ec 100644
--- a/analyzers/render/wip.py
+++ b/analysis/analyzers/render/wip.py
@@ -2,8 +2,8 @@ import json
 
 import numpy as np
 
-import analyzers
-from util.geo import calc_distance
+import analysis.analyzers
+from analysis.util.geo import calc_distance
 
 
 def time_distribution(store):
@@ -97,7 +97,7 @@ def time_distribution(store):
 
 from collections import defaultdict
 import matplotlib.pyplot as plt
-from util.meta_temp import CONFIG_NAMES
+from analysis.util.meta_temp import CONFIG_NAMES
 
 keys = [
 	"simu",
diff --git a/analyzers/settings.py b/analysis/analyzers/settings.py
similarity index 87%
rename from analyzers/settings.py
rename to analysis/analyzers/settings.py
index 295f670..36f7682 100644
--- a/analyzers/settings.py
+++ b/analysis/analyzers/settings.py
@@ -1,6 +1,6 @@
 import json
 import sys
-from sources import SOURCES
+from analysis.sources import SOURCES
 
 
 def load_source(config):
@@ -28,7 +28,7 @@ class LogSettings:
 		self.boards = json_dict['boards']
 		for mod in json_dict['analyzers']:
 			for name in json_dict['analyzers'][mod]:
-				print(mod, name)
+				print(mod, name, getattr(sys.modules[mod], name))
 				self.analyzers.append(getattr(sys.modules[mod], name))
 		self.sequences = json_dict['sequences']
 		if 'custom' in json_dict:
@@ -51,3 +51,7 @@ class LogSettings:
 
 def load_settings(file: str) -> LogSettings:
 	return LogSettings(json.load(open(file)))
+
+
+def parse_settings(config: str) -> LogSettings:
+	return LogSettings(json.loads(config))
diff --git a/biogames.json b/analysis/biogames.json
similarity index 100%
rename from biogames.json
rename to analysis/biogames.json
diff --git a/biogames2.json b/analysis/biogames2.json
similarity index 100%
rename from biogames2.json
rename to analysis/biogames2.json
diff --git a/filter_todo/pre_filter.py b/analysis/filter_todo/pre_filter.py
similarity index 100%
rename from filter_todo/pre_filter.py
rename to analysis/filter_todo/pre_filter.py
diff --git a/loaders/__init__.py b/analysis/loaders/__init__.py
similarity index 100%
rename from loaders/__init__.py
rename to analysis/loaders/__init__.py
diff --git a/loaders/biogames.py b/analysis/loaders/biogames.py
similarity index 100%
rename from loaders/biogames.py
rename to analysis/loaders/biogames.py
diff --git a/loaders/loader.py b/analysis/loaders/loader.py
similarity index 100%
rename from loaders/loader.py
rename to analysis/loaders/loader.py
diff --git a/log_analyzer.py b/analysis/log_analyzer.py
similarity index 53%
rename from log_analyzer.py
rename to analysis/log_analyzer.py
index 1e053fb..2839abd 100644
--- a/log_analyzer.py
+++ b/analysis/log_analyzer.py
@@ -2,16 +2,16 @@ import json
 import logging
 from typing import List
 
-import analyzers
-from analyzers import get_renderer, render
-from analyzers.analyzer import ResultStore
-from analyzers.analyzer.default import write_logentry_count_csv, write_simulation_flag_csv
-from analyzers.render import wip
-from analyzers.render.default import LogEntryCountCSV
-from analyzers.render.wip import time_distribution, plot_data
-from analyzers.settings import LogSettings, load_settings
-from loaders import LOADERS
-from util.processing import grep, run_analysis, src_file
+from analysis import analyzers
+from analysis.analyzers import get_renderer, render
+from analysis.analyzers.analyzer import ResultStore
+from analysis.analyzers.analyzer.default import write_logentry_count_csv, write_simulation_flag_csv
+from analysis.analyzers.render import wip
+from analysis.analyzers.render.default import LogEntryCountCSV, KMLRender
+from analysis.analyzers.render.wip import time_distribution, plot_data
+from analysis.analyzers.settings import LogSettings, load_settings, parse_settings
+from analysis.loaders import LOADERS
+from analysis.util.processing import grep, run_analysis, src_file
 
 logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG)
 log: logging.Logger = logging.getLogger(__name__)
@@ -26,36 +26,39 @@ def urach_logs(log_ids, settings):
 
 
 if __name__ == '__main__':
-	settings: LogSettings = load_settings("biogames2.json")
-	log_ids_urach: List[str] = urach_logs([
-		# "34fecf49dbaca3401d745fb467",
-		#	"44ea194de594cd8d63ac0314be",
-		#	"57c444470dbf88605433ca935c",
-		#	"78e0c545b594e82edfad55bd7f",
-		#	"91abfd4b31a5562b1c66be37d9",
-		# "597b704fe9ace475316c345903",
-		# "e01a684aa29dff9ddd9705edf8",
-		"597b704fe9ace475316c345903",
-		"e01a684aa29dff9ddd9705edf8",
-		"fbf9d64ae0bdad0de7efa3eec6",
-		# "fbf9d64ae0bdad0de7efa3eec6",
-		"fe1331481f85560681f86827ec",  # urach
-		# "fe1331481f85560681f86827ec"]
-		"fec57041458e6cef98652df625",
-	]
-		, settings)
-	log_ids_gf = grep(["9d11b749c78a57e786bf5c8d28",  # filderstadt
-					   "a192ff420b8bdd899fd28573e2",  # eichstätt
-					   "3a3d994c04b1b1d87168422309",  # stadtökologie
-					   "fe1331481f85560681f86827ec",  # urach
-					   "96f6d9cc556b42f3b2fec0a2cb7ed36e"  # oberelsbach
-					   ],
-					  "/home/clemens/git/ma/test/src",
-					  settings)
-	log_ids = src_file("/home/clemens/git/ma/test/filtered_5_actions")
+	settings = {}
+	log_ids_gf = []
+	# settings: LogSettings = load_settings("biogames2.json")
+	# log_ids_urach: List[str] = urach_logs([
+	# 	# "34fecf49dbaca3401d745fb467",
+	# 	#	"44ea194de594cd8d63ac0314be",
+	# 	#	"57c444470dbf88605433ca935c",
+	# 	#	"78e0c545b594e82edfad55bd7f",
+	# 	#	"91abfd4b31a5562b1c66be37d9",
+	# 	# "597b704fe9ace475316c345903",
+	# 	# "e01a684aa29dff9ddd9705edf8",
+	# 	"597b704fe9ace475316c345903",
+	# 	"e01a684aa29dff9ddd9705edf8",
+	# 	"fbf9d64ae0bdad0de7efa3eec6",
+	# 	# "fbf9d64ae0bdad0de7efa3eec6",
+	# 	"fe1331481f85560681f86827ec",  # urach
+	# 	# "fe1331481f85560681f86827ec"]
+	# 	"fec57041458e6cef98652df625",
+	# ]
+	# 	, settings)
+	# log_ids_gf = grep(["9d11b749c78a57e786bf5c8d28",  # filderstadt
+	# 				   "a192ff420b8bdd899fd28573e2",  # eichstätt
+	# 				   "3a3d994c04b1b1d87168422309",  # stadtökologie
+	# 				   "fe1331481f85560681f86827ec",  # urach
+	# 				   "96f6d9cc556b42f3b2fec0a2cb7ed36e"  # oberelsbach
+	# 				   ],
+	# 				  "/home/clemens/git/ma/test/src",
+	# 				  settings)
+	# log_ids = src_file("/home/clemens/git/ma/test/filtered_5_actions")
 
-	#store: ResultStore = run_analysis(log_ids_gf, settings, LOADERS)
-	#store: ResultStore = run_analysis(log_ids, settings, LOADERS)
+	if False:
+		store: ResultStore = run_analysis(log_ids_gf, settings, LOADERS)
+	# store: ResultStore = run_analysis(log_ids, settings, LOADERS)
 
 	if False:
 		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
@@ -69,7 +72,7 @@ if __name__ == '__main__':
 		# render(analyzers.ProgressAnalyzer, store.get_all())
 
 	if False:
-		from analyzers.postprocessing import graph
+		from analysis.analyzers.postprocessing import graph
 
 		g = graph.Cache(settings)
 		g.run(store)
@@ -85,7 +88,7 @@ if __name__ == '__main__':
 	if False:
 		time_distribution(store)
 
-	if True:
+	if False:
 		# spatial_data = get_data_distance(store,relative_values=False)
 		# temporal_data = get_data(store,relative_values=False)
 		# spatial_data_rel = get_data_distance(store,relative_values=True)
@@ -104,6 +107,18 @@ if __name__ == '__main__':
 		# plot_time_space_rel(combined, keys)
 		plot_data(combined, wip.keys)
 
+	if True:
+		settings: LogSettings = load_settings("../oeb_kml.json")
+		log_ids = src_file("/home/clemens/git/ma/test/oeb_2016_path")
+		log_ids = log_ids[0:2]
+		print(log_ids)
+		store: ResultStore = run_analysis(log_ids, settings, LOADERS)
+		print("render")
+		kml = KMLRender()
+		kml.render(store.get_all())
+		print("done")
+		#for cat in store.get_categories():
+		#	render(analyzers.ActivityMapper, store.get_category(cat), name=cat)
 
 # for analyzers in analyzers:
 #	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:
diff --git a/sources/__init__.py b/analysis/sources/__init__.py
similarity index 100%
rename from sources/__init__.py
rename to analysis/sources/__init__.py
diff --git a/sources/biogames.py b/analysis/sources/biogames.py
similarity index 98%
rename from sources/biogames.py
rename to analysis/sources/biogames.py
index fa39b38..ac25e10 100644
--- a/sources/biogames.py
+++ b/analysis/sources/biogames.py
@@ -26,7 +26,7 @@ class Biogames(Source):
 				raise ValueError("missing value " + i)
 		csrf_request = requests.get(kwargs['url'])
 		if csrf_request.status_code != 200:
-			raise ConnectionError("unable to obtain CSRF token (" + str(csrf_request) + ")")
+			raise ConnectionError("unable to obtain CSRF token (" + str(csrf_request) + ")", csrf_request.url)
 		self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
 		log.info("obtained CSRF token (" + self.cookies['csrftoken'] + ")")
 		login_payload = {
diff --git a/sources/source.py b/analysis/sources/source.py
similarity index 100%
rename from sources/source.py
rename to analysis/sources/source.py
diff --git a/util/__init__.py b/analysis/util/__init__.py
similarity index 100%
rename from util/__init__.py
rename to analysis/util/__init__.py
diff --git a/util/download.py b/analysis/util/download.py
similarity index 100%
rename from util/download.py
rename to analysis/util/download.py
diff --git a/util/geo.py b/analysis/util/geo.py
similarity index 100%
rename from util/geo.py
rename to analysis/util/geo.py
diff --git a/util/iter.py b/analysis/util/iter.py
similarity index 100%
rename from util/iter.py
rename to analysis/util/iter.py
diff --git a/util/meta_temp.py b/analysis/util/meta_temp.py
similarity index 94%
rename from util/meta_temp.py
rename to analysis/util/meta_temp.py
index 7efaf63..0bb928c 100644
--- a/util/meta_temp.py
+++ b/analysis/util/meta_temp.py
@@ -97,3 +97,20 @@ CONFIG_NAMES = {
 	'fe43a0f0-3dea-11e6-a065-00199963ac6e': u'Vorlagen',
 	'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771': u'Bad Urach'
 }
+
+
+KML_PATTERN="""<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2" xmlns:gx="http://www.google.com/kml/ext/2.2">
+  <Document>
+    <Placemark>
+    	<gx:MultiTrack>
+    		<gx:Track>
+    			{when}
+    			{coordinates}
+    		</gx:Track>
+    	</gx:MultiTrack>
+      {coordinates}
+    </Placemark>
+  </Document>
+</kml>
+"""
\ No newline at end of file
diff --git a/util/processing.py b/analysis/util/processing.py
similarity index 90%
rename from util/processing.py
rename to analysis/util/processing.py
index 5b3a3ff..7291f60 100644
--- a/util/processing.py
+++ b/analysis/util/processing.py
@@ -1,8 +1,8 @@
 import logging
 from typing import List
 
-from analyzers.analyzer import ResultStore, Analyzer
-from analyzers.settings import LogSettings
+from analysis.analyzers.analyzer import ResultStore, Analyzer
+from analysis.analyzers.settings import LogSettings
 
 log: logging.Logger = logging.getLogger(__name__)
 
@@ -31,6 +31,7 @@ def process_log(logfile: str, settings: LogSettings, loaders) -> List[Analyzer]:
 def run_analysis(log_ids: list, settings, loaders):
 	store: ResultStore = ResultStore()
 	for log_id in log_ids:
+		log.info("LOG_ID: "+ str(log_id))
 		for analysis in process_log(log_id, settings, loaders):
 			log.info("* Result for " + analysis.name())
 			analysis.result(store, name=log_id)
diff --git a/clients/__init__.py b/clients/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/clients/webclients.py b/clients/webclients.py
new file mode 100644
index 0000000..95f5c6c
--- /dev/null
+++ b/clients/webclients.py
@@ -0,0 +1,110 @@
+import json
+import logging
+
+import os
+import shutil
+import tempfile
+import typing
+
+import requests
+
+log: logging.Logger = logging.getLogger(__name__)
+
+
+class Client:
+	host: str = ""
+	cookies: typing.Dict[str, str] = {}
+	headers: typing.Dict[str, str] = {}
+
+	def url(self, path):
+		if self.host:
+			return self.host + path
+		return path
+
+	def get(self, url, **kwargs) -> requests.models.Response:
+		return requests.get(self.url(url), cookies=self.cookies, headers=self.headers, **kwargs)
+
+	def post(self, url, data, **kwargs) -> requests.models.Response:
+		return requests.post(self.url(url), data, cookies=self.cookies, headers=self.headers, **kwargs)
+
+	def download_file(self, url, target, **kwargs) -> bool:
+		with open(target, "wb") as out:
+			try:
+				download = self.get(url, stream=True, **kwargs)
+				shutil.copyfileobj(download.raw, out)
+			except Exception as e:
+				log.exception(e)
+				os.remove(target)
+				return False
+		return True
+
+	def download_files(self, urls, **kwargs) -> tempfile.TemporaryDirectory:
+		target = tempfile.TemporaryDirectory()
+		for path in urls:
+			filename = os.path.join(target.name, path.split("/")[-1])
+			self.download_file(path, filename, **kwargs)
+		return target
+
+
+class BiogamesClient(Client):
+	config_fields: typing.Dict[str, typing.List[str]] = {
+		'login': ('username', 'password', 'host'),
+		'session': ('sessionid', 'csrftoken', 'host'),
+	}
+	login_url: str = "/game2/auth/json-login"
+	list_url: str = "/game2/instance/log/list/"
+	headers: typing.Dict[str, str] = {'Accept': 'application/json'}
+
+	def __init__(self, **kwargs):
+		match = {j: all([i in kwargs for i in self.config_fields[j]]) for j in self.config_fields}
+		valid = filter(lambda x: match[x], match)
+		if not valid:
+			raise ValueError("missing parameter (" + str(self.config_fields) + ")")
+		self.config = kwargs
+		self.cookies = {}
+		self.host = self.config['host']
+		if 'session' in valid:
+			self.cookies = kwargs
+
+	def login(self) -> bool:
+		csrf_request = self.get(self.list_url)
+		if not csrf_request.ok:
+			raise ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")")
+		self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
+		login_payload = {
+			'username': self.config['username'],
+			'password': self.config['password'],
+			'next': '',
+			'csrfmiddlewaretoken': 'csrftoken',
+		}
+		login = self.post(self.login_url, json.dumps(login_payload))
+		if not login.ok:
+			raise ConnectionError("Unable to authenticate", login, login.text)
+		self.cookies['sessionid'] = login.cookies['sessionid']
+		print(self.cookies)
+		return True
+
+	def list(self) -> dict:
+		print(self.cookies)
+		logs = self.get(self.list_url)
+		if not logs.ok:
+			raise ConnectionError("HTTP fail", logs, logs.text)
+		return logs.json()
+
+	def load_all_logs(self) -> tempfile.TemporaryDirectory:
+		return self.download_files([i["file_url"] for i in self.list()])
+
+
+CLIENTS: typing.Dict[str, Client] = {
+	"Biogames": BiogamesClient,
+}
+
+if __name__ == '__main__':
+	# c = BiogamesClient(host="http://biodiv", username="ba", password="853451")
+	# print(c.login())
+	# print(json.dumps(c.list(), indent=1))
+	# print(type(c.load_all_logs()))
+	# print(type(c.get("/")))
+	c = BiogamesClient(host="http://biodiv", **{'csrftoken': 'IgbwP83iEibW6RE7IADIFELYdbx0dvqQ',
+												'sessionid': 'zntsj09d92tjos1b6ruqjthlzv60xdin'})
+	print(json.dumps(c.list(), indent=1))
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..fc01d0d
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,41 @@
+version: "3"
+
+services:
+ app:
+  image: docker.clkl.de/ma/celery:0.1
+  build: .
+  volumes:
+    - ./:/app
+  working_dir: /app/selector
+  command: python3 webserver.py
+  environment:
+    - PYTHONPATH=/app
+  networks:
+   - default
+   - traefik_net
+  labels:
+   - "traefik.enable=true"
+   - "traefik.port=5000"
+   - "traefik.docker.network=traefik_net"
+   - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
+
+
+ redis:
+  image: redis:4-alpine
+
+
+
+ celery:
+  image: docker.clkl.de/ma/celery:0.1
+  build: .
+  environment:
+    - PYTHONPATH=/app
+  volumes:
+    - ./:/app
+  working_dir: /app/tasks
+  command: celery -A tasks worker --loglevel=info
+
+networks:
+ traefik_net:
+  external:
+   name: traefik_net
diff --git a/requirements.txt b/requirements.txt
index 5813878..8b4ddee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,13 @@
 requests==2.18.4
 numpy==1.13.1
 matplotlib==2.1.0
-osmnx==0.6
+#osmnx==0.6
 networkx==2.0
-pydot==1.2.3
+#pydot==1.2.3
 scipy==1.0.0
-ipython==6.2.1
\ No newline at end of file
+#ipython==6.2.1
+
+flask==0.12.2
+
+celery==4.1.0
+redis==2.10.6
\ No newline at end of file
diff --git a/selector/__init__.py b/selector/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/selector/static/script.js b/selector/static/script.js
new file mode 100644
index 0000000..865286c
--- /dev/null
+++ b/selector/static/script.js
@@ -0,0 +1,4 @@
+function validateSettings() {
+    alert(document.getElementById('safety').checked);
+    return false;
+}
\ No newline at end of file
diff --git a/selector/static/style.css b/selector/static/style.css
new file mode 100644
index 0000000..9eb023c
--- /dev/null
+++ b/selector/static/style.css
@@ -0,0 +1,6 @@
+body {
+    background-color: aqua;
+}
+#data{
+    display: none;
+}
\ No newline at end of file
diff --git a/selector/templates/base.html b/selector/templates/base.html
new file mode 100644
index 0000000..2dea66f
--- /dev/null
+++ b/selector/templates/base.html
@@ -0,0 +1,5 @@
+<!doctype html>
+<title>…</title>
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+<script type="application/javascript" src="{{url_for('static', filename='script.js') }}"></script>
+{% block body %} {% endblock %}
\ No newline at end of file
diff --git a/selector/templates/games.html b/selector/templates/games.html
new file mode 100644
index 0000000..2692bb7
--- /dev/null
+++ b/selector/templates/games.html
@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+{% block body %}
+<form action="/start" method="post">
+    <div id="data"> {{logs}} </div>
+    <ul>
+        {% for log in logs %}
+            <li>
+                <input type="checkbox" name="logs" value="{{log['@id']}}">
+                {{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
+            </li>
+        <!--{{log}}-->
+        {% endfor %}
+    </ul>
+    <input type="checkbox" id="safety"><label for="safety">Confirm selection</label>
+    <input type="submit">
+
+</form>
+
+{% endblock %}
\ No newline at end of file
diff --git a/selector/templates/index.html b/selector/templates/index.html
new file mode 100644
index 0000000..5a4c6c0
--- /dev/null
+++ b/selector/templates/index.html
@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block body %}
+<form action="/login" method="post">
+    <select name="game">
+        {% for game in clients %}
+        <option>{{ game }}</option>
+        {% endfor %}
+    </select>
+    <input type="text" name="username" placeholder="username"/>
+    <input type="password" name="password" placeholder="passwort"/>
+    <input type="submit">
+</form>
+
+{% endblock %}
\ No newline at end of file
diff --git a/selector/webserver.py b/selector/webserver.py
new file mode 100644
index 0000000..de901c2
--- /dev/null
+++ b/selector/webserver.py
@@ -0,0 +1,53 @@
+import typing
+import uuid
+
+from clients.webclients import Client, CLIENTS
+
+from flask import Flask, render_template, request, redirect, session
+
+BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+
+app = Flask(__name__)
+clients: typing.Dict[str, Client] = {}
+
+@app.route("/")
+def index():
+	return render_template("index.html", clients=CLIENTS)
+
+
+@app.route("/login", methods=["POST"])
+def login():
+	game = request.form["game"]
+	if not game in CLIENTS:
+		return redirect("/")
+	client = CLIENTS[game](host=BIOGAMES_HOST, username=request.form['username'], password=request.form['password'])
+	if client.login():
+		session['logged_in'] = True
+		session['uid'] = str(uuid.uuid4())
+		session['cookies'] = client.cookies
+		session['game'] = game
+		session['host'] = BIOGAMES_HOST
+		clients[session['uid']] = client
+		return redirect("/games")
+	return redirect("/")
+
+
+@app.route("/games")
+def games():
+	if not session['logged_in']:
+		return redirect("/")
+	if session['logged_in'] and not session['uid'] in clients:
+		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
+	return render_template("games.html", logs=clients[session['uid']].list())
+
+@app.route("/start", methods=['POST'])
+def start():
+	pass
+
+@app.route("/status")
+def status():
+	pass
+
+if __name__ == '__main__':
+	app.config.update({"SECRET_KEY":"59765798-2784-11e8-8d05-db4d6f6606c9"})
+	app.run(host="0.0.0.0", debug=True)
diff --git a/static/progress/my.js b/static/progress/my.js
index 29c2dd8..309ad7a 100644
--- a/static/progress/my.js
+++ b/static/progress/my.js
@@ -1,8 +1,9 @@
-$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) {    //urach
+//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) {    //urach
 //$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-68854ffab9aa.json", function (data) {  //urach
 //$.getJSON("data/90278021-4c57-464e-90b1-d603799d07eb_07da99c9-398a-424f-99fc-2701763a63e9.json", function (data) {  //eichstätt
 //$.getJSON("data/13241906-cdae-441a-aed0-d57ebeb37cac_d33976a6-8a56-4a63-b492-fe5427dbf377.json", function (data) {  //stadtökologie
-//$.getJSON("data/5e64ce07-1c16-4d50-ac4e-b3117847ea43_2f664d7b-f0d8-42f5-8731-c034ef86703e.json", function (data) {  //filderstadt
+$.getJSON("data/5e64ce07-1c16-4d50-ac4e-b3117847ea43_2f664d7b-f0d8-42f5-8731-c034ef86703e.json", function (data) {  //filderstadt
+//$.getJSON("data/17d401a9-de21-49a2-95bc-7dafa53dda64_98edcb70-03db-4465-b185-a9c9574995ce.json", function (data) {  //oeb2016
     var images = {};
     var tiles = {
         "openstreetmap": L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
diff --git a/tasks/__init__.py b/tasks/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tasks/tasks.py b/tasks/tasks.py
new file mode 100644
index 0000000..2423c78
--- /dev/null
+++ b/tasks/tasks.py
@@ -0,0 +1,14 @@
+from celery import Celery
+#from analysis import log_analyzer as la
+
+app = Celery('tasks', backend='redis://redis', broker='redis://redis')
+
+
+@app.task
+def add(x, y):
+	return x + y
+
+#@app.task
+#def analyze(config, log_ids):
+#	settings = la.parse_settings(config)
+#	store = la.run_analysis(log_ids, settings, la.LOADERS)
\ No newline at end of file

From d9fa60dfe57c5b83484077079f0637495494d8fa Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Wed, 14 Mar 2018 18:03:13 +0100
Subject: [PATCH 12/24] replace source with clients

---
 analysis/analyzers/settings.py |  8 ++--
 analysis/sources/__init__.py   |  5 --
 analysis/sources/biogames.py   | 85 ----------------------------------
 analysis/sources/source.py     | 18 -------
 analysis/util/download.py      |  4 +-
 clients/webclients.py          |  8 +++-
 6 files changed, 13 insertions(+), 115 deletions(-)
 delete mode 100644 analysis/sources/__init__.py
 delete mode 100644 analysis/sources/biogames.py
 delete mode 100644 analysis/sources/source.py

diff --git a/analysis/analyzers/settings.py b/analysis/analyzers/settings.py
index 36f7682..17631a5 100644
--- a/analysis/analyzers/settings.py
+++ b/analysis/analyzers/settings.py
@@ -1,12 +1,12 @@
 import json
 import sys
-from analysis.sources import SOURCES
+from clients.webclients import CLIENTS
 
 
 def load_source(config):
-	if config["type"] in SOURCES:
-		source = SOURCES[config["type"]]()
-		source.connect(**config)
+	if config["type"] in CLIENTS:
+		source = CLIENTS[config["type"]](**config)
+		source.login()
 		return source
 
 
diff --git a/analysis/sources/__init__.py b/analysis/sources/__init__.py
deleted file mode 100644
index a1db87b..0000000
--- a/analysis/sources/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from .biogames import Biogames
-
-SOURCES = {
-	"Biogames": Biogames,
-}
\ No newline at end of file
diff --git a/analysis/sources/biogames.py b/analysis/sources/biogames.py
deleted file mode 100644
index ac25e10..0000000
--- a/analysis/sources/biogames.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import json
-import logging
-import typing
-from tempfile import TemporaryDirectory
-
-import os
-
-from sources.source import Source
-
-import shutil
-import requests
-
-log: logging.Logger = logging.getLogger(__name__)
-
-
-class Biogames(Source):
-	def __init__(self):
-		self.headers: typing.Dict[str, str] = {'Accept': 'application/json'}
-		self.cookies: typing.Dict[str, str] = {}
-		self.id2link: typing.Dict[str, str] = {}
-		self.host: str = None
-
-	def connect(self, **kwargs):
-		for i in ['username', 'password', 'url', 'login_url', 'host']:
-			if not i in kwargs:
-				raise ValueError("missing value " + i)
-		csrf_request = requests.get(kwargs['url'])
-		if csrf_request.status_code != 200:
-			raise ConnectionError("unable to obtain CSRF token (" + str(csrf_request) + ")", csrf_request.url)
-		self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
-		log.info("obtained CSRF token (" + self.cookies['csrftoken'] + ")")
-		login_payload = {
-			'username': kwargs['username'],
-			'password': kwargs['password'],
-			'next': '',
-			'csrfmiddlewaretoken': 'csrftoken'
-		}
-		login = requests.post(kwargs['login_url'], data=json.dumps(login_payload), cookies=self.cookies)
-		if login.status_code != 200:
-			raise ConnectionError("Unable to authenticate!", login, login.text)
-		self.cookies['sessionid'] = login.cookies['sessionid']
-		log.info("obtained sessionid (" + self.cookies['sessionid'] + ")")
-		self.url = kwargs['url']
-		self.host = kwargs['host']
-		log.info("stored url (" + self.url + ")")
-
-	def list(self):
-		logs = self.get_json(self.url)
-		log.info(len(logs))
-		for i in logs:
-			self.id2link[i["id"]] = i["link"]  # TODO
-		return logs
-
-	def get(self, ids: typing.Collection):
-		dir = TemporaryDirectory()
-		files = []
-		for i in ids:
-			url = self.id2link[i]
-			filename = os.path.join(dir.name, url.split("/")[-1])
-			file = self.download_file(url, filename)
-			if file:
-				files.append(file)
-		return dir
-
-	def download_file(self, url, filename):
-		with open(filename, "wb") as out:
-			try:
-				download = self._get(url)
-				shutil.copyfileobj(download.raw, out)
-				return filename
-			except Exception as e:
-				log.exception(e)
-				os.remove(filename)
-
-	def get_json(self, url):
-		http = self._get(url, stream=False)
-		if not http.ok:
-			raise ConnectionError("HTTP status is not OK", http.url)
-		return http.json()
-
-	def close(self):
-		pass
-
-	def _get(self, url, stream=True):
-		return requests.get(self.host + url, cookies=self.cookies, headers=self.headers, stream=stream)
diff --git a/analysis/sources/source.py b/analysis/sources/source.py
deleted file mode 100644
index 905b897..0000000
--- a/analysis/sources/source.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import typing
-
-
-class Source:
-	def connect(self, **kwargs):
-		raise NotImplementedError
-
-	def list(self):
-		raise NotImplementedError
-
-	def get(self, ids: typing.Collection):
-		raise NotImplementedError
-
-	def get_json(self, url:str) -> dict:
-		raise NotImplementedError
-
-	def close(self):
-		raise NotImplementedError
diff --git a/analysis/util/download.py b/analysis/util/download.py
index e5dfb78..0f3a0d9 100644
--- a/analysis/util/download.py
+++ b/analysis/util/download.py
@@ -18,7 +18,7 @@ def download_board(board_id, instance_config_id, sequence_id, source):
 		sequence_id=sequence_id,
 		board_id=board_id
 	)
-	board = source._get(url)
+	board = source.get(url)
 	if not board.ok:
 		raise ConnectionError()
 	data = board.json()
@@ -68,7 +68,7 @@ def get_json(source, url):
 	if url in cache:
 		return cache[url]
 	try:
-		data = source.get_json(url)
+		data = source.get(url).json()
 
 	except Exception as e:
 		print("exception", e, e.args)  # TODO: logging
diff --git a/clients/webclients.py b/clients/webclients.py
index 95f5c6c..025dbf4 100644
--- a/clients/webclients.py
+++ b/clients/webclients.py
@@ -45,6 +45,12 @@ class Client:
 			self.download_file(path, filename, **kwargs)
 		return target
 
+	def login(self):
+		pass
+
+	def list(self):
+		pass
+
 
 class BiogamesClient(Client):
 	config_fields: typing.Dict[str, typing.List[str]] = {
@@ -95,7 +101,7 @@ class BiogamesClient(Client):
 		return self.download_files([i["file_url"] for i in self.list()])
 
 
-CLIENTS: typing.Dict[str, Client] = {
+CLIENTS: typing.Dict[str, typing.Type[Client]] = {
 	"Biogames": BiogamesClient,
 }
 

From 7d93b5f6fd0fe46320133ecf912fe40314606157 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Wed, 14 Mar 2018 18:37:06 +0100
Subject: [PATCH 13/24] fix some imports

---
 analysis/analyzers/analyzer/locomotion.py |  2 +-
 analysis/util/download.py                 |  2 +-
 tasks/tasks.py                            | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/analysis/analyzers/analyzer/locomotion.py b/analysis/analyzers/analyzer/locomotion.py
index 671680a..0eede5b 100644
--- a/analysis/analyzers/analyzer/locomotion.py
+++ b/analysis/analyzers/analyzer/locomotion.py
@@ -1,4 +1,4 @@
-import util
+from analysis import util
 from . import Analyzer, LogSettings, Result, ResultStore
 
 
diff --git a/analysis/util/download.py b/analysis/util/download.py
index 0f3a0d9..3753a95 100644
--- a/analysis/util/download.py
+++ b/analysis/util/download.py
@@ -1,7 +1,7 @@
 import logging
 import os
 
-from util import json_path
+from analysis.util import json_path
 
 logger = logging.getLogger(__name__)
 
diff --git a/tasks/tasks.py b/tasks/tasks.py
index 2423c78..b173f73 100644
--- a/tasks/tasks.py
+++ b/tasks/tasks.py
@@ -1,5 +1,5 @@
 from celery import Celery
-#from analysis import log_analyzer as la
+from analysis import log_analyzer as la
 
 app = Celery('tasks', backend='redis://redis', broker='redis://redis')
 
@@ -8,7 +8,7 @@ app = Celery('tasks', backend='redis://redis', broker='redis://redis')
 def add(x, y):
 	return x + y
 
-#@app.task
-#def analyze(config, log_ids):
-#	settings = la.parse_settings(config)
-#	store = la.run_analysis(log_ids, settings, la.LOADERS)
\ No newline at end of file
+@app.task
+def analyze(config, log_ids):
+	settings = la.parse_settings(config)
+	store = la.run_analysis(log_ids, settings, la.LOADERS)
\ No newline at end of file

From 01e2433b8b684a2f8af6769a5d73b946c52da223 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Tue, 20 Mar 2018 18:01:28 +0100
Subject: [PATCH 14/24] first analysis running detached with status updates
 through redis

---
 analysis/analyzers/render/default.py |  3 ++
 analysis/analyzers/settings.py       |  2 +
 clients/webclients.py                |  9 ++--
 docker-compose.yml                   | 29 ++++++++---
 selector/static/style.css            |  3 ++
 selector/temp_config.py              | 63 +++++++++++++++++++++++
 selector/templates/games.html        | 34 +++++++++---
 selector/webserver.py                | 40 +++++++++++++--
 tasks/__init__.py                    | 65 +++++++++++++++++++++++
 tasks/tasks.py                       | 77 ++++++++++++++++++++++++++--
 10 files changed, 299 insertions(+), 26 deletions(-)
 create mode 100644 selector/temp_config.py

diff --git a/analysis/analyzers/render/default.py b/analysis/analyzers/render/default.py
index d0d94c1..6f74600 100644
--- a/analysis/analyzers/render/default.py
+++ b/analysis/analyzers/render/default.py
@@ -49,6 +49,7 @@ class KMLRender(Render):
 	result_types = [LocationAnalyzer]
 
 	def render(self, results: List[Result], name=None):
+		files = []
 		for result in self.filter(results):
 			times = ["<when>{time}</when>".format(time=format_time(entry["timestamp"])) for entry in result.get()]
 			coords = [
@@ -62,6 +63,8 @@ class KMLRender(Render):
 			print(filename)
 			with open(filename, "w") as out:
 				out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times)))
+			files.append(filename)
+		return files
 
 
 
diff --git a/analysis/analyzers/settings.py b/analysis/analyzers/settings.py
index 17631a5..269eb00 100644
--- a/analysis/analyzers/settings.py
+++ b/analysis/analyzers/settings.py
@@ -35,6 +35,8 @@ class LogSettings:
 			self.custom = json_dict['custom']
 		if "source" in json_dict:
 			self.source = load_source(json_dict['source'])
+		if "render" in json_dict:
+			self.render = json_dict['render']
 
 	def __repr__(self):
 		return str({
diff --git a/clients/webclients.py b/clients/webclients.py
index 025dbf4..13e13e9 100644
--- a/clients/webclients.py
+++ b/clients/webclients.py
@@ -75,8 +75,10 @@ class BiogamesClient(Client):
 	def login(self) -> bool:
 		csrf_request = self.get(self.list_url)
 		if not csrf_request.ok:
-			raise ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")")
-		self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
+			log.exception(ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")"))
+			return False
+		if not 'csrftoken' in self.cookies:
+			self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
 		login_payload = {
 			'username': self.config['username'],
 			'password': self.config['password'],
@@ -85,7 +87,8 @@ class BiogamesClient(Client):
 		}
 		login = self.post(self.login_url, json.dumps(login_payload))
 		if not login.ok:
-			raise ConnectionError("Unable to authenticate", login, login.text)
+			log.exception(ConnectionError("Unable to authenticate", login, login.text))
+			return False
 		self.cookies['sessionid'] = login.cookies['sessionid']
 		print(self.cookies)
 		return True
diff --git a/docker-compose.yml b/docker-compose.yml
index fc01d0d..4943cb5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -19,12 +19,6 @@ services:
    - "traefik.docker.network=traefik_net"
    - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
 
-
- redis:
-  image: redis:4-alpine
-
-
-
  celery:
   image: docker.clkl.de/ma/celery:0.1
   build: .
@@ -32,8 +26,27 @@ services:
     - PYTHONPATH=/app
   volumes:
     - ./:/app
-  working_dir: /app/tasks
-  command: celery -A tasks worker --loglevel=info
+    - ./data/results:/data/results
+  working_dir: /app
+  command: celery -A tasks.tasks worker --loglevel=info
+
+
+ redis:
+  image: redis:4-alpine
+  volumes:
+    - ./data/redis:/data
+  command: redis-server --appendonly yes
+
+ nginx:
+  image: nginx:1.13-alpine
+  volumes:
+   - ./data/results:/usr/share/nginx/html:ro
+  labels:
+   - "traefik.enable=true"
+   - "traefik.port=80"
+   - "traefik.docker.network=traefik_net"
+   - "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de"
+
 
 networks:
  traefik_net:
diff --git a/selector/static/style.css b/selector/static/style.css
index 9eb023c..4ddd162 100644
--- a/selector/static/style.css
+++ b/selector/static/style.css
@@ -3,4 +3,7 @@ body {
 }
 #data{
     display: none;
+}
+li{
+    list-style-type: none;
 }
\ No newline at end of file
diff --git a/selector/temp_config.py b/selector/temp_config.py
new file mode 100644
index 0000000..9f73f7f
--- /dev/null
+++ b/selector/temp_config.py
@@ -0,0 +1,63 @@
+KML = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  },
+  "render": [
+    "KMLRender"
+  ]
+}"""
+
+CONFIGS = {"KML": KML}#TODO
diff --git a/selector/templates/games.html b/selector/templates/games.html
index 2692bb7..3ef5a0a 100644
--- a/selector/templates/games.html
+++ b/selector/templates/games.html
@@ -1,19 +1,41 @@
 {% extends "base.html" %}
 {% block body %}
 <form action="/start" method="post">
-    <div id="data"> {{logs}} </div>
+    <div id="data"> {{logs}}</div>
     <ul>
         {% for log in logs %}
-            <li>
-                <input type="checkbox" name="logs" value="{{log['@id']}}">
-                {{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
-            </li>
+        <li>
+            <input type="checkbox" name="logs" value="{{log['@id']}}">
+            {{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
+        </li>
         <!--{{log}}-->
         {% endfor %}
     </ul>
-    <input type="checkbox" id="safety"><label for="safety">Confirm selection</label>
+    <!--input type="checkbox" id="safety"><label for="safety">Confirm selection</label-->
+    <input type="text" id="name" maxlength="128" placeholder="name" name="name"/><br>
+    <select name="config">
+        {% for config in configs %}
+        <option>{{config}}</option>
+        {% endfor %}
+    </select>
     <input type="submit">
 
 </form>
 
+<div id="results">
+    <ul>
+    {% for job in jobs %}
+        <li> {{jobs[job].status}}: "{{job}}":
+            <ul>
+                {% for r in jobs[job].results %}
+                <li>{{r}}</li>
+                {% endfor %}
+            </ul>
+
+
+        </li>
+    {% endfor %}
+    </ul>
+</div>
+
 {% endblock %}
\ No newline at end of file
diff --git a/selector/webserver.py b/selector/webserver.py
index de901c2..3c369d7 100644
--- a/selector/webserver.py
+++ b/selector/webserver.py
@@ -1,14 +1,23 @@
+import json
+import logging
 import typing
 import uuid
 
+import time
+
 from clients.webclients import Client, CLIENTS
 
 from flask import Flask, render_template, request, redirect, session
 
+from tasks import tasks
+from selector.temp_config import CONFIGS
+
 BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
 
 app = Flask(__name__)
 clients: typing.Dict[str, Client] = {}
+log: logging.Logger = logging.getLogger(__name__)
+
 
 @app.route("/")
 def index():
@@ -24,6 +33,7 @@ def login():
 	if client.login():
 		session['logged_in'] = True
 		session['uid'] = str(uuid.uuid4())
+		session['username'] = request.form['username']
 		session['cookies'] = client.cookies
 		session['game'] = game
 		session['host'] = BIOGAMES_HOST
@@ -38,16 +48,38 @@ def games():
 		return redirect("/")
 	if session['logged_in'] and not session['uid'] in clients:
 		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
-	return render_template("games.html", logs=clients[session['uid']].list())
+	job_status = json.loads(tasks.redis.get(session['username']))
+	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
+
 
 @app.route("/start", methods=['POST'])
 def start():
-	pass
+	print(str(request.form['logs']))
+	status = {
+		"status": "PENDING",
+		"submit": time.strftime("%c"),
+		"log_ids": request.form.getlist('logs'),
+		"config": request.form['config'],
+	}
+	params = {
+		"log_ids": request.form.getlist('logs'),
+		"config": CONFIGS[request.form['config']],
+		"username": session['username'],
+		"cookies": session['cookies'],
+		"host": session['host'],
+		"clientName": session['game'],
+		"name": request.form['name'],
+	}
+	tasks.status_update(session['username'], request.form['name'], status)
+	tasks.analyze.delay(**params)
+	return redirect("/games")
+
 
 @app.route("/status")
 def status():
-	pass
+	return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
+
 
 if __name__ == '__main__':
-	app.config.update({"SECRET_KEY":"59765798-2784-11e8-8d05-db4d6f6606c9"})
+	app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
 	app.run(host="0.0.0.0", debug=True)
diff --git a/tasks/__init__.py b/tasks/__init__.py
index e69de29..37e3696 100644
--- a/tasks/__init__.py
+++ b/tasks/__init__.py
@@ -0,0 +1,65 @@
+from .tasks import add, analyze
+
+__log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
+
+__config__ = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [
+    "...QuestionAnswerEvent",
+    "...SimuAnswerEvent"
+  ],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  }
+}"""
diff --git a/tasks/tasks.py b/tasks/tasks.py
index b173f73..54051c6 100644
--- a/tasks/tasks.py
+++ b/tasks/tasks.py
@@ -1,14 +1,81 @@
+import json
+import logging
+import shutil
+import uuid
+import os.path
+import os
+
+import redis as redis_lib
+import time
 from celery import Celery
 from analysis import log_analyzer as la
+from analysis.analyzers import KMLRender
+from clients.webclients import CLIENTS
+
+FLASK_DB = 1
+REDIS_HOST = "redis"
+DATA_PATH = "/app/data/results_test/"
+
+RENDERERS = {  # TODO
+	"KMLRender": KMLRender
+}
 
 app = Celery('tasks', backend='redis://redis', broker='redis://redis')
+redis = redis_lib.StrictRedis(host=REDIS_HOST, db=FLASK_DB)
+log: logging.Logger = logging.getLogger(__name__)
+
+
+def update_status(username, name, state, **kwargs):
+	status = json.loads(redis.get(username))
+	status[name][state[0]] = time.strftime("%c")
+	status[name]['status'] = state[1]
+	for i in kwargs:
+		status[name][i] = kwargs[i]
+	redis.set(username, json.dumps(status))
 
 
 @app.task
-def add(x, y):
-	return x + y
+def analyze(config, log_ids, **kwargs):
+	update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
+
+	log.info("start analysis")
+	client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
+	logs = client.list()
+	id_urls = {str(x['@id']): x['file_url'] for x in logs}
+	urls = [id_urls[i] for i in log_ids]
+	tmpdir = client.download_files(urls)
+	log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
+
+	update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
 
-@app.task
-def analyze(config, log_ids):
 	settings = la.parse_settings(config)
-	store = la.run_analysis(log_ids, settings, la.LOADERS)
\ No newline at end of file
+	store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
+	render = RENDERERS[settings.render[0]]()  # TODO
+	files = render.render(store.get_all())
+
+	uid = str(uuid.uuid4())
+	results = []
+	log.error(files)
+	os.mkdir(os.path.join(DATA_PATH, uid))
+	for file in files:
+		try:
+			head, tail = os.path.split(file)
+			target = os.path.join(DATA_PATH, uid, tail)
+			shutil.move(file, target)
+			results.append(target)
+		except FileNotFoundError as e:
+			log.exception(e)
+	tmpdir.cleanup()
+
+	update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
+
+
+def status_update(key, status_key, status):
+	record = redis.get(key)
+	if not record:
+		redis.set(key, json.dumps({status_key: status}))
+	else:
+		data = json.loads(record)
+		data[status_key] = status
+		redis.set(key, json.dumps(data))
+	redis.save()

From 412239515dd2b209d9c70e8bda3abed47f2de180 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Wed, 21 Mar 2018 13:08:02 +0100
Subject: [PATCH 15/24] activityMapper working with celery

---
 analysis/analyzers/render/biogames.py | 16 ++++---
 analysis/util/download.py             | 16 +++----
 clients/webclients.py                 |  2 +
 docker-compose.yml                    |  2 +
 selector/temp_config.py               | 67 ++++++++++++++++++++++++++-
 selector/templates/games.html         |  6 +--
 selector/webserver.py                 | 18 ++++++-
 tasks/__init__.py                     |  2 +-
 tasks/tasks.py                        | 63 +++++++++++++------------
 9 files changed, 140 insertions(+), 52 deletions(-)

diff --git a/analysis/analyzers/render/biogames.py b/analysis/analyzers/render/biogames.py
index 608d045..16e8b63 100644
--- a/analysis/analyzers/render/biogames.py
+++ b/analysis/analyzers/render/biogames.py
@@ -40,7 +40,7 @@ def __plot_or_show(name=None):
 
 
 def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
-         rotation='vertical', name=None):
+		 rotation='vertical', name=None):
 	names, datas = list(zip(*src_data))
 	# plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True)
 	rand = np.random.rand(len(datas), len(datas[0]))
@@ -53,7 +53,7 @@ def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", titl
 
 
 def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="sequential simulation retries",
-               rotation='vertical', name=None):
+			   rotation='vertical', name=None):
 	config_name = CONFIG_NAMES[name] if name in CONFIG_NAMES else "---"
 	counts_per_group = [sum(i) for i in src_data]
 	label = "{}: n={n}; # of sim runs: ⌀={avg:.2f}, median={median}".format(
@@ -152,12 +152,14 @@ class ActivityMapperRender(Render):
 
 	def render(self, results: List[Result], name=None):
 		print(os.getcwd())
+		files = []
 		for result in self.filter(results):
 			data = result.get()
-			with open(os.path.join("static", "progress", "data", data['instance'] + "_" + str(name) + ".json"),
-			          "w") as out:
+			path = os.path.join("/tmp", data['instance'] + "_" + str(name) + ".json")
+			with open(path, "w") as out:
 				json.dump(data, out, indent=1)
-		return "ok"
+			files.append(path)
+		return files
 
 
 class StoreRender(Render):
@@ -186,7 +188,7 @@ class SimulationOrderRender(Render):
 
 class SimulationGroupRender(Render):
 	def render(self, results: List[Result], name=None):
-		#data = [r.get() for r in self.filter(results)]
+		# data = [r.get() for r in self.filter(results)]
 		data = []
 		for r in self.filter(results):
 			raw = r.get()
@@ -196,6 +198,6 @@ class SimulationGroupRender(Render):
 		print(name, len(data))
 		# graph_fit(list(data), name=name)
 		graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None,
-		           name=name)
+				   name=name)
 
 	result_types = [SimulationOrderAnalyzer]
diff --git a/analysis/util/download.py b/analysis/util/download.py
index 3753a95..8ee6d89 100644
--- a/analysis/util/download.py
+++ b/analysis/util/download.py
@@ -6,12 +6,10 @@ from analysis.util import json_path
 logger = logging.getLogger(__name__)
 
 
-def download_board(board_id, instance_config_id, sequence_id, source):
-	local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
-		config_id=instance_config_id,
-		sequence_id=sequence_id,
-		board_id=board_id)
-	if os.path.exists(local_file):
+def download_board(board_id, instance_config_id, sequence_id, source, path="/data/results/"):
+	local_file = os.path.join("static", instance_config_id, sequence_id, board_id)
+	abs_path = os.path.join(path, local_file)
+	if os.path.exists(abs_path):
 		return local_file
 	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
 		config_id=instance_config_id,
@@ -20,12 +18,12 @@ def download_board(board_id, instance_config_id, sequence_id, source):
 	)
 	board = source.get(url)
 	if not board.ok:
-		raise ConnectionError()
+		raise ConnectionError(url, board, board.status_code)
 	data = board.json()
 	preview_url = json_path(data, "preview_url.medium")
 	logger.debug(preview_url)
-	os.makedirs(local_file[:-len(board_id)], exist_ok=True)
-	source.download_file(preview_url, local_file)
+	os.makedirs(abs_path[:-len(board_id)], exist_ok=True)
+	source.download_file(preview_url, abs_path)
 	return local_file
 
 
diff --git a/clients/webclients.py b/clients/webclients.py
index 13e13e9..2e2821a 100644
--- a/clients/webclients.py
+++ b/clients/webclients.py
@@ -22,9 +22,11 @@ class Client:
 		return path
 
 	def get(self, url, **kwargs) -> requests.models.Response:
+		log.info("GET " + str(url))
 		return requests.get(self.url(url), cookies=self.cookies, headers=self.headers, **kwargs)
 
 	def post(self, url, data, **kwargs) -> requests.models.Response:
+		log.info("POST " + str(url))
 		return requests.post(self.url(url), data, cookies=self.cookies, headers=self.headers, **kwargs)
 
 	def download_file(self, url, target, **kwargs) -> bool:
diff --git a/docker-compose.yml b/docker-compose.yml
index 4943cb5..dc00dbe 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -41,6 +41,8 @@ services:
   image: nginx:1.13-alpine
   volumes:
    - ./data/results:/usr/share/nginx/html:ro
+  networks:
+   - traefik_net
   labels:
    - "traefik.enable=true"
    - "traefik.port=80"
diff --git a/selector/temp_config.py b/selector/temp_config.py
index 9f73f7f..318da1f 100644
--- a/selector/temp_config.py
+++ b/selector/temp_config.py
@@ -60,4 +60,69 @@ KML = """{
   ]
 }"""
 
-CONFIGS = {"KML": KML}#TODO
+ACTIVITY = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "ActivityMapper"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  },
+  "render": [
+    "ActivityMapper"
+  ]
+}"""
+
+CONFIGS = {  # TODO
+	"KML": KML,
+	"ActivityMapper": ACTIVITY,
+}
diff --git a/selector/templates/games.html b/selector/templates/games.html
index 3ef5a0a..5f7d91a 100644
--- a/selector/templates/games.html
+++ b/selector/templates/games.html
@@ -24,17 +24,17 @@
 
 <div id="results">
     <ul>
-    {% for job in jobs %}
+        {% for job in jobs %}
         <li> {{jobs[job].status}}: "{{job}}":
             <ul>
                 {% for r in jobs[job].results %}
-                <li>{{r}}</li>
+                <li><a href="{{r | get_url}}">{{r|get_name}}</a></li>
                 {% endfor %}
             </ul>
 
 
         </li>
-    {% endfor %}
+        {% endfor %}
     </ul>
 </div>
 
diff --git a/selector/webserver.py b/selector/webserver.py
index 3c369d7..d7efdb1 100644
--- a/selector/webserver.py
+++ b/selector/webserver.py
@@ -13,6 +13,7 @@ from tasks import tasks
 from selector.temp_config import CONFIGS
 
 BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/"
 
 app = Flask(__name__)
 clients: typing.Dict[str, Client] = {}
@@ -44,11 +45,15 @@ def login():
 
 @app.route("/games")
 def games():
-	if not session['logged_in']:
+	if not ('logged_in' in session and session['logged_in']):
 		return redirect("/")
 	if session['logged_in'] and not session['uid'] in clients:
 		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
-	job_status = json.loads(tasks.redis.get(session['username']))
+	status = tasks.redis.get(session['username'])
+	if status:
+		job_status = json.loads(status)
+	else:
+		job_status = {}
 	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
 
 
@@ -80,6 +85,15 @@ def status():
 	return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
 
 
+@app.template_filter('get_url')
+def get_url(path: str):
+	return path.replace(tasks.DATA_PATH, RESULT_HOST)
+
+@app.template_filter('get_name')
+def get_url(path: str):
+	return path.replace(tasks.DATA_PATH, "")
+
+
 if __name__ == '__main__':
 	app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
 	app.run(host="0.0.0.0", debug=True)
diff --git a/tasks/__init__.py b/tasks/__init__.py
index 37e3696..913d63f 100644
--- a/tasks/__init__.py
+++ b/tasks/__init__.py
@@ -1,4 +1,4 @@
-from .tasks import add, analyze
+from .tasks import analyze
 
 __log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
 
diff --git a/tasks/tasks.py b/tasks/tasks.py
index 54051c6..198f3e7 100644
--- a/tasks/tasks.py
+++ b/tasks/tasks.py
@@ -9,15 +9,16 @@ import redis as redis_lib
 import time
 from celery import Celery
 from analysis import log_analyzer as la
-from analysis.analyzers import KMLRender
+from analysis.analyzers import KMLRender, ActivityMapperRender
 from clients.webclients import CLIENTS
 
 FLASK_DB = 1
 REDIS_HOST = "redis"
-DATA_PATH = "/app/data/results_test/"
+DATA_PATH = "/app/data/results/"
 
 RENDERERS = {  # TODO
-	"KMLRender": KMLRender
+	"KMLRender": KMLRender,
+	"ActivityMapper": ActivityMapperRender,
 }
 
 app = Celery('tasks', backend='redis://redis', broker='redis://redis')
@@ -38,36 +39,40 @@ def update_status(username, name, state, **kwargs):
 def analyze(config, log_ids, **kwargs):
 	update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
 
-	log.info("start analysis")
-	client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
-	logs = client.list()
-	id_urls = {str(x['@id']): x['file_url'] for x in logs}
-	urls = [id_urls[i] for i in log_ids]
-	tmpdir = client.download_files(urls)
-	log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
+	try:
+		log.info("start analysis")
+		client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
+		logs = client.list()
+		id_urls = {str(x['@id']): x['file_url'] for x in logs}
+		urls = [id_urls[i] for i in log_ids]
+		tmpdir = client.download_files(urls)
+		log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
 
-	update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
+		update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
 
-	settings = la.parse_settings(config)
-	store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
-	render = RENDERERS[settings.render[0]]()  # TODO
-	files = render.render(store.get_all())
+		settings = la.parse_settings(config)
+		store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
+		render = RENDERERS[settings.render[0]]()  # TODO
+		files = render.render(store.get_all())
 
-	uid = str(uuid.uuid4())
-	results = []
-	log.error(files)
-	os.mkdir(os.path.join(DATA_PATH, uid))
-	for file in files:
-		try:
-			head, tail = os.path.split(file)
-			target = os.path.join(DATA_PATH, uid, tail)
-			shutil.move(file, target)
-			results.append(target)
-		except FileNotFoundError as e:
-			log.exception(e)
-	tmpdir.cleanup()
+		uid = str(uuid.uuid4())
+		results = []
+		log.error(files)
+		os.mkdir(os.path.join(DATA_PATH, uid))
+		for file in files:
+			try:
+				head, tail = os.path.split(file)
+				target = os.path.join(DATA_PATH, uid, tail)
+				shutil.move(file, target)
+				results.append(target)
+			except FileNotFoundError as e:
+				log.exception(e)
+		tmpdir.cleanup()
 
-	update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
+		update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
+	except Exception as e:
+		log.exception(e)
+		update_status(kwargs['username'], kwargs['name'], ('abort', 'ERROR'), exception=str(e))
 
 
 def status_update(key, status_key, status):

From bba8c0719c966f19b9f4be27b5a79b1391a28aa6 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Mon, 30 Apr 2018 14:33:01 +0200
Subject: [PATCH 16/24] * update dockerfile

---
 Dockerfile                      |  8 +++----
 analysis/util/download.py       |  3 ++-
 docker-compose.yml              | 10 ++++----
 requirements.txt                |  4 ++--
 selector/temp_config.py         |  5 ++++
 selector/templates/games.html   | 18 +--------------
 selector/templates/results.html | 22 ++++++++++++++++++
 selector/webserver.py           | 41 ++++++++++++++++++++++++++-------
 8 files changed, 73 insertions(+), 38 deletions(-)
 create mode 100644 selector/templates/results.html

diff --git a/Dockerfile b/Dockerfile
index 2de600e..94f8894 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,9 +2,7 @@ FROM alpine:edge
 
 ADD ["requirements.txt", "/"]
 RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing/" >> /etc/apk/repositories && \
-    sed -i 's/numpy/#numpy/' requirements.txt && \
-    sed -i 's/scipy/#scipy/' requirements.txt && \
-    apk add --update libpng libpng-dev freetype freetype-dev g++ python3 py3-numpy python3-dev py-numpy-dev py3-scipy&& \
-    pip3 install -r requirements.txt && \
-    apk del libpng-dev freetype-dev g++ python3-dev py-numpy-dev && \
+    apk add --update --no-cache libpng libpng-dev freetype freetype-dev g++ python3 python3-dev libstdc++ openblas-dev && \
+    pip3 --no-cache-dir install -r requirements.txt && \
+    apk del libpng-dev freetype-dev g++ python3-dev openblas-dev && \
     rm requirements.txt
\ No newline at end of file
diff --git a/analysis/util/download.py b/analysis/util/download.py
index 8ee6d89..a70a81d 100644
--- a/analysis/util/download.py
+++ b/analysis/util/download.py
@@ -69,7 +69,8 @@ def get_json(source, url):
 		data = source.get(url).json()
 
 	except Exception as e:
-		print("exception", e, e.args)  # TODO: logging
+		print("exception", e, e.args)
+		logger.exception(e)
 		data = None
 	cache[url] = data
 	return data
diff --git a/docker-compose.yml b/docker-compose.yml
index dc00dbe..fe8efe9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,9 +1,10 @@
-version: "3"
+version: "2.2"
 
 services:
  app:
-  image: docker.clkl.de/ma/celery:0.1
-  build: .
+  image: docker.clkl.de/ma/celery:0.3.3
+  build: ./selector
+  cpu_count: 4
   volumes:
     - ./:/app
   working_dir: /app/selector
@@ -20,8 +21,7 @@ services:
    - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
 
  celery:
-  image: docker.clkl.de/ma/celery:0.1
-  build: .
+  image: docker.clkl.de/ma/celery:0.3.3
   environment:
     - PYTHONPATH=/app
   volumes:
diff --git a/requirements.txt b/requirements.txt
index 8b4ddee..fc7f208 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
 requests==2.18.4
-numpy==1.13.1
+numpy==1.14.2
 matplotlib==2.1.0
 #osmnx==0.6
 networkx==2.0
 #pydot==1.2.3
-scipy==1.0.0
+scipy==1.0.1
 #ipython==6.2.1
 
 flask==0.12.2
diff --git a/selector/temp_config.py b/selector/temp_config.py
index 318da1f..7875531 100644
--- a/selector/temp_config.py
+++ b/selector/temp_config.py
@@ -126,3 +126,8 @@ CONFIGS = {  # TODO
 	"KML": KML,
 	"ActivityMapper": ACTIVITY,
 }
+
+URLS = {
+	"KML": "/",
+	"ActivityMapper": "#",
+}
\ No newline at end of file
diff --git a/selector/templates/games.html b/selector/templates/games.html
index 5f7d91a..1dc2418 100644
--- a/selector/templates/games.html
+++ b/selector/templates/games.html
@@ -21,21 +21,5 @@
     <input type="submit">
 
 </form>
-
-<div id="results">
-    <ul>
-        {% for job in jobs %}
-        <li> {{jobs[job].status}}: "{{job}}":
-            <ul>
-                {% for r in jobs[job].results %}
-                <li><a href="{{r | get_url}}">{{r|get_name}}</a></li>
-                {% endfor %}
-            </ul>
-
-
-        </li>
-        {% endfor %}
-    </ul>
-</div>
-
+<a href="/results">show analysis progress/results</a>
 {% endblock %}
\ No newline at end of file
diff --git a/selector/templates/results.html b/selector/templates/results.html
new file mode 100644
index 0000000..3129a94
--- /dev/null
+++ b/selector/templates/results.html
@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{% block body %}
+
+<a href="/games">create new analysis</a>
+
+<div id="results">
+    <ul>
+        {% for job in jobs %}
+        <li> {{jobs[job].status}}: "{{job}}":
+            <ul>
+                {% for r in jobs[job].results %}
+                <li><a href="{{jobs[job] | get_prefix}}{{r | get_name}}">{{r|get_name}} {{jobs[job].start}}</a></li>
+                {% endfor %}
+            </ul>
+
+
+        </li>
+        {% endfor %}
+    </ul>
+</div>
+
+{% endblock %}
\ No newline at end of file
diff --git a/selector/webserver.py b/selector/webserver.py
index d7efdb1..ba77179 100644
--- a/selector/webserver.py
+++ b/selector/webserver.py
@@ -10,9 +10,10 @@ from clients.webclients import Client, CLIENTS
 from flask import Flask, render_template, request, redirect, session
 
 from tasks import tasks
-from selector.temp_config import CONFIGS
+from selector.temp_config import CONFIGS, URLS
 
 BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+#BIOGAMES_HOST = "http://www.biodiv2go.de"
 RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/"
 
 app = Flask(__name__)
@@ -29,7 +30,7 @@ def index():
 def login():
 	game = request.form["game"]
 	if not game in CLIENTS:
-		return redirect("/")
+		return redirect("/?invalid_game")
 	client = CLIENTS[game](host=BIOGAMES_HOST, username=request.form['username'], password=request.form['password'])
 	if client.login():
 		session['logged_in'] = True
@@ -39,12 +40,12 @@ def login():
 		session['game'] = game
 		session['host'] = BIOGAMES_HOST
 		clients[session['uid']] = client
-		return redirect("/games")
-	return redirect("/")
+		return redirect("/results")
+	return redirect("/?fail")
 
 
-@app.route("/games")
-def games():
+@app.route("/results")
+def results():
 	if not ('logged_in' in session and session['logged_in']):
 		return redirect("/")
 	if session['logged_in'] and not session['uid'] in clients:
@@ -54,7 +55,21 @@ def games():
 		job_status = json.loads(status)
 	else:
 		job_status = {}
-	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS, jobs=job_status)
+	#for job in job_status:
+	#	results = []
+	#	for path in job_status[job]['results']:
+	#		results.append(path.replace(tasks.DATA_PATH, RESULT_HOST))
+	#	print(results) #TODO???
+	return render_template("results.html",  jobs=job_status)
+
+
+@app.route("/games")
+def games():
+	if not ('logged_in' in session and session['logged_in']):
+		return redirect("/")
+	if session['logged_in'] and not session['uid'] in clients:
+		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
+	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS)
 
 
 @app.route("/start", methods=['POST'])
@@ -77,7 +92,7 @@ def start():
 	}
 	tasks.status_update(session['username'], request.form['name'], status)
 	tasks.analyze.delay(**params)
-	return redirect("/games")
+	return redirect("/results")
 
 
 @app.route("/status")
@@ -94,6 +109,16 @@ def get_url(path: str):
 	return path.replace(tasks.DATA_PATH, "")
 
 
+@app.template_filter('get_prefix')
+def get_prefix(job):
+	print(job)
+	try:
+		return RESULT_HOST + URLS[job['config']]
+	except:
+		return RESULT_HOST + "#"
+
+
+
 if __name__ == '__main__':
 	app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
 	app.run(host="0.0.0.0", debug=True)

From 4d0e5e7ac11a2333f0e3532137e354b88cba81c6 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Tue, 29 May 2018 17:21:36 +0200
Subject: [PATCH 17/24] evaluation working

---
 .gitignore                             |  2 +-
 Dockerfile                             | 10 ++--
 analysis/analyzers/__init__.py         |  2 +-
 analysis/analyzers/analyzer/default.py |  6 ++-
 analysis/analyzers/settings.py         |  4 ++
 analysis/loaders/__init__.py           |  4 +-
 analysis/loaders/neocart.py            | 70 ++++++++++++++++++++++++++
 clients/webclients.py                  | 37 +++++++++++++-
 docker-compose.yml                     | 13 +++--
 log_data.conf                          |  9 ++++
 log_data/.gitkeep                      |  0
 neocart.json                           | 34 +++++++++++++
 requirements.txt                       |  6 ++-
 selector/templates/games.html          |  2 +-
 test_neo.py                            | 41 +++++++++++++++
 15 files changed, 223 insertions(+), 17 deletions(-)
 create mode 100644 analysis/loaders/neocart.py
 create mode 100644 log_data.conf
 create mode 100644 log_data/.gitkeep
 create mode 100644 neocart.json
 create mode 100644 test_neo.py

diff --git a/.gitignore b/.gitignore
index 428a58b..8c01398 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,5 +3,5 @@ _*
 !__init__.py
 *.pyc
 logs/
-data/
+*data/
 plots/*
diff --git a/Dockerfile b/Dockerfile
index 94f8894..c197353 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,9 @@ FROM alpine:edge
 
 ADD ["requirements.txt", "/"]
 RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing/" >> /etc/apk/repositories && \
-    apk add --update --no-cache libpng libpng-dev freetype freetype-dev g++ python3 python3-dev libstdc++ openblas-dev && \
-    pip3 --no-cache-dir install -r requirements.txt && \
-    apk del libpng-dev freetype-dev g++ python3-dev openblas-dev && \
-    rm requirements.txt
\ No newline at end of file
+	apk add --update --no-cache libpng freetype python3 libstdc++ libxml2 libxslt openblas && \
+	apk add --update --no-cache --virtual .build-deps libpng-dev freetype-dev g++ python3-dev openblas-dev libxml2-dev libxslt-dev && \
+	pip3 --no-cache-dir install -r requirements.txt && \
+	apk del .build-deps && \
+	rm requirements.txt
+USER guest
\ No newline at end of file
diff --git a/analysis/analyzers/__init__.py b/analysis/analyzers/__init__.py
index 07a8186..97d33b6 100644
--- a/analysis/analyzers/__init__.py
+++ b/analysis/analyzers/__init__.py
@@ -5,7 +5,7 @@ from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer,
 	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer, \
 	SimulationFlagsAnalyzer
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
-	CategorizerStub, Store, ProgressAnalyzer
+	CategorizerStub, Store, ProgressAnalyzer, SimpleCategorizer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
 from .analyzer.mask import MaskSpatials
 from .render import Render
diff --git a/analysis/analyzers/analyzer/default.py b/analysis/analyzers/analyzer/default.py
index fce0e35..f17ae2f 100644
--- a/analysis/analyzers/analyzer/default.py
+++ b/analysis/analyzers/analyzer/default.py
@@ -90,12 +90,16 @@ class CategorizerStub(Analyzer):
 	__name__ = "Categorizer"
 
 	def result(self, store: ResultStore, name=None) -> None:
-		store.new_category(self.key)
+		store.new_category(name if name else self.key)
 
 	def __init__(self, settings: LogSettings):
 		super().__init__(settings)
 		self.key = "default"
 
+class SimpleCategorizer(CategorizerStub):
+	def process(self, entry):
+		return False
+
 
 class Store(Analyzer):
 	"""
diff --git a/analysis/analyzers/settings.py b/analysis/analyzers/settings.py
index 269eb00..61a4494 100644
--- a/analysis/analyzers/settings.py
+++ b/analysis/analyzers/settings.py
@@ -1,13 +1,17 @@
 import json
+import logging
 import sys
 from clients.webclients import CLIENTS
 
+log: logging.Logger = logging.getLogger(__name__)
 
 def load_source(config):
 	if config["type"] in CLIENTS:
 		source = CLIENTS[config["type"]](**config)
 		source.login()
 		return source
+	else:
+		log.warn(f"client {config['type']} not found!")
 
 
 class LogSettings:
diff --git a/analysis/loaders/__init__.py b/analysis/loaders/__init__.py
index 4829227..f889e09 100644
--- a/analysis/loaders/__init__.py
+++ b/analysis/loaders/__init__.py
@@ -1,8 +1,10 @@
 from .biogames import SQLiteLoader, ZipSQLiteLoader
 from .loader import JSONLoader
+from .neocart import NeoCartLoader
 
 LOADERS = {
 	"json": JSONLoader,
 	"sqlite": SQLiteLoader,
-	"zip": ZipSQLiteLoader
+	"zip": ZipSQLiteLoader,
+	"neocartographer": NeoCartLoader,
 }
diff --git a/analysis/loaders/neocart.py b/analysis/loaders/neocart.py
new file mode 100644
index 0000000..b6b0975
--- /dev/null
+++ b/analysis/loaders/neocart.py
@@ -0,0 +1,70 @@
+import logging
+from datetime import datetime
+
+from lxml import etree
+
+from .loader import Loader
+
+log = logging.getLogger(__name__)
+
+NS = {'gpx':"http://www.topografix.com/GPX/1/1"}
+
+class NeoCartLoader(Loader):
+	def load(self, file: str):
+		src = open(file, "r")
+		parser = etree.XMLParser(recover=True)
+		tree = etree.parse(src, parser=parser)
+		self.entries = []
+		for point in tree.xpath("//gpx:trkpt", namespaces=NS):
+			try: 
+				self.entries.append(self.parse_point(point))
+			except ValueError as e:
+				print(e, etree.tostring(point, pretty_print=True).decode())
+				log.exception(e)
+	
+	def parse_point(self, point):
+		raw_lat = point.xpath("@lat")[0]
+		if raw_lat.count(".") > 1:
+			log.warning(f"recreate lat/lon from: {raw_lat}")
+			log.warn(etree.tostring(point, pretty_print=True).decode())
+			start_offset = 4
+			x = raw_lat[start_offset:].index(".")
+			offset = start_offset + x
+			raw_lon = raw_lat[offset:]
+			raw_lat = raw_lat[:offset]
+		else:
+			raw_lon = point.xpath("@lon")[0]
+		lat = float(raw_lat)
+		lon = float(raw_lon)
+		times = point.xpath("gpx:time",namespaces=NS)
+		assert len(times) == 1
+		time = times[0].text
+		dt = datetime.strptime(time, "%Y-%m-%dT%H:%M:%SZ")
+		timestamp = int(dt.timestamp() * 1000) # python3.6 has no timestamp_ns (yet)
+		events = point.xpath(".//gpx:event",namespaces=NS)
+		assert 0 <= len(events) <= 1
+		event = {}
+		if events:
+			event = dict(events[0].attrib)
+			if events[0].tail and events[0].tail.strip():
+				try:
+					# base case: trailing 'geoid="0"/>'
+					key, v = events[0].tail.strip().split("=")
+					value = v.split('"')[1]
+					event[key] = value
+				except:
+					event['__tail__'] = events[0].tail.strip()
+		
+		return {
+			"location": {
+				"type": "Point",
+				"coordinates": [lon, lat]
+			},
+			"timestamp": timestamp,
+			"event": event,
+			"type": "event" if event else "location"
+		}
+
+	def get_entry(self) -> object:
+		for i in self.entries:
+			yield i
\ No newline at end of file
diff --git a/clients/webclients.py b/clients/webclients.py
index 2e2821a..f414da1 100644
--- a/clients/webclients.py
+++ b/clients/webclients.py
@@ -48,10 +48,10 @@ class Client:
 		return target
 
 	def login(self):
-		pass
+		pass #TODO
 
 	def list(self):
-		pass
+		pass #TODO
 
 
 class BiogamesClient(Client):
@@ -105,9 +105,42 @@ class BiogamesClient(Client):
 	def load_all_logs(self) -> tempfile.TemporaryDirectory:
 		return self.download_files([i["file_url"] for i in self.list()])
 
+class GeogamesClient(Client):
+	config_fields = ("host", "path")
+	
+	def __init__(self, **kwargs):
+		for field in self.config_fields:
+			if not field in kwargs:
+				raise ValueError(f"missing parameter: {field}")
+		self.host = kwargs['host']
+		self.path = kwargs['path']
+	
+	def list(self):
+		logs = self.get(self.path)
+		data = logs.json()
+		prepared_logs = []
+		for log in data:
+			players = self.get(f"{self.path}/{log['name']}/").json()
+			for player in players:
+				prepared_logs.append({
+					'@id': f"{log['name']}/{player['name']}",
+					'start_date': player['mtime'],
+					'player_group_name': player['name'],
+					'file_url': f"{self.path}/{log['name']}/{player['name']}",
+				})
+		return prepared_logs
+
+	def download_files(self, urls, **kwargs) -> tempfile.TemporaryDirectory:
+		target = tempfile.TemporaryDirectory()
+		for path in urls:
+			filename = os.path.join(target.name, "-".join
+(path.split("/")[-2:]))
+			self.download_file(path, filename, **kwargs)
+		return target
 
 CLIENTS: typing.Dict[str, typing.Type[Client]] = {
 	"Biogames": BiogamesClient,
+	"Geogames": GeogamesClient,
 }
 
 if __name__ == '__main__':
diff --git a/docker-compose.yml b/docker-compose.yml
index fe8efe9..9a91058 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,9 +2,8 @@ version: "2.2"
 
 services:
  app:
-  image: docker.clkl.de/ma/celery:0.3.3
-  build: ./selector
-  cpu_count: 4
+  image: docker.clkl.de/ma/celery:0.4.1
+  build: .
   volumes:
     - ./:/app
   working_dir: /app/selector
@@ -21,7 +20,7 @@ services:
    - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
 
  celery:
-  image: docker.clkl.de/ma/celery:0.3.3
+  image: docker.clkl.de/ma/celery:0.4.1
   environment:
     - PYTHONPATH=/app
   volumes:
@@ -49,6 +48,12 @@ services:
    - "traefik.docker.network=traefik_net"
    - "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de"
 
+ log_data:
+  image: nginx:1.13-alpine
+  volumes:
+   - ./log_data/:/srv/:ro
+   - ./log_data.conf:/etc/nginx/conf.d/log_data.conf
+
 
 networks:
  traefik_net:
diff --git a/log_data.conf b/log_data.conf
new file mode 100644
index 0000000..0c92fcb
--- /dev/null
+++ b/log_data.conf
@@ -0,0 +1,9 @@
+server {
+	listen 80;
+	server_name log_data;
+	location / {
+		root /srv/;
+		autoindex on;
+		autoindex_format json;
+	}
+}
\ No newline at end of file
diff --git a/log_data/.gitkeep b/log_data/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/neocart.json b/neocart.json
new file mode 100644
index 0000000..392d8ad
--- /dev/null
+++ b/neocart.json
@@ -0,0 +1,34 @@
+{
+  "logFormat": "neocartographer",
+  "entryType": "type",
+  "spatials": [
+    "location"
+  ],
+  "actions": [],
+  "boards": [],
+  "analyzers": {
+    "analysis.analyzers": [
+      "SimpleCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {},
+  "custom": {
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Geogames",
+    "host": "http://log_data/",
+    "path": "neocartographer"
+  },
+  "render": [
+    "KMLRender"
+  ]
+}
+
+
diff --git a/requirements.txt b/requirements.txt
index fc7f208..eae5dfa 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,5 +9,7 @@ scipy==1.0.1
 
 flask==0.12.2
 
-celery==4.1.0
-redis==2.10.6
\ No newline at end of file
+celery==4.1.1
+redis==2.10.6
+
+lxml==4.2.1
\ No newline at end of file
diff --git a/selector/templates/games.html b/selector/templates/games.html
index 1dc2418..b8acfaf 100644
--- a/selector/templates/games.html
+++ b/selector/templates/games.html
@@ -6,7 +6,7 @@
         {% for log in logs %}
         <li>
             <input type="checkbox" name="logs" value="{{log['@id']}}">
-            {{log.start_date}}: {{log.player_group_name}} (→{{log.end_date}})
+            {{log.start_date}}: {{log.player_group_name}}
         </li>
         <!--{{log}}-->
         {% endfor %}
diff --git a/test_neo.py b/test_neo.py
new file mode 100644
index 0000000..3d52d34
--- /dev/null
+++ b/test_neo.py
@@ -0,0 +1,41 @@
+
+from analysis import log_analyzer as la
+settings = la.load_settings("neocart.json")
+client = settings.source
+logs = client.list()
+id_urls = {str(x['@id']): x['file_url'] for x in logs}
+
+log_ids=['20351/playerid1430317168972.gpx','20351/playerid1430317188358.gpx']
+
+urls = [id_urls[i] for i in log_ids]
+tmpdir = client.download_files(urls)
+import os
+store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
+
+import json
+print(json.dumps(store.serializable(), indent=1))
+
+
+from analysis.analyzers import KMLRender, ActivityMapperRender
+RENDERERS = {  # TODO
+	"KMLRender": KMLRender,
+	"ActivityMapper": ActivityMapperRender,
+}
+render = RENDERERS[settings.render[0]]()
+files = render.render(store.get_all())
+DATA_PATH = "/app/data/results/"
+import uuid
+uid = str(uuid.uuid4())
+results = []
+os.mkdir(os.path.join(DATA_PATH, uid))
+import shutil
+
+for file in files:
+	try:
+		head, tail = os.path.split(file)
+		target = os.path.join(DATA_PATH, uid, tail)
+		shutil.move(file, target)
+		results.append(target)
+	except FileNotFoundError as e:
+		log.exception(e)
+tmpdir.cleanup()
\ No newline at end of file

From da5df266c3efcfd4d3b9b5dbaf9aa83ac3d0a648 Mon Sep 17 00:00:00 2001
From: agp8x <git@agp8x.org>
Date: Fri, 1 Jun 2018 10:23:39 +0200
Subject: [PATCH 18/24] update boarddurationrender

---
 analysis/analyzers/render/__init__.py | 2 +-
 analysis/analyzers/render/biogames.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/analysis/analyzers/render/__init__.py b/analysis/analyzers/render/__init__.py
index aeb6a20..24c4d2d 100644
--- a/analysis/analyzers/render/__init__.py
+++ b/analysis/analyzers/render/__init__.py
@@ -5,7 +5,7 @@ from .. import Result
 class Render:
 	result_types = []
 
-	def render(self, results: List[Result], name=None):
+	def render(self, results: List[Result], name=None) -> [str]:
 		raise NotImplementedError()
 
 	def filter(self, results: List[Result]):
diff --git a/analysis/analyzers/render/biogames.py b/analysis/analyzers/render/biogames.py
index 16e8b63..53c12c4 100644
--- a/analysis/analyzers/render/biogames.py
+++ b/analysis/analyzers/render/biogames.py
@@ -135,16 +135,16 @@ class BoardDurationHistRender(Render):
 class BoardDurationBoxRender(Render):
 	result_types = [BoardDurationAnalyzer]
 
-	def render(self, results: List[Result], name=None):
+	def render(self, results: List[Result], name=None) -> [str]:
 		data = defaultdict(list)
 		for result in self.filter(results):
-			get = result.get()
-			for board in get:
+			for board in result.get():
 				duration = board['active'] if 'active' in board else 0
 				data[board['id']].append(duration)
 		data_tuples = [(key, data[key]) for key in sorted(data)]
 		data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
-		plot(data_tuples)
+		plot(data_tuples, name=name)
+		return [name]
 
 
 class ActivityMapperRender(Render):

From 0f8863817bd657cd7c70d3b6878c5fcd8b95df34 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Fri, 8 Jun 2018 20:55:52 +0200
Subject: [PATCH 19/24] 	neue Datei:     Readme.md

---
 Readme.md | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 Readme.md

diff --git a/Readme.md b/Readme.md
new file mode 100644
index 0000000..f49c892
--- /dev/null
+++ b/Readme.md
@@ -0,0 +1,10 @@
+# Geogame Log Analyzer
+
+## log data
+
+### set mtime of gpx files to the first date:
+
+```
+for i in */*; do touch -m -d "$(head -n 15 $i|grep time | head -n 1 |cut -d">" -f 3|cut -d"<" -f1)" $i; done
+for i in */; do touch -m -d "$(head -n 15 $i/*.gpx|grep time | head -n 1 |cut -d">" -f 3|cut -d"<" -f1)" $i; done
+```
\ No newline at end of file

From f0184d7f2638fff1c05374cd9b51fa026fd8b711 Mon Sep 17 00:00:00 2001
From: agp8x <git@agp8x.org>
Date: Fri, 8 Jun 2018 21:14:07 +0200
Subject: [PATCH 20/24] PYTHONUNBUFFERED envvar

---
 docker-compose.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 9a91058..ba49ee1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.2"
+version: "3"
 
 services:
  app:
@@ -10,6 +10,7 @@ services:
   command: python3 webserver.py
   environment:
     - PYTHONPATH=/app
+    - PYTHONUNBUFFERED=1
   networks:
    - default
    - traefik_net
@@ -23,6 +24,7 @@ services:
   image: docker.clkl.de/ma/celery:0.4.1
   environment:
     - PYTHONPATH=/app
+    - PYTHONUNBUFFERED=1
   volumes:
     - ./:/app
     - ./data/results:/data/results

From ca0da300bed9ab67406a7f3fde760e02ec08274f Mon Sep 17 00:00:00 2001
From: agp8x <git@agp8x.org>
Date: Fri, 8 Jun 2018 21:17:19 +0200
Subject: [PATCH 21/24] use event's message as type

---
 analysis/loaders/neocart.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analysis/loaders/neocart.py b/analysis/loaders/neocart.py
index b6b0975..8b56e92 100644
--- a/analysis/loaders/neocart.py
+++ b/analysis/loaders/neocart.py
@@ -62,7 +62,7 @@ class NeoCartLoader(Loader):
 			},
 			"timestamp": timestamp,
 			"event": event,
-			"type": "event" if event else "location"
+			"type": event['message'] if event else "location"
 		}
 
 	def get_entry(self) -> object:

From 79b379e267bdcf06430888b9f0b58d34e26a4317 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Mon, 11 Jun 2018 15:51:08 +0200
Subject: [PATCH 22/24] improve NC integration

---
 clients/webclients.py   |  8 ++++++--
 selector/temp_config.py | 11 +++++++++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/clients/webclients.py b/clients/webclients.py
index f414da1..aa4e94c 100644
--- a/clients/webclients.py
+++ b/clients/webclients.py
@@ -106,14 +106,18 @@ class BiogamesClient(Client):
 		return self.download_files([i["file_url"] for i in self.list()])
 
 class GeogamesClient(Client):
-	config_fields = ("host", "path")
+	config_fields = ("host",)
 	
 	def __init__(self, **kwargs):
 		for field in self.config_fields:
 			if not field in kwargs:
 				raise ValueError(f"missing parameter: {field}")
 		self.host = kwargs['host']
-		self.path = kwargs['path']
+		self.path = "neocartographer"
+		self.config = kwargs
+	
+	def login(self):
+		return True
 	
 	def list(self):
 		logs = self.get(self.path)
diff --git a/selector/temp_config.py b/selector/temp_config.py
index 7875531..46de267 100644
--- a/selector/temp_config.py
+++ b/selector/temp_config.py
@@ -122,7 +122,7 @@ ACTIVITY = """{
   ]
 }"""
 
-CONFIGS = {  # TODO
+CONFIGS = {  # TODO: more
 	"KML": KML,
 	"ActivityMapper": ACTIVITY,
 }
@@ -130,4 +130,11 @@ CONFIGS = {  # TODO
 URLS = {
 	"KML": "/",
 	"ActivityMapper": "#",
-}
\ No newline at end of file
+}
+
+HOSTS = {
+	"Biogames": "http://biogames.potato.kinf.wiai.uni-bamberg.de",
+	"Geogames": "http://log_data/",
+}
+
+RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/"
\ No newline at end of file

From 465ee6f9ce1d46898da6b6d90fec79fd0e961342 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Mon, 11 Jun 2018 16:08:45 +0200
Subject: [PATCH 23/24] add missing updates to selector

---
 selector/webserver.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/selector/webserver.py b/selector/webserver.py
index ba77179..2d11542 100644
--- a/selector/webserver.py
+++ b/selector/webserver.py
@@ -10,11 +10,7 @@ from clients.webclients import Client, CLIENTS
 from flask import Flask, render_template, request, redirect, session
 
 from tasks import tasks
-from selector.temp_config import CONFIGS, URLS
-
-BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
-#BIOGAMES_HOST = "http://www.biodiv2go.de"
-RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/"
+from selector.temp_config import CONFIGS, URLS, HOSTS, RESULT_HOST
 
 app = Flask(__name__)
 clients: typing.Dict[str, Client] = {}
@@ -29,9 +25,9 @@ def index():
 @app.route("/login", methods=["POST"])
 def login():
 	game = request.form["game"]
-	if not game in CLIENTS:
+	if not game in CLIENTS or game not in HOSTS:
 		return redirect("/?invalid_game")
-	client = CLIENTS[game](host=BIOGAMES_HOST, username=request.form['username'], password=request.form['password'])
+	client = CLIENTS[game](host=HOSTS[game], username=request.form['username'], password=request.form['password'])
 	if client.login():
 		session['logged_in'] = True
 		session['uid'] = str(uuid.uuid4())

From 9b13cfdf4642168f83555c1d1c4f9cf1053593c3 Mon Sep 17 00:00:00 2001
From: Clemens Klug <clemens.klug@uni-bamberg.de>
Date: Tue, 12 Jun 2018 11:29:50 +0200
Subject: [PATCH 24/24] add a frontend reference

---
 frontend/Readme.md                     | 41 ++++++++++++++++++++++++++
 frontend/traefik/config.toml           |  8 +++++
 frontend/traefik/config_acme.toml      | 26 ++++++++++++++++
 frontend/traefik/docker-compose.yml    | 21 +++++++++++++
 selector/{temp_config.py => config.py} |  0
 selector/webserver.py                  |  2 +-
 6 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100644 frontend/Readme.md
 create mode 100644 frontend/traefik/config.toml
 create mode 100644 frontend/traefik/config_acme.toml
 create mode 100644 frontend/traefik/docker-compose.yml
 rename selector/{temp_config.py => config.py} (100%)

diff --git a/frontend/Readme.md b/frontend/Readme.md
new file mode 100644
index 0000000..8ede456
--- /dev/null
+++ b/frontend/Readme.md
@@ -0,0 +1,41 @@
+# Traefik reverse proxy for analysis framework
+
+## Usage (default: http)
+
+1. `cd traefik`
+2. `editor docker-compose.yml`
+	* Adjust the *traefik.frontend.rule* label for the traefik dashboard
+		* Default (match any):
+			* traefik.localhost
+			* traefik.potato.kinf.wiai.uni-bamberg.de
+	* Adjust port mapping
+		* Default:
+			* 80 → 80
+		* Syntax: <host-port>:<container-port>
+3. `docker-compose up -d`
+4. `cd ../..`
+5. `editor docker-compose.yml`
+	* adjust the *traefik.url.frontend.rule* labels for services *app* and *nginx*
+	* adjust the urls in *selector/config.py* accordingly
+6. `docker-compose up -d`
+7. You have a working analysis framework setup now
+	* Stop with `docker-compose down`
+	* Start with `docker-compose up -d`
+
+## Usage (https)
+
+1. Be on a host with port 80 available from the internet
+2. Follw HTTP usage above up to step 2
+3. Add acme.json volume:
+	* Uncomment the line for the acme.json volume
+	* Adjust the host path
+	* Syntax: <host-path>:<container-path>
+4. Create acme.json
+	* touch acme.json
+	* chmod 600 acme.json
+5. Activate traefiks ACME module
+	* `mv config.toml config_http.toml`
+	* `mv config_acme.toml config.toml`
+	* `editor config.toml`
+		* Adjust the *acme.email* value
+6. Continue with HTTP Usage steps 3 +
diff --git a/frontend/traefik/config.toml b/frontend/traefik/config.toml
new file mode 100644
index 0000000..8a974e9
--- /dev/null
+++ b/frontend/traefik/config.toml
@@ -0,0 +1,8 @@
+logLevel = "INFO"
+
+[web]
+address = ":8080"
+
+[docker]
+watch = true
+exposedbydefault = false
diff --git a/frontend/traefik/config_acme.toml b/frontend/traefik/config_acme.toml
new file mode 100644
index 0000000..90ffaa0
--- /dev/null
+++ b/frontend/traefik/config_acme.toml
@@ -0,0 +1,26 @@
+logLevel = "INFO"
+defaultEntryPoints = ["https", "http"]
+
+[web]
+address = ":8080"
+
+[docker]
+watch = true
+exposedbydefault = false
+
+[entryPoints]
+	[entryPoints.http]
+		address = ":80"
+		[entryPoints.http.redirect]
+			entryPoint = "https"
+	[entryPoints.https]
+		address = ":443"
+		[entryPoints.https.tls]
+
+[acme]
+	email = "tls-admin@org.example"
+	storage = "acme.json"
+	entryPoint = "https"
+	OnHostRule = true
+	[acme.httpChallenge]
+		entryPoint = "http"
\ No newline at end of file
diff --git a/frontend/traefik/docker-compose.yml b/frontend/traefik/docker-compose.yml
new file mode 100644
index 0000000..134095c
--- /dev/null
+++ b/frontend/traefik/docker-compose.yml
@@ -0,0 +1,21 @@
+version: "3"
+services:
+ traefik:
+  image: traefik:1.6
+  command: --configFile=/traefik.toml
+  volumes:
+   - ./config.toml:/traefik.toml
+#  - ./acme.json:/acme.json
+   - /var/run/docker.sock:/var/run/docker.sock
+  ports:
+   - 80:80
+  networks:
+   - net
+  labels:
+   - "traefik.enable=true"
+   - "traefik.port=8080"
+   - "traefik.frontend.rule=Host:traefik.localhost,traefik.potato.kinf.wiai.uni-bamberg.de"
+  restart: on-failure:5
+networks:
+ net:
+  driver: bridge
diff --git a/selector/temp_config.py b/selector/config.py
similarity index 100%
rename from selector/temp_config.py
rename to selector/config.py
diff --git a/selector/webserver.py b/selector/webserver.py
index 2d11542..30e3abe 100644
--- a/selector/webserver.py
+++ b/selector/webserver.py
@@ -10,7 +10,7 @@ from clients.webclients import Client, CLIENTS
 from flask import Flask, render_template, request, redirect, session
 
 from tasks import tasks
-from selector.temp_config import CONFIGS, URLS, HOSTS, RESULT_HOST
+from selector.config import CONFIGS, URLS, HOSTS, RESULT_HOST
 
 app = Flask(__name__)
 clients: typing.Dict[str, Client] = {}