Merge branch 'master' of git.clkl.de:ma/project

* add selector frontend
* basic celery backend
* client library

.gitignore

@@ -3,5 +3,5 @@ _*
 !__init__.py
 *.pyc
 logs/
-data/
+*data/
-
+plots/*

Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:edge
+
+ADD ["requirements.txt", "/"]
+RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing/" >> /etc/apk/repositories && \
+	apk add --update --no-cache libpng freetype python3 libstdc++ libxml2 libxslt openblas && \
+	apk add --update --no-cache --virtual .build-deps libpng-dev freetype-dev g++ python3-dev openblas-dev libxml2-dev libxslt-dev && \
+	pip3 --no-cache-dir install -r requirements.txt && \
+	apk del .build-deps && \
+	rm requirements.txt
+USER guest

Readme.md

@@ -0,0 +1,10 @@
+# Geogame Log Analyzer
+
+## log data
+
+### set mtime of gpx files to the first date:
+
+```
+for i in */*; do touch -m -d "$(head -n 15 $i|grep time | head -n 1 |cut -d">" -f 3|cut -d"<" -f1)" $i; done
+for i in */; do touch -m -d "$(head -n 15 $i/*.gpx|grep time | head -n 1 |cut -d">" -f 3|cut -d"<" -f1)" $i; done
+```

analysis/analyzers/__init__.py

@@ -2,15 +2,17 @@ from typing import List
 
 from .analyzer import Analyzer, Result
 from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
-	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer
+	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer, \
+	SimulationFlagsAnalyzer
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
-	CategorizerStub, Store, ProgressAnalyzer
+	CategorizerStub, Store, ProgressAnalyzer, SimpleCategorizer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
 from .analyzer.mask import MaskSpatials
 from .render import Render
 from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
-	ActivityMapperRender, StoreRender, SimulationOrderRender
+	ActivityMapperRender, StoreRender, SimulationOrderRender, SimulationGroupRender
-from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender
+from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender, LogEntryCountAnalyzerPlot, \
+	LogEntryCountCSV, KMLRender
 from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
 	LocomotionActionRatioRender
 
@@ -21,7 +23,9 @@ __MAPPING__ = {
 		LocomotionActionRelativeRender,
 		LocomotionActionRatioRender, ],
 	LogEntryCountAnalyzer: [
-		JSONRender,
+		# JSONRender,
+		LogEntryCountAnalyzerPlot,
+		LogEntryCountCSV,
 	],
 	SimulationRoundsAnalyzer: [
 		JSONRender,
@@ -37,6 +41,7 @@ __MAPPING__ = {
 	LocationAnalyzer: [
 		TrackRender,
 		HeatMapRender,
+		KMLRender,
 	],
 	ActivityMapper: [
 		ActivityMapperRender
@@ -48,8 +53,9 @@ __MAPPING__ = {
 		StoreRender
 	],
 	SimulationOrderAnalyzer: [
-		JSONRender,
+		#JSONRender,
-SimulationOrderRender
+		# SimulationOrderRender,
+		SimulationGroupRender
 	]
 }
 
@@ -60,10 +66,10 @@ def get_renderer(cls: type) -> [type]:
 	return __MAPPING__[cls]
 
 
-def render(cls: type, results: List[Result]):
+def render(cls: type, results: List[Result], name=None):
 	for r in get_renderer(cls):
 		p = r()
 		p.result_types.append(cls)
-		rendered = p.render(results)
+		rendered = p.render(results, name=name)
 		if rendered:
 			print(str(r))

analysis/analyzers/analyzer/__init__.py

@@ -2,16 +2,17 @@ import logging
 from collections import KeysView
 from typing import Type, Sized, Collection
 
-from analyzers.settings import LogSettings
+from analysis.analyzers.settings import LogSettings
 
 log: logging.Logger = logging.getLogger(__name__)
 
 
 class Result:
-	def __init__(self, analysis: Type, result: Sized):
+	def __init__(self, analysis: Type, result: Sized, name: str = None):
 		self.result = result
 		self.__analysis__ = analysis
 		log.debug("set" + str(len(self.result)))
+		self.name = name
 
 	def analysis(self):
 		return self.__analysis__
@@ -21,7 +22,8 @@ class Result:
 		return self.result
 
 	def __repr__(self):
-		return "<Result " + str(self.__analysis__) + ": " + str(type(self.result)) + " " + str(len(self.result)) + ">"
+		return "<Result " + str(self.__analysis__) + ": " + str(type(self.result)) + " " + str(
+			len(self.result)) + " for " + str(self.name) + ">"
 
 
 class ResultStore:
@@ -50,7 +52,7 @@ class ResultStore:
 		:return:
 		"""
 		result = []
-		for key in self.store:
+		for key in sorted(self.store):
 			result += self.store[key]
 		return result
 
@@ -59,13 +61,18 @@ class ResultStore:
 
 	def get_category(self, key):
 		if key not in self.store:
-			return self.entry
+			return self.entry()
+		log.error("get_category %s %s", key, len(self.store[key]))
 		return self.store[key]
 
 	def serializable(self):
 		values = {}
 		for key in self.store:
-			values[key] = [{"analysis": str(result.analysis()), "result": result.get()} for result in self.store[key]]
+			values[key] = [{
+				"analysis": str(result.analysis()),
+				"result": result.get(),
+				"name": result.name
+			} for result in self.store[key]]
 		return values
 
 
@@ -83,7 +90,7 @@ class Analyzer:
 		"""
 		raise NotImplementedError()
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, name=None) -> None:
 		raise NotImplementedError()
 
 	def name(self) -> str:

analysis/analyzers/analyzer/biogames.py

@@ -3,8 +3,8 @@ from collections import defaultdict, namedtuple, OrderedDict
 from types import SimpleNamespace
 from typing import List, NamedTuple
 
-from util import json_path, combinate
+from analysis.util import json_path, combinate
-from util.download import download_board
+from analysis.util.download import download_board, get_board_data
 from . import Result, LogSettings, Analyzer, ResultStore
 from .default import CategorizerStub, Store
 
@@ -52,6 +52,36 @@ class BoardDurationAnalyzer(Analyzer):
 		self.last = {}
 
 
+class TypedBoardDuration(Analyzer):
+	__name__ = "BoardDuration"
+
+	def result(self, store: ResultStore) -> None:
+		pass
+
+	def process(self, entry: dict) -> bool:
+		entry_type = entry[self.settings.type_field]
+		if entry_type in self.settings.boards:
+			pass
+
+	def add_board(self, entry):
+		board_data = get_board_data(self.settings.source, )
+
+	def add_location(self, entry):
+		self.track['coordinates'].append(json_path(entry, self.settings.custom['coordinates']))
+
+	def add_track(self, **props):
+		self.track['properties'] = props
+		self.tracks.append(self.track)
+		self.track = dict(self.template)
+
+	def __init__(self, settings: LogSettings):
+		super().__init__(settings)
+		self.last_board = {}
+		self.tracks = []
+		self.template = {"type": "LineString", "coordinates": [], "properties": {}}
+		self.track = dict(self.template)
+
+
 class SimulationRoundsAnalyzer(Analyzer):
 	__name__ = "SimuRounds"
 
@@ -90,7 +120,7 @@ class ActivationSequenceAnalyzer(Analyzer):
 		return False
 
 
-class BiogamesCategorizer(CategorizerStub):
+class BiogamesCategorizer(CategorizerStub):  # TODO: refactor
 	__name__ = "BiogamesCategorizer"
 
 	def __init__(self, settings: LogSettings):
@@ -105,10 +135,31 @@ class BiogamesCategorizer(CategorizerStub):
 
 class ActivityMapper(Analyzer):
 	__name__ = "ActivityMapper"
+	classes = {
+		"sequence.simulation.": "simu",
+		"sequence.question.": "question",
+		"error": "error"
+	}
+	colors = {
+		"simu": "blue",
+		"question": "orange",
+		"image": "green",
+		"audio": "red",
+		"video": "purple",
+		"other": "brown",
+		"map": "violet",
+		"error": "grey"
+	}
 
 	def __init__(self, settings: LogSettings) -> None:
 		super().__init__(settings)
 		self.store: List[self.State] = []
+		self.timeline = []
+		self.last_board = {}
+		self.last_board_type = "other"
+		self.last_coordinate = None
+		self.tracks = []
+		self.track = None
 		self.instance_config_id: str = None
 		self.filters = SimpleNamespace()
 		self.filters.start = lambda entry: combinate(self.settings.custom["sequences2"]["start"], entry)
@@ -116,52 +167,94 @@ class ActivityMapper(Analyzer):
 
 		self.State: NamedTuple = namedtuple("State", ["sequence", "events", "track", "timestamp"])
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, **kwargs) -> None:
-		instance_config_id = self.instance_config_id
+		for board in self.timeline:
-		for active_segment in self.store:  # active_segment → sequence or None (None → map active)
+			if board[self.settings.type_field] in self.settings.boards:
-			seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
+				if board["extra_data"]["activity_type"] == "simu":
-				config_id=instance_config_id,
+					board["image"] = "simu.png"
-				sequence_id=active_segment.sequence,
+					continue
-			)
+				local_file = download_board(board["board_id"], self.instance_config_id, board["sequence_id"],
-			source = self.settings.source
+											self.settings.source)
-			seq_data = source._get(seq_data_url).json()
+				if local_file:
-			# TODO: use sequence names
+					board['image'] = local_file
-			logger.warning(seq_data)
+				else:
-			for event in active_segment.events:
+					board['image'] = "ERROR_FETCHING_FILE"
-				if event[self.settings.type_field] in self.settings.boards:
+					logger.error("error downloading board! %s %s %s", self.instance_config_id, board["sequence_id"],
-					sequence_id = active_segment.sequence
+								 board["board_id"])
-					board_id = event["board_id"]
+			else:
-					local_file = download_board(board_id, instance_config_id, sequence_id, source)
+				board["image"] = "map.png"
-					if local_file is not None:
+		store.add(Result(type(self), {
-						event["image"] = local_file[16:]
+			"instance": self.instance_config_id,
-		store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]}))
+			"track": self.tracks,
+			"boards": self.timeline,
+			"colors": self.colors,
+		}))
 
 	def process(self, entry: dict) -> bool:
+		if self.track is None:
+			self.track = self.new_track(entry['timestamp'])
 		if self.instance_config_id is None:
 			if entry[self.settings.type_field] in self.settings.custom['instance_start']:
 				self.instance_config_id = json_path(entry, self.settings.custom['instance_config_id'])
-		if self.filters.start(entry):
-			self.store.append(
-				self.State(
-					sequence=json_path(entry, json_path(self.settings.custom, "sequences2.id_field")),
-					events=[],
-					track=[],
-					timestamp=entry['timestamp']))
-		elif self.filters.end(entry) or not self.store:
-			self.store.append(self.State(sequence=None, events=[], track=[], timestamp=entry['timestamp']))
 
+		self.update_board_type(entry)
 		if entry[self.settings.type_field] in self.settings.spatials:
-			self.store[-1].track.append(
+			self.add_location(entry)
-				{
+		elif entry[self.settings.type_field] in self.settings.boards:
-					'timestamp': entry['timestamp'],
+			board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
-					'coordinates': json_path(entry, "location.coordinates"),
+										entry["board_id"])
-					'accuracy': entry['accuracy']
+			entry["extra_data"] = board_data
-				}
+			entry["extra_data"]["activity_type"] = self.last_board_type
-			)
+			entry['coordinate'] = self.new_coordinate()
-		else:
+			self.timeline.append(entry)
-			self.store[-1].events.append(entry)
 		return False
 
+	def update_board_type(self, entry):
+		type = self.classify_entry(entry)
+		if not type == self.last_board_type:
+			self.add_track(activity_type=self.last_board_type, end_timestamp=entry['timestamp'])
+			self.last_board_type = type
+
+	def classify_entry(self, entry):
+		entry_type = entry[self.settings.type_field]
+		if self.filters.end(entry):
+			data = {"extra_data": {"activity_type": "map"}, "coordinate": self.new_coordinate()}
+			data.update(entry)
+			self.timeline.append(data)
+			return "map"
+		if not entry_type in self.settings.boards:
+			return self.last_board_type
+		board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"],
+									entry["board_id"])
+		for pattern in self.classes:
+			if pattern in board_data['class']:
+				return self.classes[pattern]
+		if board_data['has_video']:
+			return "video"
+		elif board_data['has_audio']:
+			return "audio"
+		elif board_data['has_image']:
+			return "image"
+		return "other"
+
+	def new_coordinate(self):
+		return {"type": "Point", "coordinates": self.last_coordinate}
+
+	def add_location(self, entry):
+		coordinates = json_path(entry, self.settings.custom['coordinates'])
+		self.track['coordinates'].append(coordinates)
+		self.last_coordinate = coordinates
+
+	def add_track(self, **props):
+		self.track['properties'].update(props)
+		self.tracks.append(self.track)
+		self.track = self.new_track(props['end_timestamp'])
+		if self.last_coordinate:
+			self.track['coordinates'].append(self.last_coordinate)
+
+	def new_track(self, timestamp):
+		return {"type": "LineString", "coordinates": [], "properties": {'start_timestamp': timestamp}}
+
 
 class BiogamesStore(Store):
 	__name__ = "BiogamesStore"
@@ -200,8 +293,8 @@ class InstanceConfig(Analyzer):
 			print(entry)
 			self.store["instance_id"] = json_path(entry, self.settings.custom["instance_config_id"])
 
-	def result(self, store: ResultStore):
+	def result(self, store: ResultStore, name=None):
-		store.add(Result(type(self), dict(self.store)))
+		store.add(Result(type(self), dict(self.store), name=name))
 
 
 class SimulationOrderAnalyzer(Analyzer):
@@ -212,8 +305,8 @@ class SimulationOrderAnalyzer(Analyzer):
 		self.store = defaultdict(lambda: -1)  # TODO verify
 		self.order = []
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, name=None) -> None:
-		store.add(Result(type(self), [self.store[sim] for sim in self.order]))
+		store.add(Result(type(self), [self.store[sim] for sim in self.order], name=name))
 
 	def process(self, entry: dict) -> bool:
 		entry_type = entry[self.settings.type_field]
@@ -224,3 +317,35 @@ class SimulationOrderAnalyzer(Analyzer):
 				if not simu_id in self.order:
 					self.order.append(simu_id)
 		return False
+
+
+class SimulationCategorizer(CategorizerStub):  # TODO: refactor categorizer
+	__name__ = "SimulationCategorizer"  # TODO: rename -.- (InstanceConfigIDCategorizer)
+
+	def process(self, entry: dict) -> bool:
+		if self.key is "default":
+			if entry[self.settings.type_field] in self.settings.custom['instance_start']:
+				try:
+					self.key = json_path(entry, self.settings.custom['instance_config_id'])
+				except KeyError as e:
+					print(entry)
+					raise e
+		return False
+
+
+class SimulationFlagsAnalyzer(Analyzer):
+	__name__ = "SimuFlags"
+
+	def __init__(self, settings: LogSettings) -> None:
+		super().__init__(settings)
+		self.store = []
+
+	def process(self, entry: dict) -> bool:
+		entry_type = entry[self.settings.type_field]
+		if entry_type in self.settings.custom['simulation_rounds']:
+			if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]:
+				self.store.append(entry)
+		return False
+
+	def result(self, store: ResultStore, name=None) -> None:
+		store.add(Result(type(self), self.store, name=name))

analysis/analyzers/analyzer/default.py

@@ -1,7 +1,7 @@
 import logging
 from collections import defaultdict, OrderedDict
 
-from util import json_path
+from analysis.util import json_path
 from . import Result, LogSettings, Analyzer, ResultStore
 
 
@@ -16,9 +16,9 @@ class LocationAnalyzer(Analyzer):
 		super().__init__(settings)
 		self.entries = []
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, **kwargs) -> None:
 		self.log.debug(len(self.entries))
-		store.add(Result(type(self), list(self.entries)))
+		store.add(Result(type(self), list(self.entries), name=kwargs['name']))
 
 	def process(self, entry: dict) -> bool:
 		if entry[self.settings.type_field] in self.settings.spatials:
@@ -28,6 +28,7 @@ class LocationAnalyzer(Analyzer):
 
 
 class LogEntryCountAnalyzer(Analyzer):
+	#TODO: flexibler: z.b. min/max lat/long
 	"""
 	count occurrences of log entry types
 	"""
@@ -88,13 +89,17 @@ class CategorizerStub(Analyzer):
 
 	__name__ = "Categorizer"
 
-	def result(self, store: ResultStore) -> None:
+	def result(self, store: ResultStore, name=None) -> None:
-		store.new_category(self.key)
+		store.new_category(name if name else self.key)
 
 	def __init__(self, settings: LogSettings):
 		super().__init__(settings)
 		self.key = "default"
 
+class SimpleCategorizer(CategorizerStub):
+	def process(self, entry):
+		return False
+
 
 class Store(Analyzer):
 	"""
@@ -136,3 +141,72 @@ class ProgressAnalyzer(Analyzer):
 		if entry[self.settings.type_field] in self.settings.boards:
 			self.board[entry["timestamp"]] = entry
 		return False
+
+
+class MetaDataAnalyzer(Analyzer):
+	"""collect metadata"""
+	__name__ = "MetaDataAnalyzer"
+
+	def result(self, store: ResultStore, name=None) -> None:
+		store.add(Result(type(self), dict(self.store)))
+
+	def process(self, entry: dict) -> bool:
+		if not "metadata" in self.settings.custom:
+			return False
+		for mdata in self.settings.custom["metadata"]:
+			key = self.settings.custom["metadata"]
+			if key in entry:
+				self.store[mdata] = json_path(entry, key)
+
+	def __init__(self, settings: LogSettings) -> None:
+		super().__init__(settings)
+		self.store = {}
+
+
+def write_logentry_count_csv(LogEntryCountCSV, store, render, analyzers):
+	global cat, data, lines, csvfile
+	LogEntryCountCSV.summary = None
+	for cat in store.get_categories():
+		data = store.get_category(cat)
+		render(analyzers.LogEntryCountAnalyzer, data, name=cat)
+	if LogEntryCountCSV.summary:
+		headers = []
+		lines = []
+		for name in LogEntryCountCSV.summary:
+			data = LogEntryCountCSV.summary[name]
+			for head in data:
+				if not head in headers:
+					headers.append(head)
+			line = [name]
+			for head in headers:
+				line.append(data[head]) if head in data else line.append(0)
+			lines.append(line)
+		import csv
+
+		with open('logentrycount.csv', 'w', newline='') as csvfile:
+			writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE)
+			writer.writerow(["name"] + [h.split(".")[-1] for h in headers])
+			for line in lines:
+				writer.writerow(line)
+
+
+def write_simulation_flag_csv(store):
+	global csvfile, result, i
+	from datetime import datetime
+	import json
+	json.dump(store.serializable(), open("simus.json", "w"), indent=2)
+	with open("simus.csv", "w") as csvfile:
+		csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n")
+		for key in store.get_store():
+			csvfile.write("{}\n".format(key))
+			for result in store.store[key]:
+				csvfile.write(",{}\n".format(result.name))
+				for i in result.get():
+					csvfile.write(",,{},{},{},{},{},{}\n".format(
+						i['answers']['@id'],
+						i['answers']['answered'],
+						len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0,
+						len(i['selected_actions']) if i['selected_actions'] else 0,
+						i['timestamp'],
+						str(datetime.fromtimestamp(i['timestamp'] / 1000))
+					))

analysis/analyzers/analyzer/locomotion.py

@@ -1,4 +1,4 @@
-import util
+from analysis import util
 from . import Analyzer, LogSettings, Result, ResultStore
 
 

analysis/analyzers/render/__init__.py

@@ -5,7 +5,7 @@ from .. import Result
 class Render:
 	result_types = []
 
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		raise NotImplementedError()
 
 	def filter(self, results: List[Result]):

analysis/analyzers/render/biogames.py

@@ -0,0 +1,203 @@
+import json
+from collections import defaultdict
+from typing import List, Tuple
+
+import matplotlib.pyplot as plt
+import os
+import numpy as np
+from scipy.interpolate import interp1d
+import networkx as nx
+import itertools
+
+from analysis.analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
+from analysis.util.meta_temp import CONFIG_NAMES
+from . import Render
+from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
+
+
+def add_edge(graph, src, dest):
+	if graph.has_edge(src, dest):
+		weight = graph.get_edge_data(src, dest)['weight'] + 1
+	else:
+		weight = 1
+	graph.add_edge(tuple(src), tuple(dest), weight=weight)
+
+
+def pairs(iterable):
+	a, b = itertools.tee(iterable)
+	next(b, None)
+	return zip(a, b)
+
+
+def __plot_or_show(name=None):
+	if name:
+		plt.savefig(name)
+	else:
+		plt.show()
+	plt.cla()
+	plt.clf()
+	plt.close()
+
+
+def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
+		 rotation='vertical', name=None):
+	names, datas = list(zip(*src_data))
+	# plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True)
+	rand = np.random.rand(len(datas), len(datas[0]))
+	plt.plot(datas + rand, linewidth=.2)
+	plt.xticks(rotation=rotation)
+	# plt.margins()
+	plt.ylabel(ylabel)
+	plt.title(title)
+	__plot_or_show(name)
+
+
+def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="sequential simulation retries",
+			   rotation='vertical', name=None):
+	config_name = CONFIG_NAMES[name] if name in CONFIG_NAMES else "---"
+	counts_per_group = [sum(i) for i in src_data]
+	label = "{}: n={n}; # of sim runs: ⌀={avg:.2f}, median={median}".format(
+		config_name,
+		n=len(src_data),
+		avg=np.mean(counts_per_group),
+		median=np.median(counts_per_group)
+	)
+	print(config_name)
+	name = "plots/{}.png".format(name)
+	g = nx.Graph()
+	for group in src_data:
+		for i in pairs(enumerate(group)):
+			add_edge(g, i[0], i[1])
+	positions = {}
+	for node in g.nodes():
+		positions[node] = node
+	widths = [x[2] / 10.0 for x in g.edges.data('weight')]
+	print(max(widths))
+	nx.draw_networkx_edges(g, positions, width=widths)
+	# rand = np.random.rand(len(datas),len(datas[0]))
+	# plt.plot(datas+rand, linewidth=.2)
+	plt.xticks(rotation=rotation)
+	# plt.margins()
+	plt.ylabel(ylabel)
+	plt.title(title)
+	plt.figtext(0.5, 0.13, label, ha="center")
+	__plot_or_show(name)
+
+
+def graph_fit(src_data, deg=5, name=None):
+	plt.title("polyfit(x,y,deg=" + str(deg) + ")")
+	for i in src_data:
+		# plt.plot(i)
+		count = len(i)
+		xp = np.linspace(0, count - 1, num=count, endpoint=True)
+		# fit = np.poly1d(np.polyfit(range(len(i)), i, deg=deg))
+		# plt.plot(xp, fit(xp), linewidth=0.1)
+		xnew = np.linspace(0, count - 1, num=count * 20, endpoint=True)
+		f = interp1d(xp, i, kind='quadratic')
+
+		plt.plot(range(count), i, '.', markersize=1)
+		plt.plot(xnew, f(xnew), linewidth=0.2)
+		__plot_or_show(name)
+
+
+class SimulationRoundsRender(Render):
+	def render(self, results: List[Result], name=None):
+		data = defaultdict(list)
+		for result in self.filter(results):
+			get = result.get()
+			for key in get:
+				data[key].append(get[key])
+		data_tuples = [(key, data[key]) for key in sorted(data)]
+		data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
+		plot(data_tuples)
+
+	result_types = [SimulationRoundsAnalyzer]
+
+
+class BoardDurationHistRender(Render):
+	result_types = [BoardDurationAnalyzer]
+
+	def render(self, results: List[Result], name=None):
+		data = []
+		for result in self.filter(results):
+			session = result.get()
+			_data = []
+			for board in session:
+				if "active" in board:
+					_data.append(board["active"])
+				else:
+					_data.append(0)
+			data.append(_data)
+		n, bins, patches = plt.hist(data, log=True)
+		plt.show()
+
+
+class BoardDurationBoxRender(Render):
+	result_types = [BoardDurationAnalyzer]
+
+	def render(self, results: List[Result], name=None):
+		data = defaultdict(list)
+		for result in self.filter(results):
+			get = result.get()
+			for board in get:
+				duration = board['active'] if 'active' in board else 0
+				data[board['id']].append(duration)
+		data_tuples = [(key, data[key]) for key in sorted(data)]
+		data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
+		plot(data_tuples)
+
+
+class ActivityMapperRender(Render):
+	result_types = [ActivityMapper]
+
+	def render(self, results: List[Result], name=None):
+		print(os.getcwd())
+		files = []
+		for result in self.filter(results):
+			data = result.get()
+			path = os.path.join("/tmp", data['instance'] + "_" + str(name) + ".json")
+			with open(path, "w") as out:
+				json.dump(data, out, indent=1)
+			files.append(path)
+		return files
+
+
+class StoreRender(Render):
+	result_types = [Store, BiogamesStore]
+
+	def render(self, results: List[Result], name=None):
+		for result in self.filter(results):
+			with open(os.path.join("static", "progress", "data", "fooo"), "w") as out:
+				json.dump(result.get(), out, indent=1)
+
+
+class SimulationOrderRender(Render):
+	def render(self, results: List[Result], name=None):
+		data = defaultdict(list)
+		for result in self.filter(results):
+			get = result.get()
+			for i, value in enumerate(get):
+				data[i].append(value)
+		# data_tuples = [(key, data[key]) for key in sorted(data)]
+		# data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
+		# plot(enumerate([r.get() for r in self.filter(results)]))
+		plot(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
+
+	result_types = [SimulationOrderAnalyzer]
+
+
+class SimulationGroupRender(Render):
+	def render(self, results: List[Result], name=None):
+		# data = [r.get() for r in self.filter(results)]
+		data = []
+		for r in self.filter(results):
+			raw = r.get()
+			if len(raw) < 6:
+				raw = [0] + raw
+			data.append(raw)
+		print(name, len(data))
+		# graph_fit(list(data), name=name)
+		graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None,
+				   name=name)
+
+	result_types = [SimulationOrderAnalyzer]

analysis/analyzers/render/default.py

@@ -0,0 +1,116 @@
+import json
+import logging
+from typing import List
+
+import datetime
+import matplotlib.pyplot as plt
+
+from analysis.analyzers import LogEntryCountAnalyzer
+from analysis.util.meta_temp import KML_PATTERN
+from . import Render, Result
+from analysis.analyzers import LocationAnalyzer
+
+log = logging.getLogger(__name__)
+
+
+class PrintRender(Render):
+	def render(self, results: List[Result], name=None):
+		print("\t" + "\n\t".join([str(r) for r in results]))
+
+
+class JSONRender(Render):
+	def render(self, results: List[Result], name=None):
+		print(json.dumps([r.get() for r in self.filter(results)], indent=1))
+
+
+class TrackRender(Render):
+	result_types = [LocationAnalyzer]
+
+	def render(self, results: List[Result], name=None):
+		data = []
+		log.debug(results)
+		for result in self.filter(results):
+			if len(result.get()) > 0:
+				data.append(
+					[[entry['location']['coordinates'][1], entry['location']['coordinates'][0]] for entry in
+					 # TODO: configurable
+					 result.get()])
+		dumps = json.dumps(data)
+		with open("track_data.js", "w") as out:
+			out.write("tracks=" + dumps + ";")
+		return dumps
+
+
+def format_time(ts):
+	return datetime.datetime.fromtimestamp(ts/1000).strftime("%Y-%m-%dT%H:%M:%S.%f")
+
+
+class KMLRender(Render):
+	result_types = [LocationAnalyzer]
+
+	def render(self, results: List[Result], name=None):
+		files = []
+		for result in self.filter(results):
+			times = ["<when>{time}</when>".format(time=format_time(entry["timestamp"])) for entry in result.get()]
+			coords = [
+				"<gx:coord>{long} {lat} 0.0</gx:coord>"
+					.format(
+					lat=entry['location']['coordinates'][1],
+					long=entry['location']['coordinates'][0])
+				for entry in result.get()
+			]
+			filename = str(result.name)+".kml"
+			print(filename)
+			with open(filename, "w") as out:
+				out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times)))
+			files.append(filename)
+		return files
+
+
+
+
+class HeatMapRender(TrackRender):
+	weight = 0.01
+
+	def render(self, results: List[Result], name=None):
+		raw = super(HeatMapRender, self).render(results)
+		data = []
+		for session in json.loads(raw):
+			data += [(entry[0], entry[1], self.weight) for entry in session]
+		dumps = json.dumps(data)
+		with open('heat_data.js', 'w') as out:
+			out.write("coords = " + dumps + ";")
+		return dumps
+
+
+class LogEntryCountAnalyzerPlot(Render):
+	result_types = [LogEntryCountAnalyzer]
+
+	def render(self, results: List[Result], name=None):
+		raw_data = list(self.filter(results))[0].get()
+		print(raw_data)
+		labels = []
+		data = []
+		for x in sorted(raw_data.items()):
+			labels.append(str(x[0]).split(".")[-1])
+			data.append(x[1])
+		plt.bar(range(len(data)), list(data))
+		plt.xticks(range(len(data)), labels, rotation="vertical")
+		plt.tight_layout()
+
+		name = "plots/{}.png".format(name)
+		plt.savefig(name)
+		plt.cla()
+		plt.clf()
+		plt.close()
+
+class LogEntryCountCSV(Render):
+	result_types = [LogEntryCountAnalyzer]
+	summary = None
+
+	def render(self, results: List[Result], name=None):
+		if self.summary is None:
+			return
+		for result in self.filter(results):
+			raw_data = result.get()
+			self.summary[name] = raw_data

analysis/analyzers/render/locomotion.py

@@ -51,18 +51,18 @@ class LocomotionActionRender(Render):
 
 
 class LocomotionActionAbsoluteRender(LocomotionActionRender):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		results = filter_results(self.filter(results), ['locomotion_sum', 'action_sum'])
 		plot(results, "time", "abs loc/action")
 
 
 class LocomotionActionRelativeRender(LocomotionActionRender):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		results = filter_results(self.filter(results), ['locomotion_relative', 'action_relative'])
 		plot(results, "fraction of time", "rel loc/action")
 
 
 class LocomotionActionRatioRender(LocomotionActionRender):
-	def render(self, results: List[Result]):
+	def render(self, results: List[Result], name=None):
 		results = filter_results(self.filter(results), ['locomotion_action_ratio'])
 		plot_line(results, ylabel="Ratio", title="Locomotion/Action Ratio")

analysis/analyzers/render/wip.py

@@ -0,0 +1,393 @@
+import json
+
+import numpy as np
+
+import analysis.analyzers
+from analysis.util.geo import calc_distance
+
+
+def time_distribution(store):
+	# json.dump(store.serializable(), open("new.json", "w"), indent=1)
+
+	keys = [
+		"simu",
+		"question",
+		"image",
+		"audio",
+		"video",
+		"other",
+		"map"
+	]
+	import matplotlib.pyplot as plt
+
+	# results = []
+
+	places = defaultdict(list)
+
+	for log in store.get_all():
+		result = defaultdict(lambda: 0)
+		for i in log.get()['track']:
+			duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
+			result[i['properties']['activity_type']] += duration
+		print(json.dumps(result, indent=4))
+		total = sum(result.values())
+		print(total)
+		percentage = defaultdict(lambda: 0)
+		minutes = defaultdict(lambda: 0)
+		for i in result:
+			percentage[i] = result[i] / total
+			minutes[i] = result[i] / 60_000
+		print(json.dumps(percentage, indent=4))
+		if not 'error' in result:
+			# places[log.get()['instance']].append(percentage)
+			places[log.get()['instance']].append(minutes)
+
+	for place in places:
+		places[place] = sorted(places[place], key=lambda item: item['map'])
+
+	dummy = [0] * len(keys)
+	results = []
+	sites = []
+	from util.meta_temp import CONFIG_NAMES
+
+	for i in places:
+		for j in places[i]:
+			ordered = []
+			for k in keys:
+				ordered.append(j[k])
+			results.append(ordered)
+		results.append(dummy)
+		sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
+
+	size = len(results)
+	ind = np.arange(size)
+	width = 0.9
+	print(results)
+	data = list(zip(*results))
+	print(data)
+	lines = []
+	bottom = [0] * len(results)
+	for i in range(0, len(data)):
+		lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+		for k, x in enumerate(data[i]):
+			bottom[k] += x
+	plt.legend(lines, keys)
+	plt.title(", ".join(sites))
+	plt.show()
+
+
+# size = len(results)
+# ind = np.arange(size)
+# width = 0.9
+# print(results)
+# data = list(zip(*results))
+# print(data)
+# lines = []
+# bottom = [0] * len(results)
+# for i in range(0, len(data)):
+#	lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+#	for k, x in enumerate(data[i]):
+#		bottom[k] += x
+# plt.legend(lines, keys)
+# plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)")
+# plt.show()
+
+# json.dump(store.serializable(), open("new.json", "w"), indent=1)
+
+
+from collections import defaultdict
+import matplotlib.pyplot as plt
+from analysis.util.meta_temp import CONFIG_NAMES
+
+keys = [
+	"simu",
+	"question",
+	"image",
+	"audio",
+	"video",
+	"other",
+	"map",
+	# "error"
+]
+
+loc_keys = [
+	"question",
+	"image",
+	"audio",
+	"video"
+]
+
+
+def get_data(store, relative_values=True, sort=True, show_errors=False):
+	places = defaultdict(list)
+
+	for log in store.get_all():
+		if not log.analysis() == analyzers.ActivityMapper:
+			continue
+		result = defaultdict(lambda: 0)
+		for i in log.get()['track']:
+			duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
+			result[i['properties']['activity_type']] += duration
+		print(json.dumps(result, indent=4))
+		total = sum(result.values())
+		print(total)
+		percentage = defaultdict(lambda: 0)
+		minutes = defaultdict(lambda: 0)
+		for i in result:
+			percentage[i] = result[i] / total
+			minutes[i] = result[i] / 60_000
+		print(json.dumps(percentage, indent=4))
+		if not 'error' in result or show_errors:
+			if relative_values:
+				places[log.get()['instance']].append(percentage)
+			else:
+				places[log.get()['instance']].append(minutes)
+	if sort:
+		for place in places:
+			places[place] = sorted(places[place], key=lambda item: item['map'])
+	return places
+
+
+whitelist = ['16fc3117-61db-4f50-b84f-81de6310206f', '5e64ce07-1c16-4d50-ac4e-b3117847ea43',
+			 '90278021-4c57-464e-90b1-d603799d07eb', 'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771']
+
+
+def get_data_distance(store, relative_values=True, sort=True, show_errors=False):
+	places = defaultdict(list)
+
+	for log in store.get_all():
+		if not log.analysis() == analyzers.ActivityMapper:
+			continue
+		result = defaultdict(lambda: 0)
+		for i in log.get()['track']:
+			coords = i['coordinates']
+			if len(coords) > 1:
+				distance = calc_distance(coords)
+				result[i['properties']['activity_type']] += distance
+		total = sum(result.values())
+		percentage = defaultdict(lambda: 0)
+		for i in result:
+			if not total == 0:
+				percentage[i] = result[i] / total
+		if not 'error' in result or show_errors:
+			if relative_values:
+				places[log.get()['instance']].append(percentage)
+			else:
+				places[log.get()['instance']].append(result)
+	if sort:
+		for place in places:
+			places[place] = sorted(places[place], key=lambda item: item['map'])
+	return places
+
+
+def get_all_data(store, sort=False, relative=True):
+	places = defaultdict(list)
+	simu_distribution = defaultdict(lambda: 0)
+	# divisiors = {"time":60_000, "space":1000000}
+	for log in store.get_all():
+		if not log.analysis() == analyzers.ActivityMapper:
+			continue
+		result = defaultdict(lambda: defaultdict(lambda: 0))
+		for i in log.get()['track']:
+			coords = i['coordinates']
+			if len(coords) > 1:
+				distance = calc_distance(coords)
+			else:
+				distance = 0.0
+			result["space"][i['properties']['activity_type']] += distance
+			duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp']
+			result["time"][i['properties']['activity_type']] += duration
+		total_space = sum(result["space"].values())
+		total_time = sum(result["time"].values())
+		percentage = defaultdict(lambda: defaultdict(lambda: 0))
+		total = defaultdict(lambda: defaultdict(lambda: 0))
+		for i in result["space"]:
+			if not total_space == 0:
+				percentage[i]["space"] = result["space"][i] / total_space
+			else:
+				percentage[i]["space"] = 0
+			if not total_time == 0:
+				percentage[i]["time"] = result["time"][i] / total_time
+			else:
+				percentage[i]["time"] = 0
+			for t in ("space", "time"):
+				# total[i][t] += (result[t][i] / divisiors[t])
+				total[i][t] += result[t][i]
+		print(percentage)
+		if not 'error' in result:
+			if relative:
+				value = percentage
+			else:
+				value = total
+			places[log.get()['instance']].append(value)
+			simus = defaultdict(lambda: 0)
+			for item in log.get()['boards']:
+				if item["extra_data"]["activity_type"] == "simu":
+					simus[item["board_id"]] += 1
+			simu_distribution[len(simus)] += 1
+
+	if sort:
+		for place in places:
+			places[place] = sorted(places[place], key=lambda item: item['map']['time'])
+	print(simu_distribution)
+	return places
+
+
+def stack_data(keys, places, type="space"):
+	divisiors = {"time": 60_000, "space": 1000}
+	# divisiors = {"time": 1, "space": 1}
+	dummy = [0] * len(keys)
+	results = []
+	sites = []
+	for i in sorted(places):
+		if not i in whitelist:
+			continue
+		place = sorted(places[i], key=lambda item: item['map'][type])
+		for j in place:
+			ordered = []
+			for k in keys:
+				if k in j:
+					ordered.append(j[k][type] / divisiors[type])
+				else:
+					ordered.append(0)
+			print(sum(ordered))
+			# if sum(ordered) > 0.9 and sum(ordered) < 4000 and sum(ordered)>10:
+			if sum(ordered) > 0.9 and sum(ordered) < 100:
+				# print(sum(ordered), 1-sum(ordered))
+				# if sum(ordered)<1:
+				#	ordered[-2] = 1-sum(ordered[:-2], ordered[-1])
+				results.append(ordered)
+		results.append(dummy)
+		sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---")
+	return results, sites
+
+
+def plot_data(places, keys):
+	results, sites = stack_data(keys, places)
+	dpi = 86.1
+	plt.figure(figsize=(1280 / dpi, 720 / dpi))
+	size = len(results)
+	print("{} elements total".format(size))
+	ind = np.arange(size)
+	width = 1
+	# print(results)
+	data = list(zip(*results))
+	# print(data)
+	lines = []
+	bottom = [0] * size
+	plt.ticklabel_format(useMathText=False)
+	for i in range(0, len(data)):
+		lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0])
+		for k, x in enumerate(data[i]):
+			bottom[k] += x
+	plt.legend(lines, keys)
+	plt.title(", ".join(sites))
+	# plt.show()
+	dpi = 86
+	plt.savefig("space_abs_{}.png".format(size), dpi=dpi, bbox_inches="tight")
+
+
+colors = {
+	"simu": "blue",
+	"question": "orange",
+	"image": "green",
+	"audio": "red",
+	"video": "purple",
+	"other": "brown",
+	"map": "violet",
+	# "error":"grey",
+	"tasks": "olive",
+}
+markers = [".", "o", "x", "s", "*", "D", "p", ",", "<", ">", "^", "v", "1", "2", "3", "4"]
+
+
+def plot_time_space(time_data, space_data, keys):
+	# assuming time_data and space_data are in same order!
+	marker = 0
+	for id in time_data:
+		for k in keys:
+			for i in range(len(time_data[id])):
+				print(time_data[id][i][k], space_data[id][i][k])
+				plt.plot(time_data[id][i][k], space_data[id][i][k], color=colors[k], marker=markers[marker])
+		marker += 1
+	plt.show()
+
+
+# plt.cla()
+# plt.clf()
+# plt.close()
+
+def group_locationbased_tasks(data):
+	for id in data:
+		for log in data[id]:
+			loc = {"space": 0, "time": 0}
+			for k in log:
+				if k in loc_keys:
+					for i in ["space", "time"]:
+						loc[i] += log[k][i]
+			log["tasks"] = loc
+
+
+def plot_time_space_rel(combined, keys):
+	groups = defaultdict(list)
+	keys = list(keys)
+	keys.remove("other")
+	for i in loc_keys:
+		keys.remove(i)
+	keys.append("tasks")
+	ids = []
+	group_locationbased_tasks(combined)
+	for k in keys:
+		for id in sorted(combined):
+			if id not in whitelist:
+				continue
+			if not id in ids:
+				ids.append(id)
+			group = 0.0
+			count = 0
+			for item in combined[id]:
+				if k in item:
+					time = item[k]["time"] / 1000
+					distance = item[k]["space"]
+					if time > 0:
+						group += (distance / time)
+						count += 1
+					else:
+						print("div by zero", distance, time)
+			if count > 0:
+				groups[k].append(group / count)
+			else:
+				groups[k].append(0.0)
+	print(ids)
+	ind = np.arange(len(ids))
+	width = .7 / len(groups)
+	print(ind)
+	print(json.dumps(groups, indent=1))
+	bars = []
+	dpi = 200
+	plt.figure(figsize=(1280 / dpi, 720 / dpi))
+	fig, ax = plt.subplots()
+	for k in groups:
+		print(groups[k])
+		if not len(groups[k]):
+			groups[k].append(0)
+		ind = ind + (width)
+		bars.append(ax.bar((ind + width * len(groups) / 2), groups[k], width, color=colors[k]))
+	ax.set_xticks(ind + width / 2)
+	ax.set_xticklabels(list([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids]))
+	kmh = plt.hlines((1 / 3.6), 0.3, 4.2, linestyles="dashed", label="1 km/h", linewidths=1)
+	plt.legend(bars + [kmh], keys + [kmh.get_label()])
+	print(combined.keys(), ids)
+	print([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids])
+	# plt.show()
+	dpi = 200
+	plt.savefig("speed2.png", dpi=dpi)
+
+
+
+
+
+# plot_time_space_rel(temporal_data_rel, spatial_data_rel, keys)
+
+# plot_data(combined, keys)
+# plot_data(get_data_distance(store,relative_values=False), keys)

analysis/analyzers/settings.py

@@ -1,13 +1,17 @@
 import json
+import logging
 import sys
-from sources import SOURCES
+from clients.webclients import CLIENTS
 
+log: logging.Logger = logging.getLogger(__name__)
 
 def load_source(config):
-	if config["type"] in SOURCES:
+	if config["type"] in CLIENTS:
-		source = SOURCES[config["type"]]()
+		source = CLIENTS[config["type"]](**config)
-		source.connect(**config)
+		source.login()
 		return source
+	else:
+		log.warn(f"client {config['type']} not found!")
 
 
 class LogSettings:
@@ -28,13 +32,15 @@ class LogSettings:
 		self.boards = json_dict['boards']
 		for mod in json_dict['analyzers']:
 			for name in json_dict['analyzers'][mod]:
-				print(mod, name)
+				print(mod, name, getattr(sys.modules[mod], name))
 				self.analyzers.append(getattr(sys.modules[mod], name))
 		self.sequences = json_dict['sequences']
 		if 'custom' in json_dict:
 			self.custom = json_dict['custom']
 		if "source" in json_dict:
 			self.source = load_source(json_dict['source'])
+		if "render" in json_dict:
+			self.render = json_dict['render']
 
 	def __repr__(self):
 		return str({
@@ -51,3 +57,7 @@ class LogSettings:
 
 def load_settings(file: str) -> LogSettings:
 	return LogSettings(json.load(open(file)))
+
+
+def parse_settings(config: str) -> LogSettings:
+	return LogSettings(json.loads(config))

analysis/biogames2.json

@@ -13,13 +13,18 @@
   ],
   "analyzers": {
     "analyzers": [
-      "BiogamesCategorizer",
+      "SimulationCategorizer",
-      "SimulationOrderAnalyzer"
+      "SimulationOrderAnalyzer",
+      "ActivityMapper"
     ]
   },
   "dis":[
       "ActivityMapper",
+      "BiogamesCategorizer",
+      "LogEntryCountAnalyzer",
+      "SimulationOrderAnalyzer",
       "ProgressAnalyzer",
+      "SimulationCategorizer",
       "InstanceConfig"],
   "disabled_analyzers": [
       "LocomotionActionAnalyzer",
@@ -62,14 +67,19 @@
         "action":"PAUSE"
       }
     },
-    "coordinates": "location.coordinates"
+    "coordinates": "location.coordinates",
+    "metadata":{
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
   },
   "source":{
     "type": "Biogames",
     "url": "http://0.0.0.0:5000/game2/instance/log/list/",
     "login_url": "http://localhost:5000/game2/auth/json-login",
-    "username": "dev",
+    "username": "ba",
-    "password": "dev",
+    "password": "853451",
     "host":"http://0.0.0.0:5000"
   }
 }

analysis/filter_todo/pre_filter.py

@@ -0,0 +1,90 @@
+import os
+from zipfile import ZipFile
+import sqlite3
+import json
+from collections import defaultdict
+
+
+def get_json(filename):
+	log = []
+	id = None
+	with ZipFile(filename) as zipf:
+		zipf.extract('instance_log.sqlite')
+		sql = sqlite3.connect('instance_log.sqlite')
+		cursor = sql.cursor()
+		for r in cursor.execute('SELECT json FROM log_entry;'):
+			entry = json.loads(r[0])
+			log.append(entry)
+			if id is None:
+				id = entry['instance_id']
+		sql.close()
+		os.remove('instance_log.sqlite')
+	return id, log
+
+
+def is_finished(log):
+	for entry in log:
+		if "action" in entry:
+			if "LogEntryInstanceAction" in entry["@class"] and entry["action"][
+				"@class"] == "de.findevielfalt.games.game2.instance.action.EndGameEnableAction" and entry['action'][
+				'enable']:
+				return True
+	return False
+
+
+def get_simus(log):
+	simus = defaultdict(lambda: 0)
+	order = []
+	actions = 0
+	for entry in log:
+		if "LogEntryQuestion" in entry["@class"]:
+			if "SimulationBoardData" in entry["answers"]["@class"]:
+				id = entry["answers"]["@id"]
+				simus[id] += 1
+				actions += 1 if entry['selected_actions'] else 0
+				if not id in order:
+					order.append(id)
+	return dict(simus), order, actions
+
+
+def simu_dist(simus):
+	dist = defaultdict(lambda: 0)
+	for instance in simus:
+		sim = simus[instance]
+		dist[len(sim)] += 1
+	return dist
+
+
+logs = {}
+finished = []
+simus = {}
+distribution = defaultdict(lambda: 0)
+finished_and_simu = defaultdict(list)
+files = {}
+actions_dist = defaultdict(list)
+with open('/home/clemens/git/ma/test/src') as src:
+	for line in src:
+		line = line.strip()
+		instance_id, log = get_json(line)
+		logs[instance_id] = log
+		files[instance_id] = line
+for id in logs:
+	simus[id] = get_simus(logs[id])
+	simu_count = len(simus[id][1])
+	distribution[simu_count] += 1
+	actions_dist[simus[id][2]].append(id)
+	if is_finished(logs[id]):
+		finished.append(id)
+		finished_and_simu[simu_count].append(id)
+print("total: ", len(logs))
+print("finished: ", len(finished))
+print("simu_dist: ", len(distribution), json.dumps(distribution, sort_keys=True))
+for i in sorted(finished_and_simu):
+	print("fin+sim" + str(i) + ": ", len(finished_and_simu[i]))
+for i in sorted(actions_dist):
+	print("actions: ", i, len(actions_dist[i]))
+print(json.dumps(actions_dist[4], sort_keys=True, indent=2))
+
+# print(finished_and_simu)
+# for instance in finished_and_simu:
+#	print(files[instance])

analysis/loaders/__init__.py

@@ -1,8 +1,10 @@
 from .biogames import SQLiteLoader, ZipSQLiteLoader
 from .loader import JSONLoader
+from .neocart import NeoCartLoader
 
 LOADERS = {
 	"json": JSONLoader,
 	"sqlite": SQLiteLoader,
-	"zip": ZipSQLiteLoader
+	"zip": ZipSQLiteLoader,
+	"neocartographer": NeoCartLoader,
 }

analysis/loaders/neocart.py

@@ -0,0 +1,70 @@
+import logging
+from datetime import datetime
+
+from lxml import etree
+
+from .loader import Loader
+
+log = logging.getLogger(__name__)
+
+NS = {'gpx':"http://www.topografix.com/GPX/1/1"}
+
+class NeoCartLoader(Loader):
+	def load(self, file: str):
+		src = open(file, "r")
+		parser = etree.XMLParser(recover=True)
+		tree = etree.parse(src, parser=parser)
+		self.entries = []
+		for point in tree.xpath("//gpx:trkpt", namespaces=NS):
+			try: 
+				self.entries.append(self.parse_point(point))
+			except ValueError as e:
+				print(e, etree.tostring(point, pretty_print=True).decode())
+				log.exception(e)
+	
+	def parse_point(self, point):
+		raw_lat = point.xpath("@lat")[0]
+		if raw_lat.count(".") > 1:
+			log.warning(f"recreate lat/lon from: {raw_lat}")
+			log.warn(etree.tostring(point, pretty_print=True).decode())
+			start_offset = 4
+			x = raw_lat[start_offset:].index(".")
+			offset = start_offset + x
+			raw_lon = raw_lat[offset:]
+			raw_lat = raw_lat[:offset]
+		else:
+			raw_lon = point.xpath("@lon")[0]
+		lat = float(raw_lat)
+		lon = float(raw_lon)
+		times = point.xpath("gpx:time",namespaces=NS)
+		assert len(times) == 1
+		time = times[0].text
+		dt = datetime.strptime(time, "%Y-%m-%dT%H:%M:%SZ")
+		timestamp = int(dt.timestamp() * 1000) # python3.6 has no timestamp_ns (yet)
+		events = point.xpath(".//gpx:event",namespaces=NS)
+		assert 0 <= len(events) <= 1
+		event = {}
+		if events:
+			event = dict(events[0].attrib)
+			if events[0].tail and events[0].tail.strip():
+				try:
+					# base case: trailing 'geoid="0"/>'
+					key, v = events[0].tail.strip().split("=")
+					value = v.split('"')[1]
+					event[key] = value
+				except:
+					event['__tail__'] = events[0].tail.strip()
+		
+		return {
+			"location": {
+				"type": "Point",
+				"coordinates": [lon, lat]
+			},
+			"timestamp": timestamp,
+			"event": event,
+			"type": "event" if event else "location"
+		}
+
+	def get_entry(self) -> object:
+		for i in self.entries:
+			yield i

analysis/log_analyzer.py

@@ -0,0 +1,134 @@
+import json
+import logging
+from typing import List
+
+from analysis import analyzers
+from analysis.analyzers import get_renderer, render
+from analysis.analyzers.analyzer import ResultStore
+from analysis.analyzers.analyzer.default import write_logentry_count_csv, write_simulation_flag_csv
+from analysis.analyzers.render import wip
+from analysis.analyzers.render.default import LogEntryCountCSV, KMLRender
+from analysis.analyzers.render.wip import time_distribution, plot_data
+from analysis.analyzers.settings import LogSettings, load_settings, parse_settings
+from analysis.loaders import LOADERS
+from analysis.util.processing import grep, run_analysis, src_file
+
+logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG)
+log: logging.Logger = logging.getLogger(__name__)
+
+logging.getLogger('requests').setLevel(logging.WARN)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+
+def urach_logs(log_ids, settings):
+	# return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
+	return ["data/{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids]
+
+
+if __name__ == '__main__':
+	settings = {}
+	log_ids_gf = []
+	# settings: LogSettings = load_settings("biogames2.json")
+	# log_ids_urach: List[str] = urach_logs([
+	# 	# "34fecf49dbaca3401d745fb467",
+	# 	#	"44ea194de594cd8d63ac0314be",
+	# 	#	"57c444470dbf88605433ca935c",
+	# 	#	"78e0c545b594e82edfad55bd7f",
+	# 	#	"91abfd4b31a5562b1c66be37d9",
+	# 	# "597b704fe9ace475316c345903",
+	# 	# "e01a684aa29dff9ddd9705edf8",
+	# 	"597b704fe9ace475316c345903",
+	# 	"e01a684aa29dff9ddd9705edf8",
+	# 	"fbf9d64ae0bdad0de7efa3eec6",
+	# 	# "fbf9d64ae0bdad0de7efa3eec6",
+	# 	"fe1331481f85560681f86827ec",  # urach
+	# 	# "fe1331481f85560681f86827ec"]
+	# 	"fec57041458e6cef98652df625",
+	# ]
+	# 	, settings)
+	# log_ids_gf = grep(["9d11b749c78a57e786bf5c8d28",  # filderstadt
+	# 				   "a192ff420b8bdd899fd28573e2",  # eichstätt
+	# 				   "3a3d994c04b1b1d87168422309",  # stadtökologie
+	# 				   "fe1331481f85560681f86827ec",  # urach
+	# 				   "96f6d9cc556b42f3b2fec0a2cb7ed36e"  # oberelsbach
+	# 				   ],
+	# 				  "/home/clemens/git/ma/test/src",
+	# 				  settings)
+	# log_ids = src_file("/home/clemens/git/ma/test/filtered_5_actions")
+
+	if False:
+		store: ResultStore = run_analysis(log_ids_gf, settings, LOADERS)
+	# store: ResultStore = run_analysis(log_ids, settings, LOADERS)
+
+	if False:
+		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
+			r().render(store.get_all())
+	if False:
+		render(analyzers.LocationAnalyzer, store.get_all())
+	# print(json.dumps(store.serializable(), indent=1))
+	if False:
+		for cat in store.get_categories():
+			render(analyzers.ActivityMapper, store.get_category(cat), name=cat)
+		# render(analyzers.ProgressAnalyzer, store.get_all())
+
+	if False:
+		from analysis.analyzers.postprocessing import graph
+
+		g = graph.Cache(settings)
+		g.run(store)
+	if False:
+		# render(analyzers.SimulationOrderAnalyzer, store.get_all())
+		for cat in store.get_categories():
+			data = store.get_category(cat)
+			render(analyzers.SimulationOrderAnalyzer, data, name=cat)
+	if False:
+		write_logentry_count_csv(LogEntryCountCSV, store, render, analyzers)
+	if False:
+		write_simulation_flag_csv(store)
+	if False:
+		time_distribution(store)
+
+	if False:
+		# spatial_data = get_data_distance(store,relative_values=False)
+		# temporal_data = get_data(store,relative_values=False)
+		# spatial_data_rel = get_data_distance(store,relative_values=True)
+		# temporal_data_rel = get_data(store,relative_values=True)
+		# temporal_data_rel = json.load(open("temporal_rel.json"))
+		# spatial_data_rel = json.load(open("spatial_rel.json"))
+		# import IPython
+		# IPython.embed()
+
+		# print(json.dumps(get_all_data(store)))
+		# json.dump(get_all_data(store), open("combined.json", "w"))
+		# combined = get_all_data(store, sort=True, relative=True)
+		# json.dump(combined, open("combined_rel.json", "w"))
+		# combined = json.load(open("combined_rel.json"))
+		combined = json.load(open("combined_total.json"))
+		# plot_time_space_rel(combined, keys)
+		plot_data(combined, wip.keys)
+
+	if True:
+		settings: LogSettings = load_settings("../oeb_kml.json")
+		log_ids = src_file("/home/clemens/git/ma/test/oeb_2016_path")
+		log_ids = log_ids[0:2]
+		print(log_ids)
+		store: ResultStore = run_analysis(log_ids, settings, LOADERS)
+		print("render")
+		kml = KMLRender()
+		kml.render(store.get_all())
+		print("done")
+		#for cat in store.get_categories():
+		#	render(analyzers.ActivityMapper, store.get_category(cat), name=cat)
+
+# for analyzers in analyzers:
+#	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:
+#		print(json.dumps(analyzers.result(), indent=2))
+
+# for analyzers in analyzers:
+#	if analyzers.name() in ["BoardDuration"]:
+#		print(json.dumps(analyzers.result(), indent=2))
+#		print(analyzers.render())
+
+# coords = analyzers[1].render()
+# with open("test.js", "w") as out:
+#	out.write("coords = "+coords)

analysis/util/download.py

@@ -0,0 +1,76 @@
+import logging
+import os
+
+from analysis.util import json_path
+
+logger = logging.getLogger(__name__)
+
+
+def download_board(board_id, instance_config_id, sequence_id, source, path="/data/results/"):
+	local_file = os.path.join("static", instance_config_id, sequence_id, board_id)
+	abs_path = os.path.join(path, local_file)
+	if os.path.exists(abs_path):
+		return local_file
+	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
+		config_id=instance_config_id,
+		sequence_id=sequence_id,
+		board_id=board_id
+	)
+	board = source.get(url)
+	if not board.ok:
+		raise ConnectionError(url, board, board.status_code)
+	data = board.json()
+	preview_url = json_path(data, "preview_url.medium")
+	logger.debug(preview_url)
+	os.makedirs(abs_path[:-len(board_id)], exist_ok=True)
+	source.download_file(preview_url, abs_path)
+	return local_file
+
+
+def get_config(source, instance_id):
+	url = "/game2/editor/config/{config_id}/".format(config_id=instance_id)
+	instance_data = get_json(source, url)
+	caches = url + "cache/"
+	cache_data = get_json(source, caches)
+
+	result = {
+		"name": instance_data["name"],
+		"id": instance_data["@id"],
+		"caches": cache_data
+	}
+	return result
+
+
+def get_board_data(source, instance_id, sequence_id, board_id):
+	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
+		config_id=instance_id,
+		sequence_id=sequence_id,
+		board_id=board_id
+	)
+	instance_data = get_json(source, url)
+	if instance_data is None:
+		return {"class": "error"}
+	result = {
+		"class": instance_data["@class"],
+	}
+	for i in ["image", "audio", "video"]:
+		key = i + "_file"
+		result["has_" + i] = bool(key in instance_data and instance_data[key])
+	return result
+
+
+cache = {}
+
+
+def get_json(source, url):
+	if url in cache:
+		return cache[url]
+	try:
+		data = source.get(url).json()
+
+	except Exception as e:
+		print("exception", e, e.args)
+		logger.exception(e)
+		data = None
+	cache[url] = data
+	return data

analysis/util/geo.py

@@ -0,0 +1,12 @@
+def calc_distance(geojson: str):
+	from shapely.geometry import LineString
+	from shapely.ops import transform
+	from functools import partial
+	import pyproj
+	import json
+	track = LineString(json.loads(geojson)['coordinates'])
+	project = partial(
+		pyproj.transform,
+		pyproj.Proj(init='EPSG:4326'),
+		pyproj.Proj(init='EPSG:32633'))
+	return transform(project, track).length

analysis/util/meta_temp.py

@@ -0,0 +1,116 @@
+CONFIG_NAMES = {
+	'06c627ac-09fb-4f03-9a70-49261adefed9': u'Kopie 2 - Filderstadt',
+	'07c3566b-371b-4627-9fa6-96fdcf421ad8': u'Stadt\xf6kologieLindlarAWG-V.1',
+	'08369b6c-f699-41ba-9313-9d6ea2d22f78': u'Schierke',
+	'0bd1bba5-cde8-40e5-ad1c-326e55bf1247': u'Simulation: Wildkatzen',
+	'0d2e711b-be77-46e1-92f5-73199626b68c': u'Kopie 2 - Simulation: Rinder',
+	'11b9793e-7b4f-41ec-98fc-e46de557ae07': u'Kopie 11 - Bad Urach',
+	'13241906-cdae-441a-aed0-d57ebeb37cac': u'Stadt\xf6kologie',
+	'14dee52a-d040-4c70-9e1f-359c7faadfab': u'Kopie 5 - Bad Urach',
+	'14e8f4be-d27e-43a4-95e1-e033950a99bd': u'Kopie 13 - Bad Urach',
+	'16fc3117-61db-4f50-b84f-81de6310206f': u'Oberelsbach',
+	'17926099-4ed3-4ca0-996d-cc577c6fdaed': u'Kopie 6 - Bad Urach',
+	'17d401a9-de21-49a2-95bc-7dafa53dda64': u'Oberelsbach 2016',
+	'1cae4e4c-6d8b-43f0-b17d-1034b213cbaf': u'Test-Born',
+	'1f56b428-7c2c-4333-8fe1-c740ccbff40f': u'Bad Gateway',
+	'1f8b9d55-3b95-4739-914a-e2eff2dc52c3': u'Kopie 2 - Bad Gateway',
+	'2b3975be-242a-4c9d-80c7-8d9a370c9fe0': u'Simulation: Rinder',
+	'2bdc24f5-c51d-41a3-9cbd-adfc3a77a5ce': u'Simulation: Landnutzung',
+	'2c7cdb5d-7012-4a06-b4c8-980ad2470f10': u'Kopie 3 - Bad Gateway',
+	'30b743e6-144c-4fd7-a055-e87680f74c27': u'Nakundu2go',
+	'31da50e3-d59f-4166-91f2-7c84454c5769': u'Kopie 4 - Taltitz',
+	'3269d0d4-dc13-46f7-b666-1db94350fcd4': u'simu rindfleisch',
+	'32ed9be7-6fc2-4c50-afdb-8f32453d8409': u'Kopie 1 - Eichstaett - Stadt\xf6kologie2',
+	'33024899-7c10-4d44-b950-38fd0c2b0e16': u'Kopie 3 - Stadt\xf6kologie',
+	'3fe38d6e-04d8-49e7-a6d9-7a5f12635369': u'Kopie 8 - Bad Urach',
+	'50716b57-e494-46e0-9449-919cecb02a3d': u'Kopie 2 - Lindlar',
+	'5436357d-9644-4e3d-a181-bb4c6c0b3055': u'Kopie 4 - Bad Urach',
+	'543ac9b8-e990-4540-8277-994c3c756f47': u'Kopie 4 - Lindlar',
+	'5544ec80-5928-41e1-ba89-c13e570bda88': u'Test - Nakundu2Go',
+	'5637a078-c931-458d-865d-adc5d0653147': u'Kopie 2 - Bad Gateway',
+	'57e079b1-0a58-4150-a358-627fc9e896cc': u'Kopie 1 - Schierke',
+	'5cb3318a-cb5f-412f-bfd6-6467101ed505': u'Eichst\xe4tt-Schafe-1',
+	'5e64ce07-1c16-4d50-ac4e-b3117847ea43': u'Filderstadt',
+	'60e77829-2686-4022-84e6-b9e8875f7ca0': u'Kopie 10 - Bad Urach',
+	'6140a24e-32c6-4872-92b8-c463468f79a2': u'Taltitz neu',
+	'63610965-7a82-471b-a11a-0f696b4d6996': u'Kopie 3 - Lindlar',
+	'6479e339-f70a-4ed7-9b9e-9884a8037d81': u'Kopie 5 - Lindlar',
+	'658e1856-d04a-4284-9fb3-95c8e89843d9': u'Simulation: Streuobst',
+	'66fd2366-5985-4cac-8777-51a83e169d93': u'Kopie 1 - Test - Garnison Stadt\xf6kologie',
+	'681b9c2a-2547-4ffd-b510-ef28f5a2d355': u'Kopie 6 - Bad Gateway',
+	'74f0bd8c-c53c-4293-b583-1d7aec98fafa': u'Simulation: Luchse',
+	'78a00aac-422c-4772-9327-3241b32cea03': u'Kopie 2 - Stadt\xf6kologie',
+	'7a056d76-5636-45cc-a0bf-0555eff6101c': u'Test - Osnabr\xfcck Stadt\xf6kologie',
+	'7bf1de94-2627-489b-a310-cbad568d2230': u'Kopie 2 - Taltitz',
+	'7ea9ff83-c015-4ede-a561-8a16a1fb0833': u'Kopie 1 - Stadt\xf6kologieLindlarAWG-V.1',
+	'81ebf491-a556-43a8-b5d7-48ee193e2636': u'Test - Oberelsbach Wildkatze',
+	'877a8c70-fe0c-464b-98c1-73f8669cabd6': u'Mittenwald',
+	'890e99b0-eeed-4a20-ac21-ea027daf16f3': u'Kopie 3 - Bad Urach',
+	'8aa01b71-2609-4a47-a14c-8c3b51905fd2': u'? (lb)',
+	'8c002b38-606b-45cd-b046-bc4641188e18': u'Kopie 7 - Bad Urach',
+	'8cf124c1-3041-4e9b-a35a-1d4e09694917': u'Kopie 1 - AAAAA Bad Gateway',
+	'8e13e952-180c-4498-8730-9691dc837515': u'Test_Eichst\xe4tt_Schafe',
+	'90278021-4c57-464e-90b1-d603799d07eb': u'Eichst\xe4tt',
+	'92fc4eef-1489-4b31-b9e1-9d9436f7f43e': u'Kopie 5 - Taltitz',
+	'98c6aed7-3632-467e-9f20-5bdc3276f616': u'Kopie 8 - Bad Gateway',
+	'995e06bf-abc4-4103-a572-9f096d71d192': u'Eichstaett - Stadt\xf6kologie',
+	'9cb95913-8245-49e6-8416-ee6635e67aab': u'Kopie 2 - Simulation: Landnutzung',
+	'9e819058-f7f9-459e-9947-84349c7d849c': u'Kopie 9 - Bad Urach',
+	'9f99c761-4fb6-4636-92da-a9627977d8b3': u'Simulation: Schafe',
+	'a5fa36f5-7531-4821-ba0e-cf8f2a502ad4': u'Garmisch',
+	'a79bb488-5fea-4bf9-9b25-395691c8e7cd': u'Kopie 1 - A kopietest docker',
+	'abdf9bd0-9b7e-4286-a25e-2cb14742db30': u'Test - Bad Urach Streuobst',
+	'ac0eb831-0f47-4cf1-a1a5-2e6a535e70e9': u'Kopie 1 - Vorlagen',
+	'ae726f64-cfa5-4f86-9538-a1e63dd914cf': u'AAAAA Bad Gateway',
+	'b307e954-5f0e-43bb-855f-d39c1a8858bd': u'Kopie 7 - Bad Gateway',
+	'b58ea5b3-b864-42e3-aaf4-1a76633c037e': u'Kopie 4 - Bad Gateway',
+	'b5c27cc6-e2bc-4288-be97-1e5bc1f6f94f': u'Kopie 1 - Simulation: Landnutzung',
+	'b623a3c8-7ff8-47b5-853e-7b08e200dd27': u'Taltitz',
+	'be1b2167-293c-4a4b-beda-28d6aa8b47a7': u'Kopie 1 - Stadt\xf6kologie berichtigt',
+	'bf7146bd-c83b-4497-b948-dd6dfc8607aa': u'Kopie 1 - Rinder Lindlar gps',
+	'c2c87501-f180-40de-af7b-1a3288c82292': u'Eichstaett - Stadt\xf6kologie2',
+	'c3016b66-3c26-4ec4-bcf1-d36be07be037': u'?? (lb)',
+	'c3598b20-e8a5-45eb-953a-2b474fd2695a': u'Test - Eichst\xe4tt Schafe',
+	'c39fe95e-2cfd-461b-8103-cfd3e2b45e67': u'??? (lb)',
+	'c46994cc-5ca7-4f9f-b548-7bd6a6fff026': u'Kopie 1 - Bad Gateway',
+	'c528e93f-3469-4de9-b05d-e809575d2999': u'????',
+	'c6606915-3b7e-48f4-adfe-cbcf3130e76a': u'Kopie 12 - Bad Urach',
+	'c8ed936c-25d6-40ea-a731-2741c1d53b48': u'Born',
+	'c9df06e1-33a7-40fc-bd3d-0eba6abba245': u'Test - Schierke Luchse',
+	'ce093332-dc98-4cfa-9ff4-47903897e84f': u'Kopie 5 - Bad Gateway',
+	'ceb78b48-495d-4761-bb61-563fa4dd41fb': u'Kopie 2 - Eichstaett - Stadt\xf6kologie2',
+	'd5712976-59fa-452c-a5e3-8b4232b5cb44': u'Kopie 1 - Garmisch Test',
+	'd7d0be7e-a0ac-4f4f-910d-c55e13a01e88': u'Test - Garmisch Rinder',
+	'db1cd7aa-878b-4c30-9234-0739498996d6': u'Bad Urach - \xfcberarbeitet',
+	'df68db4d-3d51-45f3-96ed-08429a7de3c9': u'A kopietest docker',
+	'e3b0ffce-6135-400e-9796-d1aef173aaf5': u'Kopie 3 - Taltitz',
+	'e3e86e25-4d92-11e6-b176-00199963ac6e': u'Garmisch (geschrumpft)',
+	'e7b3094d-d3c6-41c7-92e3-28638365e018': u'Born II',
+	'e7f36db3-b919-4208-8d98-b5d400b5d972': u'Kopie 15 - Bad Urach',
+	'e9f35c27-6f4f-487c-b07e-9d0ab27a7b85': u'Kopie 1 - Filderstadt',
+	'ea94b249-439b-46dd-b621-e0fbe99aa4ee': u'Stadt\xf6kologie berichtigt',
+	'ec782ab1-eb9d-43b9-a2d1-4699f8432adb': u'Kopie 2 - Bad Urach',
+	'ecfdfd0b-28be-4df2-8994-8092a7fe87b5': u'Kopie 3 - Simulation: Landnutzung',
+	'f7bb56a3-fb15-413a-9a3e-f61e22d0a7d1': u'Kopie 2 - Schierke',
+	'f8f65e9d-de9e-4f8d-8bb6-d1e4e01593a0': u'Arbeitstitel',
+	'fca28f01-ea17-4c41-8e60-4726f96dfca8': u'Kopie 1 - Test-Born',
+	'fe43a0f0-3dea-11e6-a065-00199963ac6e': u'Vorlagen',
+	'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771': u'Bad Urach'
+}
+
+
+KML_PATTERN="""<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2" xmlns:gx="http://www.google.com/kml/ext/2.2">
+  <Document>
+    <Placemark>
+    	<gx:MultiTrack>
+    		<gx:Track>
+    			{when}
+    			{coordinates}
+    		</gx:Track>
+    	</gx:MultiTrack>
+      {coordinates}
+    </Placemark>
+  </Document>
+</kml>
+"""

analysis/util/processing.py

@@ -0,0 +1,67 @@
+import logging
+from typing import List
+
+from analysis.analyzers.analyzer import ResultStore, Analyzer
+from analysis.analyzers.settings import LogSettings
+
+log: logging.Logger = logging.getLogger(__name__)
+
+
+def process_log(logfile: str, settings: LogSettings, loaders) -> List[Analyzer]:
+	loader = loaders[settings.log_format]()
+	try:
+		loader.load(logfile)
+	except BaseException as e:
+		raise RuntimeError(e)
+	analyzers: List[Analyzer] = []
+	log.debug("build analyzers")
+	for analyzer in settings.analyzers:
+		analyzers.append(analyzer(settings))
+	log.debug("process entries")
+	for entry in loader.get_entry():
+		for analyzer in analyzers:
+			try:
+				if analyzer.process(entry):
+					break
+			except KeyError as e:
+				log.exception(e)
+	return analyzers
+
+
+def run_analysis(log_ids: list, settings, loaders):
+	store: ResultStore = ResultStore()
+	for log_id in log_ids:
+		log.info("LOG_ID: "+ str(log_id))
+		for analysis in process_log(log_id, settings, loaders):
+			log.info("* Result for " + analysis.name())
+			analysis.result(store, name=log_id)
+	return store
+
+
+def load_ids(name: str):
+	log_ids = []
+	with open(name) as src:
+		for line in src:
+			line = line.strip()
+			log_ids.append(line)
+	return log_ids
+
+
+def grep(log_ids, source, settings):
+	logs = []
+	with open(source) as src:
+		lines = src.readlines()
+		for id in log_ids:
+			for line in lines:
+				if id in line:
+					logs.append(line.strip())
+	return logs
+
+
+def src_file(filename):
+	log_ids = []
+	with open(filename) as src:
+		for line in src:
+			line = line.strip()
+			log_ids.append(line)
+	return log_ids

analyzers/render/biogames.py

@@ -1,104 +0,0 @@
-import json
-from collections import defaultdict
-from typing import List, Tuple
-
-import matplotlib.pyplot as plt
-import os
-
-from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
-from . import Render
-from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
-
-
-def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
-         rotation='vertical'):
-	names, datas = list(zip(*src_data))
-	plt.boxplot(datas, labels=names)
-	plt.xticks(rotation=rotation)
-	# plt.margins()
-	plt.ylabel(ylabel)
-	plt.title(title)
-	plt.show()
-
-
-class SimulationRoundsRender(Render):
-	def render(self, results: List[Result]):
-		data = defaultdict(list)
-		for result in self.filter(results):
-			get = result.get()
-			for key in get:
-				data[key].append(get[key])
-		data_tuples = [(key, data[key]) for key in sorted(data)]
-		data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
-		plot(data_tuples)
-
-	result_types = [SimulationRoundsAnalyzer]
-
-
-class BoardDurationHistRender(Render):
-	result_types = [BoardDurationAnalyzer]
-
-	def render(self, results: List[Result]):
-		data = []
-		for result in self.filter(results):
-			session = result.get()
-			_data = []
-			for board in session:
-				if "active" in board:
-					_data.append(board["active"])
-				else:
-					_data.append(0)
-			data.append(_data)
-		n, bins, patches = plt.hist(data, log=True)
-		plt.show()
-
-
-class BoardDurationBoxRender(Render):
-	result_types = [BoardDurationAnalyzer]
-
-	def render(self, results: List[Result]):
-		data = defaultdict(list)
-		for result in self.filter(results):
-			get = result.get()
-			for board in get:
-				duration = board['active'] if 'active' in board else 0
-				data[board['id']].append(duration)
-		data_tuples = [(key, data[key]) for key in sorted(data)]
-		data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
-		plot(data_tuples)
-
-
-class ActivityMapperRender(Render):
-	result_types = [ActivityMapper]
-
-	def render(self, results: List[Result]):
-		print(os.getcwd())
-		for result in self.filter(results):
-			data = result.get()
-			with open(os.path.join("static", "progress", "data", data['instance']),"w") as out:
-				json.dump(data["store"], out, indent=1)
-		return "ok"
-
-
-class StoreRender(Render):
-	result_types = [Store, BiogamesStore]
-
-	def render(self, results: List[Result]):
-		for result in self.filter(results):
-			with open(os.path.join("static","progress","data","fooo"), "w") as out:
-				json.dump(result.get(), out, indent=1)
-
-
-class SimulationOrderRender(Render):
-	def render(self, results: List[Result]):
-		data = defaultdict(list)
-		for result in self.filter(results):
-			get = result.get()
-			for i,value in enumerate(get):
-				data[i].append(value)
-		#data_tuples = [(key, data[key]) for key in sorted(data)]
-		#data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
-		#plot(enumerate([r.get() for r in self.filter(results)]))
-		plot(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
-
-	result_types = [SimulationOrderAnalyzer]

analyzers/render/default.py

@@ -1,49 +0,0 @@
-import json
-import logging
-from typing import List
-
-from . import Render, Result
-from .. import LocationAnalyzer
-
-log = logging.getLogger(__name__)
-
-
-class PrintRender(Render):
-	def render(self, results: List[Result]):
-		print("\t" + "\n\t".join([str(r) for r in results]))
-
-
-class JSONRender(Render):
-	def render(self, results: List[Result]):
-		print(json.dumps([r.get() for r in self.filter(results)], indent=1))
-
-
-class TrackRender(Render):
-	result_types = [LocationAnalyzer]
-
-	def render(self, results: List[Result]):
-		data = []
-		log.debug(results)
-		for result in self.filter(results):
-			if len(result.get()) > 0:
-				data.append(
-					[[entry['location']['coordinates'][1], entry['location']['coordinates'][0]] for entry in  # TODO: configurable
-					 result.get()])
-		dumps = json.dumps(data)
-		with open("track_data.js", "w") as out:
-			out.write("tracks=" + dumps + ";")
-		return dumps
-
-
-class HeatMapRender(TrackRender):
-	weight = 0.01
-
-	def render(self, results: List[Result]):
-		raw = super(HeatMapRender, self).render(results)
-		data = []
-		for session in json.loads(raw):
-			data += [(entry[0], entry[1], self.weight) for entry in session]
-		dumps = json.dumps(data)
-		with open('heat_data.js', 'w') as out:
-			out.write("coords = " + dumps + ";")
-		return dumps

clients/webclients.py

@@ -0,0 +1,154 @@
+import json
+import logging
+
+import os
+import shutil
+import tempfile
+import typing
+
+import requests
+
+log: logging.Logger = logging.getLogger(__name__)
+
+
+class Client:
+	host: str = ""
+	cookies: typing.Dict[str, str] = {}
+	headers: typing.Dict[str, str] = {}
+
+	def url(self, path):
+		if self.host:
+			return self.host + path
+		return path
+
+	def get(self, url, **kwargs) -> requests.models.Response:
+		log.info("GET " + str(url))
+		return requests.get(self.url(url), cookies=self.cookies, headers=self.headers, **kwargs)
+
+	def post(self, url, data, **kwargs) -> requests.models.Response:
+		log.info("POST " + str(url))
+		return requests.post(self.url(url), data, cookies=self.cookies, headers=self.headers, **kwargs)
+
+	def download_file(self, url, target, **kwargs) -> bool:
+		with open(target, "wb") as out:
+			try:
+				download = self.get(url, stream=True, **kwargs)
+				shutil.copyfileobj(download.raw, out)
+			except Exception as e:
+				log.exception(e)
+				os.remove(target)
+				return False
+		return True
+
+	def download_files(self, urls, **kwargs) -> tempfile.TemporaryDirectory:
+		target = tempfile.TemporaryDirectory()
+		for path in urls:
+			filename = os.path.join(target.name, path.split("/")[-1])
+			self.download_file(path, filename, **kwargs)
+		return target
+
+	def login(self):
+		pass #TODO
+
+	def list(self):
+		pass #TODO
+
+
+class BiogamesClient(Client):
+	config_fields: typing.Dict[str, typing.List[str]] = {
+		'login': ('username', 'password', 'host'),
+		'session': ('sessionid', 'csrftoken', 'host'),
+	}
+	login_url: str = "/game2/auth/json-login"
+	list_url: str = "/game2/instance/log/list/"
+	headers: typing.Dict[str, str] = {'Accept': 'application/json'}
+
+	def __init__(self, **kwargs):
+		match = {j: all([i in kwargs for i in self.config_fields[j]]) for j in self.config_fields}
+		valid = filter(lambda x: match[x], match)
+		if not valid:
+			raise ValueError("missing parameter (" + str(self.config_fields) + ")")
+		self.config = kwargs
+		self.cookies = {}
+		self.host = self.config['host']
+		if 'session' in valid:
+			self.cookies = kwargs
+
+	def login(self) -> bool:
+		csrf_request = self.get(self.list_url)
+		if not csrf_request.ok:
+			log.exception(ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")"))
+			return False
+		if not 'csrftoken' in self.cookies:
+			self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
+		login_payload = {
+			'username': self.config['username'],
+			'password': self.config['password'],
+			'next': '',
+			'csrfmiddlewaretoken': 'csrftoken',
+		}
+		login = self.post(self.login_url, json.dumps(login_payload))
+		if not login.ok:
+			log.exception(ConnectionError("Unable to authenticate", login, login.text))
+			return False
+		self.cookies['sessionid'] = login.cookies['sessionid']
+		print(self.cookies)
+		return True
+
+	def list(self) -> dict:
+		print(self.cookies)
+		logs = self.get(self.list_url)
+		if not logs.ok:
+			raise ConnectionError("HTTP fail", logs, logs.text)
+		return logs.json()
+
+	def load_all_logs(self) -> tempfile.TemporaryDirectory:
+		return self.download_files([i["file_url"] for i in self.list()])
+
+class GeogamesClient(Client):
+	config_fields = ("host", "path")
+	
+	def __init__(self, **kwargs):
+		for field in self.config_fields:
+			if not field in kwargs:
+				raise ValueError(f"missing parameter: {field}")
+		self.host = kwargs['host']
+		self.path = kwargs['path']
+	
+	def list(self):
+		logs = self.get(self.path)
+		data = logs.json()
+		prepared_logs = []
+		for log in data:
+			players = self.get(f"{self.path}/{log['name']}/").json()
+			for player in players:
+				prepared_logs.append({
+					'@id': f"{log['name']}/{player['name']}",
+					'start_date': player['mtime'],
+					'player_group_name': player['name'],
+					'file_url': f"{self.path}/{log['name']}/{player['name']}",
+				})
+		return prepared_logs
+
+	def download_files(self, urls, **kwargs) -> tempfile.TemporaryDirectory:
+		target = tempfile.TemporaryDirectory()
+		for path in urls:
+			filename = os.path.join(target.name, "-".join
+(path.split("/")[-2:]))
+			self.download_file(path, filename, **kwargs)
+		return target
+
+CLIENTS: typing.Dict[str, typing.Type[Client]] = {
+	"Biogames": BiogamesClient,
+	"Geogames": GeogamesClient,
+}
+
+if __name__ == '__main__':
+	# c = BiogamesClient(host="http://biodiv", username="ba", password="853451")
+	# print(c.login())
+	# print(json.dumps(c.list(), indent=1))
+	# print(type(c.load_all_logs()))
+	# print(type(c.get("/")))
+	c = BiogamesClient(host="http://biodiv", **{'csrftoken': 'IgbwP83iEibW6RE7IADIFELYdbx0dvqQ',
+												'sessionid': 'zntsj09d92tjos1b6ruqjthlzv60xdin'})
+	print(json.dumps(c.list(), indent=1))

docker-compose.yml

@@ -0,0 +1,61 @@
+version: "2.2"
+
+services:
+ app:
+  image: docker.clkl.de/ma/celery:0.4.1
+  build: .
+  volumes:
+    - ./:/app
+  working_dir: /app/selector
+  command: python3 webserver.py
+  environment:
+    - PYTHONPATH=/app
+  networks:
+   - default
+   - traefik_net
+  labels:
+   - "traefik.enable=true"
+   - "traefik.port=5000"
+   - "traefik.docker.network=traefik_net"
+   - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de"
+
+ celery:
+  image: docker.clkl.de/ma/celery:0.4.1
+  environment:
+    - PYTHONPATH=/app
+  volumes:
+    - ./:/app
+    - ./data/results:/data/results
+  working_dir: /app
+  command: celery -A tasks.tasks worker --loglevel=info
+
+
+ redis:
+  image: redis:4-alpine
+  volumes:
+    - ./data/redis:/data
+  command: redis-server --appendonly yes
+
+ nginx:
+  image: nginx:1.13-alpine
+  volumes:
+   - ./data/results:/usr/share/nginx/html:ro
+  networks:
+   - traefik_net
+  labels:
+   - "traefik.enable=true"
+   - "traefik.port=80"
+   - "traefik.docker.network=traefik_net"
+   - "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de"
+
+ log_data:
+  image: nginx:1.13-alpine
+  volumes:
+   - ./log_data/:/srv/:ro
+   - ./log_data.conf:/etc/nginx/conf.d/log_data.conf
+
+
+networks:
+ traefik_net:
+  external:
+   name: traefik_net

log_analyzer.py

@@ -1,92 +0,0 @@
-import json
-import logging
-from typing import List
-
-import analyzers
-from analyzers import get_renderer, Analyzer, render, Store
-from analyzers.analyzer import ResultStore
-from analyzers.settings import LogSettings, load_settings
-from loaders import LOADERS
-
-logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG)
-log: logging.Logger = logging.getLogger(__name__)
-
-requests_log = logging.getLogger('requests')
-requests_log.setLevel(logging.WARN)
-
-
-def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]:
-	logfile: str = "data/inst_{id}.{format}".format(id=log_id, format=settings.log_format)
-	loader = LOADERS[settings.log_format]()
-	try:
-		loader.load(logfile)
-	except BaseException as e:
-		raise RuntimeError(e)
-	analyzers: List[Analyzer] = []
-	log.debug("build analyzers")
-	for analyzer in settings.analyzers:
-		analyzers.append(analyzer(settings))
-	log.debug("process entries")
-	for entry in loader.get_entry():
-		for analyzer in analyzers:
-			if analyzer.process(entry):
-				break
-	return analyzers
-
-
-if __name__ == '__main__':
-	settings: LogSettings = load_settings("biogames2.json")
-	log_ids: List[str] = [
-		"20d4244719404ffab0ca386c76e4b112",
-		"56d9b64144ab44e7b90bf766f3be32e3",
-		"dc2cdc28ca074715b905e4aa5badff10",
-		"e32b16998440475b994ab46d481d3e0c",
-	]
-	log_ids: List[str] = [
-		#"34fecf49dbaca3401d745fb467",
-		#	"44ea194de594cd8d63ac0314be",
-		#	"57c444470dbf88605433ca935c",
-		#	"78e0c545b594e82edfad55bd7f",
-		#	"91abfd4b31a5562b1c66be37d9",
-			"597b704fe9ace475316c345903",
-			"e01a684aa29dff9ddd9705edf8",
-			"fbf9d64ae0bdad0de7efa3eec6",
-		#	"fe1331481f85560681f86827ec",
-			"fe1331481f85560681f86827ec"]
-		#"fec57041458e6cef98652df625", ]
-	store: ResultStore = ResultStore()
-	for log_id in log_ids:
-		for analysis in process_log(log_id, settings):
-			log.info("* Result for " + analysis.name())
-			# print(analysis.result())
-			# print(analysis.render())
-			analysis.result(store)
-	if False:
-		for r in get_renderer(analyzers.LocomotionActionAnalyzer):
-			r().render(store.get_all())
-	if False:
-		render(analyzers.LocationAnalyzer, store.get_all())
-	#print(json.dumps(store.serializable(), indent=1))
-	if False:
-		render(analyzers.ActivityMapper, store.get_all())
-		render(analyzers.ProgressAnalyzer, store.get_all())
-
-	if False:
-		from analyzers.postprocessing import graph
-		g = graph.Cache(settings)
-		g.run(store)
-	if True:
-		render(analyzers.SimulationOrderAnalyzer, store.get_all())
-
-# for analyzers in analyzers:
-#	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:
-#		print(json.dumps(analyzers.result(), indent=2))
-
-# for analyzers in analyzers:
-#	if analyzers.name() in ["BoardDuration"]:
-#		print(json.dumps(analyzers.result(), indent=2))
-#		print(analyzers.render())
-
-# coords = analyzers[1].render()
-# with open("test.js", "w") as out:
-#	out.write("coords = "+coords)

log_data.conf

@@ -0,0 +1,9 @@
+server {
+	listen 80;
+	server_name log_data;
+	location / {
+		root /srv/;
+		autoindex on;
+		autoindex_format json;
+	}
+}

neocart.json

@@ -0,0 +1,34 @@
+{
+  "logFormat": "neocartographer",
+  "entryType": "type",
+  "spatials": [
+    "location"
+  ],
+  "actions": [],
+  "boards": [],
+  "analyzers": {
+    "analysis.analyzers": [
+      "SimpleCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {},
+  "custom": {
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Geogames",
+    "host": "http://log_data/",
+    "path": "neocartographer"
+  },
+  "render": [
+    "KMLRender"
+  ]
+}
+
+

requirements.txt

@@ -1,6 +1,15 @@
 requests==2.18.4
-numpy==1.13.1
+numpy==1.14.2
 matplotlib==2.1.0
-osmnx==0.6
+#osmnx==0.6
 networkx==2.0
-pydot==1.2.3
+#pydot==1.2.3
+scipy==1.0.1
+#ipython==6.2.1
+
+flask==0.12.2
+
+celery==4.1.1
+redis==2.10.6
+
+lxml==4.2.1

selector/static/script.js

@@ -0,0 +1,4 @@
+function validateSettings() {
+    alert(document.getElementById('safety').checked);
+    return false;
+}

selector/static/style.css

@@ -0,0 +1,9 @@
+body {
+    background-color: aqua;
+}
+#data{
+    display: none;
+}
+li{
+    list-style-type: none;
+}

selector/temp_config.py

@@ -0,0 +1,133 @@
+KML = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  },
+  "render": [
+    "KMLRender"
+  ]
+}"""
+
+ACTIVITY = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "ActivityMapper"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  },
+  "render": [
+    "ActivityMapper"
+  ]
+}"""
+
+CONFIGS = {  # TODO
+	"KML": KML,
+	"ActivityMapper": ACTIVITY,
+}
+
+URLS = {
+	"KML": "/",
+	"ActivityMapper": "#",
+}

selector/templates/base.html

@@ -0,0 +1,5 @@
+<!doctype html>
+<title>…</title>
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+<script type="application/javascript" src="{{url_for('static', filename='script.js') }}"></script>
+{% block body %} {% endblock %}

selector/templates/games.html

@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+{% block body %}
+<form action="/start" method="post">
+    <div id="data"> {{logs}}</div>
+    <ul>
+        {% for log in logs %}
+        <li>
+            <input type="checkbox" name="logs" value="{{log['@id']}}">
+            {{log.start_date}}: {{log.player_group_name}}
+        </li>
+        <!--{{log}}-->
+        {% endfor %}
+    </ul>
+    <!--input type="checkbox" id="safety"><label for="safety">Confirm selection</label-->
+    <input type="text" id="name" maxlength="128" placeholder="name" name="name"/><br>
+    <select name="config">
+        {% for config in configs %}
+        <option>{{config}}</option>
+        {% endfor %}
+    </select>
+    <input type="submit">
+
+</form>
+<a href="/results">show analysis progress/results</a>
+{% endblock %}

selector/templates/index.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block body %}
+<form action="/login" method="post">
+    <select name="game">
+        {% for game in clients %}
+        <option>{{ game }}</option>
+        {% endfor %}
+    </select>
+    <input type="text" name="username" placeholder="username"/>
+    <input type="password" name="password" placeholder="passwort"/>
+    <input type="submit">
+</form>
+
+{% endblock %}

selector/templates/results.html

@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{% block body %}
+
+<a href="/games">create new analysis</a>
+
+<div id="results">
+    <ul>
+        {% for job in jobs %}
+        <li> {{jobs[job].status}}: "{{job}}":
+            <ul>
+                {% for r in jobs[job].results %}
+                <li><a href="{{jobs[job] | get_prefix}}{{r | get_name}}">{{r|get_name}} {{jobs[job].start}}</a></li>
+                {% endfor %}
+            </ul>
+
+
+        </li>
+        {% endfor %}
+    </ul>
+</div>
+
+{% endblock %}

selector/webserver.py

@@ -0,0 +1,124 @@
+import json
+import logging
+import typing
+import uuid
+
+import time
+
+from clients.webclients import Client, CLIENTS
+
+from flask import Flask, render_template, request, redirect, session
+
+from tasks import tasks
+from selector.temp_config import CONFIGS, URLS
+
+BIOGAMES_HOST = "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+#BIOGAMES_HOST = "http://www.biodiv2go.de"
+RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/"
+
+app = Flask(__name__)
+clients: typing.Dict[str, Client] = {}
+log: logging.Logger = logging.getLogger(__name__)
+
+
+@app.route("/")
+def index():
+	return render_template("index.html", clients=CLIENTS)
+
+
+@app.route("/login", methods=["POST"])
+def login():
+	game = request.form["game"]
+	if not game in CLIENTS:
+		return redirect("/?invalid_game")
+	client = CLIENTS[game](host=BIOGAMES_HOST, username=request.form['username'], password=request.form['password'])
+	if client.login():
+		session['logged_in'] = True
+		session['uid'] = str(uuid.uuid4())
+		session['username'] = request.form['username']
+		session['cookies'] = client.cookies
+		session['game'] = game
+		session['host'] = BIOGAMES_HOST
+		clients[session['uid']] = client
+		return redirect("/results")
+	return redirect("/?fail")
+
+
+@app.route("/results")
+def results():
+	if not ('logged_in' in session and session['logged_in']):
+		return redirect("/")
+	if session['logged_in'] and not session['uid'] in clients:
+		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
+	status = tasks.redis.get(session['username'])
+	if status:
+		job_status = json.loads(status)
+	else:
+		job_status = {}
+	#for job in job_status:
+	#	results = []
+	#	for path in job_status[job]['results']:
+	#		results.append(path.replace(tasks.DATA_PATH, RESULT_HOST))
+	#	print(results) #TODO???
+	return render_template("results.html",  jobs=job_status)
+
+
+@app.route("/games")
+def games():
+	if not ('logged_in' in session and session['logged_in']):
+		return redirect("/")
+	if session['logged_in'] and not session['uid'] in clients:
+		clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies'])
+	return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS)
+
+
+@app.route("/start", methods=['POST'])
+def start():
+	print(str(request.form['logs']))
+	status = {
+		"status": "PENDING",
+		"submit": time.strftime("%c"),
+		"log_ids": request.form.getlist('logs'),
+		"config": request.form['config'],
+	}
+	params = {
+		"log_ids": request.form.getlist('logs'),
+		"config": CONFIGS[request.form['config']],
+		"username": session['username'],
+		"cookies": session['cookies'],
+		"host": session['host'],
+		"clientName": session['game'],
+		"name": request.form['name'],
+	}
+	tasks.status_update(session['username'], request.form['name'], status)
+	tasks.analyze.delay(**params)
+	return redirect("/results")
+
+
+@app.route("/status")
+def status():
+	return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2)
+
+
+@app.template_filter('get_url')
+def get_url(path: str):
+	return path.replace(tasks.DATA_PATH, RESULT_HOST)
+
+@app.template_filter('get_name')
+def get_url(path: str):
+	return path.replace(tasks.DATA_PATH, "")
+
+
+@app.template_filter('get_prefix')
+def get_prefix(job):
+	print(job)
+	try:
+		return RESULT_HOST + URLS[job['config']]
+	except:
+		return RESULT_HOST + "#"
+
+
+
+if __name__ == '__main__':
+	app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"})
+	app.run(host="0.0.0.0", debug=True)

sources/__init__.py

@@ -1,5 +0,0 @@
-from .biogames import Biogames
-
-SOURCES = {
-	"Biogames": Biogames,
-}

sources/biogames.py

@@ -1,82 +0,0 @@
-import json
-import logging
-import typing
-from tempfile import TemporaryDirectory
-
-import os
-
-from sources.source import Source
-
-import shutil
-import requests
-
-log: logging.Logger = logging.getLogger(__name__)
-
-
-class Biogames(Source):
-	def __init__(self):
-		self.headers: typing.Dict[str, str] = {'Accept': 'application/json'}
-		self.cookies: typing.Dict[str, str] = {}
-		self.id2link: typing.Dict[str, str] = {}
-		self.host: str = None
-
-	def connect(self, **kwargs):
-		for i in ['username', 'password', 'url', 'login_url', 'host']:
-			if not i in kwargs:
-				raise ValueError("missing value " + i)
-		csrf_request = requests.get(kwargs['url'])
-		if csrf_request.status_code != 200:
-			raise ConnectionError("unable to obtain CSRF token (" + str(csrf_request) + ")")
-		self.cookies['csrftoken'] = csrf_request.cookies['csrftoken']
-		log.info("obtained CSRF token (" + self.cookies['csrftoken'] + ")")
-		login_payload = {
-			'username': kwargs['username'],
-			'password': kwargs['password'],
-			'next': '',
-			'csrfmiddlewaretoken': 'csrftoken'
-		}
-		login = requests.post(kwargs['login_url'], data=json.dumps(login_payload), cookies=self.cookies)
-		if login.status_code != 200:
-			raise ConnectionError("Unable to authenticate!", login, login.text)
-		self.cookies['sessionid'] = login.cookies['sessionid']
-		log.info("obtained sessionid (" + self.cookies['sessionid'] + ")")
-		self.url = kwargs['url']
-		self.host = kwargs['host']
-		log.info("stored url (" + self.url + ")")
-
-	def list(self):
-		logs = self.get_json(self.url)
-		log.info(len(logs))
-		for i in logs:
-			self.id2link[i["id"]] = i["link"]  # TODO
-		return logs
-
-	def get(self, ids: typing.Collection):
-		dir = TemporaryDirectory()
-		files = []
-		for i in ids:
-			url = self.id2link[i]
-			filename = os.path.join(dir.name, url.split("/")[-1])
-			file = self.download_file(url, filename)
-			if file:
-				files.append(file)
-		return dir
-
-	def download_file(self, url, filename):
-		with open(filename, "wb") as out:
-			try:
-				download = self._get(url)
-				shutil.copyfileobj(download.raw, out)
-				return filename
-			except Exception as e:
-				log.exception(e)
-				os.remove(filename)
-
-	def get_json(self, url):
-		return self._get(url, stream=False).json()
-
-	def close(self):
-		pass
-
-	def _get(self, url, stream=True):
-		return requests.get(self.host + url, cookies=self.cookies, headers=self.headers, stream=stream)

sources/source.py

@@ -1,18 +0,0 @@
-import typing
-
-
-class Source:
-	def connect(self, **kwargs):
-		raise NotImplementedError
-
-	def list(self):
-		raise NotImplementedError
-
-	def get(self, ids: typing.Collection):
-		raise NotImplementedError
-
-	def get_json(self, url:str) -> dict:
-		raise NotImplementedError
-
-	def close(self):
-		raise NotImplementedError

static/progress/index.html

@@ -9,7 +9,17 @@
         integrity="sha512-lInM/apFSqyy1o6s89K4iQUKg6ppXEgsVxT35HbzUupEVRh2Eu9Wdl4tHj7dZO0s1uvplcYGmt3498TtHq+log=="
         crossorigin=""></script>
 
+<script src="https://rawgit.com/Leaflet/Leaflet.heat/gh-pages/dist/leaflet-heat.js"></script>
+
 <script src="my.js"></script>
-    <style>
+<link href="style.css" rel="stylesheet"/>
-       .map { width: 512px; height: 256px; }
+<main>
-    </style>
+    <div class="mapDiv" id="mainMap"></div>
+    <div class="sequenceContainer">
+        <div class="sequence"></div>
+    </div>
+</main>
+
+<!--div style="font-size:0.1px;position:absolute;bottom:0;">OSM Logo: CC-BY-SA
+    http://wiki.openstreetmap.org/wiki/File:Mag_map-120x120.png
+</div-->

static/progress/my.js

@@ -1,71 +1,103 @@
-$.getJSON("tmp3.json", function (data) {
+//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) {    //urach
-    var list = $("<ul />");
+//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-68854ffab9aa.json", function (data) {  //urach
-    var maps = {};
+//$.getJSON("data/90278021-4c57-464e-90b1-d603799d07eb_07da99c9-398a-424f-99fc-2701763a63e9.json", function (data) {  //eichstätt
-    $.each(data, function (key, value) {
+//$.getJSON("data/13241906-cdae-441a-aed0-d57ebeb37cac_d33976a6-8a56-4a63-b492-fe5427dbf377.json", function (data) {  //stadtökologie
-        //key: instance_id, value: AnlysisResult
+$.getJSON("data/5e64ce07-1c16-4d50-ac4e-b3117847ea43_2f664d7b-f0d8-42f5-8731-c034ef86703e.json", function (data) {  //filderstadt
-        //value.result.instance: InstanceConfig_id
+//$.getJSON("data/17d401a9-de21-49a2-95bc-7dafa53dda64_98edcb70-03db-4465-b185-a9c9574995ce.json", function (data) {  //oeb2016
-        // console.log(key, value[0].result.store[0].timestamp);
+    var images = {};
-        $.each(value[0].result.store, function (index, entry) {
+    var tiles = {
-            //console.log(entry);
+        "openstreetmap": L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
-            var time = new Date(entry.timestamp);
+            maxNativeZoom: 19,
-            var item = $("<li>", {html: entry.sequence + " @ " + time.toLocaleDateString() + " "+ time.toLocaleTimeString()});
+            maxZoom: 24,
-            var container = $("<p />");
+            attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a>'
-            if (entry.track.length > 0) {
+        }),
-                var mapName = "map" + index;
+        "esri sat": L.tileLayer('https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}', {
-                //console.log(mapName, entry.track.length);
+            maxNativeZoom: 19,
-                var mapContainer = $("<div />", {id: mapName, class: "map"});
+            maxZoom: 24,
-                var track = [];
+            attribution: 'Tiles &copy; Esri &mdash; Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community'
-                $.each(entry.track, function (i, elem) {
+        }),
-                    track.push([elem.coordinates[1], elem.coordinates[0]]);
+        "google sat": L.tileLayer('https://{s}.google.com/vt/lyrs=s&x={x}&y={y}&z={z}', {
-                });
+            maxNativeZoom: 20,
-                maps[mapName] = track;
+            maxZoom: 24,
+            subdomains: ['mt0', 'mt1', 'mt2', 'mt3']
+        })
+    };
+    var map = L.map("mainMap", {layers: [tiles.openstreetmap]});
 
-                /* mapContainer.ready(function () {
+    function styleTrack(feature) {
-                     var map = L.map(mapName, {maxZoom: 22});
+        var styles = {};
-                     L.control.scale().addTo(map);
+        styles.color = data.colors[feature.properties.activity_type];
-                     var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
+        return styles;
-                         attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
+    }
+
+    var highlighted = null;
+
+    function onClick(e) {
+        var start = e.target.feature.geometry.properties.start_timestamp;
+        var end = e.target.feature.geometry.properties.end_timestamp;
+        var changed = highlighted !== e.target.feature;
+        $.each(images, function (timestamp, board) {
+                if ((timestamp >= start && timestamp < end) && changed) {
+                    board.image.first().addClass("highlight");
+                } else {
+                    board.image.removeClass("highlight");
+                    highlighted = null;
+                }
+            }
+        );
+        if (changed) {
+            highlighted = e.target.feature;
+        }
+    }
+
+    var coords = [];
+
+    function onEachFeature(feature, layer) {
+        layer.setStyle(styleTrack(feature));
+        layer.on('click', onClick);
+        if (feature.coordinates.length > 1) {
+            coords = coords.concat(feature.coordinates.map(function (p) {
+                return [p[1], p[0], 0.1];
+            }));
+        }
+    }
+
+    var track = L.geoJSON(data['track'], {
+        //style: styleTrack,
+        onEachFeature: onEachFeature
     }).addTo(map);
-                     var track = [];
-                     $.each(entry.track, function (i, elem) {
-                         track.push([elem.coordinates[1], elem.coordinates[0]]);
-                     });
-                     var layer = L.polyline(track, {color: "green"});
-                     console.log(track);
-                     L.control.layers(null, [layer]).addTo(map);
-                 });*/
 
-                mapContainer.appendTo(container);
+    map.fitBounds(track.getBounds());
-            }
+
-            $.each(entry.events, function (i, event) {
+    var heat = L.heatLayer(coords);
-                if ("image" in event) {
+    L.control.layers(tiles, {"heatmap": heat}).addTo(map);
-                    $("<img />", {src: event.image, height: 200}).appendTo(container);
+
-                }
+    var list = $("<ul />");
+    var current = {
+        "pos":data["boards"][1].coordinate.coordinates
+    };
+    console.log(current);
+    var marker = L.marker([current.pos[1], current.pos[0]]).addTo(map);
+    $.each(data["boards"], function (index, entry) {
+        //console.log(index, entry);
+        var item = $("<li>", {class: entry.extra_data.activity_type});
+        var container = $("<div>", {class: "board"});
+        var image = $("<img>", {src: entry.image.replace("static/progress/", "")});
+        image.attr("data-time", entry.timestamp);
+        image.hover(function () {
+            marker.setLatLng([entry.coordinate.coordinates[1], entry.coordinate.coordinates[0]]);
+        }, function () {
+            marker.setLatLng(current.pos.coordinates[1], current.pos.coordinates[0]);
         });
+        image.click(function (e) {
+            current.board = image;
+            current.pos = entry.coordinate;
+        });
+        images[entry.timestamp] = {image: image, coordinate: entry.coordinate};
+        image.appendTo(container);
         container.appendTo(item);
         item.appendTo(list);
     });
-    });
+    current.board=images[data["boards"][1].timestamp];
-    list.appendTo("body");
+    list.appendTo(".sequence");
-    var slider = $("<input />", {type: "range" })
-    /*});
-
-    $(window).on("load", function () {*/
-    // setTimeout(function () {
-
-    //console.log(maps);
-    $.each(maps, function (mapName, track) {
-        //console.log("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa");
-        var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
-            attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
-        });
-        var map = L.map(mapName, {layers: [tiles]});
-        L.control.scale().addTo(map);
-        // console.log(mapName, track);
-        var layer = L.polyline(track, {color: "green"}).addTo(map);
-        map.fitBounds(layer.getBounds());
-        //console.log(layer)
-        //L.control.layers({"osm":tiles}, {layer]).addTo(map);
-    });
-    // }, 2000);
 });

static/progress/style.css

@@ -0,0 +1,105 @@
+/*.mapDiv {
+    width: 1024px;
+    height: 768px;
+}*/
+
+
+
+.highlight {
+    /*what a nice way to highlight*/
+    display: none;
+}
+
+.simu {
+    background-color: blue;
+}
+
+.question {
+    background-color: orange;
+}
+
+.image {
+    background-color: green;
+}
+
+.audio {
+    background-color: red;
+}
+
+.video {
+    background-color: purple;
+}
+
+.other {
+    background-color: brown;
+}
+
+.map {
+    background-color: violet;
+}
+
+.error {
+    background-color: grey;
+}
+
+
+.board {
+    width: 32px;
+    height: 32px;
+    display: inline-block;
+}
+
+.board img {
+    max-width: 32px;
+    max-height: 32px;
+    position: absolute;
+    /*bottom: 0px;*/
+}
+
+.board:hover img{
+    max-width: 205px;
+    max-height: 295px;
+    z-index: 99;
+    top: 5px;
+    right:0px;
+}
+ul {
+    list-style-type: none;
+    overflow: auto;
+    overflow-y: hidden;
+    display: inline-block;
+    /*max-width:100%;
+    margin: 0 0 1em;
+    white-space: nowrap;
+    height:200px;*/
+}
+
+li {
+    display: inline-block;
+    vertical-align: top;
+    padding: 2px;
+    margin-bottom: 2px;
+}
+
+body{
+    height: 100%;
+    padding:0;
+    margin:0;
+}
+
+main{
+    display: flex;
+    flex-direction: column;
+    height:100%;
+}
+
+.mapDiv {
+    flex-grow:1;
+}
+.sequenceContainer{
+    flex-grow: 0;
+    min-height:300px;
+    padding-right: 210px;
+    position: relative;
+
+}

tasks/__init__.py

@@ -0,0 +1,65 @@
+from .tasks import analyze
+
+__log__ = ["/app/data/008cad400ab848f729913d034a.zip"]
+
+__config__ = """{
+  "logFormat": "zip",
+  "entryType": "@class",
+  "spatials": [
+    "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation"
+  ],
+  "actions": [
+    "...QuestionAnswerEvent",
+    "...SimuAnswerEvent"
+  ],
+  "boards": [
+    "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry"
+  ],
+  "analyzers": {
+    "analysis.analyzers": [
+      "BiogamesCategorizer",
+      "LocationAnalyzer"
+    ]
+  },
+  "sequences": {
+    "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache",
+    "end": {
+      "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction",
+      "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction"
+    }
+  },
+  "custom": {
+    "simulation_rounds": [
+      "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion"
+    ],
+    "simu_data": [
+      "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData"
+    ],
+    "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance",
+    "instance_id": "instance_id",
+    "instance_config_id": "config.@id",
+    "sequences2": {
+      "id_field": "sequence_id",
+      "start": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "START"
+      },
+      "end": {
+        "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry",
+        "action": "PAUSE"
+      }
+    },
+    "coordinates": "location.coordinates",
+    "metadata": {
+      "timestamp": "timestamp",
+      "gamefield": "instance_id",
+      "user": "player_group_name"
+    }
+  },
+  "source": {
+    "type": "Biogames",
+   "username": "ba",
+    "password": "853451",
+    "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de"
+  }
+}"""

tasks/tasks.py

@@ -0,0 +1,86 @@
+import json
+import logging
+import shutil
+import uuid
+import os.path
+import os
+
+import redis as redis_lib
+import time
+from celery import Celery
+from analysis import log_analyzer as la
+from analysis.analyzers import KMLRender, ActivityMapperRender
+from clients.webclients import CLIENTS
+
+FLASK_DB = 1
+REDIS_HOST = "redis"
+DATA_PATH = "/app/data/results/"
+
+RENDERERS = {  # TODO
+	"KMLRender": KMLRender,
+	"ActivityMapper": ActivityMapperRender,
+}
+
+app = Celery('tasks', backend='redis://redis', broker='redis://redis')
+redis = redis_lib.StrictRedis(host=REDIS_HOST, db=FLASK_DB)
+log: logging.Logger = logging.getLogger(__name__)
+
+
+def update_status(username, name, state, **kwargs):
+	status = json.loads(redis.get(username))
+	status[name][state[0]] = time.strftime("%c")
+	status[name]['status'] = state[1]
+	for i in kwargs:
+		status[name][i] = kwargs[i]
+	redis.set(username, json.dumps(status))
+
+
+@app.task
+def analyze(config, log_ids, **kwargs):
+	update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING'))
+
+	try:
+		log.info("start analysis")
+		client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies'])
+		logs = client.list()
+		id_urls = {str(x['@id']): x['file_url'] for x in logs}
+		urls = [id_urls[i] for i in log_ids]
+		tmpdir = client.download_files(urls)
+		log.info(tmpdir.name, list(os.scandir(tmpdir.name)))
+
+		update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING'))
+
+		settings = la.parse_settings(config)
+		store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
+		render = RENDERERS[settings.render[0]]()  # TODO
+		files = render.render(store.get_all())
+
+		uid = str(uuid.uuid4())
+		results = []
+		log.error(files)
+		os.mkdir(os.path.join(DATA_PATH, uid))
+		for file in files:
+			try:
+				head, tail = os.path.split(file)
+				target = os.path.join(DATA_PATH, uid, tail)
+				shutil.move(file, target)
+				results.append(target)
+			except FileNotFoundError as e:
+				log.exception(e)
+		tmpdir.cleanup()
+
+		update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results)
+	except Exception as e:
+		log.exception(e)
+		update_status(kwargs['username'], kwargs['name'], ('abort', 'ERROR'), exception=str(e))
+
+
+def status_update(key, status_key, status):
+	record = redis.get(key)
+	if not record:
+		redis.set(key, json.dumps({status_key: status}))
+	else:
+		data = json.loads(record)
+		data[status_key] = status
+		redis.set(key, json.dumps(data))
+	redis.save()

test_neo.py

@@ -0,0 +1,41 @@
+
+from analysis import log_analyzer as la
+settings = la.load_settings("neocart.json")
+client = settings.source
+logs = client.list()
+id_urls = {str(x['@id']): x['file_url'] for x in logs}
+
+log_ids=['20351/playerid1430317168972.gpx','20351/playerid1430317188358.gpx']
+
+urls = [id_urls[i] for i in log_ids]
+tmpdir = client.download_files(urls)
+import os
+store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS)
+
+import json
+print(json.dumps(store.serializable(), indent=1))
+
+
+from analysis.analyzers import KMLRender, ActivityMapperRender
+RENDERERS = {  # TODO
+	"KMLRender": KMLRender,
+	"ActivityMapper": ActivityMapperRender,
+}
+render = RENDERERS[settings.render[0]]()
+files = render.render(store.get_all())
+DATA_PATH = "/app/data/results/"
+import uuid
+uid = str(uuid.uuid4())
+results = []
+os.mkdir(os.path.join(DATA_PATH, uid))
+import shutil
+
+for file in files:
+	try:
+		head, tail = os.path.split(file)
+		target = os.path.join(DATA_PATH, uid, tail)
+		shutil.move(file, target)
+		results.append(target)
+	except FileNotFoundError as e:
+		log.exception(e)
+tmpdir.cleanup()

util/download.py

@@ -1,42 +0,0 @@
-import logging
-import os
-
-from util import json_path
-
-logger = logging.getLogger(__name__)
-
-
-def download_board(board_id, instance_config_id, sequence_id, source):
-	local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
-		config_id=instance_config_id,
-		sequence_id=sequence_id,
-		board_id=board_id)
-	if os.path.exists(local_file):
-		return local_file
-	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
-		config_id=instance_config_id,
-		sequence_id=sequence_id,
-		board_id=board_id
-	)
-	board = source._get(url)
-	if not board.ok:
-		raise ConnectionError()
-	data = board.json()
-	preview_url = json_path(data, "preview_url.medium")
-	logger.debug(preview_url)
-	os.makedirs(local_file[:-len(board_id)], exist_ok=True)
-	source.download_file(preview_url, local_file)
-	return local_file
-
-
-def get_config(source, instance_id):
-	url = "/game2/editor/config/{config_id}/".format(config_id=instance_id)
-	instance_data = source.get_json(url)
-	caches = url + "cache/"
-	cache_data = source.get_json(caches)
-
-	return {
-		"name": instance_data["name"],
-		"id": instance_data["@id"],
-		"caches": cache_data
-	}