Merge branch 'master' of https://clkl.de/git/ma/project

analyzers/__init__.py

@@ -2,14 +2,14 @@ from typing import List
 
 from .analyzer import Analyzer, Result
 from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
-	BiogamesCategorizer, ActivityMapper, BiogamesStore
+	BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
 	CategorizerStub, Store, ProgressAnalyzer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
 from .analyzer.mask import MaskSpatials
 from .render import Render
 from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
-	ActivityMapperRender, StoreRender, SimulationRoundsMeanRender
+	ActivityMapperRender, StoreRender, SimulationRoundsMeanRender, SimulationOrderRender
 from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender
 from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
 	LocomotionActionRatioRender, LocomotionActionRatioHistRender
@@ -48,6 +48,10 @@ __MAPPING__ = {
 	],
 	ProgressAnalyzer: [
 		StoreRender
+	],
+	SimulationOrderAnalyzer: [
+		JSONRender,
+SimulationOrderRender
 	]
 }
 

analyzers/analyzer/__init__.py

@@ -43,7 +43,7 @@ class ResultStore:
 	def get_store(self) -> dict:
 		return dict(self.store)
 
-	def get_all(self) -> list:
+	def get_all(self) -> Collection[Result]:
 		"""
 		Throw all categories together
 		:return:

analyzers/analyzer/biogames.py

@@ -119,9 +119,7 @@ class ActivityMapper(Analyzer):
 	def result(self, store: ResultStore) -> None:
 		instance_config_id = self.instance_config_id
 		for active_segment in self.store:  # active_segment → sequence or None (None → map active)
-			host = self.settings.custom["host"]
-			seq_data_url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
-				host=host,
+			seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
 				config_id=instance_config_id,
 				sequence_id=active_segment.sequence,
 			)
@@ -138,7 +136,7 @@ class ActivityMapper(Analyzer):
 				if event[self.settings.type_field] in self.settings.boards:
 					sequence_id = active_segment.sequence
 					board_id = event["board_id"]
-					local_file = download_board(board_id, host, instance_config_id, sequence_id, source)
+					local_file = download_board(board_id, instance_config_id, sequence_id, source)
 					if local_file is not None:
 						event["image"] = local_file[16:]
 		store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]}))
@@ -181,7 +179,6 @@ class BiogamesStore(Store):
 				board_id = event["board_id"]
 				local_file = download_board(
 					board_id=board_id,
-					host=self.settings.custom["host"],
 					instance_config_id=json_path(self.store[0], self.settings.custom["instance_config_id"]),
 					sequence_id=sequence_id,
 					source=self.settings.source)
@@ -193,4 +190,42 @@ class BiogamesStore(Store):
 
 	def process(self, entry: dict) -> bool:
 		self.store.append(entry)
+		return False
+
+
+class InstanceConfig(Analyzer):
+	__name__ = "InstanceConfig"
+
+	def __init__(self, settings: LogSettings):
+		super().__init__(settings)
+		self.store = {}
+
+	def process(self, entry: dict):
+		if entry[self.settings.type_field] in self.settings.custom["instance_start"]:
+			print(entry)
+			self.store["instance_id"] = json_path(entry, self.settings.custom["instance_config_id"])
+
+	def result(self, store: ResultStore):
+		store.add(Result(type(self), dict(self.store)))
+
+
+class SimulationOrderAnalyzer(Analyzer):
+	__name__ = "SimuOrder"
+
+	def __init__(self, settings: LogSettings):
+		super().__init__(settings)
+		self.store = defaultdict(lambda: -1)  # TODO verify
+		self.order = []
+
+	def result(self, store: ResultStore) -> None:
+		store.add(Result(type(self), [self.store[sim] for sim in self.order]))
+
+	def process(self, entry: dict) -> bool:
+		entry_type = entry[self.settings.type_field]
+		if entry_type in self.settings.custom['simulation_rounds']:
+			if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]:
+				simu_id = entry['answers']["@id"]
+				self.store[simu_id] += 1
+				if not simu_id in self.order:
+					self.order.append(simu_id)
 		return False

analyzers/render/biogames.py

@@ -5,14 +5,15 @@ from typing import List, Tuple
 import matplotlib.pyplot as plt
 import os
 
-from analyzers import Store, BiogamesStore
+from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer
 from . import Render
 from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
 
-
 ONE_DAY = 24 * 60 * 60
 
-def plot(src_data: List[Tuple[str, List[int]]], title:str="simulation retries", ylabel:str="simulation rounds", xargs={}):
+
+def plot(src_data: List[Tuple[str, List[int]]], title: str = "simulation retries", ylabel: str = "simulation rounds",
+		 xargs={}):
 	names, datas = list(zip(*src_data))
 	plt.boxplot(datas, labels=names, **xargs)
 	plt.xticks(rotation='vertical')
@@ -22,9 +23,21 @@ def plot(src_data: List[Tuple[str, List[int]]], title:str="simulation retries",
 	plt.show()
 
 
+def plot_old(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries",
+			 rotation='vertical'):
+	names, datas = list(zip(*src_data))
+	plt.boxplot(datas, labels=names, **{"showfliers": True})
+	plt.xticks(rotation=rotation)
+	# plt.margins()
+	plt.ylabel(ylabel)
+	plt.title(title)
+	plt.show()
+
+
 class SimulationRoundsRender(Render):
 	result_types = [SimulationRoundsAnalyzer]
 	xargs = {}
+
 	def render(self, results: List[Result]):
 		data = defaultdict(list)
 		for result in self.filter(results):
@@ -37,6 +50,7 @@ class SimulationRoundsRender(Render):
 
 	result_types = [SimulationRoundsAnalyzer]
 
+
 class SimulationRoundsMeanRender(SimulationRoundsRender):
 	xargs = {"showfliers": False}
 
@@ -89,7 +103,7 @@ class ActivityMapperRender(Render):
 		print(os.getcwd())
 		for result in self.filter(results):
 			data = result.get()
-			with open(os.path.join("static", "progress", "data", data['instance']),"w") as out:
+			with open(os.path.join("static", "progress", "data", data['instance']), "w") as out:
 				json.dump(data["store"], out, indent=1)
 		return "ok"
 
@@ -99,5 +113,20 @@ class StoreRender(Render):
 
 	def render(self, results: List[Result]):
 		for result in self.filter(results):
-			with open(os.path.join("static","progress","data","fooo"), "w") as out:
+			with open(os.path.join("static", "progress", "data", "fooo"), "w") as out:
 				json.dump(result.get(), out, indent=1)
+
+
+class SimulationOrderRender(Render):
+	def render(self, results: List[Result]):
+		data = defaultdict(list)
+		for result in self.filter(results):
+			get = result.get()
+			for i, value in enumerate(get):
+				data[i].append(value)
+		# data_tuples = [(key, data[key]) for key in sorted(data)]
+		# data_tuples = sorted(data_tuples, key=lambda x: sum(x[1]))
+		# plot(enumerate([r.get() for r in self.filter(results)]))
+		plot_old(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None)
+
+	result_types = [SimulationOrderAnalyzer]

biogames2.json

@@ -14,15 +14,15 @@
   "analyzers": {
     "analyzers": [
       "BiogamesCategorizer",
-    "ActionSequenceAnalyzer",
-    "BoardDurationAnalyzer",
-    "LocomotionActionAnalyzer",
-    "CacheSequenceAnalyzer",
-    "SimulationRoundsAnalyzer",
-    "ActivationSequenceAnalyzer",
-      "ProgressAnalyzer"
+      "ActivityMapper",
+      "ProgressAnalyzer",
+      "SimulationOrderAnalyzer"
     ]
   },
+  "dis":[
+      "ActivityMapper",
+      "ProgressAnalyzer",
+      "InstanceConfig"],
   "disabled_analyzers": [
       "ActivityMapper",
       "LocomotionActionAnalyzer",
@@ -65,14 +65,13 @@
         "action":"PAUSE"
       }
     },
-    "host":"http://potato.kinf.wiai.uni-bamberg.de:5000",
-    "host2":"http://potato.kinf.wiai.uni-bamberg.de:5000",
     "coordinates": "location.coordinates"
   },
   "source":{
     "type": "Biogames",
     "url": "http://potato.kinf.wiai.uni-bamberg.de:5000/game2/instance/log/list/",
     "login_url": "http://potato.kinf.wiai.uni-bamberg.de:5000/game2/auth/json-login",
+    "host":"http://potato.kinf.wiai.uni-bamberg.de:5000",
     "username": "lb",
     "password": "81743"
   }

log_analyzer.py

@@ -130,9 +130,9 @@ if __name__ == '__main__':
 		#	"57c444470dbf88605433ca935c",
 		#	"78e0c545b594e82edfad55bd7f",
 		#	"91abfd4b31a5562b1c66be37d9",
-		#	"597b704fe9ace475316c345903",
-		#	"e01a684aa29dff9ddd9705edf8",
-		#	"fbf9d64ae0bdad0de7efa3eec6",
+			"597b704fe9ace475316c345903",
+			"e01a684aa29dff9ddd9705edf8",
+			"fbf9d64ae0bdad0de7efa3eec6",
 		#	"fe1331481f85560681f86827ec",
 		#	"fe1331481f85560681f86827ec"]
 		#"fec57041458e6cef98652df625", ]
@@ -165,6 +165,13 @@ if __name__ == '__main__':
 			print(str(a))
 			render(a, store.get_all())
 
+	if False:
+		from analyzers.postprocessing import graph
+		g = graph.Cache(settings)
+		g.run(store)
+	if True:
+		render(analyzers.SimulationOrderAnalyzer, store.get_all())
+
 # for analyzers in analyzers:
 #	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:
 #		print(json.dumps(analyzers.result(), indent=2))

requirements.txt

@@ -1,3 +1,6 @@
-requests
-numpy
-matplotlib
+requests==2.18.4
+numpy==1.13.1
+matplotlib==2.1.0
+osmnx==0.6
+networkx==2.0
+pydot==1.2.3

sources/biogames.py

@@ -18,9 +18,10 @@ class Biogames(Source):
 		self.headers: typing.Dict[str, str] = {'Accept': 'application/json'}
 		self.cookies: typing.Dict[str, str] = {}
 		self.id2link: typing.Dict[str, str] = {}
+		self.host: str = None
 
 	def connect(self, **kwargs):
-		for i in ['username', 'password', 'url', 'login_url']:
+		for i in ['username', 'password', 'url', 'login_url', 'host']:
 			if not i in kwargs:
 				raise ValueError("missing value " + i)
 		csrf_request = requests.get(kwargs['url'])
@@ -40,12 +41,11 @@ class Biogames(Source):
 		self.cookies['sessionid'] = login.cookies['sessionid']
 		log.info("obtained sessionid (" + self.cookies['sessionid'] + ")")
 		self.url = kwargs['url']
+		self.host = kwargs['host']
 		log.info("stored url (" + self.url + ")")
 
 	def list(self):
-		logs_query = requests.get(self.url, cookies=self.cookies, headers=self.headers)
-		log.info(logs_query.status_code)
-		logs = logs_query.json()
+		logs = self.get_json(self.url)
 		log.info(len(logs))
 		for i in logs:
 			self.id2link[i["id"]] = i["link"]  # TODO
@@ -65,15 +65,18 @@ class Biogames(Source):
 	def download_file(self, url, filename):
 		with open(filename, "wb") as out:
 			try:
-				download = requests.get(url, cookies=self.cookies, stream=True)
+				download = self._get(url)
 				shutil.copyfileobj(download.raw, out)
 				return filename
 			except Exception as e:
 				log.exception(e)
 				os.remove(filename)
 
+	def get_json(self, url):
+		return self._get(url, stream=False).json()
+
 	def close(self):
 		pass
 
-	def _get(self, url):
-		return requests.get(url, cookies=self.cookies, headers=self.headers, stream=True)
+	def _get(self, url, stream=True):
+		return requests.get(self.host + url, cookies=self.cookies, headers=self.headers, stream=stream)

sources/source.py

@@ -11,5 +11,8 @@ class Source:
 	def get(self, ids: typing.Collection):
 		raise NotImplementedError
 
+	def get_json(self, url:str) -> dict:
+		raise NotImplementedError
+
 	def close(self):
 		raise NotImplementedError

util/download.py

@@ -6,7 +6,7 @@ from util import json_path
 logger = logging.getLogger(__name__)
 
 
-def download_board(board_id, host, instance_config_id, sequence_id, source):
+def download_board(board_id, instance_config_id, sequence_id, source):
 	if sequence_id is None:
 		return "static"
 	local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
@@ -15,8 +15,7 @@ def download_board(board_id, host, instance_config_id, sequence_id, source):
 		board_id=board_id)
 	if os.path.exists(local_file):
 		return local_file
-	url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
-		host=host,
+	url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
 		config_id=instance_config_id,
 		sequence_id=sequence_id,
 		board_id=board_id
@@ -30,5 +29,18 @@ def download_board(board_id, host, instance_config_id, sequence_id, source):
 	preview_url = json_path(data, "preview_url.medium")
 	logger.debug(preview_url)
 	os.makedirs(local_file[:-len(board_id)], exist_ok=True)
-	source.download_file(host + preview_url, local_file)
+	source.download_file(preview_url, local_file)
 	return local_file
+
+
+def get_config(source, instance_id):
+	url = "/game2/editor/config/{config_id}/".format(config_id=instance_id)
+	instance_data = source.get_json(url)
+	caches = url + "cache/"
+	cache_data = source.get_json(caches)
+
+	return {
+		"name": instance_data["name"],
+		"id": instance_data["@id"],
+		"caches": cache_data
+	}