progress...

analyzers/__init__.py

@@ -2,14 +2,14 @@ from typing import List
 
 from .analyzer import Analyzer, Result
 from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
-	BiogamesCategorizer, ActivityMapper
+	BiogamesCategorizer, ActivityMapper, BiogamesStore
 from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
-	CategorizerStub, Store
+	CategorizerStub, Store, ProgressAnalyzer
 from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
 from .analyzer.mask import MaskSpatials
 from .render import Render
 from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
-	ActivityMapperRender
+	ActivityMapperRender, StoreRender
 from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender
 from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
 	LocomotionActionRatioRender
@@ -38,8 +38,14 @@ __MAPPING__ = {
 		TrackRender,
 		HeatMapRender,
 	],
-	ActivityMapper:[
+	ActivityMapper: [
 		ActivityMapperRender
+	],
+	BiogamesStore: [
+		StoreRender
+	],
+	ProgressAnalyzer: [
+		StoreRender
 	]
 }
 

analyzers/analyzer/biogames.py

@@ -1,13 +1,12 @@
 import logging
-from collections import defaultdict, namedtuple
+from collections import defaultdict, namedtuple, OrderedDict
 from types import SimpleNamespace
 from typing import List, NamedTuple
 
-import os
-
 from util import json_path, combinate
+from util.download import download_board
 from . import Result, LogSettings, Analyzer, ResultStore
-from .default import CategorizerStub
+from .default import CategorizerStub, Store
 
 logger = logging.getLogger(__name__)
 
@@ -93,6 +92,7 @@ class ActivationSequenceAnalyzer(Analyzer):
 
 class BiogamesCategorizer(CategorizerStub):
 	__name__ = "BiogamesCategorizer"
+
 	def __init__(self, settings: LogSettings):
 		super().__init__(settings)
 
@@ -105,6 +105,7 @@ class BiogamesCategorizer(CategorizerStub):
 
 class ActivityMapper(Analyzer):
 	__name__ = "ActivityMapper"
+
 	def __init__(self, settings: LogSettings) -> None:
 		super().__init__(settings)
 		self.store: List[self.State] = []
@@ -116,38 +117,26 @@ class ActivityMapper(Analyzer):
 		self.State: NamedTuple = namedtuple("State", ["sequence", "events", "track", "timestamp"])
 
 	def result(self, store: ResultStore) -> None:
+		instance_config_id = self.instance_config_id
 		for active_segment in self.store:  # active_segment → sequence or None (None → map active)
+			host = self.settings.custom["host"]
 			seq_data_url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
-						host=self.settings.custom["host"],
+				host=host,
-						config_id=self.instance_config_id,
+				config_id=instance_config_id,
-						sequence_id=active_segment.sequence,
+				sequence_id=active_segment.sequence,
-					)
+			)
-			seq_data = self.settings.source._get(seq_data_url).json()
+			source = self.settings.source
-			#TODO: use sequence names
+			seq_data = source._get(seq_data_url).json()
+			# TODO: use sequence names
+			logger.warning(seq_data)
 			for event in active_segment.events:
 				if event[self.settings.type_field] in self.settings.boards:
-					local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
+					sequence_id = active_segment.sequence
-						config_id=self.instance_config_id,
+					board_id = event["board_id"]
-						sequence_id=active_segment.sequence,
+					local_file = download_board(board_id, host, instance_config_id, sequence_id, source)
-						board_id=event["board_id"])
+					if local_file is not None:
-					event["image"] = local_file[16:]
+						event["image"] = local_file[16:]
-					if os.path.exists(local_file):
+		store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]}))
-						continue
-					url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
-						host=self.settings.custom["host"],
-						config_id=self.instance_config_id,
-						sequence_id=active_segment.sequence,
-						board_id=event["board_id"]
-					)
-					board = self.settings.source._get(url)
-					if not board.ok:
-						raise ConnectionError()
-					data = board.json()
-					preview_url = json_path(data, "preview_url.medium")
-					logger.debug(preview_url)
-					os.makedirs(local_file[:-len(event["board_id"])], exist_ok=True)
-					self.settings.source.download_file(self.settings.custom['host'] + preview_url, local_file)
-		store.add(Result(type(self), {"instance": self.instance_config_id, "store": [x._asdict() for x in self.store]}))
 
 	def process(self, entry: dict) -> bool:
 		if self.instance_config_id is None:
@@ -174,3 +163,29 @@ class ActivityMapper(Analyzer):
 		else:
 			self.store[-1].events.append(entry)
 		return False
+
+
+class BiogamesStore(Store):
+	__name__ = "BiogamesStore"
+
+	def result(self, store: ResultStore) -> None:
+		result = OrderedDict()
+		for event in self.store:
+			if event[self.settings.type_field] in self.settings.boards:
+				sequence_id = json_path(event, json_path(self.settings.custom, "sequences2.id_field"))
+				board_id = event["board_id"]
+				local_file = download_board(
+					board_id=board_id,
+					host=self.settings.custom["host"],
+					instance_config_id=json_path(self.store[0], self.settings.custom["instance_config_id"]),
+					sequence_id=sequence_id,
+					source=self.settings.source)
+				if local_file is not None:
+					event["image"] = local_file[16:]
+			result[event["timestamp"]] = event
+
+		store.add(Result(type(self), result))
+
+	def process(self, entry: dict) -> bool:
+		self.store.append(entry)
+		return False

analyzers/analyzer/default.py

@@ -1,6 +1,7 @@
 import logging
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
 
+from util import json_path
 from . import Result, LogSettings, Analyzer, ResultStore
 
 
@@ -111,3 +112,27 @@ class Store(Analyzer):
 	def __init__(self, settings: LogSettings):
 		super().__init__(settings)
 		self.store: list = []
+
+
+class ProgressAnalyzer(Analyzer):
+	"""track spatial and ingame progress"""
+	__name__ = "ProgressAnalyzer"
+
+	def __init__(self, settings: LogSettings) -> None:
+		super().__init__(settings)
+		self.spatial = OrderedDict()
+		self.board = OrderedDict()
+
+	def result(self, store: ResultStore) -> None:
+		store.add(Result(type(self), {"spatials": self.spatial, "boards": self.board}))
+
+	def process(self, entry: dict) -> bool:
+		if entry[self.settings.type_field] in self.settings.spatials:
+			self.spatial[entry["timestamp"]] = 				{
+					'timestamp': entry['timestamp'],
+					'coordinates': json_path(entry, "location.coordinates"),
+					'accuracy': entry['accuracy']
+				}
+		if entry[self.settings.type_field] in self.settings.boards:
+			self.board[entry["timestamp"]] = entry
+		return False

analyzers/render/biogames.py

@@ -1,8 +1,11 @@
+import json
 from collections import defaultdict
 from typing import List, Tuple
 
 import matplotlib.pyplot as plt
+import os
 
+from analyzers import Store, BiogamesStore
 from . import Render
 from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
 
@@ -68,4 +71,18 @@ class ActivityMapperRender(Render):
 	result_types = [ActivityMapper]
 
 	def render(self, results: List[Result]):
-		pass
+		print(os.getcwd())
+		for result in self.filter(results):
+			data = result.get()
+			with open(os.path.join("static", "progress", "data", data['instance']),"w") as out:
+				json.dump(data["store"], out, indent=1)
+		return "ok"
+
+
+class StoreRender(Render):
+	result_types = [Store, BiogamesStore]
+
+	def render(self, results: List[Result]):
+		for result in self.filter(results):
+			with open(os.path.join("static","progress","data","fooo"), "w") as out:
+				json.dump(result.get(), out, indent=1)

biogames2.json

@@ -14,7 +14,8 @@
   "analyzers": {
     "analyzers": [
       "BiogamesCategorizer",
-      "ActivityMapper"
+      "ActivityMapper",
+      "ProgressAnalyzer"
     ]
   },
   "disabled_analyzers": [
@@ -58,7 +59,8 @@
         "action":"PAUSE"
       }
     },
-    "host":"http://0.0.0.0:5000"
+    "host":"http://0.0.0.0:5000",
+    "coordinates": "location.coordinates"
   },
   "source":{
     "type": "Biogames",

log_analyzer.py

@@ -3,7 +3,7 @@ import logging
 from typing import List
 
 import analyzers
-from analyzers import get_renderer, Analyzer, render
+from analyzers import get_renderer, Analyzer, render, Store
 from analyzers.analyzer import ResultStore
 from analyzers.settings import LogSettings, load_settings
 from loaders import LOADERS
@@ -11,6 +11,9 @@ from loaders import LOADERS
 logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG)
 log: logging.Logger = logging.getLogger(__name__)
 
+requests_log = logging.getLogger('requests')
+requests_log.setLevel(logging.WARN)
+
 
 def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]:
 	logfile: str = "data/inst_{id}.{format}".format(id=log_id, format=settings.log_format)
@@ -63,7 +66,10 @@ if __name__ == '__main__':
 			r().render(store.get_all())
 	if False:
 		render(analyzers.LocationAnalyzer, store.get_all())
-	print(json.dumps(store.serializable(), indent=1))
+	#print(json.dumps(store.serializable(), indent=1))
+	if True:
+		render(analyzers.ActivityMapper, store.get_all())
+		render(analyzers.ProgressAnalyzer, store.get_all())
 
 # for analyzers in analyzers:
 #	if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:

static/progress/log.html

@@ -0,0 +1,15 @@
+<script
+  src="https://code.jquery.com/jquery-3.2.1.min.js"
+  integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+  crossorigin="anonymous"></script>
+    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.2.0/dist/leaflet.css"
+   integrity="sha512-M2wvCLH6DSRazYeZRIm1JnYyh22purTM+FDB5CsyxtQJYeKq83arPe5wgbNmcFXGqiSH2XR8dT/fJISVA1r/zQ=="
+   crossorigin=""/>
+    <script src="https://unpkg.com/leaflet@1.2.0/dist/leaflet.js"
+   integrity="sha512-lInM/apFSqyy1o6s89K4iQUKg6ppXEgsVxT35HbzUupEVRh2Eu9Wdl4tHj7dZO0s1uvplcYGmt3498TtHq+log=="
+   crossorigin=""></script>
+  
+<script src="log.js"></script>
+    <style>
+       .map { width: 512px; height: 512px; }
+    </style>

static/progress/log.js

@@ -0,0 +1,72 @@
+$.getJSON("data/fooo", function (data) {
+    var list = $("<ul />");
+    var mapC = $("<div />", {class: "map", id: "map"});
+    mapC.appendTo("body");
+    var track = [];
+    var times = [];
+    $.each(data.spatials, function (i, elem) {
+        track.push([elem.coordinates[1], elem.coordinates[0]]);
+        times.push(i);
+    });
+    var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
+            attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
+        });
+    var map = L.map("map", {layers: [tiles]});
+    L.control.scale().addTo(map);
+    var layer = L.polyline(track).addTo(map);
+    map.fitBounds(layer.getBounds());
+    $.each(data, function (key, value) {
+        //console.log(key, value);
+        //key: instance_id, value: AnlysisResult
+        //value.result.instance: InstanceConfig_id
+        // console.log(key, value[0].result.store[0].timestamp);
+        /*$.each(value[0].result.store, function (index, entry) {
+            //console.log(entry);
+            var time = new Date(entry.timestamp);
+            var item = $("<li>", {html: entry.sequence + " @ " + time.toLocaleDateString() + " "+ time.toLocaleTimeString()});
+            var container = $("<p />");
+            if (entry.track.length > 0) {
+                var mapName = "map" + index;
+                //console.log(mapName, entry.track.length);
+                var mapContainer = $("<div />", {id: mapName, class: "map"});
+                var track = [];
+                $.each(entry.track, function (i, elem) {
+                    track.push([elem.coordinates[1], elem.coordinates[0]]);
+                });
+                maps[mapName] = track;
+
+                mapContainer.appendTo(container);
+            }
+            $.each(entry.events, function (i, event) {
+                if ("image" in event) {
+                    $("<img />", {src: event.image, height: 200}).appendTo(container);
+                }
+            });
+            container.appendTo(item);
+            item.appendTo(list);
+        });*/
+    });
+    list.appendTo("body");
+    var slider = $("<input />", {type: "range", start:0,end:100});
+    slider.appendTo("body");
+    /*});
+
+    $(window).on("load", function () {*/
+    // setTimeout(function () {
+
+    //console.log(maps);
+    /*$.each(maps, function (mapName, track) {
+        //console.log("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa");
+        var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
+            attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
+        });
+        var map = L.map(mapName, {layers: [tiles]});
+        L.control.scale().addTo(map);
+        // console.log(mapName, track);
+        var layer = L.polyline(track, {color: "green"}).addTo(map);
+        map.fitBounds(layer.getBounds());
+        //console.log(layer)
+        //L.control.layers({"osm":tiles}, {layer]).addTo(map);
+    });*/
+    // }, 2000);
+});

static/progress/my.js

@@ -47,6 +47,7 @@ $.getJSON("tmp3.json", function (data) {
         });
     });
     list.appendTo("body");
+    var slider = $("<input />", {type: "range" })
     /*});
 
     $(window).on("load", function () {*/

util/download.py

@@ -0,0 +1,30 @@
+import logging
+import os
+
+from util import json_path
+
+logger = logging.getLogger(__name__)
+
+
+def download_board(board_id, host, instance_config_id, sequence_id, source):
+	local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
+		config_id=instance_config_id,
+		sequence_id=sequence_id,
+		board_id=board_id)
+	if os.path.exists(local_file):
+		return local_file
+	url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
+		host=host,
+		config_id=instance_config_id,
+		sequence_id=sequence_id,
+		board_id=board_id
+	)
+	board = source._get(url)
+	if not board.ok:
+		raise ConnectionError()
+	data = board.json()
+	preview_url = json_path(data, "preview_url.medium")
+	logger.debug(preview_url)
+	os.makedirs(local_file[:-len(board_id)], exist_ok=True)
+	source.download_file(host + preview_url, local_file)
+	return local_file