From 6bbc66d55edb6600e294756840cd95e279d366d0 Mon Sep 17 00:00:00 2001 From: agp8x Date: Mon, 16 Oct 2017 07:53:34 +0200 Subject: [PATCH] progress... --- analyzers/__init__.py | 14 ++++-- analyzers/analyzer/biogames.py | 79 ++++++++++++++++++++-------------- analyzers/analyzer/default.py | 27 +++++++++++- analyzers/render/biogames.py | 19 +++++++- biogames2.json | 6 ++- log_analyzer.py | 10 ++++- static/progress/log.html | 15 +++++++ static/progress/log.js | 72 +++++++++++++++++++++++++++++++ static/progress/my.js | 1 + util/download.py | 30 +++++++++++++ 10 files changed, 231 insertions(+), 42 deletions(-) create mode 100644 static/progress/log.html create mode 100644 static/progress/log.js create mode 100644 util/download.py diff --git a/analyzers/__init__.py b/analyzers/__init__.py index 57e2a49..1a098ff 100644 --- a/analyzers/__init__.py +++ b/analyzers/__init__.py @@ -2,14 +2,14 @@ from typing import List from .analyzer import Analyzer, Result from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \ - BiogamesCategorizer, ActivityMapper + BiogamesCategorizer, ActivityMapper, BiogamesStore from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \ - CategorizerStub, Store + CategorizerStub, Store, ProgressAnalyzer from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer from .analyzer.mask import MaskSpatials from .render import Render from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \ - ActivityMapperRender + ActivityMapperRender, StoreRender from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \ LocomotionActionRatioRender @@ -38,8 +38,14 @@ __MAPPING__ = { TrackRender, HeatMapRender, ], - ActivityMapper:[ + ActivityMapper: [ ActivityMapperRender + ], + BiogamesStore: [ + StoreRender + ], + ProgressAnalyzer: [ + StoreRender ] } diff --git a/analyzers/analyzer/biogames.py b/analyzers/analyzer/biogames.py index c1ef1a4..57fb2cb 100644 --- a/analyzers/analyzer/biogames.py +++ b/analyzers/analyzer/biogames.py @@ -1,13 +1,12 @@ import logging -from collections import defaultdict, namedtuple +from collections import defaultdict, namedtuple, OrderedDict from types import SimpleNamespace from typing import List, NamedTuple -import os - from util import json_path, combinate +from util.download import download_board from . import Result, LogSettings, Analyzer, ResultStore -from .default import CategorizerStub +from .default import CategorizerStub, Store logger = logging.getLogger(__name__) @@ -93,6 +92,7 @@ class ActivationSequenceAnalyzer(Analyzer): class BiogamesCategorizer(CategorizerStub): __name__ = "BiogamesCategorizer" + def __init__(self, settings: LogSettings): super().__init__(settings) @@ -105,6 +105,7 @@ class BiogamesCategorizer(CategorizerStub): class ActivityMapper(Analyzer): __name__ = "ActivityMapper" + def __init__(self, settings: LogSettings) -> None: super().__init__(settings) self.store: List[self.State] = [] @@ -116,38 +117,26 @@ class ActivityMapper(Analyzer): self.State: NamedTuple = namedtuple("State", ["sequence", "events", "track", "timestamp"]) def result(self, store: ResultStore) -> None: + instance_config_id = self.instance_config_id for active_segment in self.store: # active_segment → sequence or None (None → map active) + host = self.settings.custom["host"] seq_data_url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/".format( - host=self.settings.custom["host"], - config_id=self.instance_config_id, - sequence_id=active_segment.sequence, - ) - seq_data = self.settings.source._get(seq_data_url).json() - #TODO: use sequence names + host=host, + config_id=instance_config_id, + sequence_id=active_segment.sequence, + ) + source = self.settings.source + seq_data = source._get(seq_data_url).json() + # TODO: use sequence names + logger.warning(seq_data) for event in active_segment.events: if event[self.settings.type_field] in self.settings.boards: - local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format( - config_id=self.instance_config_id, - sequence_id=active_segment.sequence, - board_id=event["board_id"]) - event["image"] = local_file[16:] - if os.path.exists(local_file): - continue - url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( - host=self.settings.custom["host"], - config_id=self.instance_config_id, - sequence_id=active_segment.sequence, - board_id=event["board_id"] - ) - board = self.settings.source._get(url) - if not board.ok: - raise ConnectionError() - data = board.json() - preview_url = json_path(data, "preview_url.medium") - logger.debug(preview_url) - os.makedirs(local_file[:-len(event["board_id"])], exist_ok=True) - self.settings.source.download_file(self.settings.custom['host'] + preview_url, local_file) - store.add(Result(type(self), {"instance": self.instance_config_id, "store": [x._asdict() for x in self.store]})) + sequence_id = active_segment.sequence + board_id = event["board_id"] + local_file = download_board(board_id, host, instance_config_id, sequence_id, source) + if local_file is not None: + event["image"] = local_file[16:] + store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]})) def process(self, entry: dict) -> bool: if self.instance_config_id is None: @@ -174,3 +163,29 @@ class ActivityMapper(Analyzer): else: self.store[-1].events.append(entry) return False + + +class BiogamesStore(Store): + __name__ = "BiogamesStore" + + def result(self, store: ResultStore) -> None: + result = OrderedDict() + for event in self.store: + if event[self.settings.type_field] in self.settings.boards: + sequence_id = json_path(event, json_path(self.settings.custom, "sequences2.id_field")) + board_id = event["board_id"] + local_file = download_board( + board_id=board_id, + host=self.settings.custom["host"], + instance_config_id=json_path(self.store[0], self.settings.custom["instance_config_id"]), + sequence_id=sequence_id, + source=self.settings.source) + if local_file is not None: + event["image"] = local_file[16:] + result[event["timestamp"]] = event + + store.add(Result(type(self), result)) + + def process(self, entry: dict) -> bool: + self.store.append(entry) + return False \ No newline at end of file diff --git a/analyzers/analyzer/default.py b/analyzers/analyzer/default.py index 9d80f5d..9841162 100644 --- a/analyzers/analyzer/default.py +++ b/analyzers/analyzer/default.py @@ -1,6 +1,7 @@ import logging -from collections import defaultdict +from collections import defaultdict, OrderedDict +from util import json_path from . import Result, LogSettings, Analyzer, ResultStore @@ -111,3 +112,27 @@ class Store(Analyzer): def __init__(self, settings: LogSettings): super().__init__(settings) self.store: list = [] + + +class ProgressAnalyzer(Analyzer): + """track spatial and ingame progress""" + __name__ = "ProgressAnalyzer" + + def __init__(self, settings: LogSettings) -> None: + super().__init__(settings) + self.spatial = OrderedDict() + self.board = OrderedDict() + + def result(self, store: ResultStore) -> None: + store.add(Result(type(self), {"spatials": self.spatial, "boards": self.board})) + + def process(self, entry: dict) -> bool: + if entry[self.settings.type_field] in self.settings.spatials: + self.spatial[entry["timestamp"]] = { + 'timestamp': entry['timestamp'], + 'coordinates': json_path(entry, "location.coordinates"), + 'accuracy': entry['accuracy'] + } + if entry[self.settings.type_field] in self.settings.boards: + self.board[entry["timestamp"]] = entry + return False diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py index 655b209..7aea6ab 100644 --- a/analyzers/render/biogames.py +++ b/analyzers/render/biogames.py @@ -1,8 +1,11 @@ +import json from collections import defaultdict from typing import List, Tuple import matplotlib.pyplot as plt +import os +from analyzers import Store, BiogamesStore from . import Render from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper @@ -68,4 +71,18 @@ class ActivityMapperRender(Render): result_types = [ActivityMapper] def render(self, results: List[Result]): - pass + print(os.getcwd()) + for result in self.filter(results): + data = result.get() + with open(os.path.join("static", "progress", "data", data['instance']),"w") as out: + json.dump(data["store"], out, indent=1) + return "ok" + + +class StoreRender(Render): + result_types = [Store, BiogamesStore] + + def render(self, results: List[Result]): + for result in self.filter(results): + with open(os.path.join("static","progress","data","fooo"), "w") as out: + json.dump(result.get(), out, indent=1) diff --git a/biogames2.json b/biogames2.json index 2764b09..1bebe72 100644 --- a/biogames2.json +++ b/biogames2.json @@ -14,7 +14,8 @@ "analyzers": { "analyzers": [ "BiogamesCategorizer", - "ActivityMapper" + "ActivityMapper", + "ProgressAnalyzer" ] }, "disabled_analyzers": [ @@ -58,7 +59,8 @@ "action":"PAUSE" } }, - "host":"http://0.0.0.0:5000" + "host":"http://0.0.0.0:5000", + "coordinates": "location.coordinates" }, "source":{ "type": "Biogames", diff --git a/log_analyzer.py b/log_analyzer.py index f400885..d650e7f 100644 --- a/log_analyzer.py +++ b/log_analyzer.py @@ -3,7 +3,7 @@ import logging from typing import List import analyzers -from analyzers import get_renderer, Analyzer, render +from analyzers import get_renderer, Analyzer, render, Store from analyzers.analyzer import ResultStore from analyzers.settings import LogSettings, load_settings from loaders import LOADERS @@ -11,6 +11,9 @@ from loaders import LOADERS logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG) log: logging.Logger = logging.getLogger(__name__) +requests_log = logging.getLogger('requests') +requests_log.setLevel(logging.WARN) + def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]: logfile: str = "data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) @@ -63,7 +66,10 @@ if __name__ == '__main__': r().render(store.get_all()) if False: render(analyzers.LocationAnalyzer, store.get_all()) - print(json.dumps(store.serializable(), indent=1)) + #print(json.dumps(store.serializable(), indent=1)) + if True: + render(analyzers.ActivityMapper, store.get_all()) + render(analyzers.ProgressAnalyzer, store.get_all()) # for analyzers in analyzers: # if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]: diff --git a/static/progress/log.html b/static/progress/log.html new file mode 100644 index 0000000..3bacffc --- /dev/null +++ b/static/progress/log.html @@ -0,0 +1,15 @@ + + + + + + \ No newline at end of file diff --git a/static/progress/log.js b/static/progress/log.js new file mode 100644 index 0000000..3abbfd6 --- /dev/null +++ b/static/progress/log.js @@ -0,0 +1,72 @@ +$.getJSON("data/fooo", function (data) { + var list = $("