ma
/
project
1
0
Fork 0
Code Issues 5 Pull Requests Releases Wiki Activity

progress...

simu_flags
agp8x 2017-10-16 07:53:34 +02:00
parent 0d66cc7c88
commit 6bbc66d55e
10 changed files with 231 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
analyzers/__init__.py
View File

@ -2,14 +2,14 @@ from typing import List
from .analyzer import Analyzer, Result
from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \
BiogamesCategorizer, ActivityMapper
BiogamesCategorizer, ActivityMapper, BiogamesStore
from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \
CategorizerStub, Store
CategorizerStub, Store, ProgressAnalyzer
from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer
from .analyzer.mask import MaskSpatials
from .render import Render
from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \
ActivityMapperRender
ActivityMapperRender, StoreRender
from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender
from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \
LocomotionActionRatioRender
@ -38,8 +38,14 @@ __MAPPING__ = {
TrackRender,
HeatMapRender,
],
ActivityMapper:[
ActivityMapper: [
ActivityMapperRender
],
BiogamesStore: [
StoreRender
],
ProgressAnalyzer: [
StoreRender
]
}

79
analyzers/analyzer/biogames.py
View File

@ -1,13 +1,12 @@
import logging
from collections import defaultdict, namedtuple
from collections import defaultdict, namedtuple, OrderedDict
from types import SimpleNamespace
from typing import List, NamedTuple
import os
from util import json_path, combinate
from util.download import download_board
from . import Result, LogSettings, Analyzer, ResultStore
from .default import CategorizerStub
from .default import CategorizerStub, Store
logger = logging.getLogger(__name__)
@ -93,6 +92,7 @@ class ActivationSequenceAnalyzer(Analyzer):
class BiogamesCategorizer(CategorizerStub):
__name__ = "BiogamesCategorizer"
def __init__(self, settings: LogSettings):
super().__init__(settings)
@ -105,6 +105,7 @@ class BiogamesCategorizer(CategorizerStub):
class ActivityMapper(Analyzer):
__name__ = "ActivityMapper"
def __init__(self, settings: LogSettings) -> None:
super().__init__(settings)
self.store: List[self.State] = []
@ -116,38 +117,26 @@ class ActivityMapper(Analyzer):
self.State: NamedTuple = namedtuple("State", ["sequence", "events", "track", "timestamp"])
def result(self, store: ResultStore) -> None:
instance_config_id = self.instance_config_id
for active_segment in self.store: # active_segment → sequence or None (None → map active)
host = self.settings.custom["host"]
seq_data_url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/".format(
host=self.settings.custom["host"],
config_id=self.instance_config_id,
sequence_id=active_segment.sequence,
)
seq_data = self.settings.source._get(seq_data_url).json()
#TODO: use sequence names
host=host,
config_id=instance_config_id,
sequence_id=active_segment.sequence,
)
source = self.settings.source
seq_data = source._get(seq_data_url).json()
# TODO: use sequence names
logger.warning(seq_data)
for event in active_segment.events:
if event[self.settings.type_field] in self.settings.boards:
local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
config_id=self.instance_config_id,
sequence_id=active_segment.sequence,
board_id=event["board_id"])
event["image"] = local_file[16:]
if os.path.exists(local_file):
continue
url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
host=self.settings.custom["host"],
config_id=self.instance_config_id,
sequence_id=active_segment.sequence,
board_id=event["board_id"]
)
board = self.settings.source._get(url)
if not board.ok:
raise ConnectionError()
data = board.json()
preview_url = json_path(data, "preview_url.medium")
logger.debug(preview_url)
os.makedirs(local_file[:-len(event["board_id"])], exist_ok=True)
self.settings.source.download_file(self.settings.custom['host'] + preview_url, local_file)
store.add(Result(type(self), {"instance": self.instance_config_id, "store": [x._asdict() for x in self.store]}))
sequence_id = active_segment.sequence
board_id = event["board_id"]
local_file = download_board(board_id, host, instance_config_id, sequence_id, source)
if local_file is not None:
event["image"] = local_file[16:]
store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]}))
def process(self, entry: dict) -> bool:
if self.instance_config_id is None:
@ -174,3 +163,29 @@ class ActivityMapper(Analyzer):
else:
self.store[-1].events.append(entry)
return False
class BiogamesStore(Store):
__name__ = "BiogamesStore"
def result(self, store: ResultStore) -> None:
result = OrderedDict()
for event in self.store:
if event[self.settings.type_field] in self.settings.boards:
sequence_id = json_path(event, json_path(self.settings.custom, "sequences2.id_field"))
board_id = event["board_id"]
local_file = download_board(
board_id=board_id,
host=self.settings.custom["host"],
instance_config_id=json_path(self.store[0], self.settings.custom["instance_config_id"]),
sequence_id=sequence_id,
source=self.settings.source)
if local_file is not None:
event["image"] = local_file[16:]
result[event["timestamp"]] = event
store.add(Result(type(self), result))
def process(self, entry: dict) -> bool:
self.store.append(entry)
return False

27
analyzers/analyzer/default.py
View File

@ -1,6 +1,7 @@
import logging
from collections import defaultdict
from collections import defaultdict, OrderedDict
from util import json_path
from . import Result, LogSettings, Analyzer, ResultStore
@ -111,3 +112,27 @@ class Store(Analyzer):
def __init__(self, settings: LogSettings):
super().__init__(settings)
self.store: list = []
class ProgressAnalyzer(Analyzer):
"""track spatial and ingame progress"""
__name__ = "ProgressAnalyzer"
def __init__(self, settings: LogSettings) -> None:
super().__init__(settings)
self.spatial = OrderedDict()
self.board = OrderedDict()
def result(self, store: ResultStore) -> None:
store.add(Result(type(self), {"spatials": self.spatial, "boards": self.board}))
def process(self, entry: dict) -> bool:
if entry[self.settings.type_field] in self.settings.spatials:
self.spatial[entry["timestamp"]] = {
'timestamp': entry['timestamp'],
'coordinates': json_path(entry, "location.coordinates"),
'accuracy': entry['accuracy']
}
if entry[self.settings.type_field] in self.settings.boards:
self.board[entry["timestamp"]] = entry
return False

19
analyzers/render/biogames.py
View File

@ -1,8 +1,11 @@
import json
from collections import defaultdict
from typing import List, Tuple
import matplotlib.pyplot as plt
import os
from analyzers import Store, BiogamesStore
from . import Render
from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper
@ -68,4 +71,18 @@ class ActivityMapperRender(Render):
result_types = [ActivityMapper]
def render(self, results: List[Result]):
pass
print(os.getcwd())
for result in self.filter(results):
data = result.get()
with open(os.path.join("static", "progress", "data", data['instance']),"w") as out:
json.dump(data["store"], out, indent=1)
return "ok"
class StoreRender(Render):
result_types = [Store, BiogamesStore]
def render(self, results: List[Result]):
for result in self.filter(results):
with open(os.path.join("static","progress","data","fooo"), "w") as out:
json.dump(result.get(), out, indent=1)

6
biogames2.json
View File

@ -14,7 +14,8 @@
"analyzers": {
"analyzers": [
"BiogamesCategorizer",
"ActivityMapper"
"ActivityMapper",
"ProgressAnalyzer"
]
},
"disabled_analyzers": [
@ -58,7 +59,8 @@
"action":"PAUSE"
}
},
"host":"http://0.0.0.0:5000"
"host":"http://0.0.0.0:5000",
"coordinates": "location.coordinates"
},
"source":{
"type": "Biogames",

10
log_analyzer.py
View File

@ -3,7 +3,7 @@ import logging
from typing import List
import analyzers
from analyzers import get_renderer, Analyzer, render
from analyzers import get_renderer, Analyzer, render, Store
from analyzers.analyzer import ResultStore
from analyzers.settings import LogSettings, load_settings
from loaders import LOADERS
@ -11,6 +11,9 @@ from loaders import LOADERS
logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG)
log: logging.Logger = logging.getLogger(__name__)
requests_log = logging.getLogger('requests')
requests_log.setLevel(logging.WARN)
def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]:
logfile: str = "data/inst_{id}.{format}".format(id=log_id, format=settings.log_format)
@ -63,7 +66,10 @@ if __name__ == '__main__':
r().render(store.get_all())
if False:
render(analyzers.LocationAnalyzer, store.get_all())
print(json.dumps(store.serializable(), indent=1))
#print(json.dumps(store.serializable(), indent=1))
if True:
render(analyzers.ActivityMapper, store.get_all())
render(analyzers.ProgressAnalyzer, store.get_all())
# for analyzers in analyzers:
# if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]:

15
static/progress/log.html Normal file
View File

@ -0,0 +1,15 @@
<script
src="https://code.jquery.com/jquery-3.2.1.min.js"
integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
crossorigin="anonymous"></script>
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.2.0/dist/leaflet.css"
integrity="sha512-M2wvCLH6DSRazYeZRIm1JnYyh22purTM+FDB5CsyxtQJYeKq83arPe5wgbNmcFXGqiSH2XR8dT/fJISVA1r/zQ=="
crossorigin=""/>
<script src="https://unpkg.com/leaflet@1.2.0/dist/leaflet.js"
integrity="sha512-lInM/apFSqyy1o6s89K4iQUKg6ppXEgsVxT35HbzUupEVRh2Eu9Wdl4tHj7dZO0s1uvplcYGmt3498TtHq+log=="
crossorigin=""></script>
<script src="log.js"></script>
<style>
.map { width: 512px; height: 512px; }
</style>

72
static/progress/log.js Normal file
View File

@ -0,0 +1,72 @@
$.getJSON("data/fooo", function (data) {
var list = $("<ul />");
var mapC = $("<div />", {class: "map", id: "map"});
mapC.appendTo("body");
var track = [];
var times = [];
$.each(data.spatials, function (i, elem) {
track.push([elem.coordinates[1], elem.coordinates[0]]);
times.push(i);
});
var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
});
var map = L.map("map", {layers: [tiles]});
L.control.scale().addTo(map);
var layer = L.polyline(track).addTo(map);
map.fitBounds(layer.getBounds());
$.each(data, function (key, value) {
//console.log(key, value);
//key: instance_id, value: AnlysisResult
//value.result.instance: InstanceConfig_id
// console.log(key, value[0].result.store[0].timestamp);
/*$.each(value[0].result.store, function (index, entry) {
//console.log(entry);
var time = new Date(entry.timestamp);
var item = $("<li>", {html: entry.sequence + " @ " + time.toLocaleDateString() + " "+ time.toLocaleTimeString()});
var container = $("<p />");
if (entry.track.length > 0) {
var mapName = "map" + index;
//console.log(mapName, entry.track.length);
var mapContainer = $("<div />", {id: mapName, class: "map"});
var track = [];
$.each(entry.track, function (i, elem) {
track.push([elem.coordinates[1], elem.coordinates[0]]);
});
maps[mapName] = track;
mapContainer.appendTo(container);
}
$.each(entry.events, function (i, event) {
if ("image" in event) {
$("<img />", {src: event.image, height: 200}).appendTo(container);
}
});
container.appendTo(item);
item.appendTo(list);
});*/
});
list.appendTo("body");
var slider = $("<input />", {type: "range", start:0,end:100});
slider.appendTo("body");
/*});
$(window).on("load", function () {*/
// setTimeout(function () {
//console.log(maps);
/*$.each(maps, function (mapName, track) {
//console.log("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa");
var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
attribution: '&copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors',
});
var map = L.map(mapName, {layers: [tiles]});
L.control.scale().addTo(map);
// console.log(mapName, track);
var layer = L.polyline(track, {color: "green"}).addTo(map);
map.fitBounds(layer.getBounds());
//console.log(layer)
//L.control.layers({"osm":tiles}, {layer]).addTo(map);
});*/
// }, 2000);
});

1
static/progress/my.js
View File

@ -47,6 +47,7 @@ $.getJSON("tmp3.json", function (data) {
});
});
list.appendTo("body");
var slider = $("<input />", {type: "range" })
/*});
$(window).on("load", function () {*/

30
util/download.py Normal file
View File

@ -0,0 +1,30 @@
import logging
import os
from util import json_path
logger = logging.getLogger(__name__)
def download_board(board_id, host, instance_config_id, sequence_id, source):
local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format(
config_id=instance_config_id,
sequence_id=sequence_id,
board_id=board_id)
if os.path.exists(local_file):
return local_file
url = "{host}/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format(
host=host,
config_id=instance_config_id,
sequence_id=sequence_id,
board_id=board_id
)
board = source._get(url)
if not board.ok:
raise ConnectionError()
data = board.json()
preview_url = json_path(data, "preview_url.medium")
logger.debug(preview_url)
os.makedirs(local_file[:-len(board_id)], exist_ok=True)
source.download_file(host + preview_url, local_file)
return local_file