diff --git a/.gitignore b/.gitignore index 3b2357b..8c01398 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ _* !__init__.py *.pyc logs/ -data/ - +*data/ +plots/* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c197353 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:edge + +ADD ["requirements.txt", "/"] +RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing/" >> /etc/apk/repositories && \ + apk add --update --no-cache libpng freetype python3 libstdc++ libxml2 libxslt openblas && \ + apk add --update --no-cache --virtual .build-deps libpng-dev freetype-dev g++ python3-dev openblas-dev libxml2-dev libxslt-dev && \ + pip3 --no-cache-dir install -r requirements.txt && \ + apk del .build-deps && \ + rm requirements.txt +USER guest \ No newline at end of file diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..f49c892 --- /dev/null +++ b/Readme.md @@ -0,0 +1,10 @@ +# Geogame Log Analyzer + +## log data + +### set mtime of gpx files to the first date: + +``` +for i in */*; do touch -m -d "$(head -n 15 $i|grep time | head -n 1 |cut -d">" -f 3|cut -d"<" -f1)" $i; done +for i in */; do touch -m -d "$(head -n 15 $i/*.gpx|grep time | head -n 1 |cut -d">" -f 3|cut -d"<" -f1)" $i; done +``` \ No newline at end of file diff --git a/analysis/__init__.py b/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/analyzers/__init__.py b/analysis/analyzers/__init__.py similarity index 76% rename from analyzers/__init__.py rename to analysis/analyzers/__init__.py index c4b0ff6..808e2da 100644 --- a/analyzers/__init__.py +++ b/analysis/analyzers/__init__.py @@ -2,15 +2,17 @@ from typing import List from .analyzer import Analyzer, Result from .analyzer.biogames import BoardDurationAnalyzer, SimulationRoundsAnalyzer, ActivationSequenceAnalyzer, \ - BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer + BiogamesCategorizer, ActivityMapper, BiogamesStore, InstanceConfig, SimulationOrderAnalyzer, SimulationCategorizer, \ + SimulationFlagsAnalyzer from .analyzer.default import LogEntryCountAnalyzer, LocationAnalyzer, LogEntrySequenceAnalyzer, ActionSequenceAnalyzer, \ - CategorizerStub, Store, ProgressAnalyzer + CategorizerStub, Store, ProgressAnalyzer, SimpleCategorizer from .analyzer.locomotion import LocomotionActionAnalyzer, CacheSequenceAnalyzer from .analyzer.mask import MaskSpatials from .render import Render from .render.biogames import SimulationRoundsRender, BoardDurationHistRender, BoardDurationBoxRender, \ - ActivityMapperRender, StoreRender, SimulationRoundsMeanRender, SimulationOrderRender -from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender + ActivityMapperRender, StoreRender, SimulationOrderRender, SimulationGroupRender +from .render.default import PrintRender, JSONRender, TrackRender, HeatMapRender, LogEntryCountAnalyzerPlot, \ + LogEntryCountCSV, KMLRender from .render.locomotion import LocomotionActionRelativeRender, LocomotionActionAbsoluteRender, \ LocomotionActionRatioRender, LocomotionActionRatioHistRender @@ -23,7 +25,9 @@ __MAPPING__ = { LocomotionActionRatioHistRender, ], LogEntryCountAnalyzer: [ - JSONRender, + # JSONRender, + LogEntryCountAnalyzerPlot, + LogEntryCountCSV, ], SimulationRoundsAnalyzer: [ SimulationRoundsRender, @@ -39,6 +43,7 @@ __MAPPING__ = { LocationAnalyzer: [ TrackRender, HeatMapRender, + KMLRender, ], ActivityMapper: [ ActivityMapperRender @@ -50,8 +55,9 @@ __MAPPING__ = { StoreRender ], SimulationOrderAnalyzer: [ - JSONRender, -SimulationOrderRender + #JSONRender, + # SimulationOrderRender, + SimulationGroupRender ] } @@ -62,10 +68,10 @@ def get_renderer(cls: type) -> [type]: return __MAPPING__[cls] -def render(cls: type, results: List[Result]): +def render(cls: type, results: List[Result], name=None): for r in get_renderer(cls): p = r() p.result_types.append(cls) - rendered = p.render(results) + rendered = p.render(results, name=name) if rendered: print(str(r)) diff --git a/analyzers/analyzer/__init__.py b/analysis/analyzers/analyzer/__init__.py similarity index 76% rename from analyzers/analyzer/__init__.py rename to analysis/analyzers/analyzer/__init__.py index 39050ba..4e87e2a 100644 --- a/analyzers/analyzer/__init__.py +++ b/analysis/analyzers/analyzer/__init__.py @@ -2,16 +2,17 @@ import logging from collections import KeysView from typing import Type, Sized, Collection -from analyzers.settings import LogSettings +from analysis.analyzers.settings import LogSettings log: logging.Logger = logging.getLogger(__name__) class Result: - def __init__(self, analysis: Type, result: Sized): + def __init__(self, analysis: Type, result: Sized, name: str = None): self.result = result self.__analysis__ = analysis log.debug("set" + str(len(self.result))) + self.name = name def analysis(self): return self.__analysis__ @@ -20,7 +21,8 @@ class Result: return self.result def __repr__(self): - return "" + return "" class ResultStore: @@ -49,7 +51,7 @@ class ResultStore: :return: """ result = [] - for key in self.store: + for key in sorted(self.store): result += self.store[key] return result @@ -58,13 +60,18 @@ class ResultStore: def get_category(self, key): if key not in self.store: - return self.entry + return self.entry() + log.error("get_category %s %s", key, len(self.store[key])) return self.store[key] def serializable(self): values = {} for key in self.store: - values[key] = [{"analysis": str(result.analysis()), "result": result.get()} for result in self.store[key]] + values[key] = [{ + "analysis": str(result.analysis()), + "result": result.get(), + "name": result.name + } for result in self.store[key]] return values @@ -82,7 +89,7 @@ class Analyzer: """ raise NotImplementedError() - def result(self, store: ResultStore) -> None: + def result(self, store: ResultStore, name=None) -> None: raise NotImplementedError() def name(self) -> str: diff --git a/analyzers/analyzer/biogames.py b/analysis/analyzers/analyzer/biogames.py similarity index 50% rename from analyzers/analyzer/biogames.py rename to analysis/analyzers/analyzer/biogames.py index 96ad8f5..aebf3e5 100644 --- a/analyzers/analyzer/biogames.py +++ b/analysis/analyzers/analyzer/biogames.py @@ -3,8 +3,8 @@ from collections import defaultdict, namedtuple, OrderedDict from types import SimpleNamespace from typing import List, NamedTuple -from util import json_path, combinate -from util.download import download_board +from analysis.util import json_path, combinate +from analysis.util.download import download_board, get_board_data from . import Result, LogSettings, Analyzer, ResultStore from .default import CategorizerStub, Store @@ -52,6 +52,36 @@ class BoardDurationAnalyzer(Analyzer): self.last = {} +class TypedBoardDuration(Analyzer): + __name__ = "BoardDuration" + + def result(self, store: ResultStore) -> None: + pass + + def process(self, entry: dict) -> bool: + entry_type = entry[self.settings.type_field] + if entry_type in self.settings.boards: + pass + + def add_board(self, entry): + board_data = get_board_data(self.settings.source, ) + + def add_location(self, entry): + self.track['coordinates'].append(json_path(entry, self.settings.custom['coordinates'])) + + def add_track(self, **props): + self.track['properties'] = props + self.tracks.append(self.track) + self.track = dict(self.template) + + def __init__(self, settings: LogSettings): + super().__init__(settings) + self.last_board = {} + self.tracks = [] + self.template = {"type": "LineString", "coordinates": [], "properties": {}} + self.track = dict(self.template) + + class SimulationRoundsAnalyzer(Analyzer): __name__ = "SimuRounds" @@ -90,7 +120,7 @@ class ActivationSequenceAnalyzer(Analyzer): return False -class BiogamesCategorizer(CategorizerStub): +class BiogamesCategorizer(CategorizerStub): # TODO: refactor __name__ = "BiogamesCategorizer" def __init__(self, settings: LogSettings): @@ -105,10 +135,31 @@ class BiogamesCategorizer(CategorizerStub): class ActivityMapper(Analyzer): __name__ = "ActivityMapper" + classes = { + "sequence.simulation.": "simu", + "sequence.question.": "question", + "error": "error" + } + colors = { + "simu": "blue", + "question": "orange", + "image": "green", + "audio": "red", + "video": "purple", + "other": "brown", + "map": "violet", + "error": "grey" + } def __init__(self, settings: LogSettings) -> None: super().__init__(settings) self.store: List[self.State] = [] + self.timeline = [] + self.last_board = {} + self.last_board_type = "other" + self.last_coordinate = None + self.tracks = [] + self.track = None self.instance_config_id: str = None self.filters = SimpleNamespace() self.filters.start = lambda entry: combinate(self.settings.custom["sequences2"]["start"], entry) @@ -116,57 +167,94 @@ class ActivityMapper(Analyzer): self.State: NamedTuple = namedtuple("State", ["sequence", "events", "track", "timestamp"]) - def result(self, store: ResultStore) -> None: - instance_config_id = self.instance_config_id - for active_segment in self.store: # active_segment → sequence or None (None → map active) - seq_data_url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/".format( - config_id=instance_config_id, - sequence_id=active_segment.sequence, - ) - source = self.settings.source - seq_data = source._get(seq_data_url) - if not seq_data.ok: - logger.error("HTTP ERROR:", seq_data) - seq_data = {} + def result(self, store: ResultStore, **kwargs) -> None: + for board in self.timeline: + if board[self.settings.type_field] in self.settings.boards: + if board["extra_data"]["activity_type"] == "simu": + board["image"] = "simu.png" + continue + local_file = download_board(board["board_id"], self.instance_config_id, board["sequence_id"], + self.settings.source) + if local_file: + board['image'] = local_file + else: + board['image'] = "ERROR_FETCHING_FILE" + logger.error("error downloading board! %s %s %s", self.instance_config_id, board["sequence_id"], + board["board_id"]) else: - seq_data = seq_data.json() - # TODO: use sequence names - logger.warning(seq_data) - for event in active_segment.events: - if event[self.settings.type_field] in self.settings.boards: - sequence_id = active_segment.sequence - board_id = event["board_id"] - local_file = download_board(board_id, instance_config_id, sequence_id, source) - if local_file is not None: - event["image"] = local_file[16:] - store.add(Result(type(self), {"instance": instance_config_id, "store": [x._asdict() for x in self.store]})) + board["image"] = "map.png" + store.add(Result(type(self), { + "instance": self.instance_config_id, + "track": self.tracks, + "boards": self.timeline, + "colors": self.colors, + })) def process(self, entry: dict) -> bool: + if self.track is None: + self.track = self.new_track(entry['timestamp']) if self.instance_config_id is None: if entry[self.settings.type_field] in self.settings.custom['instance_start']: self.instance_config_id = json_path(entry, self.settings.custom['instance_config_id']) - if self.filters.start(entry): - self.store.append( - self.State( - sequence=json_path(entry, json_path(self.settings.custom, "sequences2.id_field")), - events=[], - track=[], - timestamp=entry['timestamp'])) - elif self.filters.end(entry) or not self.store: - self.store.append(self.State(sequence=None, events=[], track=[], timestamp=entry['timestamp'])) + self.update_board_type(entry) if entry[self.settings.type_field] in self.settings.spatials: - self.store[-1].track.append( - { - 'timestamp': entry['timestamp'], - 'coordinates': json_path(entry, "location.coordinates"), - 'accuracy': entry['accuracy'] - } - ) - else: - self.store[-1].events.append(entry) + self.add_location(entry) + elif entry[self.settings.type_field] in self.settings.boards: + board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"], + entry["board_id"]) + entry["extra_data"] = board_data + entry["extra_data"]["activity_type"] = self.last_board_type + entry['coordinate'] = self.new_coordinate() + self.timeline.append(entry) return False + def update_board_type(self, entry): + type = self.classify_entry(entry) + if not type == self.last_board_type: + self.add_track(activity_type=self.last_board_type, end_timestamp=entry['timestamp']) + self.last_board_type = type + + def classify_entry(self, entry): + entry_type = entry[self.settings.type_field] + if self.filters.end(entry): + data = {"extra_data": {"activity_type": "map"}, "coordinate": self.new_coordinate()} + data.update(entry) + self.timeline.append(data) + return "map" + if not entry_type in self.settings.boards: + return self.last_board_type + board_data = get_board_data(self.settings.source, self.instance_config_id, entry["sequence_id"], + entry["board_id"]) + for pattern in self.classes: + if pattern in board_data['class']: + return self.classes[pattern] + if board_data['has_video']: + return "video" + elif board_data['has_audio']: + return "audio" + elif board_data['has_image']: + return "image" + return "other" + + def new_coordinate(self): + return {"type": "Point", "coordinates": self.last_coordinate} + + def add_location(self, entry): + coordinates = json_path(entry, self.settings.custom['coordinates']) + self.track['coordinates'].append(coordinates) + self.last_coordinate = coordinates + + def add_track(self, **props): + self.track['properties'].update(props) + self.tracks.append(self.track) + self.track = self.new_track(props['end_timestamp']) + if self.last_coordinate: + self.track['coordinates'].append(self.last_coordinate) + + def new_track(self, timestamp): + return {"type": "LineString", "coordinates": [], "properties": {'start_timestamp': timestamp}} + class BiogamesStore(Store): __name__ = "BiogamesStore" @@ -205,8 +293,8 @@ class InstanceConfig(Analyzer): print(entry) self.store["instance_id"] = json_path(entry, self.settings.custom["instance_config_id"]) - def result(self, store: ResultStore): - store.add(Result(type(self), dict(self.store))) + def result(self, store: ResultStore, name=None): + store.add(Result(type(self), dict(self.store), name=name)) class SimulationOrderAnalyzer(Analyzer): @@ -217,8 +305,8 @@ class SimulationOrderAnalyzer(Analyzer): self.store = defaultdict(lambda: -1) # TODO verify self.order = [] - def result(self, store: ResultStore) -> None: - store.add(Result(type(self), [self.store[sim] for sim in self.order])) + def result(self, store: ResultStore, name=None) -> None: + store.add(Result(type(self), [self.store[sim] for sim in self.order], name=name)) def process(self, entry: dict) -> bool: entry_type = entry[self.settings.type_field] @@ -228,4 +316,36 @@ class SimulationOrderAnalyzer(Analyzer): self.store[simu_id] += 1 if not simu_id in self.order: self.order.append(simu_id) - return False \ No newline at end of file + return False + + +class SimulationCategorizer(CategorizerStub): # TODO: refactor categorizer + __name__ = "SimulationCategorizer" # TODO: rename -.- (InstanceConfigIDCategorizer) + + def process(self, entry: dict) -> bool: + if self.key is "default": + if entry[self.settings.type_field] in self.settings.custom['instance_start']: + try: + self.key = json_path(entry, self.settings.custom['instance_config_id']) + except KeyError as e: + print(entry) + raise e + return False + + +class SimulationFlagsAnalyzer(Analyzer): + __name__ = "SimuFlags" + + def __init__(self, settings: LogSettings) -> None: + super().__init__(settings) + self.store = [] + + def process(self, entry: dict) -> bool: + entry_type = entry[self.settings.type_field] + if entry_type in self.settings.custom['simulation_rounds']: + if entry["answers"][self.settings.type_field] in self.settings.custom["simu_data"]: + self.store.append(entry) + return False + + def result(self, store: ResultStore, name=None) -> None: + store.add(Result(type(self), self.store, name=name)) diff --git a/analyzers/analyzer/default.py b/analysis/analyzers/analyzer/default.py similarity index 54% rename from analyzers/analyzer/default.py rename to analysis/analyzers/analyzer/default.py index 9841162..f17ae2f 100644 --- a/analyzers/analyzer/default.py +++ b/analysis/analyzers/analyzer/default.py @@ -1,7 +1,7 @@ import logging from collections import defaultdict, OrderedDict -from util import json_path +from analysis.util import json_path from . import Result, LogSettings, Analyzer, ResultStore @@ -16,9 +16,9 @@ class LocationAnalyzer(Analyzer): super().__init__(settings) self.entries = [] - def result(self, store: ResultStore) -> None: + def result(self, store: ResultStore, **kwargs) -> None: self.log.debug(len(self.entries)) - store.add(Result(type(self), list(self.entries))) + store.add(Result(type(self), list(self.entries), name=kwargs['name'])) def process(self, entry: dict) -> bool: if entry[self.settings.type_field] in self.settings.spatials: @@ -28,6 +28,7 @@ class LocationAnalyzer(Analyzer): class LogEntryCountAnalyzer(Analyzer): + #TODO: flexibler: z.b. min/max lat/long """ count occurrences of log entry types """ @@ -88,13 +89,17 @@ class CategorizerStub(Analyzer): __name__ = "Categorizer" - def result(self, store: ResultStore) -> None: - store.new_category(self.key) + def result(self, store: ResultStore, name=None) -> None: + store.new_category(name if name else self.key) def __init__(self, settings: LogSettings): super().__init__(settings) self.key = "default" +class SimpleCategorizer(CategorizerStub): + def process(self, entry): + return False + class Store(Analyzer): """ @@ -136,3 +141,72 @@ class ProgressAnalyzer(Analyzer): if entry[self.settings.type_field] in self.settings.boards: self.board[entry["timestamp"]] = entry return False + + +class MetaDataAnalyzer(Analyzer): + """collect metadata""" + __name__ = "MetaDataAnalyzer" + + def result(self, store: ResultStore, name=None) -> None: + store.add(Result(type(self), dict(self.store))) + + def process(self, entry: dict) -> bool: + if not "metadata" in self.settings.custom: + return False + for mdata in self.settings.custom["metadata"]: + key = self.settings.custom["metadata"] + if key in entry: + self.store[mdata] = json_path(entry, key) + + def __init__(self, settings: LogSettings) -> None: + super().__init__(settings) + self.store = {} + + +def write_logentry_count_csv(LogEntryCountCSV, store, render, analyzers): + global cat, data, lines, csvfile + LogEntryCountCSV.summary = None + for cat in store.get_categories(): + data = store.get_category(cat) + render(analyzers.LogEntryCountAnalyzer, data, name=cat) + if LogEntryCountCSV.summary: + headers = [] + lines = [] + for name in LogEntryCountCSV.summary: + data = LogEntryCountCSV.summary[name] + for head in data: + if not head in headers: + headers.append(head) + line = [name] + for head in headers: + line.append(data[head]) if head in data else line.append(0) + lines.append(line) + import csv + + with open('logentrycount.csv', 'w', newline='') as csvfile: + writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE) + writer.writerow(["name"] + [h.split(".")[-1] for h in headers]) + for line in lines: + writer.writerow(line) + + +def write_simulation_flag_csv(store): + global csvfile, result, i + from datetime import datetime + import json + json.dump(store.serializable(), open("simus.json", "w"), indent=2) + with open("simus.csv", "w") as csvfile: + csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n") + for key in store.get_store(): + csvfile.write("{}\n".format(key)) + for result in store.store[key]: + csvfile.write(",{}\n".format(result.name)) + for i in result.get(): + csvfile.write(",,{},{},{},{},{},{}\n".format( + i['answers']['@id'], + i['answers']['answered'], + len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0, + len(i['selected_actions']) if i['selected_actions'] else 0, + i['timestamp'], + str(datetime.fromtimestamp(i['timestamp'] / 1000)) + )) \ No newline at end of file diff --git a/analyzers/analyzer/locomotion.py b/analysis/analyzers/analyzer/locomotion.py similarity index 99% rename from analyzers/analyzer/locomotion.py rename to analysis/analyzers/analyzer/locomotion.py index 55705f1..7c330b4 100644 --- a/analyzers/analyzer/locomotion.py +++ b/analysis/analyzers/analyzer/locomotion.py @@ -1,5 +1,4 @@ -import logging -import util +from analysis import util from . import Analyzer, LogSettings, Result, ResultStore log: logging.Logger = logging.getLogger(__name__) diff --git a/analyzers/analyzer/mask.py b/analysis/analyzers/analyzer/mask.py similarity index 100% rename from analyzers/analyzer/mask.py rename to analysis/analyzers/analyzer/mask.py diff --git a/analyzers/render/__init__.py b/analysis/analyzers/render/__init__.py similarity index 84% rename from analyzers/render/__init__.py rename to analysis/analyzers/render/__init__.py index 25b87fc..24c4d2d 100644 --- a/analyzers/render/__init__.py +++ b/analysis/analyzers/render/__init__.py @@ -5,7 +5,7 @@ from .. import Result class Render: result_types = [] - def render(self, results: List[Result]): + def render(self, results: List[Result], name=None) -> [str]: raise NotImplementedError() def filter(self, results: List[Result]): diff --git a/analysis/analyzers/render/biogames.py b/analysis/analyzers/render/biogames.py new file mode 100644 index 0000000..53c12c4 --- /dev/null +++ b/analysis/analyzers/render/biogames.py @@ -0,0 +1,203 @@ +import json +from collections import defaultdict +from typing import List, Tuple + +import matplotlib.pyplot as plt +import os +import numpy as np +from scipy.interpolate import interp1d +import networkx as nx +import itertools + +from analysis.analyzers import Store, BiogamesStore, SimulationOrderAnalyzer +from analysis.util.meta_temp import CONFIG_NAMES +from . import Render +from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper + + +def add_edge(graph, src, dest): + if graph.has_edge(src, dest): + weight = graph.get_edge_data(src, dest)['weight'] + 1 + else: + weight = 1 + graph.add_edge(tuple(src), tuple(dest), weight=weight) + + +def pairs(iterable): + a, b = itertools.tee(iterable) + next(b, None) + return zip(a, b) + + +def __plot_or_show(name=None): + if name: + plt.savefig(name) + else: + plt.show() + plt.cla() + plt.clf() + plt.close() + + +def plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries", + rotation='vertical', name=None): + names, datas = list(zip(*src_data)) + # plt.boxplot(datas, labels=names, showfliers=False, showmeans=True, meanline=True) + rand = np.random.rand(len(datas), len(datas[0])) + plt.plot(datas + rand, linewidth=.2) + plt.xticks(rotation=rotation) + # plt.margins() + plt.ylabel(ylabel) + plt.title(title) + __plot_or_show(name) + + +def graph_plot(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="sequential simulation retries", + rotation='vertical', name=None): + config_name = CONFIG_NAMES[name] if name in CONFIG_NAMES else "---" + counts_per_group = [sum(i) for i in src_data] + label = "{}: n={n}; # of sim runs: ⌀={avg:.2f}, median={median}".format( + config_name, + n=len(src_data), + avg=np.mean(counts_per_group), + median=np.median(counts_per_group) + ) + print(config_name) + name = "plots/{}.png".format(name) + g = nx.Graph() + for group in src_data: + for i in pairs(enumerate(group)): + add_edge(g, i[0], i[1]) + positions = {} + for node in g.nodes(): + positions[node] = node + widths = [x[2] / 10.0 for x in g.edges.data('weight')] + print(max(widths)) + nx.draw_networkx_edges(g, positions, width=widths) + # rand = np.random.rand(len(datas),len(datas[0])) + # plt.plot(datas+rand, linewidth=.2) + plt.xticks(rotation=rotation) + # plt.margins() + plt.ylabel(ylabel) + plt.title(title) + plt.figtext(0.5, 0.13, label, ha="center") + __plot_or_show(name) + + +def graph_fit(src_data, deg=5, name=None): + plt.title("polyfit(x,y,deg=" + str(deg) + ")") + for i in src_data: + # plt.plot(i) + count = len(i) + xp = np.linspace(0, count - 1, num=count, endpoint=True) + # fit = np.poly1d(np.polyfit(range(len(i)), i, deg=deg)) + # plt.plot(xp, fit(xp), linewidth=0.1) + xnew = np.linspace(0, count - 1, num=count * 20, endpoint=True) + f = interp1d(xp, i, kind='quadratic') + + plt.plot(range(count), i, '.', markersize=1) + plt.plot(xnew, f(xnew), linewidth=0.2) + __plot_or_show(name) + + +class SimulationRoundsRender(Render): + def render(self, results: List[Result], name=None): + data = defaultdict(list) + for result in self.filter(results): + get = result.get() + for key in get: + data[key].append(get[key]) + data_tuples = [(key, data[key]) for key in sorted(data)] + data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) + plot(data_tuples) + + result_types = [SimulationRoundsAnalyzer] + + +class BoardDurationHistRender(Render): + result_types = [BoardDurationAnalyzer] + + def render(self, results: List[Result], name=None): + data = [] + for result in self.filter(results): + session = result.get() + _data = [] + for board in session: + if "active" in board: + _data.append(board["active"]) + else: + _data.append(0) + data.append(_data) + n, bins, patches = plt.hist(data, log=True) + plt.show() + + +class BoardDurationBoxRender(Render): + result_types = [BoardDurationAnalyzer] + + def render(self, results: List[Result], name=None) -> [str]: + data = defaultdict(list) + for result in self.filter(results): + for board in result.get(): + duration = board['active'] if 'active' in board else 0 + data[board['id']].append(duration) + data_tuples = [(key, data[key]) for key in sorted(data)] + data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) + plot(data_tuples, name=name) + return [name] + + +class ActivityMapperRender(Render): + result_types = [ActivityMapper] + + def render(self, results: List[Result], name=None): + print(os.getcwd()) + files = [] + for result in self.filter(results): + data = result.get() + path = os.path.join("/tmp", data['instance'] + "_" + str(name) + ".json") + with open(path, "w") as out: + json.dump(data, out, indent=1) + files.append(path) + return files + + +class StoreRender(Render): + result_types = [Store, BiogamesStore] + + def render(self, results: List[Result], name=None): + for result in self.filter(results): + with open(os.path.join("static", "progress", "data", "fooo"), "w") as out: + json.dump(result.get(), out, indent=1) + + +class SimulationOrderRender(Render): + def render(self, results: List[Result], name=None): + data = defaultdict(list) + for result in self.filter(results): + get = result.get() + for i, value in enumerate(get): + data[i].append(value) + # data_tuples = [(key, data[key]) for key in sorted(data)] + # data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) + # plot(enumerate([r.get() for r in self.filter(results)])) + plot(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None) + + result_types = [SimulationOrderAnalyzer] + + +class SimulationGroupRender(Render): + def render(self, results: List[Result], name=None): + # data = [r.get() for r in self.filter(results)] + data = [] + for r in self.filter(results): + raw = r.get() + if len(raw) < 6: + raw = [0] + raw + data.append(raw) + print(name, len(data)) + # graph_fit(list(data), name=name) + graph_plot(list(data), ylabel="simulation retries", title="sequential simulation retries", rotation=None, + name=name) + + result_types = [SimulationOrderAnalyzer] diff --git a/analysis/analyzers/render/default.py b/analysis/analyzers/render/default.py new file mode 100644 index 0000000..6f74600 --- /dev/null +++ b/analysis/analyzers/render/default.py @@ -0,0 +1,116 @@ +import json +import logging +from typing import List + +import datetime +import matplotlib.pyplot as plt + +from analysis.analyzers import LogEntryCountAnalyzer +from analysis.util.meta_temp import KML_PATTERN +from . import Render, Result +from analysis.analyzers import LocationAnalyzer + +log = logging.getLogger(__name__) + + +class PrintRender(Render): + def render(self, results: List[Result], name=None): + print("\t" + "\n\t".join([str(r) for r in results])) + + +class JSONRender(Render): + def render(self, results: List[Result], name=None): + print(json.dumps([r.get() for r in self.filter(results)], indent=1)) + + +class TrackRender(Render): + result_types = [LocationAnalyzer] + + def render(self, results: List[Result], name=None): + data = [] + log.debug(results) + for result in self.filter(results): + if len(result.get()) > 0: + data.append( + [[entry['location']['coordinates'][1], entry['location']['coordinates'][0]] for entry in + # TODO: configurable + result.get()]) + dumps = json.dumps(data) + with open("track_data.js", "w") as out: + out.write("tracks=" + dumps + ";") + return dumps + + +def format_time(ts): + return datetime.datetime.fromtimestamp(ts/1000).strftime("%Y-%m-%dT%H:%M:%S.%f") + + +class KMLRender(Render): + result_types = [LocationAnalyzer] + + def render(self, results: List[Result], name=None): + files = [] + for result in self.filter(results): + times = ["{time}".format(time=format_time(entry["timestamp"])) for entry in result.get()] + coords = [ + "{long} {lat} 0.0" + .format( + lat=entry['location']['coordinates'][1], + long=entry['location']['coordinates'][0]) + for entry in result.get() + ] + filename = str(result.name)+".kml" + print(filename) + with open(filename, "w") as out: + out.write(KML_PATTERN.format(name=str(result.name), coordinates="\n".join(coords), when="\n".join(times))) + files.append(filename) + return files + + + + +class HeatMapRender(TrackRender): + weight = 0.01 + + def render(self, results: List[Result], name=None): + raw = super(HeatMapRender, self).render(results) + data = [] + for session in json.loads(raw): + data += [(entry[0], entry[1], self.weight) for entry in session] + dumps = json.dumps(data) + with open('heat_data.js', 'w') as out: + out.write("coords = " + dumps + ";") + return dumps + + +class LogEntryCountAnalyzerPlot(Render): + result_types = [LogEntryCountAnalyzer] + + def render(self, results: List[Result], name=None): + raw_data = list(self.filter(results))[0].get() + print(raw_data) + labels = [] + data = [] + for x in sorted(raw_data.items()): + labels.append(str(x[0]).split(".")[-1]) + data.append(x[1]) + plt.bar(range(len(data)), list(data)) + plt.xticks(range(len(data)), labels, rotation="vertical") + plt.tight_layout() + + name = "plots/{}.png".format(name) + plt.savefig(name) + plt.cla() + plt.clf() + plt.close() + +class LogEntryCountCSV(Render): + result_types = [LogEntryCountAnalyzer] + summary = None + + def render(self, results: List[Result], name=None): + if self.summary is None: + return + for result in self.filter(results): + raw_data = result.get() + self.summary[name] = raw_data \ No newline at end of file diff --git a/analyzers/render/locomotion.py b/analysis/analyzers/render/locomotion.py similarity index 91% rename from analyzers/render/locomotion.py rename to analysis/analyzers/render/locomotion.py index 96a71c1..b3ae942 100644 --- a/analyzers/render/locomotion.py +++ b/analysis/analyzers/render/locomotion.py @@ -62,20 +62,19 @@ class LocomotionActionRender(Render): class LocomotionActionAbsoluteRender(LocomotionActionRender): - def render(self, results: List[Result]): - results = filter_results(self.filter(results), ['locomotion_sum', 'action_sum'], sort) + def render(self, results: List[Result], name=None): + results = filter_results(self.filter(results), ['locomotion_sum', 'action_sum']) plot(results, "time", "abs loc/action") class LocomotionActionRelativeRender(LocomotionActionRender): - def render(self, results: List[Result]): - results = filter_results(self.filter(results), ['locomotion_relative', 'action_relative'], sort) - log.error(json.dumps(results[-10:], indent=2)) + def render(self, results: List[Result], name=None): + results = filter_results(self.filter(results), ['locomotion_relative', 'action_relative']) plot(results, "fraction of time", "rel loc/action") class LocomotionActionRatioRender(LocomotionActionRender): - def render(self, results: List[Result]): + def render(self, results: List[Result], name=None): results = filter_results(self.filter(results), ['locomotion_action_ratio']) plot_line(results, ylabel="Ratio", title="Locomotion/Action Ratio") diff --git a/analysis/analyzers/render/wip.py b/analysis/analyzers/render/wip.py new file mode 100644 index 0000000..19249ec --- /dev/null +++ b/analysis/analyzers/render/wip.py @@ -0,0 +1,393 @@ +import json + +import numpy as np + +import analysis.analyzers +from analysis.util.geo import calc_distance + + +def time_distribution(store): + # json.dump(store.serializable(), open("new.json", "w"), indent=1) + + keys = [ + "simu", + "question", + "image", + "audio", + "video", + "other", + "map" + ] + import matplotlib.pyplot as plt + + # results = [] + + places = defaultdict(list) + + for log in store.get_all(): + result = defaultdict(lambda: 0) + for i in log.get()['track']: + duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp'] + result[i['properties']['activity_type']] += duration + print(json.dumps(result, indent=4)) + total = sum(result.values()) + print(total) + percentage = defaultdict(lambda: 0) + minutes = defaultdict(lambda: 0) + for i in result: + percentage[i] = result[i] / total + minutes[i] = result[i] / 60_000 + print(json.dumps(percentage, indent=4)) + if not 'error' in result: + # places[log.get()['instance']].append(percentage) + places[log.get()['instance']].append(minutes) + + for place in places: + places[place] = sorted(places[place], key=lambda item: item['map']) + + dummy = [0] * len(keys) + results = [] + sites = [] + from util.meta_temp import CONFIG_NAMES + + for i in places: + for j in places[i]: + ordered = [] + for k in keys: + ordered.append(j[k]) + results.append(ordered) + results.append(dummy) + sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---") + + size = len(results) + ind = np.arange(size) + width = 0.9 + print(results) + data = list(zip(*results)) + print(data) + lines = [] + bottom = [0] * len(results) + for i in range(0, len(data)): + lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0]) + for k, x in enumerate(data[i]): + bottom[k] += x + plt.legend(lines, keys) + plt.title(", ".join(sites)) + plt.show() + + +# size = len(results) +# ind = np.arange(size) +# width = 0.9 +# print(results) +# data = list(zip(*results)) +# print(data) +# lines = [] +# bottom = [0] * len(results) +# for i in range(0, len(data)): +# lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0]) +# for k, x in enumerate(data[i]): +# bottom[k] += x +# plt.legend(lines, keys) +# plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)") +# plt.show() + +# json.dump(store.serializable(), open("new.json", "w"), indent=1) + + +from collections import defaultdict +import matplotlib.pyplot as plt +from analysis.util.meta_temp import CONFIG_NAMES + +keys = [ + "simu", + "question", + "image", + "audio", + "video", + "other", + "map", + # "error" +] + +loc_keys = [ + "question", + "image", + "audio", + "video" +] + + +def get_data(store, relative_values=True, sort=True, show_errors=False): + places = defaultdict(list) + + for log in store.get_all(): + if not log.analysis() == analyzers.ActivityMapper: + continue + result = defaultdict(lambda: 0) + for i in log.get()['track']: + duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp'] + result[i['properties']['activity_type']] += duration + print(json.dumps(result, indent=4)) + total = sum(result.values()) + print(total) + percentage = defaultdict(lambda: 0) + minutes = defaultdict(lambda: 0) + for i in result: + percentage[i] = result[i] / total + minutes[i] = result[i] / 60_000 + print(json.dumps(percentage, indent=4)) + if not 'error' in result or show_errors: + if relative_values: + places[log.get()['instance']].append(percentage) + else: + places[log.get()['instance']].append(minutes) + if sort: + for place in places: + places[place] = sorted(places[place], key=lambda item: item['map']) + return places + + +whitelist = ['16fc3117-61db-4f50-b84f-81de6310206f', '5e64ce07-1c16-4d50-ac4e-b3117847ea43', + '90278021-4c57-464e-90b1-d603799d07eb', 'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771'] + + +def get_data_distance(store, relative_values=True, sort=True, show_errors=False): + places = defaultdict(list) + + for log in store.get_all(): + if not log.analysis() == analyzers.ActivityMapper: + continue + result = defaultdict(lambda: 0) + for i in log.get()['track']: + coords = i['coordinates'] + if len(coords) > 1: + distance = calc_distance(coords) + result[i['properties']['activity_type']] += distance + total = sum(result.values()) + percentage = defaultdict(lambda: 0) + for i in result: + if not total == 0: + percentage[i] = result[i] / total + if not 'error' in result or show_errors: + if relative_values: + places[log.get()['instance']].append(percentage) + else: + places[log.get()['instance']].append(result) + if sort: + for place in places: + places[place] = sorted(places[place], key=lambda item: item['map']) + return places + + +def get_all_data(store, sort=False, relative=True): + places = defaultdict(list) + simu_distribution = defaultdict(lambda: 0) + # divisiors = {"time":60_000, "space":1000000} + for log in store.get_all(): + if not log.analysis() == analyzers.ActivityMapper: + continue + result = defaultdict(lambda: defaultdict(lambda: 0)) + for i in log.get()['track']: + coords = i['coordinates'] + if len(coords) > 1: + distance = calc_distance(coords) + else: + distance = 0.0 + result["space"][i['properties']['activity_type']] += distance + duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp'] + result["time"][i['properties']['activity_type']] += duration + total_space = sum(result["space"].values()) + total_time = sum(result["time"].values()) + percentage = defaultdict(lambda: defaultdict(lambda: 0)) + total = defaultdict(lambda: defaultdict(lambda: 0)) + for i in result["space"]: + if not total_space == 0: + percentage[i]["space"] = result["space"][i] / total_space + else: + percentage[i]["space"] = 0 + if not total_time == 0: + percentage[i]["time"] = result["time"][i] / total_time + else: + percentage[i]["time"] = 0 + for t in ("space", "time"): + # total[i][t] += (result[t][i] / divisiors[t]) + total[i][t] += result[t][i] + print(percentage) + if not 'error' in result: + if relative: + value = percentage + else: + value = total + places[log.get()['instance']].append(value) + simus = defaultdict(lambda: 0) + for item in log.get()['boards']: + if item["extra_data"]["activity_type"] == "simu": + simus[item["board_id"]] += 1 + simu_distribution[len(simus)] += 1 + + if sort: + for place in places: + places[place] = sorted(places[place], key=lambda item: item['map']['time']) + print(simu_distribution) + return places + + +def stack_data(keys, places, type="space"): + divisiors = {"time": 60_000, "space": 1000} + # divisiors = {"time": 1, "space": 1} + dummy = [0] * len(keys) + results = [] + sites = [] + for i in sorted(places): + if not i in whitelist: + continue + place = sorted(places[i], key=lambda item: item['map'][type]) + for j in place: + ordered = [] + for k in keys: + if k in j: + ordered.append(j[k][type] / divisiors[type]) + else: + ordered.append(0) + print(sum(ordered)) + # if sum(ordered) > 0.9 and sum(ordered) < 4000 and sum(ordered)>10: + if sum(ordered) > 0.9 and sum(ordered) < 100: + # print(sum(ordered), 1-sum(ordered)) + # if sum(ordered)<1: + # ordered[-2] = 1-sum(ordered[:-2], ordered[-1]) + results.append(ordered) + results.append(dummy) + sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---") + return results, sites + + +def plot_data(places, keys): + results, sites = stack_data(keys, places) + dpi = 86.1 + plt.figure(figsize=(1280 / dpi, 720 / dpi)) + size = len(results) + print("{} elements total".format(size)) + ind = np.arange(size) + width = 1 + # print(results) + data = list(zip(*results)) + # print(data) + lines = [] + bottom = [0] * size + plt.ticklabel_format(useMathText=False) + for i in range(0, len(data)): + lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0]) + for k, x in enumerate(data[i]): + bottom[k] += x + plt.legend(lines, keys) + plt.title(", ".join(sites)) + # plt.show() + dpi = 86 + plt.savefig("space_abs_{}.png".format(size), dpi=dpi, bbox_inches="tight") + + +colors = { + "simu": "blue", + "question": "orange", + "image": "green", + "audio": "red", + "video": "purple", + "other": "brown", + "map": "violet", + # "error":"grey", + "tasks": "olive", +} +markers = [".", "o", "x", "s", "*", "D", "p", ",", "<", ">", "^", "v", "1", "2", "3", "4"] + + +def plot_time_space(time_data, space_data, keys): + # assuming time_data and space_data are in same order! + marker = 0 + for id in time_data: + for k in keys: + for i in range(len(time_data[id])): + print(time_data[id][i][k], space_data[id][i][k]) + plt.plot(time_data[id][i][k], space_data[id][i][k], color=colors[k], marker=markers[marker]) + marker += 1 + plt.show() + + +# plt.cla() +# plt.clf() +# plt.close() + +def group_locationbased_tasks(data): + for id in data: + for log in data[id]: + loc = {"space": 0, "time": 0} + for k in log: + if k in loc_keys: + for i in ["space", "time"]: + loc[i] += log[k][i] + log["tasks"] = loc + + +def plot_time_space_rel(combined, keys): + groups = defaultdict(list) + keys = list(keys) + keys.remove("other") + for i in loc_keys: + keys.remove(i) + keys.append("tasks") + ids = [] + group_locationbased_tasks(combined) + for k in keys: + for id in sorted(combined): + if id not in whitelist: + continue + if not id in ids: + ids.append(id) + group = 0.0 + count = 0 + for item in combined[id]: + if k in item: + time = item[k]["time"] / 1000 + distance = item[k]["space"] + if time > 0: + group += (distance / time) + count += 1 + else: + print("div by zero", distance, time) + if count > 0: + groups[k].append(group / count) + else: + groups[k].append(0.0) + print(ids) + ind = np.arange(len(ids)) + width = .7 / len(groups) + print(ind) + print(json.dumps(groups, indent=1)) + bars = [] + dpi = 200 + plt.figure(figsize=(1280 / dpi, 720 / dpi)) + fig, ax = plt.subplots() + for k in groups: + print(groups[k]) + if not len(groups[k]): + groups[k].append(0) + ind = ind + (width) + bars.append(ax.bar((ind + width * len(groups) / 2), groups[k], width, color=colors[k])) + ax.set_xticks(ind + width / 2) + ax.set_xticklabels(list([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids])) + kmh = plt.hlines((1 / 3.6), 0.3, 4.2, linestyles="dashed", label="1 km/h", linewidths=1) + plt.legend(bars + [kmh], keys + [kmh.get_label()]) + print(combined.keys(), ids) + print([CONFIG_NAMES[i] if i in CONFIG_NAMES else "---" for i in ids]) + # plt.show() + dpi = 200 + plt.savefig("speed2.png", dpi=dpi) + + + + + +# plot_time_space_rel(temporal_data_rel, spatial_data_rel, keys) + +# plot_data(combined, keys) +# plot_data(get_data_distance(store,relative_values=False), keys) diff --git a/analyzers/settings.py b/analysis/analyzers/settings.py similarity index 71% rename from analyzers/settings.py rename to analysis/analyzers/settings.py index 295f670..61a4494 100644 --- a/analyzers/settings.py +++ b/analysis/analyzers/settings.py @@ -1,13 +1,17 @@ import json +import logging import sys -from sources import SOURCES +from clients.webclients import CLIENTS +log: logging.Logger = logging.getLogger(__name__) def load_source(config): - if config["type"] in SOURCES: - source = SOURCES[config["type"]]() - source.connect(**config) + if config["type"] in CLIENTS: + source = CLIENTS[config["type"]](**config) + source.login() return source + else: + log.warn(f"client {config['type']} not found!") class LogSettings: @@ -28,13 +32,15 @@ class LogSettings: self.boards = json_dict['boards'] for mod in json_dict['analyzers']: for name in json_dict['analyzers'][mod]: - print(mod, name) + print(mod, name, getattr(sys.modules[mod], name)) self.analyzers.append(getattr(sys.modules[mod], name)) self.sequences = json_dict['sequences'] if 'custom' in json_dict: self.custom = json_dict['custom'] if "source" in json_dict: self.source = load_source(json_dict['source']) + if "render" in json_dict: + self.render = json_dict['render'] def __repr__(self): return str({ @@ -51,3 +57,7 @@ class LogSettings: def load_settings(file: str) -> LogSettings: return LogSettings(json.load(open(file))) + + +def parse_settings(config: str) -> LogSettings: + return LogSettings(json.loads(config)) diff --git a/biogames.json b/analysis/biogames.json similarity index 100% rename from biogames.json rename to analysis/biogames.json diff --git a/biogames2.json b/analysis/biogames2.json similarity index 77% rename from biogames2.json rename to analysis/biogames2.json index 3a76e5b..6056bb4 100644 --- a/biogames2.json +++ b/analysis/biogames2.json @@ -13,15 +13,18 @@ ], "analyzers": { "analyzers": [ - "BiogamesCategorizer", - "ActivityMapper", - "ProgressAnalyzer", - "SimulationOrderAnalyzer" + "SimulationCategorizer", + "SimulationOrderAnalyzer", + "ActivityMapper" ] }, "dis":[ "ActivityMapper", + "BiogamesCategorizer", + "LogEntryCountAnalyzer", + "SimulationOrderAnalyzer", "ProgressAnalyzer", + "SimulationCategorizer", "InstanceConfig"], "disabled_analyzers": [ "ActivityMapper", @@ -65,14 +68,19 @@ "action":"PAUSE" } }, - "coordinates": "location.coordinates" + "coordinates": "location.coordinates", + "metadata":{ + "timestamp": "timestamp", + "gamefield": "instance_id", + "user": "player_group_name" + } }, "source":{ "type": "Biogames", - "url": "http://potato.kinf.wiai.uni-bamberg.de:5000/game2/instance/log/list/", - "login_url": "http://potato.kinf.wiai.uni-bamberg.de:5000/game2/auth/json-login", - "host":"http://potato.kinf.wiai.uni-bamberg.de:5000", - "username": "lb", - "password": "81743" + "url": "http://0.0.0.0:5000/game2/instance/log/list/", + "login_url": "http://localhost:5000/game2/auth/json-login", + "username": "ba", + "password": "853451", + "host":"http://0.0.0.0:5000" } } \ No newline at end of file diff --git a/analysis/filter_todo/pre_filter.py b/analysis/filter_todo/pre_filter.py new file mode 100644 index 0000000..8cc3c98 --- /dev/null +++ b/analysis/filter_todo/pre_filter.py @@ -0,0 +1,90 @@ +import os +from zipfile import ZipFile +import sqlite3 +import json +from collections import defaultdict + + +def get_json(filename): + log = [] + id = None + with ZipFile(filename) as zipf: + zipf.extract('instance_log.sqlite') + sql = sqlite3.connect('instance_log.sqlite') + cursor = sql.cursor() + for r in cursor.execute('SELECT json FROM log_entry;'): + entry = json.loads(r[0]) + log.append(entry) + if id is None: + id = entry['instance_id'] + sql.close() + os.remove('instance_log.sqlite') + return id, log + + +def is_finished(log): + for entry in log: + if "action" in entry: + if "LogEntryInstanceAction" in entry["@class"] and entry["action"][ + "@class"] == "de.findevielfalt.games.game2.instance.action.EndGameEnableAction" and entry['action'][ + 'enable']: + return True + return False + + +def get_simus(log): + simus = defaultdict(lambda: 0) + order = [] + actions = 0 + for entry in log: + if "LogEntryQuestion" in entry["@class"]: + if "SimulationBoardData" in entry["answers"]["@class"]: + id = entry["answers"]["@id"] + simus[id] += 1 + actions += 1 if entry['selected_actions'] else 0 + if not id in order: + order.append(id) + return dict(simus), order, actions + + +def simu_dist(simus): + dist = defaultdict(lambda: 0) + for instance in simus: + sim = simus[instance] + dist[len(sim)] += 1 + return dist + + +logs = {} +finished = [] +simus = {} +distribution = defaultdict(lambda: 0) +finished_and_simu = defaultdict(list) +files = {} +actions_dist = defaultdict(list) +with open('/home/clemens/git/ma/test/src') as src: + for line in src: + line = line.strip() + instance_id, log = get_json(line) + logs[instance_id] = log + files[instance_id] = line +for id in logs: + simus[id] = get_simus(logs[id]) + simu_count = len(simus[id][1]) + distribution[simu_count] += 1 + actions_dist[simus[id][2]].append(id) + if is_finished(logs[id]): + finished.append(id) + finished_and_simu[simu_count].append(id) +print("total: ", len(logs)) +print("finished: ", len(finished)) +print("simu_dist: ", len(distribution), json.dumps(distribution, sort_keys=True)) +for i in sorted(finished_and_simu): + print("fin+sim" + str(i) + ": ", len(finished_and_simu[i])) +for i in sorted(actions_dist): + print("actions: ", i, len(actions_dist[i])) +print(json.dumps(actions_dist[4], sort_keys=True, indent=2)) + +# print(finished_and_simu) +# for instance in finished_and_simu: +# print(files[instance]) diff --git a/loaders/__init__.py b/analysis/loaders/__init__.py similarity index 60% rename from loaders/__init__.py rename to analysis/loaders/__init__.py index 4829227..f889e09 100644 --- a/loaders/__init__.py +++ b/analysis/loaders/__init__.py @@ -1,8 +1,10 @@ from .biogames import SQLiteLoader, ZipSQLiteLoader from .loader import JSONLoader +from .neocart import NeoCartLoader LOADERS = { "json": JSONLoader, "sqlite": SQLiteLoader, - "zip": ZipSQLiteLoader + "zip": ZipSQLiteLoader, + "neocartographer": NeoCartLoader, } diff --git a/loaders/biogames.py b/analysis/loaders/biogames.py similarity index 100% rename from loaders/biogames.py rename to analysis/loaders/biogames.py diff --git a/loaders/loader.py b/analysis/loaders/loader.py similarity index 100% rename from loaders/loader.py rename to analysis/loaders/loader.py diff --git a/analysis/loaders/neocart.py b/analysis/loaders/neocart.py new file mode 100644 index 0000000..8b56e92 --- /dev/null +++ b/analysis/loaders/neocart.py @@ -0,0 +1,70 @@ +import logging +from datetime import datetime + +from lxml import etree + +from .loader import Loader + +log = logging.getLogger(__name__) + +NS = {'gpx':"http://www.topografix.com/GPX/1/1"} + +class NeoCartLoader(Loader): + def load(self, file: str): + src = open(file, "r") + parser = etree.XMLParser(recover=True) + tree = etree.parse(src, parser=parser) + self.entries = [] + for point in tree.xpath("//gpx:trkpt", namespaces=NS): + try: + self.entries.append(self.parse_point(point)) + except ValueError as e: + print(e, etree.tostring(point, pretty_print=True).decode()) + log.exception(e) + + def parse_point(self, point): + raw_lat = point.xpath("@lat")[0] + if raw_lat.count(".") > 1: + log.warning(f"recreate lat/lon from: {raw_lat}") + log.warn(etree.tostring(point, pretty_print=True).decode()) + start_offset = 4 + x = raw_lat[start_offset:].index(".") + offset = start_offset + x + raw_lon = raw_lat[offset:] + raw_lat = raw_lat[:offset] + else: + raw_lon = point.xpath("@lon")[0] + lat = float(raw_lat) + lon = float(raw_lon) + times = point.xpath("gpx:time",namespaces=NS) + assert len(times) == 1 + time = times[0].text + dt = datetime.strptime(time, "%Y-%m-%dT%H:%M:%SZ") + timestamp = int(dt.timestamp() * 1000) # python3.6 has no timestamp_ns (yet) + events = point.xpath(".//gpx:event",namespaces=NS) + assert 0 <= len(events) <= 1 + event = {} + if events: + event = dict(events[0].attrib) + if events[0].tail and events[0].tail.strip(): + try: + # base case: trailing 'geoid="0"/>' + key, v = events[0].tail.strip().split("=") + value = v.split('"')[1] + event[key] = value + except: + event['__tail__'] = events[0].tail.strip() + + return { + "location": { + "type": "Point", + "coordinates": [lon, lat] + }, + "timestamp": timestamp, + "event": event, + "type": event['message'] if event else "location" + } + + def get_entry(self) -> object: + for i in self.entries: + yield i \ No newline at end of file diff --git a/analysis/log_analyzer.py b/analysis/log_analyzer.py new file mode 100644 index 0000000..2839abd --- /dev/null +++ b/analysis/log_analyzer.py @@ -0,0 +1,134 @@ +import json +import logging +from typing import List + +from analysis import analyzers +from analysis.analyzers import get_renderer, render +from analysis.analyzers.analyzer import ResultStore +from analysis.analyzers.analyzer.default import write_logentry_count_csv, write_simulation_flag_csv +from analysis.analyzers.render import wip +from analysis.analyzers.render.default import LogEntryCountCSV, KMLRender +from analysis.analyzers.render.wip import time_distribution, plot_data +from analysis.analyzers.settings import LogSettings, load_settings, parse_settings +from analysis.loaders import LOADERS +from analysis.util.processing import grep, run_analysis, src_file + +logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG) +log: logging.Logger = logging.getLogger(__name__) + +logging.getLogger('requests').setLevel(logging.WARN) +logging.getLogger("urllib3").setLevel(logging.WARNING) + + +def urach_logs(log_ids, settings): + # return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids] + return ["data/{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids] + + +if __name__ == '__main__': + settings = {} + log_ids_gf = [] + # settings: LogSettings = load_settings("biogames2.json") + # log_ids_urach: List[str] = urach_logs([ + # # "34fecf49dbaca3401d745fb467", + # # "44ea194de594cd8d63ac0314be", + # # "57c444470dbf88605433ca935c", + # # "78e0c545b594e82edfad55bd7f", + # # "91abfd4b31a5562b1c66be37d9", + # # "597b704fe9ace475316c345903", + # # "e01a684aa29dff9ddd9705edf8", + # "597b704fe9ace475316c345903", + # "e01a684aa29dff9ddd9705edf8", + # "fbf9d64ae0bdad0de7efa3eec6", + # # "fbf9d64ae0bdad0de7efa3eec6", + # "fe1331481f85560681f86827ec", # urach + # # "fe1331481f85560681f86827ec"] + # "fec57041458e6cef98652df625", + # ] + # , settings) + # log_ids_gf = grep(["9d11b749c78a57e786bf5c8d28", # filderstadt + # "a192ff420b8bdd899fd28573e2", # eichstätt + # "3a3d994c04b1b1d87168422309", # stadtökologie + # "fe1331481f85560681f86827ec", # urach + # "96f6d9cc556b42f3b2fec0a2cb7ed36e" # oberelsbach + # ], + # "/home/clemens/git/ma/test/src", + # settings) + # log_ids = src_file("/home/clemens/git/ma/test/filtered_5_actions") + + if False: + store: ResultStore = run_analysis(log_ids_gf, settings, LOADERS) + # store: ResultStore = run_analysis(log_ids, settings, LOADERS) + + if False: + for r in get_renderer(analyzers.LocomotionActionAnalyzer): + r().render(store.get_all()) + if False: + render(analyzers.LocationAnalyzer, store.get_all()) + # print(json.dumps(store.serializable(), indent=1)) + if False: + for cat in store.get_categories(): + render(analyzers.ActivityMapper, store.get_category(cat), name=cat) + # render(analyzers.ProgressAnalyzer, store.get_all()) + + if False: + from analysis.analyzers.postprocessing import graph + + g = graph.Cache(settings) + g.run(store) + if False: + # render(analyzers.SimulationOrderAnalyzer, store.get_all()) + for cat in store.get_categories(): + data = store.get_category(cat) + render(analyzers.SimulationOrderAnalyzer, data, name=cat) + if False: + write_logentry_count_csv(LogEntryCountCSV, store, render, analyzers) + if False: + write_simulation_flag_csv(store) + if False: + time_distribution(store) + + if False: + # spatial_data = get_data_distance(store,relative_values=False) + # temporal_data = get_data(store,relative_values=False) + # spatial_data_rel = get_data_distance(store,relative_values=True) + # temporal_data_rel = get_data(store,relative_values=True) + # temporal_data_rel = json.load(open("temporal_rel.json")) + # spatial_data_rel = json.load(open("spatial_rel.json")) + # import IPython + # IPython.embed() + + # print(json.dumps(get_all_data(store))) + # json.dump(get_all_data(store), open("combined.json", "w")) + # combined = get_all_data(store, sort=True, relative=True) + # json.dump(combined, open("combined_rel.json", "w")) + # combined = json.load(open("combined_rel.json")) + combined = json.load(open("combined_total.json")) + # plot_time_space_rel(combined, keys) + plot_data(combined, wip.keys) + + if True: + settings: LogSettings = load_settings("../oeb_kml.json") + log_ids = src_file("/home/clemens/git/ma/test/oeb_2016_path") + log_ids = log_ids[0:2] + print(log_ids) + store: ResultStore = run_analysis(log_ids, settings, LOADERS) + print("render") + kml = KMLRender() + kml.render(store.get_all()) + print("done") + #for cat in store.get_categories(): + # render(analyzers.ActivityMapper, store.get_category(cat), name=cat) + +# for analyzers in analyzers: +# if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]: +# print(json.dumps(analyzers.result(), indent=2)) + +# for analyzers in analyzers: +# if analyzers.name() in ["BoardDuration"]: +# print(json.dumps(analyzers.result(), indent=2)) +# print(analyzers.render()) + +# coords = analyzers[1].render() +# with open("test.js", "w") as out: +# out.write("coords = "+coords) diff --git a/util/__init__.py b/analysis/util/__init__.py similarity index 100% rename from util/__init__.py rename to analysis/util/__init__.py diff --git a/analysis/util/download.py b/analysis/util/download.py new file mode 100644 index 0000000..a70a81d --- /dev/null +++ b/analysis/util/download.py @@ -0,0 +1,76 @@ +import logging +import os + +from analysis.util import json_path + +logger = logging.getLogger(__name__) + + +def download_board(board_id, instance_config_id, sequence_id, source, path="/data/results/"): + local_file = os.path.join("static", instance_config_id, sequence_id, board_id) + abs_path = os.path.join(path, local_file) + if os.path.exists(abs_path): + return local_file + url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( + config_id=instance_config_id, + sequence_id=sequence_id, + board_id=board_id + ) + board = source.get(url) + if not board.ok: + raise ConnectionError(url, board, board.status_code) + data = board.json() + preview_url = json_path(data, "preview_url.medium") + logger.debug(preview_url) + os.makedirs(abs_path[:-len(board_id)], exist_ok=True) + source.download_file(preview_url, abs_path) + return local_file + + +def get_config(source, instance_id): + url = "/game2/editor/config/{config_id}/".format(config_id=instance_id) + instance_data = get_json(source, url) + caches = url + "cache/" + cache_data = get_json(source, caches) + + result = { + "name": instance_data["name"], + "id": instance_data["@id"], + "caches": cache_data + } + return result + + +def get_board_data(source, instance_id, sequence_id, board_id): + url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( + config_id=instance_id, + sequence_id=sequence_id, + board_id=board_id + ) + instance_data = get_json(source, url) + if instance_data is None: + return {"class": "error"} + result = { + "class": instance_data["@class"], + } + for i in ["image", "audio", "video"]: + key = i + "_file" + result["has_" + i] = bool(key in instance_data and instance_data[key]) + return result + + +cache = {} + + +def get_json(source, url): + if url in cache: + return cache[url] + try: + data = source.get(url).json() + + except Exception as e: + print("exception", e, e.args) + logger.exception(e) + data = None + cache[url] = data + return data diff --git a/analysis/util/geo.py b/analysis/util/geo.py new file mode 100644 index 0000000..f50d6c5 --- /dev/null +++ b/analysis/util/geo.py @@ -0,0 +1,12 @@ +def calc_distance(geojson: str): + from shapely.geometry import LineString + from shapely.ops import transform + from functools import partial + import pyproj + import json + track = LineString(json.loads(geojson)['coordinates']) + project = partial( + pyproj.transform, + pyproj.Proj(init='EPSG:4326'), + pyproj.Proj(init='EPSG:32633')) + return transform(project, track).length \ No newline at end of file diff --git a/util/iter.py b/analysis/util/iter.py similarity index 100% rename from util/iter.py rename to analysis/util/iter.py diff --git a/analysis/util/meta_temp.py b/analysis/util/meta_temp.py new file mode 100644 index 0000000..0bb928c --- /dev/null +++ b/analysis/util/meta_temp.py @@ -0,0 +1,116 @@ +CONFIG_NAMES = { + '06c627ac-09fb-4f03-9a70-49261adefed9': u'Kopie 2 - Filderstadt', + '07c3566b-371b-4627-9fa6-96fdcf421ad8': u'Stadt\xf6kologieLindlarAWG-V.1', + '08369b6c-f699-41ba-9313-9d6ea2d22f78': u'Schierke', + '0bd1bba5-cde8-40e5-ad1c-326e55bf1247': u'Simulation: Wildkatzen', + '0d2e711b-be77-46e1-92f5-73199626b68c': u'Kopie 2 - Simulation: Rinder', + '11b9793e-7b4f-41ec-98fc-e46de557ae07': u'Kopie 11 - Bad Urach', + '13241906-cdae-441a-aed0-d57ebeb37cac': u'Stadt\xf6kologie', + '14dee52a-d040-4c70-9e1f-359c7faadfab': u'Kopie 5 - Bad Urach', + '14e8f4be-d27e-43a4-95e1-e033950a99bd': u'Kopie 13 - Bad Urach', + '16fc3117-61db-4f50-b84f-81de6310206f': u'Oberelsbach', + '17926099-4ed3-4ca0-996d-cc577c6fdaed': u'Kopie 6 - Bad Urach', + '17d401a9-de21-49a2-95bc-7dafa53dda64': u'Oberelsbach 2016', + '1cae4e4c-6d8b-43f0-b17d-1034b213cbaf': u'Test-Born', + '1f56b428-7c2c-4333-8fe1-c740ccbff40f': u'Bad Gateway', + '1f8b9d55-3b95-4739-914a-e2eff2dc52c3': u'Kopie 2 - Bad Gateway', + '2b3975be-242a-4c9d-80c7-8d9a370c9fe0': u'Simulation: Rinder', + '2bdc24f5-c51d-41a3-9cbd-adfc3a77a5ce': u'Simulation: Landnutzung', + '2c7cdb5d-7012-4a06-b4c8-980ad2470f10': u'Kopie 3 - Bad Gateway', + '30b743e6-144c-4fd7-a055-e87680f74c27': u'Nakundu2go', + '31da50e3-d59f-4166-91f2-7c84454c5769': u'Kopie 4 - Taltitz', + '3269d0d4-dc13-46f7-b666-1db94350fcd4': u'simu rindfleisch', + '32ed9be7-6fc2-4c50-afdb-8f32453d8409': u'Kopie 1 - Eichstaett - Stadt\xf6kologie2', + '33024899-7c10-4d44-b950-38fd0c2b0e16': u'Kopie 3 - Stadt\xf6kologie', + '3fe38d6e-04d8-49e7-a6d9-7a5f12635369': u'Kopie 8 - Bad Urach', + '50716b57-e494-46e0-9449-919cecb02a3d': u'Kopie 2 - Lindlar', + '5436357d-9644-4e3d-a181-bb4c6c0b3055': u'Kopie 4 - Bad Urach', + '543ac9b8-e990-4540-8277-994c3c756f47': u'Kopie 4 - Lindlar', + '5544ec80-5928-41e1-ba89-c13e570bda88': u'Test - Nakundu2Go', + '5637a078-c931-458d-865d-adc5d0653147': u'Kopie 2 - Bad Gateway', + '57e079b1-0a58-4150-a358-627fc9e896cc': u'Kopie 1 - Schierke', + '5cb3318a-cb5f-412f-bfd6-6467101ed505': u'Eichst\xe4tt-Schafe-1', + '5e64ce07-1c16-4d50-ac4e-b3117847ea43': u'Filderstadt', + '60e77829-2686-4022-84e6-b9e8875f7ca0': u'Kopie 10 - Bad Urach', + '6140a24e-32c6-4872-92b8-c463468f79a2': u'Taltitz neu', + '63610965-7a82-471b-a11a-0f696b4d6996': u'Kopie 3 - Lindlar', + '6479e339-f70a-4ed7-9b9e-9884a8037d81': u'Kopie 5 - Lindlar', + '658e1856-d04a-4284-9fb3-95c8e89843d9': u'Simulation: Streuobst', + '66fd2366-5985-4cac-8777-51a83e169d93': u'Kopie 1 - Test - Garnison Stadt\xf6kologie', + '681b9c2a-2547-4ffd-b510-ef28f5a2d355': u'Kopie 6 - Bad Gateway', + '74f0bd8c-c53c-4293-b583-1d7aec98fafa': u'Simulation: Luchse', + '78a00aac-422c-4772-9327-3241b32cea03': u'Kopie 2 - Stadt\xf6kologie', + '7a056d76-5636-45cc-a0bf-0555eff6101c': u'Test - Osnabr\xfcck Stadt\xf6kologie', + '7bf1de94-2627-489b-a310-cbad568d2230': u'Kopie 2 - Taltitz', + '7ea9ff83-c015-4ede-a561-8a16a1fb0833': u'Kopie 1 - Stadt\xf6kologieLindlarAWG-V.1', + '81ebf491-a556-43a8-b5d7-48ee193e2636': u'Test - Oberelsbach Wildkatze', + '877a8c70-fe0c-464b-98c1-73f8669cabd6': u'Mittenwald', + '890e99b0-eeed-4a20-ac21-ea027daf16f3': u'Kopie 3 - Bad Urach', + '8aa01b71-2609-4a47-a14c-8c3b51905fd2': u'? (lb)', + '8c002b38-606b-45cd-b046-bc4641188e18': u'Kopie 7 - Bad Urach', + '8cf124c1-3041-4e9b-a35a-1d4e09694917': u'Kopie 1 - AAAAA Bad Gateway', + '8e13e952-180c-4498-8730-9691dc837515': u'Test_Eichst\xe4tt_Schafe', + '90278021-4c57-464e-90b1-d603799d07eb': u'Eichst\xe4tt', + '92fc4eef-1489-4b31-b9e1-9d9436f7f43e': u'Kopie 5 - Taltitz', + '98c6aed7-3632-467e-9f20-5bdc3276f616': u'Kopie 8 - Bad Gateway', + '995e06bf-abc4-4103-a572-9f096d71d192': u'Eichstaett - Stadt\xf6kologie', + '9cb95913-8245-49e6-8416-ee6635e67aab': u'Kopie 2 - Simulation: Landnutzung', + '9e819058-f7f9-459e-9947-84349c7d849c': u'Kopie 9 - Bad Urach', + '9f99c761-4fb6-4636-92da-a9627977d8b3': u'Simulation: Schafe', + 'a5fa36f5-7531-4821-ba0e-cf8f2a502ad4': u'Garmisch', + 'a79bb488-5fea-4bf9-9b25-395691c8e7cd': u'Kopie 1 - A kopietest docker', + 'abdf9bd0-9b7e-4286-a25e-2cb14742db30': u'Test - Bad Urach Streuobst', + 'ac0eb831-0f47-4cf1-a1a5-2e6a535e70e9': u'Kopie 1 - Vorlagen', + 'ae726f64-cfa5-4f86-9538-a1e63dd914cf': u'AAAAA Bad Gateway', + 'b307e954-5f0e-43bb-855f-d39c1a8858bd': u'Kopie 7 - Bad Gateway', + 'b58ea5b3-b864-42e3-aaf4-1a76633c037e': u'Kopie 4 - Bad Gateway', + 'b5c27cc6-e2bc-4288-be97-1e5bc1f6f94f': u'Kopie 1 - Simulation: Landnutzung', + 'b623a3c8-7ff8-47b5-853e-7b08e200dd27': u'Taltitz', + 'be1b2167-293c-4a4b-beda-28d6aa8b47a7': u'Kopie 1 - Stadt\xf6kologie berichtigt', + 'bf7146bd-c83b-4497-b948-dd6dfc8607aa': u'Kopie 1 - Rinder Lindlar gps', + 'c2c87501-f180-40de-af7b-1a3288c82292': u'Eichstaett - Stadt\xf6kologie2', + 'c3016b66-3c26-4ec4-bcf1-d36be07be037': u'?? (lb)', + 'c3598b20-e8a5-45eb-953a-2b474fd2695a': u'Test - Eichst\xe4tt Schafe', + 'c39fe95e-2cfd-461b-8103-cfd3e2b45e67': u'??? (lb)', + 'c46994cc-5ca7-4f9f-b548-7bd6a6fff026': u'Kopie 1 - Bad Gateway', + 'c528e93f-3469-4de9-b05d-e809575d2999': u'????', + 'c6606915-3b7e-48f4-adfe-cbcf3130e76a': u'Kopie 12 - Bad Urach', + 'c8ed936c-25d6-40ea-a731-2741c1d53b48': u'Born', + 'c9df06e1-33a7-40fc-bd3d-0eba6abba245': u'Test - Schierke Luchse', + 'ce093332-dc98-4cfa-9ff4-47903897e84f': u'Kopie 5 - Bad Gateway', + 'ceb78b48-495d-4761-bb61-563fa4dd41fb': u'Kopie 2 - Eichstaett - Stadt\xf6kologie2', + 'd5712976-59fa-452c-a5e3-8b4232b5cb44': u'Kopie 1 - Garmisch Test', + 'd7d0be7e-a0ac-4f4f-910d-c55e13a01e88': u'Test - Garmisch Rinder', + 'db1cd7aa-878b-4c30-9234-0739498996d6': u'Bad Urach - \xfcberarbeitet', + 'df68db4d-3d51-45f3-96ed-08429a7de3c9': u'A kopietest docker', + 'e3b0ffce-6135-400e-9796-d1aef173aaf5': u'Kopie 3 - Taltitz', + 'e3e86e25-4d92-11e6-b176-00199963ac6e': u'Garmisch (geschrumpft)', + 'e7b3094d-d3c6-41c7-92e3-28638365e018': u'Born II', + 'e7f36db3-b919-4208-8d98-b5d400b5d972': u'Kopie 15 - Bad Urach', + 'e9f35c27-6f4f-487c-b07e-9d0ab27a7b85': u'Kopie 1 - Filderstadt', + 'ea94b249-439b-46dd-b621-e0fbe99aa4ee': u'Stadt\xf6kologie berichtigt', + 'ec782ab1-eb9d-43b9-a2d1-4699f8432adb': u'Kopie 2 - Bad Urach', + 'ecfdfd0b-28be-4df2-8994-8092a7fe87b5': u'Kopie 3 - Simulation: Landnutzung', + 'f7bb56a3-fb15-413a-9a3e-f61e22d0a7d1': u'Kopie 2 - Schierke', + 'f8f65e9d-de9e-4f8d-8bb6-d1e4e01593a0': u'Arbeitstitel', + 'fca28f01-ea17-4c41-8e60-4726f96dfca8': u'Kopie 1 - Test-Born', + 'fe43a0f0-3dea-11e6-a065-00199963ac6e': u'Vorlagen', + 'ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771': u'Bad Urach' +} + + +KML_PATTERN=""" + + + + + + {when} + {coordinates} + + + {coordinates} + + + +""" \ No newline at end of file diff --git a/analysis/util/processing.py b/analysis/util/processing.py new file mode 100644 index 0000000..7291f60 --- /dev/null +++ b/analysis/util/processing.py @@ -0,0 +1,67 @@ +import logging +from typing import List + +from analysis.analyzers.analyzer import ResultStore, Analyzer +from analysis.analyzers.settings import LogSettings + +log: logging.Logger = logging.getLogger(__name__) + + +def process_log(logfile: str, settings: LogSettings, loaders) -> List[Analyzer]: + loader = loaders[settings.log_format]() + try: + loader.load(logfile) + except BaseException as e: + raise RuntimeError(e) + analyzers: List[Analyzer] = [] + log.debug("build analyzers") + for analyzer in settings.analyzers: + analyzers.append(analyzer(settings)) + log.debug("process entries") + for entry in loader.get_entry(): + for analyzer in analyzers: + try: + if analyzer.process(entry): + break + except KeyError as e: + log.exception(e) + return analyzers + + +def run_analysis(log_ids: list, settings, loaders): + store: ResultStore = ResultStore() + for log_id in log_ids: + log.info("LOG_ID: "+ str(log_id)) + for analysis in process_log(log_id, settings, loaders): + log.info("* Result for " + analysis.name()) + analysis.result(store, name=log_id) + return store + + +def load_ids(name: str): + log_ids = [] + with open(name) as src: + for line in src: + line = line.strip() + log_ids.append(line) + return log_ids + + +def grep(log_ids, source, settings): + logs = [] + with open(source) as src: + lines = src.readlines() + for id in log_ids: + for line in lines: + if id in line: + logs.append(line.strip()) + return logs + + +def src_file(filename): + log_ids = [] + with open(filename) as src: + for line in src: + line = line.strip() + log_ids.append(line) + return log_ids diff --git a/analyzers/render/biogames.py b/analyzers/render/biogames.py deleted file mode 100644 index e62bcfc..0000000 --- a/analyzers/render/biogames.py +++ /dev/null @@ -1,132 +0,0 @@ -import json -from collections import defaultdict -from typing import List, Tuple - -import matplotlib.pyplot as plt -import os - -from analyzers import Store, BiogamesStore, SimulationOrderAnalyzer -from . import Render -from .. import Result, SimulationRoundsAnalyzer, BoardDurationAnalyzer, ActivityMapper - -ONE_DAY = 24 * 60 * 60 - - -def plot(src_data: List[Tuple[str, List[int]]], title: str = "simulation retries", ylabel: str = "simulation rounds", - xargs={}): - names, datas = list(zip(*src_data)) - plt.boxplot(datas, labels=names, **xargs) - plt.xticks(rotation='vertical') - # plt.margins() - plt.ylabel(ylabel) - plt.title(title) - plt.show() - - -def plot_old(src_data: List[Tuple[str, List[int]]], ylabel="simulation rounds", title="simulation retries", - rotation='vertical'): - names, datas = list(zip(*src_data)) - plt.boxplot(datas, labels=names, **{"showfliers": True}) - plt.xticks(rotation=rotation) - # plt.margins() - plt.ylabel(ylabel) - plt.title(title) - plt.show() - - -class SimulationRoundsRender(Render): - result_types = [SimulationRoundsAnalyzer] - xargs = {} - - def render(self, results: List[Result]): - data = defaultdict(list) - for result in self.filter(results): - get = result.get() - for key in get: - data[key].append(get[key]) - data_tuples = [(key, data[key]) for key in sorted(data)] - data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) - plot(data_tuples, xargs=self.xargs) - - result_types = [SimulationRoundsAnalyzer] - - -class SimulationRoundsMeanRender(SimulationRoundsRender): - xargs = {"showfliers": False} - - -class BoardDurationHistRender(Render): - result_types = [BoardDurationAnalyzer] - - def render(self, results: List[Result]): - plt.title("board display duration") - plt.xlabel("display time") - plt.ylabel("# of boards") - data = [] - for result in self.filter(results): - session = result.get() - _data = [] - for board in session: - if "active" in board: - value = board["active"] - if value > ONE_DAY: - _data.append(0) - else: - _data.append(value) - else: - _data.append(0) - data.append(_data) - n, bins, patches = plt.hist(data, log=True) - plt.show() - - -class BoardDurationBoxRender(Render): - result_types = [BoardDurationAnalyzer] - - def render(self, results: List[Result]): - data = defaultdict(list) - for result in self.filter(results): - get = result.get() - for board in get: - duration = board['active'] if 'active' in board else 0 - if duration < ONE_DAY: - data[board['id']].append(duration) - data_tuples = [(key, data[key]) for key in sorted(data)] - data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) - plot(data_tuples, "board duration", "display time", xargs={"showfliers": False}) - - -class ActivityMapperRender(Render): - result_types = [ActivityMapper] - - def render(self, results: List[Result]): - print(os.getcwd()) - for result in self.filter(results): - data = result.get() - with open(os.path.join("static", "progress", "data", data['instance']), "w") as out: - json.dump(data["store"], out, indent=1) - return "ok" - - -class StoreRender(Render): - result_types = [Store, BiogamesStore] - - def render(self, results: List[Result]): - for result in self.filter(results): - with open(os.path.join("static", "progress", "data", "fooo"), "w") as out: - json.dump(result.get(), out, indent=1) - - -class SimulationOrderRender(Render): - def render(self, results: List[Result]): - data = defaultdict(list) - for result in self.filter(results): - get = result.get() - for i, value in enumerate(get): - data[i].append(value) - # data_tuples = [(key, data[key]) for key in sorted(data)] - # data_tuples = sorted(data_tuples, key=lambda x: sum(x[1])) - # plot(enumerate([r.get() for r in self.filter(results)])) - plot_old(list(data.items()), ylabel="simulation retries", title="sequential simulation retries", rotation=None) - - result_types = [SimulationOrderAnalyzer] diff --git a/analyzers/render/default.py b/analyzers/render/default.py deleted file mode 100644 index 044fdf3..0000000 --- a/analyzers/render/default.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -import logging -from typing import List - -from . import Render, Result -from .. import LocationAnalyzer - -log = logging.getLogger(__name__) - - -class PrintRender(Render): - def render(self, results: List[Result]): - print("\t" + "\n\t".join([str(r) for r in results])) - - -class JSONRender(Render): - def render(self, results: List[Result]): - print(json.dumps([r.get() for r in self.filter(results)], indent=1)) - - -class TrackRender(Render): - result_types = [LocationAnalyzer] - - def render(self, results: List[Result]): - data = [] - log.debug(results) - for result in self.filter(results): - if len(result.get()) > 0: - data.append( - [[entry['location']['coordinates'][1], entry['location']['coordinates'][0]] for entry in # TODO: configurable - result.get()]) - dumps = json.dumps(data) - with open("track_data.js", "w") as out: - out.write("tracks=" + dumps + ";") - return dumps - - -class HeatMapRender(TrackRender): - weight = 0.01 - - def render(self, results: List[Result]): - raw = super(HeatMapRender, self).render(results) - data = [] - for session in json.loads(raw): - data += [(entry[0], entry[1], self.weight) for entry in session] - dumps = json.dumps(data) - with open('heat_data.js', 'w') as out: - out.write("coords = " + dumps + ";") - return dumps diff --git a/clients/__init__.py b/clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clients/webclients.py b/clients/webclients.py new file mode 100644 index 0000000..aa4e94c --- /dev/null +++ b/clients/webclients.py @@ -0,0 +1,158 @@ +import json +import logging + +import os +import shutil +import tempfile +import typing + +import requests + +log: logging.Logger = logging.getLogger(__name__) + + +class Client: + host: str = "" + cookies: typing.Dict[str, str] = {} + headers: typing.Dict[str, str] = {} + + def url(self, path): + if self.host: + return self.host + path + return path + + def get(self, url, **kwargs) -> requests.models.Response: + log.info("GET " + str(url)) + return requests.get(self.url(url), cookies=self.cookies, headers=self.headers, **kwargs) + + def post(self, url, data, **kwargs) -> requests.models.Response: + log.info("POST " + str(url)) + return requests.post(self.url(url), data, cookies=self.cookies, headers=self.headers, **kwargs) + + def download_file(self, url, target, **kwargs) -> bool: + with open(target, "wb") as out: + try: + download = self.get(url, stream=True, **kwargs) + shutil.copyfileobj(download.raw, out) + except Exception as e: + log.exception(e) + os.remove(target) + return False + return True + + def download_files(self, urls, **kwargs) -> tempfile.TemporaryDirectory: + target = tempfile.TemporaryDirectory() + for path in urls: + filename = os.path.join(target.name, path.split("/")[-1]) + self.download_file(path, filename, **kwargs) + return target + + def login(self): + pass #TODO + + def list(self): + pass #TODO + + +class BiogamesClient(Client): + config_fields: typing.Dict[str, typing.List[str]] = { + 'login': ('username', 'password', 'host'), + 'session': ('sessionid', 'csrftoken', 'host'), + } + login_url: str = "/game2/auth/json-login" + list_url: str = "/game2/instance/log/list/" + headers: typing.Dict[str, str] = {'Accept': 'application/json'} + + def __init__(self, **kwargs): + match = {j: all([i in kwargs for i in self.config_fields[j]]) for j in self.config_fields} + valid = filter(lambda x: match[x], match) + if not valid: + raise ValueError("missing parameter (" + str(self.config_fields) + ")") + self.config = kwargs + self.cookies = {} + self.host = self.config['host'] + if 'session' in valid: + self.cookies = kwargs + + def login(self) -> bool: + csrf_request = self.get(self.list_url) + if not csrf_request.ok: + log.exception(ConnectionError("Unable to obtain CSRF token (" + str(csrf_request) + ")")) + return False + if not 'csrftoken' in self.cookies: + self.cookies['csrftoken'] = csrf_request.cookies['csrftoken'] + login_payload = { + 'username': self.config['username'], + 'password': self.config['password'], + 'next': '', + 'csrfmiddlewaretoken': 'csrftoken', + } + login = self.post(self.login_url, json.dumps(login_payload)) + if not login.ok: + log.exception(ConnectionError("Unable to authenticate", login, login.text)) + return False + self.cookies['sessionid'] = login.cookies['sessionid'] + print(self.cookies) + return True + + def list(self) -> dict: + print(self.cookies) + logs = self.get(self.list_url) + if not logs.ok: + raise ConnectionError("HTTP fail", logs, logs.text) + return logs.json() + + def load_all_logs(self) -> tempfile.TemporaryDirectory: + return self.download_files([i["file_url"] for i in self.list()]) + +class GeogamesClient(Client): + config_fields = ("host",) + + def __init__(self, **kwargs): + for field in self.config_fields: + if not field in kwargs: + raise ValueError(f"missing parameter: {field}") + self.host = kwargs['host'] + self.path = "neocartographer" + self.config = kwargs + + def login(self): + return True + + def list(self): + logs = self.get(self.path) + data = logs.json() + prepared_logs = [] + for log in data: + players = self.get(f"{self.path}/{log['name']}/").json() + for player in players: + prepared_logs.append({ + '@id': f"{log['name']}/{player['name']}", + 'start_date': player['mtime'], + 'player_group_name': player['name'], + 'file_url': f"{self.path}/{log['name']}/{player['name']}", + }) + return prepared_logs + + def download_files(self, urls, **kwargs) -> tempfile.TemporaryDirectory: + target = tempfile.TemporaryDirectory() + for path in urls: + filename = os.path.join(target.name, "-".join +(path.split("/")[-2:])) + self.download_file(path, filename, **kwargs) + return target + +CLIENTS: typing.Dict[str, typing.Type[Client]] = { + "Biogames": BiogamesClient, + "Geogames": GeogamesClient, +} + +if __name__ == '__main__': + # c = BiogamesClient(host="http://biodiv", username="ba", password="853451") + # print(c.login()) + # print(json.dumps(c.list(), indent=1)) + # print(type(c.load_all_logs())) + # print(type(c.get("/"))) + c = BiogamesClient(host="http://biodiv", **{'csrftoken': 'IgbwP83iEibW6RE7IADIFELYdbx0dvqQ', + 'sessionid': 'zntsj09d92tjos1b6ruqjthlzv60xdin'}) + print(json.dumps(c.list(), indent=1)) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ba49ee1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,63 @@ +version: "3" + +services: + app: + image: docker.clkl.de/ma/celery:0.4.1 + build: . + volumes: + - ./:/app + working_dir: /app/selector + command: python3 webserver.py + environment: + - PYTHONPATH=/app + - PYTHONUNBUFFERED=1 + networks: + - default + - traefik_net + labels: + - "traefik.enable=true" + - "traefik.port=5000" + - "traefik.docker.network=traefik_net" + - "traefik.url.frontend.rule=Host:select.ma.potato.kinf.wiai.uni-bamberg.de" + + celery: + image: docker.clkl.de/ma/celery:0.4.1 + environment: + - PYTHONPATH=/app + - PYTHONUNBUFFERED=1 + volumes: + - ./:/app + - ./data/results:/data/results + working_dir: /app + command: celery -A tasks.tasks worker --loglevel=info + + + redis: + image: redis:4-alpine + volumes: + - ./data/redis:/data + command: redis-server --appendonly yes + + nginx: + image: nginx:1.13-alpine + volumes: + - ./data/results:/usr/share/nginx/html:ro + networks: + - traefik_net + labels: + - "traefik.enable=true" + - "traefik.port=80" + - "traefik.docker.network=traefik_net" + - "traefik.url.frontend.rule=Host:results.ma.potato.kinf.wiai.uni-bamberg.de" + + log_data: + image: nginx:1.13-alpine + volumes: + - ./log_data/:/srv/:ro + - ./log_data.conf:/etc/nginx/conf.d/log_data.conf + + +networks: + traefik_net: + external: + name: traefik_net diff --git a/frontend/Readme.md b/frontend/Readme.md new file mode 100644 index 0000000..8ede456 --- /dev/null +++ b/frontend/Readme.md @@ -0,0 +1,41 @@ +# Traefik reverse proxy for analysis framework + +## Usage (default: http) + +1. `cd traefik` +2. `editor docker-compose.yml` + * Adjust the *traefik.frontend.rule* label for the traefik dashboard + * Default (match any): + * traefik.localhost + * traefik.potato.kinf.wiai.uni-bamberg.de + * Adjust port mapping + * Default: + * 80 → 80 + * Syntax: : +3. `docker-compose up -d` +4. `cd ../..` +5. `editor docker-compose.yml` + * adjust the *traefik.url.frontend.rule* labels for services *app* and *nginx* + * adjust the urls in *selector/config.py* accordingly +6. `docker-compose up -d` +7. You have a working analysis framework setup now + * Stop with `docker-compose down` + * Start with `docker-compose up -d` + +## Usage (https) + +1. Be on a host with port 80 available from the internet +2. Follw HTTP usage above up to step 2 +3. Add acme.json volume: + * Uncomment the line for the acme.json volume + * Adjust the host path + * Syntax: : +4. Create acme.json + * touch acme.json + * chmod 600 acme.json +5. Activate traefiks ACME module + * `mv config.toml config_http.toml` + * `mv config_acme.toml config.toml` + * `editor config.toml` + * Adjust the *acme.email* value +6. Continue with HTTP Usage steps 3 + diff --git a/frontend/traefik/config.toml b/frontend/traefik/config.toml new file mode 100644 index 0000000..8a974e9 --- /dev/null +++ b/frontend/traefik/config.toml @@ -0,0 +1,8 @@ +logLevel = "INFO" + +[web] +address = ":8080" + +[docker] +watch = true +exposedbydefault = false diff --git a/frontend/traefik/config_acme.toml b/frontend/traefik/config_acme.toml new file mode 100644 index 0000000..90ffaa0 --- /dev/null +++ b/frontend/traefik/config_acme.toml @@ -0,0 +1,26 @@ +logLevel = "INFO" +defaultEntryPoints = ["https", "http"] + +[web] +address = ":8080" + +[docker] +watch = true +exposedbydefault = false + +[entryPoints] + [entryPoints.http] + address = ":80" + [entryPoints.http.redirect] + entryPoint = "https" + [entryPoints.https] + address = ":443" + [entryPoints.https.tls] + +[acme] + email = "tls-admin@org.example" + storage = "acme.json" + entryPoint = "https" + OnHostRule = true + [acme.httpChallenge] + entryPoint = "http" \ No newline at end of file diff --git a/frontend/traefik/docker-compose.yml b/frontend/traefik/docker-compose.yml new file mode 100644 index 0000000..134095c --- /dev/null +++ b/frontend/traefik/docker-compose.yml @@ -0,0 +1,21 @@ +version: "3" +services: + traefik: + image: traefik:1.6 + command: --configFile=/traefik.toml + volumes: + - ./config.toml:/traefik.toml +# - ./acme.json:/acme.json + - /var/run/docker.sock:/var/run/docker.sock + ports: + - 80:80 + networks: + - net + labels: + - "traefik.enable=true" + - "traefik.port=8080" + - "traefik.frontend.rule=Host:traefik.localhost,traefik.potato.kinf.wiai.uni-bamberg.de" + restart: on-failure:5 +networks: + net: + driver: bridge diff --git a/log_analyzer.py b/log_analyzer.py deleted file mode 100644 index d9fe24a..0000000 --- a/log_analyzer.py +++ /dev/null @@ -1,186 +0,0 @@ -import json -import logging -from typing import List - -import analyzers -from analyzers import get_renderer, Analyzer, render, Store -from analyzers.analyzer import ResultStore -from analyzers.settings import LogSettings, load_settings -from loaders import LOADERS - -logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG) -log: logging.Logger = logging.getLogger(__name__) - -requests_log = logging.getLogger('requests') -requests_log.setLevel(logging.WARN) - - -def process_log(log_id: str, settings: LogSettings) -> List[Analyzer]: - logfile: str = "data/urach/inst_{id}.{format}".format(id=log_id, format=settings.log_format) - loader = LOADERS[settings.log_format]() - try: - loader.load(logfile) - except BaseException as e: - raise RuntimeError(e) - analyzers: List[Analyzer] = [] - log.debug("build analyzers") - for analyzer in settings.analyzers: - analyzers.append(analyzer(settings)) - log.debug("process entries") - for entry in loader.get_entry(): - for analyzer in analyzers: - if analyzer.process(entry): - break - return analyzers - - -if __name__ == '__main__': - settings: LogSettings = load_settings("biogames2.json") - log_ids: List[str] = [ - "0ff5e4d75a7c4c5683dd025aaac7de0d", -"1d80340985c7411aa78f9629aef91351", -"02c537ce1a514cb09eb387e7993f51a4", -"2cf1f8a490ac45e7bb035acc2de6bbc9", -"2f1f7713a6324fe580dbc67d4254bda5", -"3b5e4a43472f468196cc6b6a26c984a5", -"4a6e165e1e85421aaa681a68b6448f6b", -"4a2079607e9f41e9a070b32bf5a6b321", -"4ad0de379b36424e91bc709139a2538f", -"4b52635a0473424da7f9593098a38b11", -"04bf6dbbf5f846ad8d505faea8be7f45", -"5a1ae8dfa1f34469b14e910f8794285d", -"5ca377951ae440c7b3c479dc6786da52", -"5dd7e3d04dce41279b14d32f42c2052d", -"6f6f2edb053b409197f77b0322fc2c10", -"7a5f314c866f4413ae3a9eb5a0bd544a", -"7b25d0c0d2a44d31a93b8c8a84ea9cbb", -"7b697ac0aa1a480698261b62807cd447", -"7c7e3b819ec44b2994dadfad40508d56", -"7c7860ecf8904a57ad8829298570382e", -"7d96db101a254161b30078b421e5c4e2", -"09a641efdb6e47a4b7be5ef697d57cee", -"9c08fb690c5e42ec8add80305c78476a", -"9c699184e5314c5192000b998912f9c3", -"34eff87a077a45cdb64a0b1001098d16", -"35a755e2c73c48dda77f32c1e65c16c5", -"36c686cc3493405c8ce3df9f0b9e9f0b", -"36e350a44a744e08b491cef368c92aae", -"39e9e1625a7a426f81b387e56a142ea3", -"46d18a1237e34e689eeb47a24bd1f1a7", -"46fdee17dffe4e4b9ac3dd9dee7d3860", -"52ded2824b6c40559a923fc772148a8d", -"59bcafe4041d4c919585bac2be39fbe4", -"63e1a33bb1bd474bb8cffe6af54f6fe2", -"63fb0ee5bdd74c9d8d1a11958e6cbcc0", -"74f47e7694cf4bd2b965790479d5c4e6", -"76a20e83af5d4edab295f4f93d436c1c", -"0080e8e0364d4ed69fb349f95810ca4e", -"122a2dbc4aa94fd9a5535385a4c5c624", -"173c95bca9024eb5897d6f196d7b35e3", -"203f9b1e3113486c87dee0cc7ab2ec3e", -"292cf93f6ad147b5babbe9fd0842672d", -"510f86e65deb43bdad3f30ca55366fa7", -"683a0ab03fee437790ea06770e850f0d", -"698a03dca3ab41969c5ba91c4ac95012", -"788ddde1fa5e49498363907f54ab525f", -"798a8620c6aa4a369d31cf07357176e8", -"974c98d5426446c9af6be608d69e8cea", -"977cd2ae146e4b5cb53b284c5b58bee6", -"1887ed68b88f49a686d8d433b7f439a0", -"1307199d0d4f4f8f8dd019fa7e4dc9a8", -"58865640dc73482984f08b5ef728683c", -"83150656b4cf47409c2a8c971d78c359", -"a17f6affa68140aaaaded61c7dd310e0", -"a39c7ff84f624237a8e649b56577aedc", -"a85e76d834d64fc4986e6eea45cf48cf", -"a3549e5a2f5e4bcea5af97af0e86ea60", -#"a57903d4d27b47a0afa4f198a7bf9741", -"b1cf964e987f4387a3c377c329623a89", -"b59d67957ea946ecab7ce57ba395f825", -"b606f8c1c00e4edb8fe8e3afc6b790cb", -"b55222002dff47afb9cde94deb2d776f", -"b845757019654c20bf4f11a8aa2964f4", -"bac9268aca25471587a274baa8cb0aef", -"c999941878a04ca4b96b00308fdf1d47", -"d0e41604aa1c4a50a06fdc867e3e0883", -"d2c2a43cce4a4ebfa7008423e108fff4", -"d20c062e155147af9aaaee477e68e209", -"d31b742005d04f8e85845ad0fb9582fc", -"d6821c791abd40cf9d90ebf01ce287b8", -"d49515a15647455db263a5b1a963bb5d", -"da2ecef74b0b42dfb4599aa231580dcf", -"dbb281fdff3849eeb7ff61d343172dd9", -"dbd5e5ea53bf4f5db3dd99582322f915", -"de8a9a1996934ae98daa93dc2425509f", -"e0c802e9e50146fc93346d0f6e865612", -"e6b34adce1534ae08af70613c0ef729d", -"e55630984dcb4b87aa13080d0ef23e15", -"eed665d75dfa4a619d5a74a6cc104341", -"f7d0033ee7124baf974f171a571e5e0a", -"f303f2c550704a178b7c56417d44c329", -"f1106d463759429e8a4dda41fda512f8", -"f8045427e89349deb279d3c12e9c0462", -"feb8cf484f604e4a8137a8d00e7226b8", -"ff935be0480c4c6cac56ebff9021895f", -"ffae0d9d78124b6f808cef1771e78179", - ] - #log_ids: List[str] = [ - #"34fecf49dbaca3401d745fb467", - # "44ea194de594cd8d63ac0314be", - # "57c444470dbf88605433ca935c", - # "78e0c545b594e82edfad55bd7f", - # "91abfd4b31a5562b1c66be37d9", - "597b704fe9ace475316c345903", - "e01a684aa29dff9ddd9705edf8", - "fbf9d64ae0bdad0de7efa3eec6", - # "fe1331481f85560681f86827ec", - # "fe1331481f85560681f86827ec"] - #"fec57041458e6cef98652df625", ] - store: ResultStore = ResultStore() - for log_id in log_ids: - log.debug(log_id) - for analysis in process_log(log_id, settings): - log.info("* Result for " + analysis.name()) - # print(analysis.result()) - # print(analysis.render()) - analysis.result(store) - if False: - for r in get_renderer(analyzers.LocomotionActionAnalyzer): - r().render(store.get_all()) - if False: - render(analyzers.LocationAnalyzer, store.get_all()) - #print(json.dumps(store.serializable(), indent=1)) - if False: - render(analyzers.ActivityMapper, store.get_all()) - render(analyzers.ProgressAnalyzer, store.get_all()) - if True: - for a in [analyzers.BiogamesCategorizer, - analyzers.ActivityMapper, - analyzers.LogEntrySequenceAnalyzer, - analyzers.ActionSequenceAnalyzer, - analyzers.BoardDurationAnalyzer, - analyzers.LocomotionActionAnalyzer, - analyzers.SimulationRoundsAnalyzer, - analyzers.ProgressAnalyzer]: - print(str(a)) - render(a, store.get_all()) - - if False: - from analyzers.postprocessing import graph - g = graph.Cache(settings) - g.run(store) - if True: - render(analyzers.SimulationOrderAnalyzer, store.get_all()) - -# for analyzers in analyzers: -# if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]: -# print(json.dumps(analyzers.result(), indent=2)) - -# for analyzers in analyzers: -# if analyzers.name() in ["BoardDuration"]: -# print(json.dumps(analyzers.result(), indent=2)) -# print(analyzers.render()) - -# coords = analyzers[1].render() -# with open("test.js", "w") as out: -# out.write("coords = "+coords) diff --git a/log_data.conf b/log_data.conf new file mode 100644 index 0000000..0c92fcb --- /dev/null +++ b/log_data.conf @@ -0,0 +1,9 @@ +server { + listen 80; + server_name log_data; + location / { + root /srv/; + autoindex on; + autoindex_format json; + } +} \ No newline at end of file diff --git a/log_data/.gitkeep b/log_data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/neocart.json b/neocart.json new file mode 100644 index 0000000..392d8ad --- /dev/null +++ b/neocart.json @@ -0,0 +1,34 @@ +{ + "logFormat": "neocartographer", + "entryType": "type", + "spatials": [ + "location" + ], + "actions": [], + "boards": [], + "analyzers": { + "analysis.analyzers": [ + "SimpleCategorizer", + "LocationAnalyzer" + ] + }, + "sequences": {}, + "custom": { + "coordinates": "location.coordinates", + "metadata": { + "timestamp": "timestamp", + "gamefield": "instance_id", + "user": "player_group_name" + } + }, + "source": { + "type": "Geogames", + "host": "http://log_data/", + "path": "neocartographer" + }, + "render": [ + "KMLRender" + ] +} + + diff --git a/requirements.txt b/requirements.txt index dc88295..eae5dfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,15 @@ requests==2.18.4 -numpy==1.13.1 +numpy==1.14.2 matplotlib==2.1.0 -osmnx==0.6 +#osmnx==0.6 networkx==2.0 -pydot==1.2.3 \ No newline at end of file +#pydot==1.2.3 +scipy==1.0.1 +#ipython==6.2.1 + +flask==0.12.2 + +celery==4.1.1 +redis==2.10.6 + +lxml==4.2.1 \ No newline at end of file diff --git a/selector/__init__.py b/selector/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/selector/config.py b/selector/config.py new file mode 100644 index 0000000..46de267 --- /dev/null +++ b/selector/config.py @@ -0,0 +1,140 @@ +KML = """{ + "logFormat": "zip", + "entryType": "@class", + "spatials": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation" + ], + "actions": [], + "boards": [ + "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry" + ], + "analyzers": { + "analysis.analyzers": [ + "BiogamesCategorizer", + "LocationAnalyzer" + ] + }, + "sequences": { + "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache", + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction", + "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction" + } + }, + "custom": { + "simulation_rounds": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion" + ], + "simu_data": [ + "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData" + ], + "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance", + "instance_id": "instance_id", + "instance_config_id": "config.@id", + "sequences2": { + "id_field": "sequence_id", + "start": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "START" + }, + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "PAUSE" + } + }, + "coordinates": "location.coordinates", + "metadata": { + "timestamp": "timestamp", + "gamefield": "instance_id", + "user": "player_group_name" + } + }, + "source": { + "type": "Biogames", + "username": "ba", + "password": "853451", + "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de" + }, + "render": [ + "KMLRender" + ] +}""" + +ACTIVITY = """{ + "logFormat": "zip", + "entryType": "@class", + "spatials": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation" + ], + "actions": [], + "boards": [ + "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry" + ], + "analyzers": { + "analysis.analyzers": [ + "BiogamesCategorizer", + "ActivityMapper" + ] + }, + "sequences": { + "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache", + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction", + "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction" + } + }, + "custom": { + "simulation_rounds": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion" + ], + "simu_data": [ + "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData" + ], + "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance", + "instance_id": "instance_id", + "instance_config_id": "config.@id", + "sequences2": { + "id_field": "sequence_id", + "start": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "START" + }, + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "PAUSE" + } + }, + "coordinates": "location.coordinates", + "metadata": { + "timestamp": "timestamp", + "gamefield": "instance_id", + "user": "player_group_name" + } + }, + "source": { + "type": "Biogames", + "username": "ba", + "password": "853451", + "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de" + }, + "render": [ + "ActivityMapper" + ] +}""" + +CONFIGS = { # TODO: more + "KML": KML, + "ActivityMapper": ACTIVITY, +} + +URLS = { + "KML": "/", + "ActivityMapper": "#", +} + +HOSTS = { + "Biogames": "http://biogames.potato.kinf.wiai.uni-bamberg.de", + "Geogames": "http://log_data/", +} + +RESULT_HOST = "http://results.ma.potato.kinf.wiai.uni-bamberg.de/" \ No newline at end of file diff --git a/selector/static/script.js b/selector/static/script.js new file mode 100644 index 0000000..865286c --- /dev/null +++ b/selector/static/script.js @@ -0,0 +1,4 @@ +function validateSettings() { + alert(document.getElementById('safety').checked); + return false; +} \ No newline at end of file diff --git a/selector/static/style.css b/selector/static/style.css new file mode 100644 index 0000000..4ddd162 --- /dev/null +++ b/selector/static/style.css @@ -0,0 +1,9 @@ +body { + background-color: aqua; +} +#data{ + display: none; +} +li{ + list-style-type: none; +} \ No newline at end of file diff --git a/selector/templates/base.html b/selector/templates/base.html new file mode 100644 index 0000000..2dea66f --- /dev/null +++ b/selector/templates/base.html @@ -0,0 +1,5 @@ + +… + + +{% block body %} {% endblock %} \ No newline at end of file diff --git a/selector/templates/games.html b/selector/templates/games.html new file mode 100644 index 0000000..b8acfaf --- /dev/null +++ b/selector/templates/games.html @@ -0,0 +1,25 @@ +{% extends "base.html" %} +{% block body %} +
+
{{logs}}
+
    + {% for log in logs %} +
  • + + {{log.start_date}}: {{log.player_group_name}} +
    • + + {% endfor %} +
+ +
+ + + +
+show analysis progress/results +{% endblock %} \ No newline at end of file diff --git a/selector/templates/index.html b/selector/templates/index.html new file mode 100644 index 0000000..5a4c6c0 --- /dev/null +++ b/selector/templates/index.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} +{% block body %} +
+ + + + +
+ +{% endblock %} \ No newline at end of file diff --git a/selector/templates/results.html b/selector/templates/results.html new file mode 100644 index 0000000..3129a94 --- /dev/null +++ b/selector/templates/results.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} +{% block body %} + +create new analysis + +
+
    + {% for job in jobs %} +
  • {{jobs[job].status}}: "{{job}}": + + + +
    • + {% endfor %} +
+
+ +{% endblock %} \ No newline at end of file diff --git a/selector/webserver.py b/selector/webserver.py new file mode 100644 index 0000000..30e3abe --- /dev/null +++ b/selector/webserver.py @@ -0,0 +1,120 @@ +import json +import logging +import typing +import uuid + +import time + +from clients.webclients import Client, CLIENTS + +from flask import Flask, render_template, request, redirect, session + +from tasks import tasks +from selector.config import CONFIGS, URLS, HOSTS, RESULT_HOST + +app = Flask(__name__) +clients: typing.Dict[str, Client] = {} +log: logging.Logger = logging.getLogger(__name__) + + +@app.route("/") +def index(): + return render_template("index.html", clients=CLIENTS) + + +@app.route("/login", methods=["POST"]) +def login(): + game = request.form["game"] + if not game in CLIENTS or game not in HOSTS: + return redirect("/?invalid_game") + client = CLIENTS[game](host=HOSTS[game], username=request.form['username'], password=request.form['password']) + if client.login(): + session['logged_in'] = True + session['uid'] = str(uuid.uuid4()) + session['username'] = request.form['username'] + session['cookies'] = client.cookies + session['game'] = game + session['host'] = BIOGAMES_HOST + clients[session['uid']] = client + return redirect("/results") + return redirect("/?fail") + + +@app.route("/results") +def results(): + if not ('logged_in' in session and session['logged_in']): + return redirect("/") + if session['logged_in'] and not session['uid'] in clients: + clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies']) + status = tasks.redis.get(session['username']) + if status: + job_status = json.loads(status) + else: + job_status = {} + #for job in job_status: + # results = [] + # for path in job_status[job]['results']: + # results.append(path.replace(tasks.DATA_PATH, RESULT_HOST)) + # print(results) #TODO??? + return render_template("results.html", jobs=job_status) + + +@app.route("/games") +def games(): + if not ('logged_in' in session and session['logged_in']): + return redirect("/") + if session['logged_in'] and not session['uid'] in clients: + clients[session['uid']] = CLIENTS[session['game']](host=session['host'], **session['cookies']) + return render_template("games.html", logs=clients[session['uid']].list(), configs=CONFIGS) + + +@app.route("/start", methods=['POST']) +def start(): + print(str(request.form['logs'])) + status = { + "status": "PENDING", + "submit": time.strftime("%c"), + "log_ids": request.form.getlist('logs'), + "config": request.form['config'], + } + params = { + "log_ids": request.form.getlist('logs'), + "config": CONFIGS[request.form['config']], + "username": session['username'], + "cookies": session['cookies'], + "host": session['host'], + "clientName": session['game'], + "name": request.form['name'], + } + tasks.status_update(session['username'], request.form['name'], status) + tasks.analyze.delay(**params) + return redirect("/results") + + +@app.route("/status") +def status(): + return json.dumps(json.loads(tasks.redis.get(session['username'])), indent=2) + + +@app.template_filter('get_url') +def get_url(path: str): + return path.replace(tasks.DATA_PATH, RESULT_HOST) + +@app.template_filter('get_name') +def get_url(path: str): + return path.replace(tasks.DATA_PATH, "") + + +@app.template_filter('get_prefix') +def get_prefix(job): + print(job) + try: + return RESULT_HOST + URLS[job['config']] + except: + return RESULT_HOST + "#" + + + +if __name__ == '__main__': + app.config.update({"SECRET_KEY": "59765798-2784-11e8-8d05-db4d6f6606c9"}) + app.run(host="0.0.0.0", debug=True) diff --git a/sources/__init__.py b/sources/__init__.py deleted file mode 100644 index a1db87b..0000000 --- a/sources/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .biogames import Biogames - -SOURCES = { - "Biogames": Biogames, -} \ No newline at end of file diff --git a/sources/biogames.py b/sources/biogames.py deleted file mode 100644 index e63a328..0000000 --- a/sources/biogames.py +++ /dev/null @@ -1,82 +0,0 @@ -import json -import logging -import typing -from tempfile import TemporaryDirectory - -import os - -from sources.source import Source - -import shutil -import requests - -log: logging.Logger = logging.getLogger(__name__) - - -class Biogames(Source): - def __init__(self): - self.headers: typing.Dict[str, str] = {'Accept': 'application/json'} - self.cookies: typing.Dict[str, str] = {} - self.id2link: typing.Dict[str, str] = {} - self.host: str = None - - def connect(self, **kwargs): - for i in ['username', 'password', 'url', 'login_url', 'host']: - if not i in kwargs: - raise ValueError("missing value " + i) - csrf_request = requests.get(kwargs['url']) - if csrf_request.status_code != 200: - raise ConnectionError("unable to obtain CSRF token (" + str(csrf_request) + ")") - self.cookies['csrftoken'] = csrf_request.cookies['csrftoken'] - log.info("obtained CSRF token (" + self.cookies['csrftoken'] + ")") - login_payload = { - 'username': kwargs['username'], - 'password': kwargs['password'], - 'next': '', - 'csrfmiddlewaretoken': 'csrftoken' - } - login = requests.post(kwargs['login_url'], data=json.dumps(login_payload), cookies=self.cookies) - if login.status_code != 200: - raise ConnectionError("Unable to authenticate!", login, login.text) - self.cookies['sessionid'] = login.cookies['sessionid'] - log.info("obtained sessionid (" + self.cookies['sessionid'] + ")") - self.url = kwargs['url'] - self.host = kwargs['host'] - log.info("stored url (" + self.url + ")") - - def list(self): - logs = self.get_json(self.url) - log.info(len(logs)) - for i in logs: - self.id2link[i["id"]] = i["link"] # TODO - return logs - - def get(self, ids: typing.Collection): - dir = TemporaryDirectory() - files = [] - for i in ids: - url = self.id2link[i] - filename = os.path.join(dir.name, url.split("/")[-1]) - file = self.download_file(url, filename) - if file: - files.append(file) - return dir - - def download_file(self, url, filename): - with open(filename, "wb") as out: - try: - download = self._get(url) - shutil.copyfileobj(download.raw, out) - return filename - except Exception as e: - log.exception(e) - os.remove(filename) - - def get_json(self, url): - return self._get(url, stream=False).json() - - def close(self): - pass - - def _get(self, url, stream=True): - return requests.get(self.host + url, cookies=self.cookies, headers=self.headers, stream=stream) diff --git a/sources/source.py b/sources/source.py deleted file mode 100644 index 905b897..0000000 --- a/sources/source.py +++ /dev/null @@ -1,18 +0,0 @@ -import typing - - -class Source: - def connect(self, **kwargs): - raise NotImplementedError - - def list(self): - raise NotImplementedError - - def get(self, ids: typing.Collection): - raise NotImplementedError - - def get_json(self, url:str) -> dict: - raise NotImplementedError - - def close(self): - raise NotImplementedError diff --git a/static/progress/data/.gitkeep b/static/progress/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/static/progress/docker-compose.yml b/static/progress/docker-compose.yml new file mode 100644 index 0000000..6e4a7f6 --- /dev/null +++ b/static/progress/docker-compose.yml @@ -0,0 +1,8 @@ +version: "2" +services: + http: + image: httpd:alpine + volumes: + - ./:/usr/local/apache2/htdocs + ports: + - 5001:80 diff --git a/static/progress/index.html b/static/progress/index.html index efade76..25464d6 100644 --- a/static/progress/index.html +++ b/static/progress/index.html @@ -1,15 +1,25 @@ - - - + src="https://code.jquery.com/jquery-3.2.1.min.js" + integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4=" + crossorigin="anonymous"> + + + + + - \ No newline at end of file + +
+
+
+
+
+
+ + \ No newline at end of file diff --git a/static/progress/my.js b/static/progress/my.js index fec1c8c..309ad7a 100644 --- a/static/progress/my.js +++ b/static/progress/my.js @@ -1,71 +1,103 @@ -$.getJSON("tmp3.json", function (data) { - var list = $("
    "); - var maps = {}; - $.each(data, function (key, value) { - //key: instance_id, value: AnlysisResult - //value.result.instance: InstanceConfig_id - // console.log(key, value[0].result.store[0].timestamp); - $.each(value[0].result.store, function (index, entry) { - //console.log(entry); - var time = new Date(entry.timestamp); - var item = $("
  • ", {html: entry.sequence + " @ " + time.toLocaleDateString() + " "+ time.toLocaleTimeString()}); - var container = $("

    "); - if (entry.track.length > 0) { - var mapName = "map" + index; - //console.log(mapName, entry.track.length); - var mapContainer = $("

    ", {id: mapName, class: "map"}); - var track = []; - $.each(entry.track, function (i, elem) { - track.push([elem.coordinates[1], elem.coordinates[0]]); - }); - maps[mapName] = track; +//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_03b9b6b4-c8ab-4182-8902-1620eebe8889.json", function (data) { //urach +//$.getJSON("data/ff8f1e8f-6cf5-4a7b-835b-5e2226c1e771_de7df5b5-edd5-4070-840f-68854ffab9aa.json", function (data) { //urach +//$.getJSON("data/90278021-4c57-464e-90b1-d603799d07eb_07da99c9-398a-424f-99fc-2701763a63e9.json", function (data) { //eichstätt +//$.getJSON("data/13241906-cdae-441a-aed0-d57ebeb37cac_d33976a6-8a56-4a63-b492-fe5427dbf377.json", function (data) { //stadtökologie +$.getJSON("data/5e64ce07-1c16-4d50-ac4e-b3117847ea43_2f664d7b-f0d8-42f5-8731-c034ef86703e.json", function (data) { //filderstadt +//$.getJSON("data/17d401a9-de21-49a2-95bc-7dafa53dda64_98edcb70-03db-4465-b185-a9c9574995ce.json", function (data) { //oeb2016 + var images = {}; + var tiles = { + "openstreetmap": L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', { + maxNativeZoom: 19, + maxZoom: 24, + attribution: '© OpenStreetMap' + }), + "esri sat": L.tileLayer('https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}', { + maxNativeZoom: 19, + maxZoom: 24, + attribution: 'Tiles © Esri — Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community' + }), + "google sat": L.tileLayer('https://{s}.google.com/vt/lyrs=s&x={x}&y={y}&z={z}', { + maxNativeZoom: 20, + maxZoom: 24, + subdomains: ['mt0', 'mt1', 'mt2', 'mt3'] + }) + }; + var map = L.map("mainMap", {layers: [tiles.openstreetmap]}); - /* mapContainer.ready(function () { - var map = L.map(mapName, {maxZoom: 22}); - L.control.scale().addTo(map); - var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', { - attribution: '© OpenStreetMap contributors', - }).addTo(map); - var track = []; - $.each(entry.track, function (i, elem) { - track.push([elem.coordinates[1], elem.coordinates[0]]); - }); - var layer = L.polyline(track, {color: "green"}); - console.log(track); - L.control.layers(null, [layer]).addTo(map); - });*/ + function styleTrack(feature) { + var styles = {}; + styles.color = data.colors[feature.properties.activity_type]; + return styles; + } - mapContainer.appendTo(container); - } - $.each(entry.events, function (i, event) { - if ("image" in event) { - $("", {src: event.image, height: 200}).appendTo(container); + var highlighted = null; + + function onClick(e) { + var start = e.target.feature.geometry.properties.start_timestamp; + var end = e.target.feature.geometry.properties.end_timestamp; + var changed = highlighted !== e.target.feature; + $.each(images, function (timestamp, board) { + if ((timestamp >= start && timestamp < end) && changed) { + board.image.first().addClass("highlight"); + } else { + board.image.removeClass("highlight"); + highlighted = null; } - }); - container.appendTo(item); - item.appendTo(list); - }); - }); - list.appendTo("body"); - var slider = $("", {type: "range" }) - /*}); + } + ); + if (changed) { + highlighted = e.target.feature; + } + } - $(window).on("load", function () {*/ - // setTimeout(function () { + var coords = []; - //console.log(maps); - $.each(maps, function (mapName, track) { - //console.log("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa"); - var tiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', { - attribution: '© OpenStreetMap contributors', + function onEachFeature(feature, layer) { + layer.setStyle(styleTrack(feature)); + layer.on('click', onClick); + if (feature.coordinates.length > 1) { + coords = coords.concat(feature.coordinates.map(function (p) { + return [p[1], p[0], 0.1]; + })); + } + } + + var track = L.geoJSON(data['track'], { + //style: styleTrack, + onEachFeature: onEachFeature + }).addTo(map); + + map.fitBounds(track.getBounds()); + + var heat = L.heatLayer(coords); + L.control.layers(tiles, {"heatmap": heat}).addTo(map); + + var list = $("
      "); + var current = { + "pos":data["boards"][1].coordinate.coordinates + }; + console.log(current); + var marker = L.marker([current.pos[1], current.pos[0]]).addTo(map); + $.each(data["boards"], function (index, entry) { + //console.log(index, entry); + var item = $("
    • ", {class: entry.extra_data.activity_type}); + var container = $("
      ", {class: "board"}); + var image = $("", {src: entry.image.replace("static/progress/", "")}); + image.attr("data-time", entry.timestamp); + image.hover(function () { + marker.setLatLng([entry.coordinate.coordinates[1], entry.coordinate.coordinates[0]]); + }, function () { + marker.setLatLng(current.pos.coordinates[1], current.pos.coordinates[0]); }); - var map = L.map(mapName, {layers: [tiles]}); - L.control.scale().addTo(map); - // console.log(mapName, track); - var layer = L.polyline(track, {color: "green"}).addTo(map); - map.fitBounds(layer.getBounds()); - //console.log(layer) - //L.control.layers({"osm":tiles}, {layer]).addTo(map); + image.click(function (e) { + current.board = image; + current.pos = entry.coordinate; + }); + images[entry.timestamp] = {image: image, coordinate: entry.coordinate}; + image.appendTo(container); + container.appendTo(item); + item.appendTo(list); }); - // }, 2000); + current.board=images[data["boards"][1].timestamp]; + list.appendTo(".sequence"); }); \ No newline at end of file diff --git a/static/progress/style.css b/static/progress/style.css new file mode 100644 index 0000000..d33b09e --- /dev/null +++ b/static/progress/style.css @@ -0,0 +1,105 @@ +/*.mapDiv { + width: 1024px; + height: 768px; +}*/ + + + +.highlight { + /*what a nice way to highlight*/ + display: none; +} + +.simu { + background-color: blue; +} + +.question { + background-color: orange; +} + +.image { + background-color: green; +} + +.audio { + background-color: red; +} + +.video { + background-color: purple; +} + +.other { + background-color: brown; +} + +.map { + background-color: violet; +} + +.error { + background-color: grey; +} + + +.board { + width: 32px; + height: 32px; + display: inline-block; +} + +.board img { + max-width: 32px; + max-height: 32px; + position: absolute; + /*bottom: 0px;*/ +} + +.board:hover img{ + max-width: 205px; + max-height: 295px; + z-index: 99; + top: 5px; + right:0px; +} +ul { + list-style-type: none; + overflow: auto; + overflow-y: hidden; + display: inline-block; + /*max-width:100%; + margin: 0 0 1em; + white-space: nowrap; + height:200px;*/ +} + +li { + display: inline-block; + vertical-align: top; + padding: 2px; + margin-bottom: 2px; +} + +body{ + height: 100%; + padding:0; + margin:0; +} + +main{ + display: flex; + flex-direction: column; + height:100%; +} + +.mapDiv { + flex-grow:1; +} +.sequenceContainer{ + flex-grow: 0; + min-height:300px; + padding-right: 210px; + position: relative; + +} \ No newline at end of file diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 0000000..913d63f --- /dev/null +++ b/tasks/__init__.py @@ -0,0 +1,65 @@ +from .tasks import analyze + +__log__ = ["/app/data/008cad400ab848f729913d034a.zip"] + +__config__ = """{ + "logFormat": "zip", + "entryType": "@class", + "spatials": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryLocation" + ], + "actions": [ + "...QuestionAnswerEvent", + "...SimuAnswerEvent" + ], + "boards": [ + "de.findevielfalt.games.game2.instance.log.entry.ShowBoardLogEntry" + ], + "analyzers": { + "analysis.analyzers": [ + "BiogamesCategorizer", + "LocationAnalyzer" + ] + }, + "sequences": { + "start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryCache", + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.LogEntryInstanceAction", + "action.@class": "de.findevielfalt.games.game2.instance.action.CacheEnableAction" + } + }, + "custom": { + "simulation_rounds": [ + "de.findevielfalt.games.game2.instance.log.entry.LogEntryQuestion" + ], + "simu_data": [ + "de.findevielfalt.games.game2.instance.data.sequence.simulation.SimulationBoardData" + ], + "instance_start": "de.findevielfalt.games.game2.instance.log.entry.LogEntryStartInstance", + "instance_id": "instance_id", + "instance_config_id": "config.@id", + "sequences2": { + "id_field": "sequence_id", + "start": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "START" + }, + "end": { + "@class": "de.findevielfalt.games.game2.instance.log.entry.ShowSequenceLogEntry", + "action": "PAUSE" + } + }, + "coordinates": "location.coordinates", + "metadata": { + "timestamp": "timestamp", + "gamefield": "instance_id", + "user": "player_group_name" + } + }, + "source": { + "type": "Biogames", + "username": "ba", + "password": "853451", + "host": "http://biogames.potato.kinf.wiai.uni-bamberg.de" + } +}""" diff --git a/tasks/tasks.py b/tasks/tasks.py new file mode 100644 index 0000000..198f3e7 --- /dev/null +++ b/tasks/tasks.py @@ -0,0 +1,86 @@ +import json +import logging +import shutil +import uuid +import os.path +import os + +import redis as redis_lib +import time +from celery import Celery +from analysis import log_analyzer as la +from analysis.analyzers import KMLRender, ActivityMapperRender +from clients.webclients import CLIENTS + +FLASK_DB = 1 +REDIS_HOST = "redis" +DATA_PATH = "/app/data/results/" + +RENDERERS = { # TODO + "KMLRender": KMLRender, + "ActivityMapper": ActivityMapperRender, +} + +app = Celery('tasks', backend='redis://redis', broker='redis://redis') +redis = redis_lib.StrictRedis(host=REDIS_HOST, db=FLASK_DB) +log: logging.Logger = logging.getLogger(__name__) + + +def update_status(username, name, state, **kwargs): + status = json.loads(redis.get(username)) + status[name][state[0]] = time.strftime("%c") + status[name]['status'] = state[1] + for i in kwargs: + status[name][i] = kwargs[i] + redis.set(username, json.dumps(status)) + + +@app.task +def analyze(config, log_ids, **kwargs): + update_status(kwargs['username'], kwargs['name'], ('load', 'LOADING')) + + try: + log.info("start analysis") + client = CLIENTS[kwargs['clientName']](host=kwargs['host'], **kwargs['cookies']) + logs = client.list() + id_urls = {str(x['@id']): x['file_url'] for x in logs} + urls = [id_urls[i] for i in log_ids] + tmpdir = client.download_files(urls) + log.info(tmpdir.name, list(os.scandir(tmpdir.name))) + + update_status(kwargs['username'], kwargs['name'], ('start', 'RUNNING')) + + settings = la.parse_settings(config) + store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS) + render = RENDERERS[settings.render[0]]() # TODO + files = render.render(store.get_all()) + + uid = str(uuid.uuid4()) + results = [] + log.error(files) + os.mkdir(os.path.join(DATA_PATH, uid)) + for file in files: + try: + head, tail = os.path.split(file) + target = os.path.join(DATA_PATH, uid, tail) + shutil.move(file, target) + results.append(target) + except FileNotFoundError as e: + log.exception(e) + tmpdir.cleanup() + + update_status(kwargs['username'], kwargs['name'], ('done', 'FINISHED'), results=results) + except Exception as e: + log.exception(e) + update_status(kwargs['username'], kwargs['name'], ('abort', 'ERROR'), exception=str(e)) + + +def status_update(key, status_key, status): + record = redis.get(key) + if not record: + redis.set(key, json.dumps({status_key: status})) + else: + data = json.loads(record) + data[status_key] = status + redis.set(key, json.dumps(data)) + redis.save() diff --git a/test_neo.py b/test_neo.py new file mode 100644 index 0000000..3d52d34 --- /dev/null +++ b/test_neo.py @@ -0,0 +1,41 @@ + +from analysis import log_analyzer as la +settings = la.load_settings("neocart.json") +client = settings.source +logs = client.list() +id_urls = {str(x['@id']): x['file_url'] for x in logs} + +log_ids=['20351/playerid1430317168972.gpx','20351/playerid1430317188358.gpx'] + +urls = [id_urls[i] for i in log_ids] +tmpdir = client.download_files(urls) +import os +store = la.run_analysis([p.path for p in os.scandir(tmpdir.name)], settings, la.LOADERS) + +import json +print(json.dumps(store.serializable(), indent=1)) + + +from analysis.analyzers import KMLRender, ActivityMapperRender +RENDERERS = { # TODO + "KMLRender": KMLRender, + "ActivityMapper": ActivityMapperRender, +} +render = RENDERERS[settings.render[0]]() +files = render.render(store.get_all()) +DATA_PATH = "/app/data/results/" +import uuid +uid = str(uuid.uuid4()) +results = [] +os.mkdir(os.path.join(DATA_PATH, uid)) +import shutil + +for file in files: + try: + head, tail = os.path.split(file) + target = os.path.join(DATA_PATH, uid, tail) + shutil.move(file, target) + results.append(target) + except FileNotFoundError as e: + log.exception(e) +tmpdir.cleanup() \ No newline at end of file diff --git a/util/download.py b/util/download.py deleted file mode 100644 index 703b658..0000000 --- a/util/download.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -import os - -from util import json_path - -logger = logging.getLogger(__name__) - - -def download_board(board_id, instance_config_id, sequence_id, source): - if sequence_id is None: - return "static" - local_file = "static/progress/images/{config_id}/{sequence_id}/{board_id}".format( - config_id=instance_config_id, - sequence_id=sequence_id, - board_id=board_id) - if os.path.exists(local_file): - return local_file - url = "/game2/editor/config/{config_id}/sequence/{sequence_id}/board/{board_id}/".format( - config_id=instance_config_id, - sequence_id=sequence_id, - board_id=board_id - ) - board = source._get(url) - if not board.ok: - #raise ConnectionError() - logger.error("HTTP FAIL:", board) - return "static" - data = board.json() - preview_url = json_path(data, "preview_url.medium") - logger.debug(preview_url) - os.makedirs(local_file[:-len(board_id)], exist_ok=True) - source.download_file(preview_url, local_file) - return local_file - - -def get_config(source, instance_id): - url = "/game2/editor/config/{config_id}/".format(config_id=instance_id) - instance_data = source.get_json(url) - caches = url + "cache/" - cache_data = source.get_json(caches) - - return { - "name": instance_data["name"], - "id": instance_data["@id"], - "caches": cache_data - }