import json import logging from typing import List import numpy as np import analyzers from analyzers import get_renderer, Analyzer, render, Store from analyzers.analyzer import ResultStore from analyzers.render.default import LogEntryCountCSV from analyzers.settings import LogSettings, load_settings from loaders import LOADERS logging.basicConfig(format='%(levelname)s %(name)s:%(message)s', level=logging.DEBUG) log: logging.Logger = logging.getLogger(__name__) logging.getLogger('requests').setLevel(logging.WARN) logging.getLogger("urllib3").setLevel(logging.WARNING) def process_log(logfile: str, settings: LogSettings) -> List[Analyzer]: loader = LOADERS[settings.log_format]() try: loader.load(logfile) except BaseException as e: raise RuntimeError(e) analyzers: List[Analyzer] = [] log.debug("build analyzers") for analyzer in settings.analyzers: analyzers.append(analyzer(settings)) log.debug("process entries") for entry in loader.get_entry(): for analyzer in analyzers: try: if analyzer.process(entry): break except KeyError as e: log.exception(e) return analyzers def run_analysis(log_ids: list, settings): store: ResultStore = ResultStore() for log_id in log_ids: for analysis in process_log(log_id, settings): log.info("* Result for " + analysis.name()) analysis.result(store, name=log_id) return store def load_ids(name: str): log_ids = [] with open(name) as src: for line in src: line = line.strip() log_ids.append(line) return log_ids def urach_logs(log_ids, settings): return ["data/inst_{id}.{format}".format(id=log_id, format=settings.log_format) for log_id in log_ids] def write_logentry_count_csv(): global cat, data, lines, csvfile LogEntryCountCSV.summary = None for cat in store.get_categories(): data = store.get_category(cat) render(analyzers.LogEntryCountAnalyzer, data, name=cat) if LogEntryCountCSV.summary: headers = [] lines = [] for name in LogEntryCountCSV.summary: data = LogEntryCountCSV.summary[name] for head in data: if not head in headers: headers.append(head) line = [name] for head in headers: line.append(data[head]) if head in data else line.append(0) lines.append(line) import csv with open('logentrycount.csv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE) writer.writerow(["name"] + [h.split(".")[-1] for h in headers]) for line in lines: writer.writerow(line) def write_simulation_flag_csv(): global csvfile, result, i from datetime import datetime json.dump(store.serializable(), open("simus.json", "w"), indent=2) with open("simus.csv", "w") as csvfile: csvfile.write("instanceconfig,log,simu,answered,universe_state,selected_actions,timestamp,time\n") for key in store.get_store(): csvfile.write("{}\n".format(key)) for result in store.store[key]: csvfile.write(",{}\n".format(result.name)) for i in result.get(): csvfile.write(",,{},{},{},{},{},{}\n".format( i['answers']['@id'], i['answers']['answered'], len(i['answers']['universe_state']) if i['answers']['universe_state'] else 0, len(i['selected_actions']) if i['selected_actions'] else 0, i['timestamp'], str(datetime.fromtimestamp(i['timestamp'] / 1000)) )) if __name__ == '__main__': settings: LogSettings = load_settings("biogames2.json") log_ids_urach: List[str] = urach_logs([ # "34fecf49dbaca3401d745fb467", # "44ea194de594cd8d63ac0314be", # "57c444470dbf88605433ca935c", # "78e0c545b594e82edfad55bd7f", # "91abfd4b31a5562b1c66be37d9", # "597b704fe9ace475316c345903", # "e01a684aa29dff9ddd9705edf8", # "fbf9d64ae0bdad0de7efa3eec6", "fe1331481f85560681f86827ec", # "fe1331481f85560681f86827ec"] "fec57041458e6cef98652df625", ] ,settings) store: ResultStore = run_analysis(log_ids_urach, settings) if False: for r in get_renderer(analyzers.LocomotionActionAnalyzer): r().render(store.get_all()) if False: render(analyzers.LocationAnalyzer, store.get_all()) # print(json.dumps(store.serializable(), indent=1)) if True: for cat in store.get_categories(): render(analyzers.ActivityMapper, store.get_category(cat), name=cat) # render(analyzers.ProgressAnalyzer, store.get_all()) if False: from analyzers.postprocessing import graph g = graph.Cache(settings) g.run(store) if False: # render(analyzers.SimulationOrderAnalyzer, store.get_all()) for cat in store.get_categories(): data = store.get_category(cat) render(analyzers.SimulationOrderAnalyzer, data, name=cat) if False: write_logentry_count_csv() if False: write_simulation_flag_csv() def calc_distance(geojson: str): from shapely.geometry import LineString from shapely.ops import transform from functools import partial import pyproj track = LineString(json.loads(geojson)['coordinates']) project = partial( pyproj.transform, pyproj.Proj(init='EPSG:4326'), pyproj.Proj(init='EPSG:32633')) return transform(project, track).length if False: # json.dump(store.serializable(), open("new.json", "w"), indent=1) from collections import defaultdict keys = [ "simu", "question", "image", "audio", "video", "other", "map" ] import matplotlib.pyplot as plt # results = [] places = defaultdict(list) for log in store.get_all(): result = defaultdict(lambda: 0) for i in log.get()['track']: duration = i['properties']['end_timestamp'] - i['properties']['start_timestamp'] result[i['properties']['activity_type']] += duration print(json.dumps(result, indent=4)) total = sum(result.values()) print(total) percentage = defaultdict(lambda: 0) minutes = defaultdict(lambda: 0) for i in result: percentage[i] = result[i] / total minutes[i] = result[i] / 60_000 print(json.dumps(percentage, indent=4)) if not 'error' in result: # places[log.get()['instance']].append(percentage) places[log.get()['instance']].append(minutes) for place in places: places[place] = sorted(places[place], key=lambda item: item['map']) dummy = [0] * len(keys) results = [] sites = [] from util.meta_temp import CONFIG_NAMES for i in places: for j in places[i]: ordered = [] for k in keys: ordered.append(j[k]) results.append(ordered) results.append(dummy) sites.append(CONFIG_NAMES[i] if i in CONFIG_NAMES else "---") size = len(results) ind = np.arange(size) width = 0.9 print(results) data = list(zip(*results)) print(data) lines = [] bottom = [0] * len(results) for i in range(0, len(data)): lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0]) for k, x in enumerate(data[i]): bottom[k] += x plt.legend(lines, keys) plt.title(", ".join(sites)) plt.show() # size = len(results) # ind = np.arange(size) # width = 0.9 # print(results) # data = list(zip(*results)) # print(data) # lines = [] # bottom = [0] * len(results) # for i in range(0, len(data)): # lines.append(plt.bar(ind, data[i], bottom=bottom, width=width)[0]) # for k, x in enumerate(data[i]): # bottom[k] += x # plt.legend(lines, keys) # plt.title("Zwei Spiele in Filderstadt (t1=237min; t2=67min)") # plt.show() # for analyzers in analyzers: # if analyzers.name() in ["LogEntryCount", "ActionSequenceAnalyzer"]: # print(json.dumps(analyzers.result(), indent=2)) # for analyzers in analyzers: # if analyzers.name() in ["BoardDuration"]: # print(json.dumps(analyzers.result(), indent=2)) # print(analyzers.render()) # coords = analyzers[1].render() # with open("test.js", "w") as out: # out.write("coords = "+coords)