diff --git a/docker-compose.yml b/docker-compose.yml index b92b5ba..59b286a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,22 +5,38 @@ services: # command: influxd -config /etc/influxdb/influxdb.conf # ports: # - "8086:8086" + restart: on-failure:5 volumes: - ./data/influx:/var/lib/influxdb/ -# - "./influxdb.conf:/etc/influxdb/influxdb.con:ro" + grafana: image: grafana/grafana:5.0.4 ports: - "3000:3000" volumes: -# - "./grafana_etc/:/etc/grafana/" - "./data/grafana/:/var/lib/grafana/" environment: - "GF_SECURITY_ADMIN_USER=root" - "GF_SECURITY_ADMIN_PASSWORD=clkl" + restart: on-failure:5 + networks:exhub.uni-bamberg.de + - traefik_net + - default + labels: + - "traefik.enable=true" + - "traefik.docker.network=traefik_net" + - "traefik.http.frontend.rule=Host:rz.potato.kinf.wiai.uni-bamberg.de" + measurement: - image: docker.clkl.de/uni/wiai/rz/measurement:0.1 + image: docker.clkl.de/uni/wiai/rz/measurement:0.2 build: . - command: python measure.py -o /app/log/results.csv -e /app/log/exceptions.log "http://vc.uni-bamberg.de" "https://vc.uni-bamberg.de" "https://fn2stud.zuv.uni-bamberg.de/FN2AUTH/FN2AuthServlet?op=Login" "https://vc.uni-bamberg.de/moodle/" + command: python measure.py -e /app/log/exceptions.log --http "http://vc.uni-bamberg.de" "https://vc.uni-bamberg.de" "https://fn2stud.zuv.uni-bamberg.de/FN2AUTH/FN2AuthServlet?op=Login" "https://vc.uni-bamberg.de/moodle/" "http://univis.uni-bamberg.de/" "https://univis.uni-bamberg.de/" --smtp exhub.uni-bamberg.de --pop3 mailex.uni-bamberg.de --imap mailex.uni-bamberg.de volumes: - - ./log:/app/log \ No newline at end of file + - ./log:/app/log + restart: on-failure:5 + + +networks: + traefik_net: + external: + name: traefik_net \ No newline at end of file diff --git a/src/measure.py b/src/measure.py index b1d945e..33f3d02 100644 --- a/src/measure.py +++ b/src/measure.py @@ -1,5 +1,6 @@ import argparse import logging +import sys import time import threading @@ -8,100 +9,93 @@ from datetime import datetime import requests import schedule +from sources import measure_http, measure_imap, measure_pop, measure_smtp + INFLUX_URL = "http://influxdb:8086/write?db=mydb" log = logging.getLogger(__name__) INFLUX_CHARS = (",", "=", " ") + def influx_escape(string): for c in INFLUX_CHARS: string = string.replace(c, "\\"+c) return string -def send_data(date, status, time, url): - data = "response_time,url={url},status={status} value={time} {timestamp}".format( +def send_data(date, status, time, url, protocol): + data = "response_time,url={url},status={status},protocol={protocol} value={time} {timestamp}".format( url=influx_escape(url), - status=status, + status=influx_escape(str(status)), time=time, - timestamp=int(date.timestamp()*1e9) + timestamp=int(date.timestamp()*1e9), + protocol=influx_escape(protocol) ) r=requests.post(INFLUX_URL, data=data) - print(r) log.warn(str(r) + " " + str(r.text)) + class Measurement(threading.Thread): - def __init__(self, url, log, start_date): + def __init__(self, measure_fn, url, exlog): threading.Thread.__init__(self) self.url = url - self.log = log - self.start_date = start_date + self.measure_fn = measure_fn + self.exlog = exlog def run(self): + start_date = datetime.now() url = self.url - log = self.log - try: - response = requests.get(url, allow_redirects=False, timeout=256) - csv = "{date},{status},{time},{url}".format( - date=self.start_date, - status=response.status_code, - time=response.elapsed.total_seconds(), - url=url - ) - datalog.info(csv) - print("send data") - send_data( - date=self.start_date, - status=response.status_code, - time=response.elapsed.total_seconds(), - url=url - ) - except requests.exceptions.ConnectionError as e: - csv = "{date},{status},{time},{url}".format( - date=self.start_date, - status="TIMEOUT > 256", - time="-1", - url=url - ) - datalog.info(csv) - send_data( - date=self.start_date, - status="-1", - time=256, - url=url - ) - exlog.exception(e) + result = self.measure_fn(url, exlog) + send_data( + date=start_date, + url=url, + status=result.status, + time=result.time, + protocol=result.protocol + ) -def run_measurements(urls): - for url in args.urls: - Measurement(url, log, datetime.now()).start() +def run_measurements(targets): + for measure_fn in targets: + for url in targets[measure_fn]: + Measurement(measure_fn, url, exlog).start() if __name__ == "__main__": logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s:%(message)s', level=logging.INFO, datefmt="%Y.%m.%d %H:%M:%S") parser = argparse.ArgumentParser(description="Measure HTTP/HTTPS response time") - parser.add_argument("--output", "-o", default="results.csv", help="Output file") - parser.add_argument("--exceptions", "-e", default="exceptions.log", help="Exception log file") - parser.add_argument("--interval", "-i", default=30, help="Interval", type=int) - parser.add_argument("urls", nargs="+", help="URLs to measure") + parser.add_argument("--exceptions", "-e", default=None, help="Exception log file (Default: disabled)") + parser.add_argument("--interval", "-i", default=30, help="Interval (seconds)", type=int) + parser.add_argument("--http", "-u", nargs="*", help="HTTP/HTTPS urls to measure") + parser.add_argument("--smtp", "-s", nargs="*", help="SMTP STARTSSL hosts to measure") + parser.add_argument("--pop3", "-p", nargs="*", help="POP3 SSL hosts to measure") + parser.add_argument("--imap", "-m", nargs="*", help="IMAP SSL hosts to measure") args = parser.parse_args() - filehandler = logging.FileHandler(args.output) - file_formatter = logging.Formatter("%(message)s") - filehandler.setFormatter(file_formatter) - datalog = logging.getLogger("data") - datalog.addHandler(filehandler) - - - filehandler2 = logging.FileHandler(args.exceptions) - file_formatter2 = logging.Formatter("%(asctime)s %(levelname)s %(name)s:%(message)s") - filehandler2.setFormatter(file_formatter2) exlog = logging.getLogger("exceptions") - exlog.addHandler(filehandler2) + if args.exceptions: + filehandler = logging.FileHandler(args.exceptions) + file_formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s:%(message)s") + filehandler.setFormatter(file_formatter) + exlog.addHandler(filehandler) - schedule.every(args.interval).seconds.do(lambda: run_measurements(args.urls)) + targets = {} - run_measurements(args.urls) + if args.imap: + targets[measure_imap] = args.imap + if args.pop3: + targets[measure_pop] = args.pop3 + if args.smtp: + targets[measure_smtp] = args.smtp + if args.http: + targets[measure_http] = args.http + + if not targets: + parser.print_help() + sys.exit(1) + + schedule.every(args.interval).seconds.do(lambda: run_measurements(targets)) + + run_measurements(targets) while True: try: diff --git a/src/sources.py b/src/sources.py new file mode 100644 index 0000000..a67c755 --- /dev/null +++ b/src/sources.py @@ -0,0 +1,88 @@ +import socket +import time + +from collections import namedtuple +from imaplib import IMAP4_SSL +from poplib import POP3_SSL +from smtplib import SMTP + +import requests + + +Result = namedtuple("Result", ['time', 'status', 'protocol']) + +def measure_http(url, exlog, timeout=256): + protocol = url.strip().split(":")[0] + try: + response = requests.get(url, allow_redirects=False, timeout=256) + result = Result( + time=response.elapsed.total_seconds(), + status=response.status_code, + protocol=protocol + ) + except requests.exceptions.ConnectionError as e: + exlog.exception(e) + result = Result( + time=timeout, + status="-1", + protocol=protocol + ) + return result + +def measure_smtp(host, exlog, timeout=256): + smtp = SMTP(timeout=timeout) + try: + before = time.time() + status, text = smtp.connect(host) + smtp.starttls() + smtp.noop() + smtp.quit() + after = time.time() + duration = after - before + except Exception as e: + exlog.exception(e) + duration = timeout + status = "-1" + return Result( + time=duration, + status=status, + protocol="smtp" + ) + +def measure_pop(host, exlog, timeout=256): + status = 0 + try: + before = time.time() + pop = POP3_SSL(host, timeout=timeout) + pop.quit() + after = time.time() + duration = after - before + except Exception as e: + exlog.exception(e) + duration = timeout + status = "-1" + return Result( + time=duration, + status=status, + protocol="pop3" + ) + +def measure_imap(host, exlog, timeout=256): + socket.setdefaulttimeout(timeout) + status = 0 + try: + before = time.time() + imap = IMAP4_SSL(host) + imap.noop() + imap.shutdown() + after = time.time() + duration = after - before + except Exception as e: + exlog.exception(e) + duration = timeout + status = "-1" + return Result( + time=duration, + status=status, + protocol="imap" + )