add IMAP/POP3/SMTP

docker-compose.yml

@@ -5,22 +5,38 @@ services:
 #  command: influxd -config /etc/influxdb/influxdb.conf
 #  ports:
 #   - "8086:8086"
+  restart: on-failure:5
   volumes:
    - ./data/influx:/var/lib/influxdb/
-#   - "./influxdb.conf:/etc/influxdb/influxdb.con:ro"
+
  grafana:
   image: grafana/grafana:5.0.4
   ports:
    - "3000:3000"
   volumes:
-#   - "./grafana_etc/:/etc/grafana/"
    - "./data/grafana/:/var/lib/grafana/"
   environment:
    - "GF_SECURITY_ADMIN_USER=root"
    - "GF_SECURITY_ADMIN_PASSWORD=clkl"
+  restart: on-failure:5
+  networks:exhub.uni-bamberg.de
+   - traefik_net
+   - default
+  labels:
+   - "traefik.enable=true"
+   - "traefik.docker.network=traefik_net"
+   - "traefik.http.frontend.rule=Host:rz.potato.kinf.wiai.uni-bamberg.de"
+
  measurement:
-  image: docker.clkl.de/uni/wiai/rz/measurement:0.1
+  image: docker.clkl.de/uni/wiai/rz/measurement:0.2
   build: .
-  command: python measure.py -o /app/log/results.csv -e /app/log/exceptions.log "http://vc.uni-bamberg.de" "https://vc.uni-bamberg.de" "https://fn2stud.zuv.uni-bamberg.de/FN2AUTH/FN2AuthServlet?op=Login" "https://vc.uni-bamberg.de/moodle/"
+  command: python measure.py -e /app/log/exceptions.log --http "http://vc.uni-bamberg.de" "https://vc.uni-bamberg.de" "https://fn2stud.zuv.uni-bamberg.de/FN2AUTH/FN2AuthServlet?op=Login" "https://vc.uni-bamberg.de/moodle/" "http://univis.uni-bamberg.de/" "https://univis.uni-bamberg.de/" --smtp exhub.uni-bamberg.de --pop3 mailex.uni-bamberg.de --imap mailex.uni-bamberg.de
   volumes:
    - ./log:/app/log
+  restart: on-failure:5
+
+
+networks:
+ traefik_net:
+  external:
+   name: traefik_net

src/measure.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import sys
 import time
 import threading
 
@@ -8,100 +9,93 @@ from datetime import datetime
 import requests
 import schedule
 
+from sources import measure_http, measure_imap, measure_pop, measure_smtp
+
 INFLUX_URL = "http://influxdb:8086/write?db=mydb"
 
 log = logging.getLogger(__name__)
 
 INFLUX_CHARS = (",", "=", " ")
+
 def influx_escape(string):
 	for c in INFLUX_CHARS:
 		string = string.replace(c, "\\"+c)
 	return string
 
-def send_data(date, status, time, url):
+def send_data(date, status, time, url, protocol):
-	data = "response_time,url={url},status={status} value={time} {timestamp}".format(
+	data = "response_time,url={url},status={status},protocol={protocol} value={time} {timestamp}".format(
 		url=influx_escape(url),
-		status=status,
+		status=influx_escape(str(status)),
 		time=time,
-		timestamp=int(date.timestamp()*1e9)
+		timestamp=int(date.timestamp()*1e9),
+		protocol=influx_escape(protocol)
 	)
 	r=requests.post(INFLUX_URL, data=data)
-	print(r)
 	log.warn(str(r) + " " + str(r.text))
 
+
 class Measurement(threading.Thread):
-	def __init__(self, url, log, start_date):
+	def __init__(self, measure_fn, url, exlog):
 		threading.Thread.__init__(self)
 		self.url = url
-		self.log = log
+		self.measure_fn = measure_fn
-		self.start_date = start_date
+		self.exlog = exlog
 	
 	def run(self):
+		start_date = datetime.now()
 		url = self.url
-		log = self.log
+		result = self.measure_fn(url, exlog)
-		try:
+		send_data(
-			response = requests.get(url, allow_redirects=False, timeout=256)
+			date=start_date,
-			csv = "{date},{status},{time},{url}".format(
+			url=url,
-				date=self.start_date,
+			status=result.status,
-				status=response.status_code,
+			time=result.time,
-				time=response.elapsed.total_seconds(),
+			protocol=result.protocol
-				url=url
+		)
-			)
-			datalog.info(csv)
-			print("send data")
-			send_data(
-				date=self.start_date,
-				status=response.status_code,
-				time=response.elapsed.total_seconds(),
-				url=url
-			)
-		except requests.exceptions.ConnectionError as e:
-			csv = "{date},{status},{time},{url}".format(
-				date=self.start_date,
-				status="TIMEOUT > 256",
-				time="-1",
-				url=url
-			)
-			datalog.info(csv)
-			send_data(
-				date=self.start_date,
-				status="-1",
-				time=256,
-				url=url
-			)
-			exlog.exception(e)
 		
-def run_measurements(urls):
+def run_measurements(targets):
-	for url in args.urls:
+	for measure_fn in targets:
-		Measurement(url, log, datetime.now()).start()
+		for url in targets[measure_fn]:
+			Measurement(measure_fn, url, exlog).start()
 
 
 if __name__ == "__main__":
 	logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s:%(message)s', level=logging.INFO, datefmt="%Y.%m.%d %H:%M:%S")
 	
 	parser = argparse.ArgumentParser(description="Measure HTTP/HTTPS response time")
-	parser.add_argument("--output", "-o", default="results.csv", help="Output file")
+	parser.add_argument("--exceptions", "-e", default=None, help="Exception log file (Default: disabled)")
-	parser.add_argument("--exceptions", "-e", default="exceptions.log", help="Exception log file")
+	parser.add_argument("--interval", "-i", default=30, help="Interval (seconds)", type=int)
-	parser.add_argument("--interval", "-i", default=30, help="Interval", type=int)
+	parser.add_argument("--http", "-u", nargs="*", help="HTTP/HTTPS urls to measure")
-	parser.add_argument("urls", nargs="+", help="URLs to measure")
+	parser.add_argument("--smtp", "-s", nargs="*", help="SMTP STARTSSL hosts to measure")
+	parser.add_argument("--pop3", "-p", nargs="*", help="POP3 SSL hosts to measure")
+	parser.add_argument("--imap", "-m", nargs="*", help="IMAP SSL hosts to measure")
 	args = parser.parse_args()
 	
-	filehandler = logging.FileHandler(args.output)
-	file_formatter = logging.Formatter("%(message)s")
-	filehandler.setFormatter(file_formatter)
-	datalog = logging.getLogger("data")
-	datalog.addHandler(filehandler)
-	
-	
-	filehandler2 = logging.FileHandler(args.exceptions)
-	file_formatter2 = logging.Formatter("%(asctime)s %(levelname)s %(name)s:%(message)s")
-	filehandler2.setFormatter(file_formatter2)
 	exlog = logging.getLogger("exceptions")
-	exlog.addHandler(filehandler2)
+	if args.exceptions:
+		filehandler = logging.FileHandler(args.exceptions)
+		file_formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s:%(message)s")
+		filehandler.setFormatter(file_formatter)
+		exlog.addHandler(filehandler)
 	
-	schedule.every(args.interval).seconds.do(lambda: run_measurements(args.urls))
+	targets = {}
 	
-	run_measurements(args.urls)
+	if args.imap:
+		targets[measure_imap] = args.imap
+	if args.pop3:
+		targets[measure_pop] = args.pop3
+	if args.smtp:
+		targets[measure_smtp] = args.smtp
+	if args.http:
+		targets[measure_http] = args.http
+	
+	if not targets:
+		parser.print_help()
+		sys.exit(1)
+	
+	schedule.every(args.interval).seconds.do(lambda: run_measurements(targets))
+	
+	run_measurements(targets)
 	
 	while True:
 		try:

src/sources.py

@@ -0,0 +1,88 @@
+import socket
+import time
+
+from collections import namedtuple
+from imaplib import IMAP4_SSL
+from poplib import POP3_SSL
+from smtplib import SMTP
+
+import requests
+
+
+Result = namedtuple("Result", ['time', 'status', 'protocol'])
+
+def measure_http(url, exlog, timeout=256):
+	protocol = url.strip().split(":")[0]
+	try:
+		response = requests.get(url, allow_redirects=False, timeout=256)
+		result = Result(
+			time=response.elapsed.total_seconds(),
+			status=response.status_code,
+			protocol=protocol
+		)
+	except requests.exceptions.ConnectionError as e:
+		exlog.exception(e)
+		result = Result(
+			time=timeout,
+			status="-1",
+			protocol=protocol
+		)
+	return result
+
+def measure_smtp(host, exlog, timeout=256):
+	smtp = SMTP(timeout=timeout)
+	try:
+		before = time.time()
+		status, text = smtp.connect(host)
+		smtp.starttls()
+		smtp.noop()
+		smtp.quit()
+		after = time.time()
+		duration = after - before
+	except Exception as e:
+		exlog.exception(e)
+		duration = timeout
+		status = "-1"
+	return Result(
+		time=duration,
+		status=status,
+		protocol="smtp"
+	)
+
+def measure_pop(host, exlog, timeout=256):
+	status = 0
+	try:
+		before = time.time()
+		pop = POP3_SSL(host, timeout=timeout)
+		pop.quit()
+		after = time.time()
+		duration = after - before
+	except Exception as e:
+		exlog.exception(e)
+		duration = timeout
+		status = "-1"
+	return Result(
+		time=duration,
+		status=status,
+		protocol="pop3"
+	)
+
+def measure_imap(host, exlog, timeout=256):
+	socket.setdefaulttimeout(timeout)
+	status = 0
+	try:
+		before = time.time()
+		imap = IMAP4_SSL(host)
+		imap.noop()
+		imap.shutdown()
+		after = time.time()
+		duration = after - before
+	except Exception as e:
+		exlog.exception(e)
+		duration = timeout
+		status = "-1"
+	return Result(
+		time=duration,
+		status=status,
+		protocol="imap"
+	)