update base images

* importer
** logging
** one request per callback

Dockerfile

@@ -1,9 +0,0 @@
-FROM python:3.7-rc-alpine
-
-RUN pip install tinkerforge requests
-
-WORKDIR /app
-
-COPY . .
-
-CMD ["python3.7", "/app/import.py"]

docker-compose.yml

@@ -1,7 +1,30 @@
-version: "2"
+version: "3"
+
 services:
  import:
-  build: .
+  build: importer
-  image: docker.clkl.de/weather/import:0.1
+  image: docker.clkl.de/weather/import:0.3.1
   environment:
    - PYTHONUNBUFFERED=1
+   - TF_HOST=192.168.2.160
+   - TF_PORT=4223
+   - TF_ID=DYC
+   - INFLUXDB_DB=mydb
+ influxdb:
+  image: influxdb:1.7-alpine
+  command: influxd -config /etc/influxdb/influxdb.conf
+  volumes:
+   - "./influxdb/:/var/lib/influxdb"
+  environment:
+   - INFLUXDB_DB=mydb # Y U NO WORK?!
+   #- INFLUXDB_USER=user
+   #- INLFUXDB_USER_PASSWORD=<secret>
+ grafana:
+  image: grafana/grafana
+  ports:
+   - "3000:3000"
+  volumes:
+   - "./grafana_lib/:/var/lib/grafana/"
+
+
+# docker-compose exec influxdb influx_inspect export -database mydb -out /var/lib/influxdb/backup-text-dump -datadir /var/lib/influxdb/data -waldir /var/lib/influxdb/wal

import.py

@@ -1,44 +0,0 @@
-HOST = "192.168.2.160"
-PORT = 4223
-UID = "DYC"
-URL = "http://192.168.2.30:8086/write?db=mydb"
-
-import time
-
-import requests
-
-from tinkerforge.ip_connection import IPConnection
-from tinkerforge.bricklet_outdoor_weather import BrickletOutdoorWeather
-
-def asdf(type, identifier, **kwargs):
-	print(time.strftime("%c"), type, identifier, kwargs)
-	time_ns = time.time_ns()
-	for key in kwargs:
-		data = "{unit},type={type},identifier={id} value={value} {time_ns}".format(unit=key, type=type, id=identifier, value=kwargs[key], time_ns=time_ns)
-		try:
-			r = requests.post(URL, data=data)
-			print(r, r.status_code)
-		except Exception as e:
-			print(e)
-
-def cb_station(identifier, temperature, humidity, wind_speed, gust_speed, rain, wind_direction, battery_low):
-        asdf(type="station", identifier=identifier, temperature=temperature, humidity=humidity, wind_speed=wind_speed, gust_speed=gust_speed, rain=rain, wind_direction=wind_direction, battery_low=battery_low)
-def cb_sensor(identifier, temperature, humidity):
-        asdf(type="sensor", identifier=identifier, temperature=temperature, humidity=humidity)
-
-if __name__ == "__main__":
-	print("starting…")
-	ipcon = IPConnection()
-	ow = BrickletOutdoorWeather(UID, ipcon)
-	ipcon.connect(HOST, PORT)
-	ow.set_station_callback_configuration(True)
-	ow.set_sensor_callback_configuration(True)
-	ow.register_callback(ow.CALLBACK_STATION_DATA, cb_station)
-	ow.register_callback(ow.CALLBACK_SENSOR_DATA, cb_sensor)
-	print("now we play the waiting game…")
-	while True:
-		try:
-			time.sleep(60*60)
-		except Exception:
-			print("…")
-	ipcon.disconnect()

importer/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.8-alpine
+
+WORKDIR /app
+
+COPY . .
+
+RUN pip install -r requirements.txt
+
+CMD ["python3.8", "/app/import.py"]

importer/import.py

@@ -0,0 +1,58 @@
+import logging
+import time
+
+from os import getenv
+
+import requests
+
+from tinkerforge.ip_connection import IPConnection
+from tinkerforge.bricklet_outdoor_weather import BrickletOutdoorWeather
+
+log = logging.getLogger(__name__)
+
+
+HOST = getenv("TF_HOST", "192.168.2.160")
+PORT = int(getenv("TF_PORT", 4223))
+UID = getenv("TF_ID", "DYC")
+URL = getenv("INFLUX_URL", "http://influxdb:8086")
+DB = getenv("INFLUXDB_DB", "mydb")
+
+
+
+def influx(type, identifier, **kwargs):
+	log.info(f"{type}, {identifier}, {kwargs}")
+	time_ns = time.time_ns()
+	data = []
+	for unit in kwargs:
+		data.append(f"{unit},type={type},identifier={identifier} value={kwargs[unit]} {time_ns}")
+	try:
+		url = f"{URL}/write?db={DB}"
+		r = requests.post(url, data="\n".join(data))
+		log.info(f"{r}, {r.text}")
+	except Exception as e:
+		log.exception(e)
+
+def cb_station(identifier, temperature, humidity, wind_speed, gust_speed, rain, wind_direction, battery_low):
+        influx(type="station", identifier=identifier, temperature=temperature, humidity=humidity, wind_speed=wind_speed, gust_speed=gust_speed, rain=rain, wind_direction=wind_direction, battery_low=battery_low)
+def cb_sensor(identifier, temperature, humidity):
+        influx(type="sensor", identifier=identifier, temperature=temperature, humidity=humidity)
+
+if __name__ == "__main__":
+	logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
+	log.info("starting…")
+	ipcon = IPConnection()
+	ow = BrickletOutdoorWeather(UID, ipcon)
+	ipcon.connect(HOST, PORT)
+	ow.set_station_callback_configuration(True)
+	ow.set_sensor_callback_configuration(True)
+	ow.register_callback(ow.CALLBACK_STATION_DATA, cb_station)
+	ow.register_callback(ow.CALLBACK_SENSOR_DATA, cb_sensor)
+	log.info("try to create influx db…")
+	requests.post(f"{URL}/query?q=CREATE DATABASE {DB}")
+	log.info("now we play the waiting game…")
+	while True:
+		try:
+			time.sleep(60*60)
+		except Exception:
+			log.warning("…")
+	ipcon.disconnect()

importer/requirements.txt

@@ -0,0 +1,2 @@
+tinkerforge==2.1.26
+requests==2.23.0

snapshotter/.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+data/
+env

snapshotter/docker-compose.yml

@@ -0,0 +1,11 @@
+version: "2"
+services:
+ test:
+  build: src
+  image: docker.clkl.de/weather/snapshot:0.2
+  volumes:
+   - ./src/:/app
+   - ./data:/data
+  env_file: env
+  working_dir: /app
+  command: python3 scheduler.py

snapshotter/env.sample

@@ -0,0 +1,8 @@
+USER=grafana_user
+PASSWORD=grafana_password
+
+SSH_USER=ssh_user
+SSH_PASSWORD=ssh_pass
+SSH_HOST=ssh_target
+
+PYTHONUNBUFFERED=1

snapshotter/src/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.12
+ENV timezone Europe/Berlin
+ADD ["requirements.txt", "/"]
+RUN apk add --update --no-cache python3 libssl1.1 py3-pip && \
+	apk add --no-cache --virtual .build-deps g++ python3-dev libffi-dev openssl-dev make tzdata && \
+	cp /usr/share/zoneinfo/$timezone /etc/localtime && \
+	echo "$timezone" >  /etc/timezone && \
+	pip3 install --upgrade pip && \
+	pip3 install -r requirements.txt && \
+	apk del .build-deps

snapshotter/src/recover.py

@@ -0,0 +1,19 @@
+from datetime import date, timedelta, datetime as dt
+import snapshot
+
+def update(start, end):
+	auth = snapshot.auth_from_env()
+	config = snapshot.ssh_from_env()
+	offset = timedelta(days=1)
+	while start <= end:
+		print(start)
+		day = snapshot.today(today=start)
+		print(day)
+		snapshot._update(auth, config, day=day)
+		start += offset
+
+
+if __name__ == "__main__":
+	start = dt(2018, 8 , 3)
+	end = dt(2018, 8 , 3)
+	update(start, end)

snapshotter/src/requirements.txt

@@ -0,0 +1,3 @@
+requests
+fs.sshfs
+schedule

snapshotter/src/scheduler.py

@@ -0,0 +1,26 @@
+import logging
+import time
+
+import schedule
+
+import snapshot
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
+
+log = logging
+
+def setup(interval=10):
+	schedule.every(interval).minutes.do(snapshot.update)
+
+if __name__ == "__main__":
+	interval = 10
+	sleep = (interval*60) / 10.0
+	setup(interval)
+	snapshot.update()
+	while True:
+		try:
+			time.sleep(sleep)
+			log.info("run")
+			schedule.run_pending()
+		except Exception as e:
+			log.exception(e)

snapshotter/src/snapshot.py

@@ -0,0 +1,71 @@
+from collections import namedtuple
+from datetime import date, timedelta, datetime as dt
+import logging
+import os
+import shutil
+import tempfile
+
+import fs
+import requests
+
+log = logging
+
+Day = namedtuple("Day", ["start", "end", "str"])
+SSHConfig = namedtuple("SSHConfig", ["user", "password", "host", "port", "dir"])
+
+URL = 'http://192.168.2.30:3000/render/d-solo/j4171pFmk/in-vs-outdoor?orgId=1&panelId=2&from={start}&to={to}&width=1000&height=500&tz=Europe%2FBerlin'
+
+
+def today(today=dt.today()):
+	t = today.timetuple()
+	delta = timedelta(hours=t.tm_hour, minutes=t.tm_min, seconds=t.tm_sec)
+	start = today - delta
+	end = start + timedelta(days=1)
+	log.debug(f"{start}, {end}")
+	name = start.strftime("temp_%d.%m.%Y.png")
+	start = int(start.timestamp() * 1000)
+	end = int(end.timestamp() * 1000)
+	return Day(start=start, end=end, str=name)
+
+def download(url, target, **kwargs):
+	response = requests.get(url, stream=True, **kwargs)
+	if not response.ok:
+		raise ValueError("HTTP not OK", response.status_code, response.text)
+	response.raw.decode_content = True
+	with open(target, "wb") as out:
+		shutil.copyfileobj(response.raw, out)
+
+def upload(local_file, filename, config):
+	url = f"ssh://{config.user}:{config.password}@{config.host}:{config.port}/{config.dir}"
+	with fs.open_fs(url) as remote:
+		return fs.copy.copy_file("/", local_file, remote, filename)
+
+def _update(auth, ssh_config, day=today()):
+	url = URL.format(start=day.start, to=day.end)
+	log.debug(url)
+	with tempfile.NamedTemporaryFile() as tmp:
+		download(url, tmp.name, auth=auth)
+		result = upload(tmp.name, day.str, ssh_config)
+		log.debug(f"{day.str}, {result}")
+
+def auth_from_env():
+	return (os.getenv("USER"), os.getenv("PASSWORD"))
+
+def ssh_from_env():
+	return SSHConfig(user=os.getenv("SSH_USER"), password=os.getenv("SSH_PASSWORD"), host=os.getenv("SSH_HOST"), port=os.getenv("SSH_PORT", 22), dir=os.getenv("SSH_DIR", "/"))
+
+def update(save=False):
+	log.info("run update")
+	try:
+		auth = auth_from_env()
+		config = ssh_from_env()
+		_update(auth, config)
+	except Exception as e:
+		if save:
+			log.e(e)
+		else:
+			raise e
+
+if __name__ == "__main__":
+	logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
+	update()