Update Readme

Setup using root
Use pip to install pipenv

.gitignore

@@ -1 +1,2 @@
-data/
+data/
+config.json

README.md

@@ -1,11 +1,59 @@
-# datalogger
+# Datalogger
 
 ESP8266 running Micropython collecting temperature humidity and light level. Pushing data to a Mosquito server. A python script to ingest the data into an influx database. Grafana to display the data.
 
-## Setting up ampy
+This is a work in progress and is not suitable for production
 
-`ampy` is a application that allows terminal access to connected micropython devices.
+## Server Setup
 
-## typology
+This assumes you are using Ubuntu-20.04
 
-![](docs/typology)
+### Install packages
+
+`sudo su -`
+
+`wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -`
+
+`echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list`
+
+`wget -qO- https://repos.influxdata.com/influxdb.key | gpg --dearmor > /etc/apt/trusted.gpg.d/influxdb.gpg`
+
+`export DISTRIB_ID=$(lsb_release -si); export DISTRIB_CODENAME=$(lsb_release -sc)`
+
+`echo "deb [signed-by=/etc/apt/trusted.gpg.d/influxdb.gpg] https://repos.influxdata.com/${DISTRIB_ID,,} ${DISTRIB_CODENAME} stable" > /etc/apt/sources.list.d/influxdb.list`
+
+`sudo apt update`
+
+`sudo apt install python3-pip nginx mosquitto grafana influxdb python3-certbot-nginx`
+
+`pip3 install pipenv`
+
+### Setup Nginx
+
+
+
+
+
+## Esp8266 Setup
+
+### Install tools
+
+```pip3 install esptool adafruit-ampy```
+
+### Download Esp8266 Firmware
+
+https://micropython.org/resources/firmware/esp8266-20210418-v1.15.bin
+
+### Install firmware
+
+```esptool.py --port <port> erase_flash```
+
+```esptool.py --port <port> --baud 1000000 write_flash --flash_size=4MB -fm dio 0 <download_folder>/esp8266-20210418-v1.15.bin```
+
+### Edit config file
+
+```cp uP/config.json.sample config.json```
+
+### Copy libraries, config and code to esp8266
+
+```ampy -p <port> put uP /```

mqttinflux/Pipfile

@@ -0,0 +1,18 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+influxdb = "*"
+paho-mqtt = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.8"
+
+[scripts]
+dev = 'bash -c "export DEBUG=1 && ls main.py mqtt-influx.ini | entr -rc python main.py"'
+prod = "python main.py"
+

mqttinflux/Pipfile.lock

@@ -0,0 +1,125 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "3634ff9d7c9021daf28f9f4b5e9ad6762f3c20398e73a53a00a691b6534813c3"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.8"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.python.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+            ],
+            "version": "==2020.12.5"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
+            ],
+            "version": "==4.0.0"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "version": "==2.10"
+        },
+        "influxdb": {
+            "hashes": [
+                "sha256:46f85e7b04ee4b3dee894672be6a295c94709003a7ddea8820deec2ac4d8b27a",
+                "sha256:65040a1f53d1a2a4f88a677e89e3a98189a7d30cf2ab61c318aaa89733280747"
+            ],
+            "index": "pypi",
+            "version": "==5.3.1"
+        },
+        "msgpack": {
+            "hashes": [
+                "sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9",
+                "sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841",
+                "sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439",
+                "sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694",
+                "sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a",
+                "sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f",
+                "sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e",
+                "sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1",
+                "sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c",
+                "sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b",
+                "sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759",
+                "sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326",
+                "sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc",
+                "sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192",
+                "sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83",
+                "sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06",
+                "sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e",
+                "sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9",
+                "sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33",
+                "sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54",
+                "sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f",
+                "sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887",
+                "sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009",
+                "sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2",
+                "sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c",
+                "sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87",
+                "sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984",
+                "sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6"
+            ],
+            "version": "==1.0.2"
+        },
+        "paho-mqtt": {
+            "hashes": [
+                "sha256:9feb068e822be7b3a116324e01fb6028eb1d66412bf98595ae72698965cb1cae"
+            ],
+            "index": "pypi",
+            "version": "==1.5.1"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
+                "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
+            ],
+            "version": "==2.8.1"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
+                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
+            ],
+            "version": "==2021.1"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
+            ],
+            "version": "==2.25.1"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "version": "==1.16.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
+                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
+            ],
+            "version": "==1.26.4"
+        }
+    },
+    "develop": {}
+}

mqttinflux/config.json.sample

@@ -0,0 +1,14 @@
+{
+    "mqtt": {
+        "host": "",
+        "port": 8883,
+        "user": "",
+        "password": ""
+        "topics": []
+    },
+    "influx": {
+        "host": "localhost",
+        "port": 8086,
+        "database": ""
+    }
+}

mqttinflux/main.py

@@ -1,49 +1,88 @@
 import paho.mqtt.client as mqtt
 from influxdb import InfluxDBClient
+from influxdb.exceptions import InfluxDBClientError
+from influxdb.exceptions import InfluxDBServerError
 import datetime
 import logging
 import json
 import os
- 
- 
-def on_connect(client, userdata, flags, rc):
-    for topic in topics:
-        client.subscribe(topic)
+import ssl
+import traceback
+import os
+import configparser
+
+class App:
+    def __init__(self):
+        with open("mqtt-influx.json", 'r') as f:
+            self.config = json.load(f)
+        debug = os.environ.get("DEBUG")
+        if debug:
+            level = logging.INFO
+        else: 
+            level = logging.WARNING
+        logging.basicConfig(level=level)
 
 
-def on_message(client, userdata, msg):
-    current_time = datetime.datetime.utcnow().isoformat()
-    data = json.loads(msg.payload)
-    print(data)
-    json_body = [
-        {
-            "measurement": msg.topic,
-            "tags": {},
-            "time": current_time,
-            "fields": data
-        }
-    ]
-    logging.info(json_body)
-    influx_client.write_points(json_body)
-   
-if 'TOPICS' not in os.environ:
-    print("At least one topic is needed.")
-    exit(1)
-topics = os.environ['TOPICS'].split(' ')   
-print(topics)
+        self.mqttInit()
+        self.influxInit()
 
-logging.basicConfig(level=logging.INFO)
-influx_client = InfluxDBClient(os.environ['INFLUX_HOST'], int(os.environ['INFLUX_PORT']), database=os.environ["INFLUX_DB"])
-influx_client.create_database(os.environ["INFLUX_DB"])
+    def mqttInit(self):
+        self.topics = ["sensors"] 
+        self.client = mqtt.Client()
+        self.client.username_pw_set(self.config["mqtt"]["user"], 
+            self.config["mqtt"]["password"])
+        self.client.tls_set(cert_reqs=ssl.CERT_REQUIRED,
+            tls_version=ssl.PROTOCOL_TLS)
+        self.client.on_connect = self.on_connect
+        self.client.on_message = self.on_message
+
+        try:
+            self.client.connect(self.config["mqtt"]["host"], self.config["mqtt"]["port"], 60)
+        except:
+            logging.critical("Failed to connect to MQTT server")
+            exit(1)
+
+    def influxInit(self):
+        try:
+            self.influx_client = InfluxDBClient(self.config["influx"]["host"],
+                self.config["influx"]["port"], 
+                database=self.config["influx"]["database"])
+            self.influx_client.create_database("sensors")
+        except Exception as e:
+            logging.critical("Failed to connect to database %s", e)
+            exit(1)
+    
+    def start(self):
+        logging.info("Starting")
+        self.client.loop_forever()
+
+    def on_connect(self, client, userdata, flags, rc):
+        for topic in self.config["mqtt"]["topics"]:
+            client.subscribe(topic)
+
+    def on_message(self, client, userdata, msg):
+        current_time = datetime.datetime.utcnow().isoformat()
+        data = json.loads(msg.payload)
+        json_body = [
+            {
+                "measurement": msg.topic,
+                "tags": {},
+                "time": current_time,
+                "fields": data
+            }
+        ]
+        logging.info(json_body)
+        try: 
+            self.influx_client.write_points(json_body)
+        except InfluxDBClientError as e:
+            logging.error("Influx write error %s", e)
+        except Exception as e:
+            logging.critical("Influx server error %s", e)
+            traceback.print_exc()
+            exit(1)
 
 
-client = mqtt.Client()
- 
-client.on_connect = on_connect
-client.on_message = on_message
- 
-client.connect(os.environ['MQTT_HOST'], int(os.environ["MQTT_PORT"]), 60)
+if __name__ == "__main__":
+    app = App()
+    app.start()
 
-print("Starting")
- 
-client.loop_forever()

mqttinflux/mqtt-influx.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=MQTT Influx ingest
+
+[Service]
+WorkingDirectory=/opt/mqtt-influx
+Restart=on-failure
+ExecStart=/usr/local/bin/pipenv run python /opt/mqtt-influx/main.py
+
+[Install]
+WantedBy=default.target

uP/main.py

@@ -2,26 +2,37 @@ from machine import Pin
 from time import sleep
 import machine
 import dht
-from umqtt.simple import MQTTClient
+from umqtt.robust import MQTTClient
 import json
 import gc
 
+with open("config.json", 'r') as f:
+    config = json.load(f)
+    
+print(config)
+
 led = Pin(2, Pin.OUT)
-d = dht.DHT11(Pin(16))
+d = dht.DHT11(Pin(12))
 adc = machine.ADC(0)
-server="10.1.1.162"
-c = MQTTClient("umqtt_client", server)
+
+c = MQTTClient("umqtt_client", config["mqtt"]["server"],ssl=True, user=config["mqtt"]["user"], password=config["mqtt"]["password"])
 gc.enable()
 
 while True:
     led(0)
     d.measure()
-    data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': 1}
+    data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': config["id"]}
     print(data)
     led(1)
-    c.connect()
-    c.publish(b"sensors", json.dumps(data))
-    c.disconnect()
+    try:
+        c.connect()
+        c.publish(b"sensors", json.dumps(data))
+    finally:
+        c.disconnect()
     gc.collect()
-    sleep(2)
+    
+    
+
+
+
 

uP/mqtt.py

@@ -1,37 +0,0 @@
-from machine import Pin
-from time import sleep
-import machine
-import dht
-from umqtt.robust import MQTTClient
-import json
-import gc
-
-with open("config.json", 'r') as f:
-    config = json.load(f)
-    
-print(config)
-
-led = Pin(2, Pin.OUT)
-d = dht.DHT11(Pin(12))
-adc = machine.ADC(0)
-
-c = MQTTClient("umqtt_client", config["mqtt"]["server"],ssl=True, user=config["mqtt"]["user"], password=config["mqtt"]["password"])
-gc.enable()
-
-while True:
-    led(0)
-    d.measure()
-    data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': config["id"]}
-    print(data)
-    led(1)
-    c.connect()
-    c.publish(b"sensors", json.dumps(data))
-    c.disconnect()
-    gc.collect()
- #   wdt.feed()
-    #sleep(1)
-    
-
-
-
-