added to readme and created dev branch

2021-05-28 13:19:01 +12:00
11 changed files with 89 additions and 318 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,2 +1 @@
 data/
 config.json
--- a/README.md
+++ b/README.md
@ -1,59 +1,11 @@
-# Datalogger
+# datalogger
 ESP8266 running Micropython collecting temperature humidity and light level. Pushing data to a Mosquito server. A python script to ingest the data into an influx database. Grafana to display the data.
-This is a work in progress and is not suitable for production
+## Setting up ampy
-## Server Setup
+`ampy` is a application that allows terminal access to connected micropython devices.
-This assumes you are using Ubuntu-20.04
+## typology
-### Install packages
+![](docs/typology)
 `sudo su -`
 `wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -`
 `echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list`
 `wget -qO- https://repos.influxdata.com/influxdb.key | gpg --dearmor > /etc/apt/trusted.gpg.d/influxdb.gpg`
 `export DISTRIB_ID=$(lsb_release -si); export DISTRIB_CODENAME=$(lsb_release -sc)`
 `echo "deb [signed-by=/etc/apt/trusted.gpg.d/influxdb.gpg] https://repos.influxdata.com/${DISTRIB_ID,,} ${DISTRIB_CODENAME} stable" > /etc/apt/sources.list.d/influxdb.list`
 `sudo apt update`
 `sudo apt install python3-pip nginx mosquitto grafana influxdb python3-certbot-nginx`
 `pip3 install pipenv`
 ### Setup Nginx
 ## Esp8266 Setup
 ### Install tools
 ```pip3 install esptool adafruit-ampy```
 ### Download Esp8266 Firmware
 https://micropython.org/resources/firmware/esp8266-20210418-v1.15.bin
 ### Install firmware
 ```esptool.py --port <port> erase_flash```
 ```esptool.py --port <port> --baud 1000000 write_flash --flash_size=4MB -fm dio 0 <download_folder>/esp8266-20210418-v1.15.bin```
 ### Edit config file
 ```cp uP/config.json.sample config.json```
 ### Copy libraries, config and code to esp8266
 ```ampy -p <port> put uP /```
--- a/docs/typology.png
+++ b/docs/typology.png
--- a/config/mosquitto.conf
+++ b/config/mosquitto.conf
--- a/mqttinflux/Pipfile
+++ b/mqttinflux/Pipfile
@ -1,18 +0,0 @@
 [[source]]
 url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 [packages]
 influxdb = "*"
 paho-mqtt = "*"
 [dev-packages]
 [requires]
 python_version = "3.8"
 [scripts]
 dev = 'bash -c "export DEBUG=1 && ls main.py mqtt-influx.ini | entr -rc python main.py"'
 prod = "python main.py"
--- a/mqttinflux/Pipfile.lock
+++ b/mqttinflux/Pipfile.lock
@ -1,125 +0,0 @@
 {
    "_meta": {
        "hash": {
            "sha256": "3634ff9d7c9021daf28f9f4b5e9ad6762f3c20398e73a53a00a691b6534813c3"
        },
        "pipfile-spec": 6,
        "requires": {
            "python_version": "3.8"
        },
        "sources": [
            {
                "name": "pypi",
                "url": "https://pypi.python.org/simple",
                "verify_ssl": true
            }
        ]
    },
    "default": {
        "certifi": {
            "hashes": [
                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
            ],
            "version": "==2020.12.5"
        },
        "chardet": {
            "hashes": [
                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
            ],
            "version": "==4.0.0"
        },
        "idna": {
            "hashes": [
                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
            ],
            "version": "==2.10"
        },
        "influxdb": {
            "hashes": [
                "sha256:46f85e7b04ee4b3dee894672be6a295c94709003a7ddea8820deec2ac4d8b27a",
                "sha256:65040a1f53d1a2a4f88a677e89e3a98189a7d30cf2ab61c318aaa89733280747"
            ],
            "index": "pypi",
            "version": "==5.3.1"
        },
        "msgpack": {
            "hashes": [
                "sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9",
                "sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841",
                "sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439",
                "sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694",
                "sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a",
                "sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f",
                "sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e",
                "sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1",
                "sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c",
                "sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b",
                "sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759",
                "sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326",
                "sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc",
                "sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192",
                "sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83",
                "sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06",
                "sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e",
                "sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9",
                "sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33",
                "sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54",
                "sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f",
                "sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887",
                "sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009",
                "sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2",
                "sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c",
                "sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87",
                "sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984",
                "sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6"
            ],
            "version": "==1.0.2"
        },
        "paho-mqtt": {
            "hashes": [
                "sha256:9feb068e822be7b3a116324e01fb6028eb1d66412bf98595ae72698965cb1cae"
            ],
            "index": "pypi",
            "version": "==1.5.1"
        },
        "python-dateutil": {
            "hashes": [
                "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
                "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
            ],
            "version": "==2.8.1"
        },
        "pytz": {
            "hashes": [
                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
            ],
            "version": "==2021.1"
        },
        "requests": {
            "hashes": [
                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
            ],
            "version": "==2.25.1"
        },
        "six": {
            "hashes": [
                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
            ],
            "version": "==1.16.0"
        },
        "urllib3": {
            "hashes": [
                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
            ],
            "version": "==1.26.4"
        }
    },
    "develop": {}
 }
--- a/mqttinflux/config.json.sample
+++ b/mqttinflux/config.json.sample
@ -1,14 +0,0 @@
 {
    "mqtt": {
        "host": "",
        "port": 8883,
        "user": "",
        "password": ""
        "topics": []
    },
    "influx": {
        "host": "localhost",
        "port": 8086,
        "database": ""
    }
 }
--- a/mqttinflux/main.py
+++ b/mqttinflux/main.py
@ -1,88 +1,49 @@
 import paho.mqtt.client as mqtt
 from influxdb import InfluxDBClient
 from influxdb.exceptions import InfluxDBClientError
 from influxdb.exceptions import InfluxDBServerError
 import datetime
 import logging
 import json
 import os
 import ssl
 import traceback
 import os
 import configparser
 class App:
    def __init__(self):
        with open("mqtt-influx.json", 'r') as f:
            self.config = json.load(f)
        debug = os.environ.get("DEBUG")
        if debug:
            level = logging.INFO
        else: 
            level = logging.WARNING
        logging.basicConfig(level=level)
-        self.mqttInit()
+def on_connect(client, userdata, flags, rc):
-        self.influxInit()
+    for topic in topics:
-
+        client.subscribe(topic)
    def mqttInit(self):
        self.topics = ["sensors"] 
        self.client = mqtt.Client()
        self.client.username_pw_set(self.config["mqtt"]["user"], 
            self.config["mqtt"]["password"])
        self.client.tls_set(cert_reqs=ssl.CERT_REQUIRED,
            tls_version=ssl.PROTOCOL_TLS)
        self.client.on_connect = self.on_connect
        self.client.on_message = self.on_message
        try:
            self.client.connect(self.config["mqtt"]["host"], self.config["mqtt"]["port"], 60)
        except:
            logging.critical("Failed to connect to MQTT server")
            exit(1)
    def influxInit(self):
        try:
            self.influx_client = InfluxDBClient(self.config["influx"]["host"],
                self.config["influx"]["port"], 
                database=self.config["influx"]["database"])
            self.influx_client.create_database("sensors")
        except Exception as e:
            logging.critical("Failed to connect to database %s", e)
            exit(1)
    def start(self):
        logging.info("Starting")
        self.client.loop_forever()
    def on_connect(self, client, userdata, flags, rc):
        for topic in self.config["mqtt"]["topics"]:
            client.subscribe(topic)
    def on_message(self, client, userdata, msg):
        current_time = datetime.datetime.utcnow().isoformat()
        data = json.loads(msg.payload)
        json_body = [
            {
                "measurement": msg.topic,
                "tags": {},
                "time": current_time,
                "fields": data
            }
        ]
        logging.info(json_body)
        try: 
            self.influx_client.write_points(json_body)
        except InfluxDBClientError as e:
            logging.error("Influx write error %s", e)
        except Exception as e:
            logging.critical("Influx server error %s", e)
            traceback.print_exc()
            exit(1)
-if __name__ == "__main__":
+def on_message(client, userdata, msg):
-    app = App()
+    current_time = datetime.datetime.utcnow().isoformat()
-    app.start()
+    data = json.loads(msg.payload)
    print(data)
    json_body = [
        {
            "measurement": msg.topic,
            "tags": {},
            "time": current_time,
            "fields": data
        }
    ]
    logging.info(json_body)
    influx_client.write_points(json_body)
 if 'TOPICS' not in os.environ:
    print("At least one topic is needed.")
    exit(1)
 topics = os.environ['TOPICS'].split(' ')   
 print(topics)
 logging.basicConfig(level=logging.INFO)
 influx_client = InfluxDBClient(os.environ['INFLUX_HOST'], int(os.environ['INFLUX_PORT']), database=os.environ["INFLUX_DB"])
 influx_client.create_database(os.environ["INFLUX_DB"])
 client = mqtt.Client()
 client.on_connect = on_connect
 client.on_message = on_message
 client.connect(os.environ['MQTT_HOST'], int(os.environ["MQTT_PORT"]), 60)
 print("Starting")
 client.loop_forever()
--- a/mqttinflux/mqtt-influx.service
+++ b/mqttinflux/mqtt-influx.service
@ -1,10 +0,0 @@
 [Unit]
 Description=MQTT Influx ingest
 [Service]
 WorkingDirectory=/opt/mqtt-influx
 Restart=on-failure
 ExecStart=/usr/local/bin/pipenv run python /opt/mqtt-influx/main.py
 [Install]
 WantedBy=default.target
--- a/uP/main.py
+++ b/uP/main.py
@ -2,37 +2,26 @@ from machine import Pin
 from time import sleep
 import machine
 import dht
-from umqtt.robust import MQTTClient
+from umqtt.simple import MQTTClient
 import json
 import gc
 with open("config.json", 'r') as f:
    config = json.load(f)
 print(config)
 led = Pin(2, Pin.OUT)
-d = dht.DHT11(Pin(12))
+d = dht.DHT11(Pin(16))
 adc = machine.ADC(0)
-
+server="10.1.1.162"
-c = MQTTClient("umqtt_client", config["mqtt"]["server"],ssl=True, user=config["mqtt"]["user"], password=config["mqtt"]["password"])
+c = MQTTClient("umqtt_client", server)
 gc.enable()
 while True:
    led(0)
    d.measure()
-    data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': config["id"]}
+    data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': 1}
    print(data)
    led(1)
-    try:
+    c.connect()
-        c.connect()
+    c.publish(b"sensors", json.dumps(data))
-        c.publish(b"sensors", json.dumps(data))
+    c.disconnect()
    finally:
        c.disconnect()
    gc.collect()
-    
+    sleep(2)
--- a/uP/mqtt.py
+++ b/uP/mqtt.py
@ -0,0 +1,37 @@
 from machine import Pin
 from time import sleep
 import machine
 import dht
 from umqtt.robust import MQTTClient
 import json
 import gc
 with open("config.json", 'r') as f:
    config = json.load(f)
 print(config)
 led = Pin(2, Pin.OUT)
 d = dht.DHT11(Pin(12))
 adc = machine.ADC(0)
 c = MQTTClient("umqtt_client", config["mqtt"]["server"],ssl=True, user=config["mqtt"]["user"], password=config["mqtt"]["password"])
 gc.enable()
 while True:
    led(0)
    d.measure()
    data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': config["id"]}
    print(data)
    led(1)
    c.connect()
    c.publish(b"sensors", json.dumps(data))
    c.disconnect()
    gc.collect()
 #   wdt.feed()
    #sleep(1)