Compare commits
1 Commits
master
...
shivan-dev
| Author | SHA1 | Date |
|---|---|---|
 Shivan Sivakumaran
|
bf604c5e30 |
|
|
@ -1,2 +1 @@
|
|||
data/
|
||||
config.json
|
||||
data/
|
||||
58
README.md
58
README.md
|
|
@ -1,59 +1,11 @@
|
|||
# Datalogger
|
||||
# datalogger
|
||||
|
||||
ESP8266 running Micropython collecting temperature humidity and light level. Pushing data to a Mosquito server. A python script to ingest the data into an influx database. Grafana to display the data.
|
||||
|
||||
This is a work in progress and is not suitable for production
|
||||
## Setting up ampy
|
||||
|
||||
## Server Setup
|
||||
`ampy` is a application that allows terminal access to connected micropython devices.
|
||||
|
||||
This assumes you are using Ubuntu-20.04
|
||||
## typology
|
||||
|
||||
### Install packages
|
||||
|
||||
`sudo su -`
|
||||
|
||||
`wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -`
|
||||
|
||||
`echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list`
|
||||
|
||||
`wget -qO- https://repos.influxdata.com/influxdb.key | gpg --dearmor > /etc/apt/trusted.gpg.d/influxdb.gpg`
|
||||
|
||||
`export DISTRIB_ID=$(lsb_release -si); export DISTRIB_CODENAME=$(lsb_release -sc)`
|
||||
|
||||
`echo "deb [signed-by=/etc/apt/trusted.gpg.d/influxdb.gpg] https://repos.influxdata.com/${DISTRIB_ID,,} ${DISTRIB_CODENAME} stable" > /etc/apt/sources.list.d/influxdb.list`
|
||||
|
||||
`sudo apt update`
|
||||
|
||||
`sudo apt install python3-pip nginx mosquitto grafana influxdb python3-certbot-nginx`
|
||||
|
||||
`pip3 install pipenv`
|
||||
|
||||
### Setup Nginx
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Esp8266 Setup
|
||||
|
||||
### Install tools
|
||||
|
||||
```pip3 install esptool adafruit-ampy```
|
||||
|
||||
### Download Esp8266 Firmware
|
||||
|
||||
https://micropython.org/resources/firmware/esp8266-20210418-v1.15.bin
|
||||
|
||||
### Install firmware
|
||||
|
||||
```esptool.py --port <port> erase_flash```
|
||||
|
||||
```esptool.py --port <port> --baud 1000000 write_flash --flash_size=4MB -fm dio 0 <download_folder>/esp8266-20210418-v1.15.bin```
|
||||
|
||||
### Edit config file
|
||||
|
||||
```cp uP/config.json.sample config.json```
|
||||
|
||||
### Copy libraries, config and code to esp8266
|
||||
|
||||
```ampy -p <port> put uP /```
|
||||
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 365 KiB After Width: | Height: | Size: 365 KiB |
|
|
@ -1,18 +0,0 @@
|
|||
[[source]]
|
||||
url = "https://pypi.python.org/simple"
|
||||
verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
influxdb = "*"
|
||||
paho-mqtt = "*"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
|
||||
[scripts]
|
||||
dev = 'bash -c "export DEBUG=1 && ls main.py mqtt-influx.ini | entr -rc python main.py"'
|
||||
prod = "python main.py"
|
||||
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "3634ff9d7c9021daf28f9f4b5e9ad6762f3c20398e73a53a00a691b6534813c3"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.8"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.python.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
|
||||
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
|
||||
],
|
||||
"version": "==2020.12.5"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
|
||||
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
|
||||
],
|
||||
"version": "==4.0.0"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||
],
|
||||
"version": "==2.10"
|
||||
},
|
||||
"influxdb": {
|
||||
"hashes": [
|
||||
"sha256:46f85e7b04ee4b3dee894672be6a295c94709003a7ddea8820deec2ac4d8b27a",
|
||||
"sha256:65040a1f53d1a2a4f88a677e89e3a98189a7d30cf2ab61c318aaa89733280747"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.3.1"
|
||||
},
|
||||
"msgpack": {
|
||||
"hashes": [
|
||||
"sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9",
|
||||
"sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841",
|
||||
"sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439",
|
||||
"sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694",
|
||||
"sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a",
|
||||
"sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f",
|
||||
"sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e",
|
||||
"sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1",
|
||||
"sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c",
|
||||
"sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b",
|
||||
"sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759",
|
||||
"sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326",
|
||||
"sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc",
|
||||
"sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192",
|
||||
"sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83",
|
||||
"sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06",
|
||||
"sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e",
|
||||
"sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9",
|
||||
"sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33",
|
||||
"sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54",
|
||||
"sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f",
|
||||
"sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887",
|
||||
"sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009",
|
||||
"sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2",
|
||||
"sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c",
|
||||
"sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87",
|
||||
"sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984",
|
||||
"sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6"
|
||||
],
|
||||
"version": "==1.0.2"
|
||||
},
|
||||
"paho-mqtt": {
|
||||
"hashes": [
|
||||
"sha256:9feb068e822be7b3a116324e01fb6028eb1d66412bf98595ae72698965cb1cae"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.5.1"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
|
||||
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
|
||||
],
|
||||
"version": "==2.8.1"
|
||||
},
|
||||
"pytz": {
|
||||
"hashes": [
|
||||
"sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
|
||||
"sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
|
||||
],
|
||||
"version": "==2021.1"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
|
||||
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
|
||||
],
|
||||
"version": "==2.25.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
|
||||
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
|
||||
],
|
||||
"version": "==1.16.0"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
|
||||
"sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
|
||||
],
|
||||
"version": "==1.26.4"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
{
|
||||
"mqtt": {
|
||||
"host": "",
|
||||
"port": 8883,
|
||||
"user": "",
|
||||
"password": ""
|
||||
"topics": []
|
||||
},
|
||||
"influx": {
|
||||
"host": "localhost",
|
||||
"port": 8086,
|
||||
"database": ""
|
||||
}
|
||||
}
|
||||
|
|
@ -1,88 +1,49 @@
|
|||
import paho.mqtt.client as mqtt
|
||||
from influxdb import InfluxDBClient
|
||||
from influxdb.exceptions import InfluxDBClientError
|
||||
from influxdb.exceptions import InfluxDBServerError
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import traceback
|
||||
import os
|
||||
import configparser
|
||||
|
||||
class App:
|
||||
def __init__(self):
|
||||
with open("mqtt-influx.json", 'r') as f:
|
||||
self.config = json.load(f)
|
||||
debug = os.environ.get("DEBUG")
|
||||
if debug:
|
||||
level = logging.INFO
|
||||
else:
|
||||
level = logging.WARNING
|
||||
logging.basicConfig(level=level)
|
||||
|
||||
|
||||
def on_connect(client, userdata, flags, rc):
|
||||
for topic in topics:
|
||||
client.subscribe(topic)
|
||||
|
||||
|
||||
self.mqttInit()
|
||||
self.influxInit()
|
||||
def on_message(client, userdata, msg):
|
||||
current_time = datetime.datetime.utcnow().isoformat()
|
||||
data = json.loads(msg.payload)
|
||||
print(data)
|
||||
json_body = [
|
||||
{
|
||||
"measurement": msg.topic,
|
||||
"tags": {},
|
||||
"time": current_time,
|
||||
"fields": data
|
||||
}
|
||||
]
|
||||
logging.info(json_body)
|
||||
influx_client.write_points(json_body)
|
||||
|
||||
if 'TOPICS' not in os.environ:
|
||||
print("At least one topic is needed.")
|
||||
exit(1)
|
||||
topics = os.environ['TOPICS'].split(' ')
|
||||
print(topics)
|
||||
|
||||
def mqttInit(self):
|
||||
self.topics = ["sensors"]
|
||||
self.client = mqtt.Client()
|
||||
self.client.username_pw_set(self.config["mqtt"]["user"],
|
||||
self.config["mqtt"]["password"])
|
||||
self.client.tls_set(cert_reqs=ssl.CERT_REQUIRED,
|
||||
tls_version=ssl.PROTOCOL_TLS)
|
||||
self.client.on_connect = self.on_connect
|
||||
self.client.on_message = self.on_message
|
||||
|
||||
try:
|
||||
self.client.connect(self.config["mqtt"]["host"], self.config["mqtt"]["port"], 60)
|
||||
except:
|
||||
logging.critical("Failed to connect to MQTT server")
|
||||
exit(1)
|
||||
|
||||
def influxInit(self):
|
||||
try:
|
||||
self.influx_client = InfluxDBClient(self.config["influx"]["host"],
|
||||
self.config["influx"]["port"],
|
||||
database=self.config["influx"]["database"])
|
||||
self.influx_client.create_database("sensors")
|
||||
except Exception as e:
|
||||
logging.critical("Failed to connect to database %s", e)
|
||||
exit(1)
|
||||
|
||||
def start(self):
|
||||
logging.info("Starting")
|
||||
self.client.loop_forever()
|
||||
|
||||
def on_connect(self, client, userdata, flags, rc):
|
||||
for topic in self.config["mqtt"]["topics"]:
|
||||
client.subscribe(topic)
|
||||
|
||||
def on_message(self, client, userdata, msg):
|
||||
current_time = datetime.datetime.utcnow().isoformat()
|
||||
data = json.loads(msg.payload)
|
||||
json_body = [
|
||||
{
|
||||
"measurement": msg.topic,
|
||||
"tags": {},
|
||||
"time": current_time,
|
||||
"fields": data
|
||||
}
|
||||
]
|
||||
logging.info(json_body)
|
||||
try:
|
||||
self.influx_client.write_points(json_body)
|
||||
except InfluxDBClientError as e:
|
||||
logging.error("Influx write error %s", e)
|
||||
except Exception as e:
|
||||
logging.critical("Influx server error %s", e)
|
||||
traceback.print_exc()
|
||||
exit(1)
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
influx_client = InfluxDBClient(os.environ['INFLUX_HOST'], int(os.environ['INFLUX_PORT']), database=os.environ["INFLUX_DB"])
|
||||
influx_client.create_database(os.environ["INFLUX_DB"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app = App()
|
||||
app.start()
|
||||
client = mqtt.Client()
|
||||
|
||||
client.on_connect = on_connect
|
||||
client.on_message = on_message
|
||||
|
||||
client.connect(os.environ['MQTT_HOST'], int(os.environ["MQTT_PORT"]), 60)
|
||||
|
||||
print("Starting")
|
||||
|
||||
client.loop_forever()
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[Unit]
|
||||
Description=MQTT Influx ingest
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/opt/mqtt-influx
|
||||
Restart=on-failure
|
||||
ExecStart=/usr/local/bin/pipenv run python /opt/mqtt-influx/main.py
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
29
uP/main.py
29
uP/main.py
|
|
@ -2,37 +2,26 @@ from machine import Pin
|
|||
from time import sleep
|
||||
import machine
|
||||
import dht
|
||||
from umqtt.robust import MQTTClient
|
||||
from umqtt.simple import MQTTClient
|
||||
import json
|
||||
import gc
|
||||
|
||||
with open("config.json", 'r') as f:
|
||||
config = json.load(f)
|
||||
|
||||
print(config)
|
||||
|
||||
led = Pin(2, Pin.OUT)
|
||||
d = dht.DHT11(Pin(12))
|
||||
d = dht.DHT11(Pin(16))
|
||||
adc = machine.ADC(0)
|
||||
|
||||
c = MQTTClient("umqtt_client", config["mqtt"]["server"],ssl=True, user=config["mqtt"]["user"], password=config["mqtt"]["password"])
|
||||
server="10.1.1.162"
|
||||
c = MQTTClient("umqtt_client", server)
|
||||
gc.enable()
|
||||
|
||||
while True:
|
||||
led(0)
|
||||
d.measure()
|
||||
data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': config["id"]}
|
||||
data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': 1}
|
||||
print(data)
|
||||
led(1)
|
||||
try:
|
||||
c.connect()
|
||||
c.publish(b"sensors", json.dumps(data))
|
||||
finally:
|
||||
c.disconnect()
|
||||
c.connect()
|
||||
c.publish(b"sensors", json.dumps(data))
|
||||
c.disconnect()
|
||||
gc.collect()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
sleep(2)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
from machine import Pin
|
||||
from time import sleep
|
||||
import machine
|
||||
import dht
|
||||
from umqtt.robust import MQTTClient
|
||||
import json
|
||||
import gc
|
||||
|
||||
with open("config.json", 'r') as f:
|
||||
config = json.load(f)
|
||||
|
||||
print(config)
|
||||
|
||||
led = Pin(2, Pin.OUT)
|
||||
d = dht.DHT11(Pin(12))
|
||||
adc = machine.ADC(0)
|
||||
|
||||
c = MQTTClient("umqtt_client", config["mqtt"]["server"],ssl=True, user=config["mqtt"]["user"], password=config["mqtt"]["password"])
|
||||
gc.enable()
|
||||
|
||||
while True:
|
||||
led(0)
|
||||
d.measure()
|
||||
data = {'temp': d.temperature(), 'humid': d.humidity(),'light': adc.read(), 'id': config["id"]}
|
||||
print(data)
|
||||
led(1)
|
||||
c.connect()
|
||||
c.publish(b"sensors", json.dumps(data))
|
||||
c.disconnect()
|
||||
gc.collect()
|
||||
# wdt.feed()
|
||||
#sleep(1)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue