Dutch Smart Meter (DSM) P1 to InfluxDB 2.0 and pvoutput.org

dsm_reader.py

#!/usr/bin/env python
# Store  DSMR telegrams from P1 into influxdb 2.0 and pvoutput.org
# EN-IEC 62056-21, Part 21: direct local data exchange, 2002-05

# WARNING: influxdb will crash/become a memory hog after collecting a few months of data !!!

#- Requires python2+
#- Install deps
#  # pacman -S python-requests python-pytz
#- Create script
#  # nano /root/dsm_reader.py && chmod +x /root/dsm_reader.py
#- Create config
#  # echo "auth_token='abcdefg'" > /root/dsm_reader_config.py

# ToDo: handle influxPi4B socat[137394]: HTTPSConnectionPool(host='pvoutput.org', port=443): Max retries exceeded with url: /service/

import logging
import os
import re
import sys
import threading
import time
import requests
import urllib3
from datetime import datetime
from pytz import timezone, utc
from dsm_reader_config import auth_token, pvoutput_token_6, pvoutput_sid_6

print_output = 0
url = 'http://127.0.0.1:8086/api/v2/write'
urlpv = 'https://pvoutput.org/service/r2/addstatus.jsp'
fp = sys.stdin
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

table = [
0x0000, 0xc0c1, 0xc181, 0x0140, 0xc301, 0x03c0, 0x0280, 0xc241,
0xc601, 0x06c0, 0x0780, 0xc741, 0x0500, 0xc5c1, 0xc481, 0x0440,
0xcc01, 0x0cc0, 0x0d80, 0xcd41, 0x0f00, 0xcfc1, 0xce81, 0x0e40,
0x0a00, 0xcac1, 0xcb81, 0x0b40, 0xc901, 0x09c0, 0x0880, 0xc841,
0xd801, 0x18c0, 0x1980, 0xd941, 0x1b00, 0xdbc1, 0xda81, 0x1a40,
0x1e00, 0xdec1, 0xdf81, 0x1f40, 0xdd01, 0x1dc0, 0x1c80, 0xdc41,
0x1400, 0xd4c1, 0xd581, 0x1540, 0xd701, 0x17c0, 0x1680, 0xd641,
0xd201, 0x12c0, 0x1380, 0xd341, 0x1100, 0xd1c1, 0xd081, 0x1040,
0xf001, 0x30c0, 0x3180, 0xf141, 0x3300, 0xf3c1, 0xf281, 0x3240,
0x3600, 0xf6c1, 0xf781, 0x3740, 0xf501, 0x35c0, 0x3480, 0xf441,
0x3c00, 0xfcc1, 0xfd81, 0x3d40, 0xff01, 0x3fc0, 0x3e80, 0xfe41,
0xfa01, 0x3ac0, 0x3b80, 0xfb41, 0x3900, 0xf9c1, 0xf881, 0x3840,
0x2800, 0xe8c1, 0xe981, 0x2940, 0xeb01, 0x2bc0, 0x2a80, 0xea41,
0xee01, 0x2ec0, 0x2f80, 0xef41, 0x2d00, 0xedc1, 0xec81, 0x2c40,
0xe401, 0x24c0, 0x2580, 0xe541, 0x2700, 0xe7c1, 0xe681, 0x2640,
0x2200, 0xe2c1, 0xe381, 0x2340, 0xe101, 0x21c0, 0x2080, 0xe041,
0xa001, 0x60c0, 0x6180, 0xa141, 0x6300, 0xa3c1, 0xa281, 0x6240,
0x6600, 0xa6c1, 0xa781, 0x6740, 0xa501, 0x65c0, 0x6480, 0xa441,
0x6c00, 0xacc1, 0xad81, 0x6d40, 0xaf01, 0x6fc0, 0x6e80, 0xae41,
0xaa01, 0x6ac0, 0x6b80, 0xab41, 0x6900, 0xa9c1, 0xa881, 0x6840,
0x7800, 0xb8c1, 0xb981, 0x7940, 0xbb01, 0x7bc0, 0x7a80, 0xba41,
0xbe01, 0x7ec0, 0x7f80, 0xbf41, 0x7d00, 0xbdc1, 0xbc81, 0x7c40,
0xb401, 0x74c0, 0x7580, 0xb541, 0x7700, 0xb7c1, 0xb681, 0x7640,
0x7200, 0xb2c1, 0xb381, 0x7340, 0xb101, 0x71c0, 0x7080, 0xb041,
0x5000, 0x90c1, 0x9181, 0x5140, 0x9301, 0x53c0, 0x5280, 0x9241,
0x9601, 0x56c0, 0x5780, 0x9741, 0x5500, 0x95c1, 0x9481, 0x5440,
0x9c01, 0x5cc0, 0x5d80, 0x9d41, 0x5f00, 0x9fc1, 0x9e81, 0x5e40,
0x5a00, 0x9ac1, 0x9b81, 0x5b40, 0x9901, 0x59c0, 0x5880, 0x9841,
0x8801, 0x48c0, 0x4980, 0x8941, 0x4b00, 0x8bc1, 0x8a81, 0x4a40,
0x4e00, 0x8ec1, 0x8f81, 0x4f40, 0x8d01, 0x4dc0, 0x4c80, 0x8c41,
0x4400, 0x84c1, 0x8581, 0x4540, 0x8701, 0x47c0, 0x4680, 0x8641,
0x8201, 0x42c0, 0x4380, 0x8341, 0x4100, 0x81c1, 0x8081, 0x4040
]

obis_objects = {
    '1-3:0.2.8': 'version_info',
    '0-0:1.0.0': 'datetime_stamp', # '210404113641S'
    '0-0:96.1.1': 'equipment_id_electricity', # '4530303331303033303031363939353135' ___tag
    '1-0:1.8.1': 'mr_toclient_t1', # '000009.691' uint _________________________________fields
    '1-0:1.8.2': 'mr_toclient_t2', # '000014.551' uint
    '1-0:2.8.1': 'mr_byclient_t1', # '002502.491' uint
    '1-0:2.8.2': 'mr_byclient_t2', # '005920.901' uint
    '0-0:96.14.0': 'tariff_indicator', # '0001' uint
    '1-0:1.7.0': 'actual_power_delivered', # '00.000' * 1000 uint
    '1-0:2.7.0': 'actual_power_received', # '10.806' * 1000 uint
    '0-0:96.7.21': 'total_pf', # '00220' uint
    '0-0:96.7.9': 'total_long_pf', # '00034' uint
    '1-0:99.97.0': 'pf_event_log',
    '1-0:32.32.0': 'volt_sag_l1', # '00024' uint
    '1-0:52.32.0': 'volt_sag_l2', # '00024' uint
    '1-0:72.32.0': 'volt_sag_l3', # '00024' uint
    '1-0:32.36.0': 'volt_swell_l1', # '00000' uint
    '1-0:52.36.0': 'volt_swell_l2', # '00000' uint
    '1-0:72.36.0': 'volt_swell_l3', # '00000' uint
    '0-0:96.13.1': 'text_message_codes',
    '0-0:96.13.0': 'text_message', # []
    '1-0:32.7.0': 'inst_volt_l1', # '231.0' float
    '1-0:52.7.0': 'inst_volt_l2', # '232.0' float
    '1-0:72.7.0': 'inst_volt_l3', # '231.0' float
    '1-0:31.7.0': 'inst_current_l1', # '015' uint
    '1-0:51.7.0': 'inst_current_l2', # '015' uint
    '1-0:71.7.0': 'inst_current_l3', # '016' uint
    '1-0:21.7.0': 'inst_power_l1_p', # '00.000' * 1000 uint
    '1-0:41.7.0': 'inst_power_l2_p', # '00.000' * 1000 uint
    '1-0:61.7.0': 'inst_power_l3_p', # '00.000' * 1000 uint
    '1-0:22.7.0': 'inst_power_l1_m', # '03.543' * 1000 uint
    '1-0:42.7.0': 'inst_power_l2_m', # '03.552' * 1000 uint
    '1-0:62.7.0': 'inst_power_l3_m', # '03.710' * 1000 uint
    '0-1:96.1.0': 'equipment_id_gas',
    '0-1:24.2.1': 'gas_toclient',
    '0-1:24.1.0': 'device_type',
}

def crc16(s):
    crc = 0x00

    for ch in s:
        crc = (crc >> 8) ^ table[(crc ^ ord(ch)) & 0xff]

    return crc

def obis_map(ref, value):
    if ref in obis_objects:
        ref = obis_objects[ref]

    return ref, value

def parse(buf):
    values = {}

    for line in buf.split():
        mo = re.findall('^([0-9.:-]+)(.+)', line.rstrip())
	# example mo=[('1-0:62.7.0', '(04.003*kW)')]

        for match in mo:
            pair = obis_map(*match)
            values.update([pair])
            if print_output == 2:
                print('%s: %s' % pair)

    return values

def verify_and_parse(buf):
    crc = int(buf[-1][1:5], 16)
    data = ''.join(buf[:-1]) + '!'

    if crc == crc16(data):
        return parse(data)

def parse_value(s):
    t = re.findall('\(([0-9.-]+[SW]?).*?\)', s)

    if re.search('^[0-9]+[.][0-9][0-9][0-9]$', t[0]):
        t[0] = str(int(float(t[0]) * 1000)) + 'u'
    elif re.search('^[0-9]{1,33}$', t[0]):
        t[0] = str(int(t[0])) + 'u'

    return t

def parse_ts(s):
    tm, tz = s[:-1], s[-1]
    is_dst = (tz == 'S')

    #if tm.startswith('23'):
    #    tm = '16' + tm[2:]

    date = datetime.strptime(tm, '%y%m%d%H%M%S')
    ams_tz = timezone('Europe/Amsterdam')
    date = ams_tz.localize(date, is_dst=is_dst)
    #date = date.astimezone(utc)

    unix = time.mktime(date.timetuple())

    return int(unix)

def handle(values):
    global arr_vals, pts, hivolt

    pts = parse_value(values['datetime_stamp'])[0]
    ts = parse_ts(pts)
    equip_id_elec = parse_value(values['equipment_id_electricity'])[0]

    keys = ['mr_toclient_t1', 'mr_toclient_t2', 'mr_byclient_t1', 'mr_byclient_t2', 'tariff_indicator', 'actual_power_delivered', 'actual_power_received', 'total_pf', 'total_long_pf', 'volt_sag_l1', 'volt_sag_l2', 'volt_sag_l3', 'volt_swell_l1', 'volt_swell_l2', 'volt_swell_l3', 'inst_volt_l1', 'inst_volt_l2', 'inst_volt_l3', 'inst_current_l1', 'inst_current_l2', 'inst_current_l3', 'inst_power_l1_p', 'inst_power_l2_p', 'inst_power_l3_p', 'inst_power_l1_m', 'inst_power_l2_m', 'inst_power_l3_m']

    #gas_ts, gas_value = parse_value(values['gas_toclient'])
    #gas_ts = parse_ts(gas_ts)

    arr_vals = lambda v: ','.join('%s=%s' % (k, parse_value(values[k])[0]) for k in v)

    # remember highest voltage of any phase
    for t in range (1, 3):
        hivolt = max(hivolt, float(arr_vals(['inst_volt_l' + str(t)])[13:]))

    # post selected keys
    postdata  = '%s %s %d\n' % ('measurement,equipment_id_electricity=' + equip_id_elec, arr_vals(keys), ts)
    #postdata += '%s gas_toclient=%s %d\n' % ('gas', gas_value, gas_ts)

    if print_output:
        print(postdata)

    # use InfluxDB 1.x compatibility API - Line protocol // db = bucket, precision = seconds
    try:
        r = requests.post(url,
                          params={'org': 'Organization', 'bucket': 'dsm', 'precision': 's'},
                          data=postdata,
                          headers={'Authorization': 'Token ' + auth_token},
                          timeout=0.9)
    except requests.exceptions.Timeout:
        # Log request to file
        filename = '/root/insert-failed-influx-posts.sh'
        try:
            with open(filename, 'r') as file:
                file.seek(0, os.SEEK_END)
                filesize = file.tell()
        except FileNotFoundError:
            filesize = 0
	
	with open(filename, 'a') as file:
            if filesize == 0:
                file.write('#!/bin/sh' + "\n")
            file.write('curl -X POST "' + url + '?org=Organization&bucket=dsm&precision=s" ' + \
                       '-H "Authorization: Token ' + auth_token + '" ' + \
                       '-H "Content-Type: application/x-www-form-urlencoded" ' + \
                       '-d "' + postdata.replace('\n','') + '"' + "\n")
    except requests.exceptions.ConnectionError:
        print('Connection refused')
    except requests.exceptions.RequestException as e:
        print(str(e))
    else:
        if print_output:
            print(str(r))

def post_pvoutput():
    global arr_vals, pts, hivolt

    threading.Timer(300.0, post_pvoutput).start()
    if pts:
        d = 'd=20' + pts[:6]
        t = 't=' + pts[6:8] + ':' + pts[8:10]
        v1 = 'v1=' + str(int(arr_vals(['mr_byclient_t1'])[15:-1]) + int(arr_vals(['mr_byclient_t2'])[15:-1]))
        v3 = 'v3=' + str(int(arr_vals(['mr_toclient_t1'])[15:-1]) + int(arr_vals(['mr_toclient_t2'])[15:-1]))
        v6 = 'v6=' + str(hivolt)
        hivolt = 0.0
        postdata = "&".join([d, t, v1, v3, v6, 'c1=1'])

	# Try posting data 9 times
        for i in range(1, 9):
            try:
                r = requests.post(
                    urlpv,
                    data=postdata,
                    headers={
                        'Content-Type': 'application/x-www-form-urlencoded',
                        'X-Pvoutput-Apikey': pvoutput_token_6,
                        'X-Pvoutput-SystemId': pvoutput_sid_6,
                        'Accept': 'text/plain'
                    },
                    timeout=30)
                if print_output:
                    print(r.text)
                    #print(postdata)
                if r.status_code == 200:
                    return
                if 400 <= r.status_code < 500:
                    warningMsg = ("Unable to connect to pvoutput.org - Reason: " + r.reason)
                    logging.warning(warningMsg)
                    return
            except requests.exceptions.RequestException as e:
                print(str(e))
            else:
                if print_output:
                    print(str(r))

if __name__ == '__main__':
    buf = []
    hivolt = 0.0
    called_once = False

    while True:
        line = fp.readline().strip('\x00')

        if line.startswith('/'):
            buf = [line]
        else:
            buf.append(line)

        if line.startswith('!'):
            values = verify_and_parse(buf)

            if values:
                handle(values)

                if called_once == False:
                     called_once = True
                     post_pvoutput() # threaded

dsm_reader.service.d

# nano /lib/systemd/system/dsm_reader.service
[Unit]
Description=Read smart meter P1 data and store the numeric values in Influxdb and pvoutput
Requires=influxdb2-bin
After=influxdb2-bin

[Install]
WantedBy=multi-user.target

[Service]
ExecStart=/usr/bin/socat -u FILE:/dev/ttyUSB0,b115200,crtscts,cfmakeraw EXEC:/root/dsm_reader.py
Restart=always
RestartSec=3

help.txt

EN-IEC 62056-21, Part 21: direct local data exchange, 2002-05

- Requires python2+
- Install deps
  # pacman -S python-requests python-pytz
- Create script
  # nano /root/dsm_reader.py && chmod +x /root/dsm_reader.py
- Create config
  # echo "auth_token='abcdefg'" > /root/dsm_reader_config.py
- Start
  # systemctl enable dsm_reader.service && systemctl start dsm_reader
- ToDo: 
  1. modify service file to a template unit file ([email protected]), where the serial device becomes variable
  2. use this name@ to create a separate backlog file per serial device?
  3. auto insert backlog to influxdb
  4. implement retries, with backoff? like 
     - solution 4 from https://izziswift.com/can-i-set-max_retries-for-requests-request/
     - https://github.com/pvl7/emu-to-pvoutput/blob/2edf7fff8d1af180fa203968b2d33acb80485dec/rainforest-to-pvoutput.py#L88
  5. also backlog pvoutput temp. errors, note: The date parameter must not be older than 14 days from the current date