Merge branch 'development03' into eth

src/CHANGES.md

@@ -1,34 +1,14 @@
 # Development Changes
 
-## 0.6.7 - 2023-04-13
+## 0.6.12 - 2023-04-28
-* merge PR #883, improved store of settings and javascript, thx @tastendruecker123
+* improved MqTT
-* support `.` and `,` as floating point seperator in setup #881
+* fix menu active item
 
-## 0.6.6 - 2023-04-12
+## 0.6.11 - 2023-04-27
-* increased distance for `import` button in mobile view #879
+* added MqTT class for publishing all values in Arduino `loop`
-* changed `led_high_active` to `bool` #879
 
-## 0.6.5 - 2023-04-11
+## 0.6.10 - HMS
-* fix #845 MqTT subscription for `ctrl/power/[IV-ID]` was missing
+* Version available in `HMS` branch
-* merge PR #876, check JSON settings during read for existance
-* **NOTE:** incompatible change: renamed `led_high_active` to `act_high`, maybe setting must be changed after update
-* merge PR #861 do not send channel metric if channel is disabled
 
-## 0.6.4 - 2023-04-06
+## 0.6.9
-* merge PR #846, improved NRF24 communication and MI, thx @beegee3 & @rejoe2
+* last Relaese
-* merge PR #859, fix burger menu height, thx @ThomasPohl
-
-## 0.6.3 - 2023-04-04
-* fix login, password length was not checked #852
-* merge PR #854 optimize browser caching, thx @tastendruecker123 #828
-* fix WiFi reconnect not working #851
-* updated issue templates #822
-
-## 0.6.2 - 2023-04-04
-* fix login from multiple clients #819
-* fix login screen on small displays
-
-## 0.6.1 - 2023-04-01
-* merge LED fix - LED1 shows MqTT state, LED configureable active high/low #839
-* only publish new inverter data #826
-* potential fix of WiFi hostname during boot up #752

src/app.cpp

@@ -199,6 +199,8 @@ void app::regularTickers(void) {
     if (mConfig->plugin.display.type != 0)
         everySec(std::bind(&DisplayType::tickerSecond, &mDisplay), "disp");
     every(std::bind(&PubSerialType::tick, &mPubSerial), mConfig->serial.interval, "uart");
+
+    // every([this]() {mPayload.simulation();}, 15, "simul");
 }
 
 //-----------------------------------------------------------------------------

src/defines.h

@@ -1,5 +1,5 @@
 //-----------------------------------------------------------------------------
-// 2023 Ahoy, https://www.mikrocontroller.net/topic/525778
+// 2023 Ahoy, https://ahoydtu.de
 // Creative Commons - https://creativecommons.org/licenses/by-nc-sa/4.0/deed
 //-----------------------------------------------------------------------------
 
@@ -13,7 +13,7 @@
 //-------------------------------------
 #define VERSION_MAJOR       0
 #define VERSION_MINOR       6
-#define VERSION_PATCH       8
+#define VERSION_PATCH       12
 
 //-------------------------------------
 typedef struct {

src/hm/hmPayload.h

@@ -70,12 +70,35 @@ class HmPayload {
             }
         }
 
+        /*void simulation() {
+            uint8_t pay[] = {
+                0x00, 0x01, 0x01, 0x24, 0x02, 0x28, 0x02, 0x33,
+                0x06, 0x49, 0x06, 0x6a, 0x00, 0x05, 0x5f, 0x1b,
+                0x00, 0x06, 0x66, 0x9a, 0x03, 0xfd, 0x04, 0x0b,
+                0x01, 0x23, 0x02, 0x28, 0x02, 0x28, 0x06, 0x41,
+                0x06, 0x43, 0x00, 0x05, 0xdc, 0x2c, 0x00, 0x06,
+                0x2e, 0x3f, 0x04, 0x01, 0x03, 0xfb, 0x09, 0x78,
+                0x13, 0x86, 0x18, 0x15, 0x00, 0xcf, 0x00, 0xfe,
+                0x03, 0xe7, 0x01, 0x42, 0x00, 0x03
+            };
+
+            Inverter<> *iv = mSys->getInverterByPos(0);
+            record_t<> *rec = iv->getRecordStruct(0x0b);
+            rec->ts = *mTimestamp;
+            for (uint8_t i = 0; i < rec->length; i++) {
+                iv->addValue(i, pay, rec);
+                yield();
+            }
+            iv->doCalculations();
+            notify(0x0b);
+        }*/
+
         void zeroYieldDay(Inverter<> *iv) {
             DPRINTLN(DBG_DEBUG, F("zeroYieldDay"));
             record_t<> *rec = iv->getRecordStruct(RealTimeRunData_Debug);
             uint8_t pos;
-            for(uint8_t ch = 0; ch < iv->channels; ch++) {
+            for(uint8_t ch = 0; ch <= iv->channels; ch++) {
-                pos = iv->getPosByChFld(CH0, FLD_YD, rec);
+                pos = iv->getPosByChFld(ch, FLD_YD, rec);
                 iv->setValue(pos, rec, 0.0f);
             }
         }

src/platformio.ini

@@ -16,17 +16,6 @@ include_dir = .
 framework = arduino
 board_build.filesystem = littlefs
 upload_speed = 921600
-
-;build_flags =
-; ;;;;; Possible Debug options ;;;;;;
-; https://docs.platformio.org/en/latest/platforms/espressif8266.html#debug-level
-    ;-DDEBUG_ESP_PORT=Serial
-    ;-DDEBUG_ESP_CORE
-    ;-DDEBUG_ESP_WIFI
-    ;-DDEBUG_ESP_HTTP_CLIENT
-    ;-DDEBUG_ESP_HTTP_SERVER
-    ;-DDEBUG_ESP_OOM
-
 monitor_speed = 115200
 
 extra_scripts =
@@ -38,9 +27,9 @@ lib_deps =
     nrf24/RF24 @ ^1.4.5
     paulstoffregen/Time @ ^1.6.1
     https://github.com/bertmelis/espMqttClient#v1.4.2
-    bblanchon/ArduinoJson @ ^6.21.0
+    bblanchon/ArduinoJson @ ^6.21.2
     https://github.com/JChristensen/Timezone @ ^1.2.4
-    olikraus/U8g2 @ ^2.34.16
+    olikraus/U8g2 @ ^2.34.17
     zinggjm/GxEPD2 @ ^1.5.0
 
 
@@ -95,7 +84,13 @@ platform = espressif8266
 board = esp8285
 board_build.ldscript = eagle.flash.1m64.ld
 board_build.f_cpu = 80000000L
-build_flags = -DDEBUG_LEVEL=DBG_DEBUG -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_OOM -DDEBUG_ESP_PORT=Serial
+build_flags = -DDEBUG_LEVEL=DBG_DEBUG
+    -DDEBUG_ESP_CORE
+    -DDEBUG_ESP_WIFI
+    -DDEBUG_ESP_HTTP_CLIENT
+    -DDEBUG_ESP_HTTP_SERVER
+    -DDEBUG_ESP_OOM
+    -DDEBUG_ESP_PORT=Serial
 build_type = debug
 monitor_filters =
     ;default   ; Remove typical terminal control codes from input
@@ -103,7 +98,7 @@ monitor_filters =
     log2file  ; Log data to a file “platformio-device-monitor-*.log” located in the current working directory
 
 [env:esp32-wroom32-release]
-platform = espressif32
+platform = espressif32@>=6.1.0
 board = lolin_d32
 build_flags = -D RELEASE -std=gnu++14
 build_unflags = -std=gnu++11
@@ -114,9 +109,11 @@ monitor_filters =
     esp32_exception_decoder
 
 [env:esp32-wroom32-release-prometheus]
-platform = espressif32
+platform = espressif32@>=6.1.0
 board = lolin_d32
-build_flags = -D RELEASE -std=gnu++14 -DENABLE_PROMETHEUS_EP
+build_flags = -D RELEASE
+    -std=gnu++14
+    -DENABLE_PROMETHEUS_EP
 build_unflags = -std=gnu++11
 monitor_filters =
     ;default   ; Remove typical terminal control codes from input
@@ -125,9 +122,16 @@ monitor_filters =
     esp32_exception_decoder
 
 [env:esp32-wroom32-debug]
-platform = espressif32
+platform = espressif32@>=6.1.0
 board = lolin_d32
-build_flags = -DDEBUG_LEVEL=DBG_DEBUG -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_OOM -DDEBUG_ESP_PORT=Serial -std=gnu++14
+build_flags = -DDEBUG_LEVEL=DBG_DEBUG
+    -DDEBUG_ESP_CORE
+    -DDEBUG_ESP_WIFI
+    -DDEBUG_ESP_HTTP_CLIENT
+    -DDEBUG_ESP_HTTP_SERVER
+    -DDEBUG_ESP_OOM
+    -DDEBUG_ESP_PORT=Serial
+    -std=gnu++14
 build_unflags = -std=gnu++11
 build_type = debug
 monitor_filters =
@@ -180,7 +184,7 @@ monitor_filters =
     esp32_exception_decoder
 
 [env:opendtufusionv1-release]
-platform = espressif32
+platform = espressif32@>=6.1.0
 board = esp32-s3-devkitc-1
 upload_protocol = esp-builtin
 upload_speed = 115200

src/publisher/pubMqtt.h

@@ -22,6 +22,7 @@
 #include "../hm/hmSystem.h"
 
 #include "pubMqttDefs.h"
+#include "pubMqttIvData.h"
 
 #define QOS_0   0
 
@@ -63,6 +64,10 @@ class PubMqtt {
             mUtcTimestamp    = utcTs;
             mIntervalTimeout = 1;
 
+            mSendIvData.setup(sys, utcTs, &mSendList);
+            mSendIvData.setPublishFunc([this](const char *subTopic, const char *payload, bool retained) {
+                publish(subTopic, payload, retained);
+            });
             mDiscovery.running = false;
 
             snprintf(mLwtTopic, MQTT_TOPIC_LEN + 5, "%s/mqtt", mCfgMqtt->topic);
@@ -88,6 +93,8 @@ class PubMqtt {
         }
 
         void loop() {
+            mSendIvData.loop();
+
             #if defined(ESP8266)
             mClient.loop();
             yield();
@@ -562,89 +569,7 @@ class PubMqtt {
             if(mSendList.empty())
                 return;
 
-            float total[4];
+            mSendIvData.start();
-            bool RTRDataHasBeenSent = false;
-
-            while(!mSendList.empty()) {
-                memset(total, 0, sizeof(float) * 4);
-                uint8_t curInfoCmd = mSendList.front();
-
-                if ((curInfoCmd != RealTimeRunData_Debug) || !RTRDataHasBeenSent) { // send RTR Data only once
-                    bool sendTotals = (curInfoCmd == RealTimeRunData_Debug);
-
-                    for (uint8_t id = 0; id < mSys->getNumInverters(); id++) {
-                        Inverter<> *iv = mSys->getInverterByPos(id);
-                        if (NULL == iv)
-                            continue; // skip to next inverter
-                        if (!iv->config->enabled)
-                            continue; // skip to next inverter
-
-                        // send RTR Data only if status is available
-                        if ((curInfoCmd != RealTimeRunData_Debug) || (MQTT_STATUS_NOT_AVAIL_NOT_PROD != mLastIvState[id]))
-                            sendData(iv, curInfoCmd);
-
-                        // calculate total values for RealTimeRunData_Debug
-                        if (sendTotals) {
-                            record_t<> *rec = iv->getRecordStruct(curInfoCmd);
-
-                            sendTotals &= (iv->getLastTs(rec) > 0);
-                            if (sendTotals) {
-                                for (uint8_t i = 0; i < rec->length; i++) {
-                                    if (CH0 == rec->assign[i].ch) {
-                                        switch (rec->assign[i].fieldId) {
-                                            case FLD_PAC:
-                                                total[0] += iv->getValue(i, rec);
-                                                break;
-                                            case FLD_YT:
-                                                total[1] += iv->getValue(i, rec);
-                                                break;
-                                            case FLD_YD:
-                                                total[2] += iv->getValue(i, rec);
-                                                break;
-                                            case FLD_PDC:
-                                                total[3] += iv->getValue(i, rec);
-                                                break;
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                        yield();
-                    }
-
-                    if (sendTotals) {
-                        uint8_t fieldId;
-                        for (uint8_t i = 0; i < 4; i++) {
-                            bool retained = true;
-                            switch (i) {
-                                default:
-                                case 0:
-                                    fieldId = FLD_PAC;
-                                    retained = false;
-                                    break;
-                                case 1:
-                                    fieldId = FLD_YT;
-                                    break;
-                                case 2:
-                                    fieldId = FLD_YD;
-                                    break;
-                                case 3:
-                                    fieldId = FLD_PDC;
-                                    retained = false;
-                                    break;
-                            }
-                            snprintf(mSubTopic, 32 + MAX_NAME_LENGTH, "total/%s", fields[fieldId]);
-                            snprintf(mVal, 40, "%g", ah::round3(total[i]));
-                            publish(mSubTopic, mVal, retained);
-                        }
-                        RTRDataHasBeenSent = true;
-                        yield();
-                    }
-                }
-
-                mSendList.pop(); // remove from list once all inverters were processed
-            }
-
             mLastAnyAvail = anyAvail;
         }
 
@@ -655,6 +580,8 @@ class PubMqtt {
         #endif
 
         HMSYSTEM *mSys;
+        PubMqttIvData<HMSYSTEM> mSendIvData;
+
         uint32_t *mUtcTimestamp;
         uint32_t mRxCnt, mTxCnt;
         std::queue<uint8_t> mSendList;

src/publisher/pubMqttIvData.h

@@ -0,0 +1,208 @@
+//-----------------------------------------------------------------------------
+// 2023 Ahoy, https://ahoydtu.de
+// Creative Commons - https://creativecommons.org/licenses/by-nc-sa/4.0/deed
+//-----------------------------------------------------------------------------
+
+#ifndef __PUB_MQTT_IV_DATA_H__
+#define __PUB_MQTT_IV_DATA_H__
+
+#include "../utils/dbg.h"
+#include "../hm/hmSystem.h"
+#include "pubMqttDefs.h"
+
+typedef std::function<void(const char *subTopic, const char *payload, bool retained)> pubMqttPublisherType;
+
+template<class HMSYSTEM>
+class PubMqttIvData {
+    public:
+        void setup(HMSYSTEM *sys, uint32_t *utcTs, std::queue<uint8_t> *sendList) {
+            mSys          = sys;
+            mUtcTimestamp = utcTs;
+            mSendList     = sendList;
+            mState        = IDLE;
+
+            memset(mIvLastRTRpub, 0, MAX_NUM_INVERTERS * 4);
+            mRTRDataHasBeenSent = false;
+
+            mTable[IDLE]        = &PubMqttIvData::stateIdle;
+            mTable[START]       = &PubMqttIvData::stateStart;
+            mTable[FIND_NXT_IV] = &PubMqttIvData::stateFindNxtIv;
+            mTable[SEND_DATA]   = &PubMqttIvData::stateSend;
+            mTable[SEND_TOTALS] = &PubMqttIvData::stateSendTotals;
+        }
+
+        void loop() {
+            (this->*mTable[mState])();
+            yield();
+        }
+
+        bool start(void) {
+            if(IDLE != mState)
+                return false;
+
+            mRTRDataHasBeenSent = false;
+            mState = START;
+            return true;
+        }
+
+        void setPublishFunc(pubMqttPublisherType cb) {
+            mPublish = cb;
+        }
+
+    private:
+        enum State {IDLE, START, FIND_NXT_IV, SEND_DATA, SEND_TOTALS, NUM_STATES};
+        typedef void (PubMqttIvData::*StateFunction)();
+
+        void stateIdle() {
+            ; // nothing to do
+        }
+
+        void stateStart() {
+            mLastIvId = 0;
+            if(!mSendList->empty()) {
+                mCmd = mSendList->front();
+
+                if((RealTimeRunData_Debug != mCmd) || !mRTRDataHasBeenSent) {
+                    mSendTotals = (RealTimeRunData_Debug == mCmd);
+                    memset(mTotal, 0, sizeof(float) * 4);
+                    mState = FIND_NXT_IV;
+                } else
+                    mSendList->pop();
+            } else
+                mState = IDLE;
+        }
+
+        void stateFindNxtIv() {
+            bool found = false;
+
+            for (; mLastIvId < mSys->getNumInverters(); mLastIvId++) {
+                mIv = mSys->getInverterByPos(mLastIvId);
+                if (NULL != mIv) {
+                    if (mIv->config->enabled) {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+
+            mLastIvId++;
+
+            mPos = 0;
+            if(found)
+                mState = SEND_DATA;
+            else if(mSendTotals)
+                mState = SEND_TOTALS;
+            else {
+                mSendList->pop();
+                mState = START;
+            }
+        }
+
+        void stateSend() {
+            record_t<> *rec = mIv->getRecordStruct(mCmd);
+            uint32_t lastTs = mIv->getLastTs(rec);
+            bool pubData = (lastTs > 0);
+            if (mCmd == RealTimeRunData_Debug)
+                pubData &= (lastTs != mIvLastRTRpub[mIv->id]);
+
+            if (pubData) {
+                if(mPos < rec->length) {
+                    bool retained = false;
+                    if (mCmd == RealTimeRunData_Debug) {
+                        switch (rec->assign[mPos].fieldId) {
+                            case FLD_YT:
+                            case FLD_YD:
+                                if ((rec->assign[mPos].ch == CH0) && (!mIv->isProducing(*mUtcTimestamp))) { // avoids returns to 0 on restart
+                                    mPos++;
+                                    return;
+                                }
+                                retained = true;
+                                break;
+                        }
+
+                        // calculate total values for RealTimeRunData_Debug
+                        if (CH0 == rec->assign[mPos].ch) {
+                            switch (rec->assign[mPos].fieldId) {
+                                case FLD_PAC:
+                                    mTotal[0] += mIv->getValue(mPos, rec);
+                                    break;
+                                case FLD_YT:
+                                    mTotal[1] += mIv->getValue(mPos, rec);
+                                    break;
+                                case FLD_YD:
+                                    mTotal[2] += mIv->getValue(mPos, rec);
+                                    break;
+                                case FLD_PDC:
+                                    mTotal[3] += mIv->getValue(mPos, rec);
+                                    break;
+                            }
+                        }
+                    } else
+                        mIvLastRTRpub[mIv->id] = lastTs;
+
+                    snprintf(mSubTopic, 32 + MAX_NAME_LENGTH, "%s/ch%d/%s", mIv->config->name, rec->assign[mPos].ch, fields[rec->assign[mPos].fieldId]);
+                    snprintf(mVal, 40, "%g", ah::round3(mIv->getValue(mPos, rec)));
+                    mPublish(mSubTopic, mVal, retained);
+                    mPos++;
+                } else
+                    mState = FIND_NXT_IV;
+            } else
+                mState = FIND_NXT_IV;
+        }
+
+        void stateSendTotals() {
+            uint8_t fieldId;
+            if(mPos < 4) {
+                bool retained = true;
+                switch (mPos) {
+                    default:
+                    case 0:
+                        fieldId = FLD_PAC;
+                        retained = false;
+                        break;
+                    case 1:
+                        fieldId = FLD_YT;
+                        break;
+                    case 2:
+                        fieldId = FLD_YD;
+                        break;
+                    case 3:
+                        fieldId = FLD_PDC;
+                        retained = false;
+                        break;
+                }
+                snprintf(mSubTopic, 32 + MAX_NAME_LENGTH, "total/%s", fields[fieldId]);
+                snprintf(mVal, 40, "%g", ah::round3(mTotal[mPos]));
+                mPublish(mSubTopic, mVal, retained);
+                mPos++;
+            } else {
+                mSendList->pop();
+                mState = START;
+            }
+
+            mRTRDataHasBeenSent = true;
+        }
+
+        HMSYSTEM *mSys;
+        uint32_t *mUtcTimestamp;
+        pubMqttPublisherType mPublish;
+        State mState;
+        StateFunction mTable[NUM_STATES];
+
+        uint8_t mCmd;
+        uint8_t mLastIvId;
+        bool mSendTotals;
+        float mTotal[4];
+
+        Inverter<> *mIv;
+        uint8_t mPos;
+        uint32_t mIvLastRTRpub[MAX_NUM_INVERTERS];
+        bool mRTRDataHasBeenSent;
+
+        char mSubTopic[32 + MAX_NAME_LENGTH + 1];
+        char mVal[40];
+
+        std::queue<uint8_t> *mSendList;
+};
+
+#endif /*__PUB_MQTT_IV_DATA_H__*/

src/web/html/api.js

@@ -78,7 +78,7 @@ function parseNav(obj) {
         if(i == 2)
             continue;
         var l = document.getElementById("nav"+i);
-        if(window.location.pathname == "/" + l.href.split('/').pop())
+        if(window.location.pathname == "/" + l.href.substring(0, l.href.indexOf("?")).split('/').pop())
             l.classList.add("active");
 
         if(obj["menu_protEn"]) {