Merge pull request #390 from sparkfun/Fix_#362

Fix #362

.github/workflows/compile-rtk-everywhere.yml

@@ -68,6 +68,12 @@ jobs:
       - name: Install platform
         run: arduino-cli core install esp32:esp32@${{ env.CORE_VERSION }}
 
+      - name: Get IDF version
+        run: |
+          cd /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs
+          IDF_VERSION=$(ls | grep idf-release)
+          echo "ESP_IDF=${IDF_VERSION}" >> "$GITHUB_ENV"
+
       - name: Get Known Libraries
         run: arduino-cli lib install
           [email protected]
@@ -95,6 +101,14 @@ jobs:
           cd Firmware/RTK_Everywhere/Patch/
           cp BleSerial.cpp /home/runner/Arduino/libraries/ESP32_BleSerial/src/BleSerial.cpp
 
+      - name: Patch libmbedtls
+        run: |
+          cd Firmware/RTK_Everywhere/Patch/
+          cp libmbedtls.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedtls.a
+          cp libmbedtls_2.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedtls_2.a
+          cp libmbedcrypto.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedcrypto.a
+          cp libmbedx509.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedx509.a
+
       - name: Setup Python
         uses: actions/setup-python@v4
         with:

.github/workflows/non-release-build.yml

@@ -68,9 +68,15 @@ jobs:
       - name: Install platform
         run: arduino-cli core install esp32:esp32@${{ env.CORE_VERSION }}
 
+      - name: Get IDF version
+        run: |
+          cd /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs
+          IDF_VERSION=$(ls | grep idf-release)
+          echo "ESP_IDF=${IDF_VERSION}" >> "$GITHUB_ENV"
+
       - name: Get Known Libraries
         run: arduino-cli lib install
-          ArduinoJson@6.19.4
+          ArduinoJson@7.0.4
           [email protected]
           [email protected]
           "ESP32-OTA-Pull"@1.0.0
@@ -95,6 +101,14 @@ jobs:
           cd Firmware/RTK_Everywhere/Patch/
           cp BleSerial.cpp /home/runner/Arduino/libraries/ESP32_BleSerial/src/BleSerial.cpp
 
+      - name: Patch libmbedtls
+        run: |
+          cd Firmware/RTK_Everywhere/Patch/
+          cp libmbedtls.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedtls.a
+          cp libmbedtls_2.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedtls_2.a
+          cp libmbedcrypto.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedcrypto.a
+          cp libmbedx509.a /home/runner/.arduino15/packages/esp32/tools/esp32-arduino-libs/${{ env.ESP_IDF }}/esp32/lib/libmbedx509.a
+
       - name: Setup Python
         uses: actions/setup-python@v4
         with:

Firmware/RTK_Everywhere/Begin.ino

@@ -683,6 +683,7 @@ void beginGnssUart()
     length = settings.gnssHandlerBufferSize + (rbOffsetEntries * sizeof(RING_BUFFER_OFFSET));
     ringBuffer = nullptr;
 
+    // Never freed...
     if (rbOffsetArray == nullptr)
     {
         if (online.psram == true)

Firmware/RTK_Everywhere/Form.ino

@@ -271,6 +271,7 @@ bool startWebServer(bool startWiFi = true, int httpPort = 80)
                 MDNS.addService("http", "tcp", 80); // Add service to MDNS-SD
         }
 
+        // Freed by stopWebServer
         if (online.psram == true)
             incomingSettings = (char *)ps_malloc(AP_CONFIG_SETTING_SIZE);
         else
@@ -284,6 +285,7 @@ bool startWebServer(bool startWiFi = true, int httpPort = 80)
         memset(incomingSettings, 0, AP_CONFIG_SETTING_SIZE);
 
         // Pre-load settings CSV
+        // Freed by stopWebServer
         if (online.psram == true)
             settingsCSV = (char *)ps_malloc(AP_CONFIG_SETTING_SIZE);
         else

Firmware/RTK_Everywhere/MQTT_Client.ino

@@ -112,7 +112,8 @@ const char MQTT_TOPIC_ASSISTNOW[] = "/pp/ubx/mga"; // AssistNow (MGA) topic
 
 static MqttClient *mqttClient;
 
-static char *mqttClientCertificateBuffer; // Buffer for client certificate
+static char *mqttClientCertificateBuffer = nullptr; // Buffer for client certificate
+static char *mqttClientPrivateKeyBuffer = nullptr; // Buffer for client private key
 
 // Throttle the time between connection attempts
 static int mqttClientConnectionAttempts; // Count the number of connection attempts between restarts
@@ -121,8 +122,6 @@ static int mqttClientConnectionAttemptsTotal; // Count the number of connection
 
 static volatile uint32_t mqttClientLastDataReceived; // Last time data was received via MQTT
 
-static char *mqttClientPrivateKeyBuffer; // Buffer for client private key
-
 static RTKNetworkSecureClient *mqttSecureClient;
 
 static volatile uint8_t mqttClientState = MQTT_CLIENT_OFF;
@@ -564,15 +563,20 @@ void mqttClientUpdate()
         }
 
         // Allocate the buffers
+        // Freed by mqttClientShutdown / mqttClientStop
         if (online.psram == true)
         {
-            mqttClientCertificateBuffer = (char *)ps_malloc(MQTT_CERT_SIZE);
-            mqttClientPrivateKeyBuffer = (char *)ps_malloc(MQTT_CERT_SIZE);
+            if (!mqttClientCertificateBuffer)
+                mqttClientCertificateBuffer = (char *)ps_malloc(MQTT_CERT_SIZE);
+            if (!mqttClientPrivateKeyBuffer)
+                mqttClientPrivateKeyBuffer = (char *)ps_malloc(MQTT_CERT_SIZE);
         }
         else
         {
-            mqttClientCertificateBuffer = (char *)malloc(MQTT_CERT_SIZE);
-            mqttClientPrivateKeyBuffer = (char *)malloc(MQTT_CERT_SIZE);
+            if (!mqttClientCertificateBuffer)
+                mqttClientCertificateBuffer = (char *)malloc(MQTT_CERT_SIZE);
+            if (!mqttClientPrivateKeyBuffer)
+                mqttClientPrivateKeyBuffer = (char *)malloc(MQTT_CERT_SIZE);
         }
 
         if ((!mqttClientCertificateBuffer) || (!mqttClientPrivateKeyBuffer))

Firmware/RTK_Everywhere/PointPerfectLibrary.ino

@@ -132,6 +132,7 @@ void beginPPL()
     reportHeapNow(false);
 
     // PPL_MAX_RTCM_BUFFER is 3345 bytes so we create it on the heap
+    // Freed by stopPPL()
     if (pplRtcmBuffer == nullptr)
     {
         if (online.psram == true)

Firmware/RTK_Everywhere/RTK_Everywhere.ino

@@ -78,6 +78,8 @@
 #include <MqttClient.h>   //http://librarymanager/All#ArduinoMqttClient by Arduino v0.1.8
 #endif  // COMPILE_NETWORK
 
+bool RTK_CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC = false; // Flag used by the special build of libmbedtls (libmbedcrypto) to select external memory
+
 #ifdef COMPILE_ETHERNET
 #include <ETH.h>
 #endif                                      // COMPILE_ETHERNET
@@ -483,7 +485,7 @@ const int btReadTaskStackSize = 4000;
 // Array of start-of-sentence offsets into the ring buffer
 #define AMOUNT_OF_RING_BUFFER_DATA_TO_DISCARD (settings.gnssHandlerBufferSize >> 2)
 #define AVERAGE_SENTENCE_LENGTH_IN_BYTES 32
-RING_BUFFER_OFFSET *rbOffsetArray;
+RING_BUFFER_OFFSET *rbOffsetArray = nullptr;
 uint16_t rbOffsetEntries;
 
 uint8_t *ringBuffer; // Buffer for reading from F9P. At 230400bps, 23040 bytes/s. If SD blocks for 250ms, we need 23040

Firmware/RTK_Everywhere/System.ino

@@ -12,7 +12,7 @@ void beginPsram()
             systemPrintf("PSRAM Size (bytes): %d\r\n", ESP.getPsramSize());
             if (ESP.getPsramSize() > 0)
             {
-                online.psram = true;
+                RTK_CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC = online.psram = true;
 
                 heap_caps_malloc_extmem_enable(
                     settings.psramMallocLevel); // Use PSRAM for memory requests larger than X bytes

Firmware/RTK_Everywhere/menuCommands.ino

@@ -2681,9 +2681,14 @@ bool commandIndexFill()
     }
     commandCount += COMMAND_COUNT - 1;
 
-    // Allocate the command array
+    // Allocate the command array. Never freed
     length = commandCount * sizeof(*commandIndex);
-    commandIndex = (int16_t *)malloc(length);
+
+    if (online.psram == true)
+        commandIndex = (int16_t *)ps_malloc(length);
+    else
+        commandIndex = (int16_t *)malloc(length);
+
     if (!commandIndex)
     {
         // Failed to allocate the commandIndex

Firmware/RTK_Everywhere/menuPP.ino

@@ -556,6 +556,8 @@ ZtpResponse pointperfectTryZtpToken(String &ztpRequest)
                         strncpy(tempHolderPtr, (const char *)((*jsonZtp)["privateKey"]), MQTT_CERT_SIZE - 1);
                         recordFile("privateKey", tempHolderPtr, strlen(tempHolderPtr));
 
+                        free(tempHolderPtr); // Clean up. Done with tempHolderPtr
+
                         // Validate the keys
                         if (!checkCertificates())
                         {
@@ -733,7 +735,10 @@ bool checkCertificates()
         free(keyContents);
 
     if (settings.debugPpCertificate)
-        systemPrintln("Stored certificates are valid!");
+    {
+        systemPrintf("Stored certificates are %svalid\r\n", validCertificates ? "" : "NOT ");
+    }
+
     return (validCertificates);
 }
 
@@ -817,6 +822,7 @@ bool pointperfectUpdateKeys()
     do
     {
         // Allocate the buffers
+        // Freed outside the do loop
         if (online.psram == true)
         {
             certificateContents = (char *)ps_malloc(MQTT_CERT_SIZE);
@@ -830,8 +836,6 @@ bool pointperfectUpdateKeys()
 
         if ((!certificateContents) || (!keyContents))
         {
-            if (certificateContents)
-                free(certificateContents);
             systemPrintln("Failed to allocate content buffers!");
             break;
         }
@@ -859,6 +863,8 @@ bool pointperfectUpdateKeys()
         // Configure the MQTT client
         menuppMqttClient->setId(settings.pointPerfectClientID);
         menuppMqttClient->onMessage(mqttCallback);
+        menuppMqttClient->setKeepAliveInterval(10 * 1000);
+        menuppMqttClient->setConnectionTimeout( 5 * 1000);
 
         // Attempt to the MQTT broker
         systemPrintf("Attempting to connect to MQTT broker: %s\r\n", settings.pointPerfectBrokerHost);
@@ -934,6 +940,8 @@ bool pointperfectUpdateKeys()
         menuppMqttClient = nullptr;
     }
 
+    secureClient.stop();
+
     // Free the content buffers
     if (keyContents)
         free(keyContents);