Changing partition size via OTA

[TMSio73]

HI, i would you like change patition table via OTA.
I have change mi default.csv in:
# Name, Type, SubType, Offset, Size, Flags
nvs, data, nvs, 0x9000, 0x5000,
otadata, data, ota, 0xe000, 0x2000,
app0, app, ota_0, 0x10000, 0x1E0000,
app1, app, ota_1, 0x1F0000,0x1E0000,
eeprom, data, 0x99, 0x3F0000,0x1000,
spiffs, data, spiffs, 0x3F1000,0xF000,



board.txt:
esp32.name=ESP32 Dev Module

esp32.upload.tool=esptool
esp32.upload.maximum_size=1966080
esp32.upload.maximum_data_size=294912
esp32.upload.wait_for_upload_port=true

But when upload my sketch (restore.bin) via OTA no change made.
I use Arduino 1.8.5 on windows 10
My schetch

Code: Select all

void start_connection() {
  //ledflash (5, 100);
  WiFi.mode(WIFI_OFF);
  delay (15000);
  static uint16_t attempt = 0;
  uint8_t i = 0;
  WiFi.begin("xxxxxxxxxxxx", "xxxxxxxxx");
  WiFi.setHostname("ESP32");
  while (WiFi.status() != WL_CONNECTED && i++ < 70)
  {
    delay(200);
    if (debug == true)
    {
      Serial.print(".");
    }
   // ledflash (1, 400);
  }

  ++attempt;
  if (debug == true)
  {
    Serial.println("");
  }

  if (i == 71) {
    if (debug == true)
    {
      Serial.print("Connection: TIMEOUT on attempt: ");
      Serial.println(attempt);
    }
    if (attempt % 2 == 0)
      //Serial.println("Check if access point available or SSID and Password\r\n");
      //Serial.println ( "Scrittura errore 2 ");
      errore = 2;
    delay(100);
    return ;
  }

  if (debug == true)
  {
    Serial.println("");
    Serial.println("WiFi connected.");
    Serial.println("IP address: ");
    Serial.println(WiFi.localIP());

  }


  execOTA(bin1, mail_to);

}

void execOTA(String path, char* mail_to) {
  int len = strlen(mail_to);
  if (debug == true)
  {
    Serial.println ( "Scrittura errore 5 ");
  }
  delay(100);
  //Serial.println("Connecting to: " + String(host));
  // Connect to S3
  if (client.connect(host.c_str(), port)) {
    // Connection Succeed.
    // Fecthing the bin

    //Serial.println("Fetching Bin: " + String(path));
    // Serial.println("");

    // Get the contents of the bin file
    client.print(String("GET ") + path + " HTTP/1.1\r\n" +
                 "Host: " + host + "\r\n" +
                 "Cache-Control: no-cache\r\n" +
                 "Connection: close\r\n\r\n");

    // Check what is being sent
    //    Serial.print(String("GET ") + bin + " HTTP/1.1\r\n" +
    //                 "Host: " + host + "\r\n" +
    //                 "Cache-Control: no-cache\r\n" +
    //                 "Connection: close\r\n\r\n");

    unsigned long timeout = millis();
    while (client.available() == 0) {
      if (millis() - timeout > 35000) {
        if (debug == true)
        {
          Serial.println(">>> Client Timeout !");
          Serial.println ( "Scrittura errore 2 ");
        }
        errore = 2;
        delay(100);
        client.stop();
        return;
      }
    }

    // Once the response is available,
    // check stuff

    while (client.available()) {
      // read line till /n
      String line = client.readStringUntil('\n');
      // remove space, to check if the line is end of headers
      line.trim();
      //Serial.println(line);
      // if the the line is empty,
      // this is end of headers
      // break the while and feed the
      // remaining `client` to the
      // Update.writeStream();
      if (!line.length()) {
        //headers ended
        break; // and get the OTA started
      }

      // Check if the HTTP Response is 200
      // else break and Exit Update
      if (line.startsWith("HTTP/1.1")) {
        if (line.indexOf("200") < 0) {
          if (debug == true)
          {
            Serial.println("Got a non 200 status code from server. Exiting OTA Update.");
          }
          errore = 1;
          break;
        }
      }

      // extract headers here
      // Start with content length
      if (line.startsWith("Content-Length: ")) {
        contentLength = atoi((getHeaderValue(line, "Content-Length: ")).c_str());
        //Serial.println("Got " + String(contentLength) + " bytes from server");
      }

      // Next, the content type
      if (line.startsWith("Content-Type: ")) {
        contentType = getHeaderValue(line, "Content-Type: ");
        //Serial.println("Got " + contentType + " payload.");
        if (contentType == "application/octet-stream" || contentType == "application/macbinary") {
          isValidContentType = true;
        }
      }
    }
  } else {
    // Connect to S3 failed
    // May be try?
    // Probably a choppy network?
    if (debug == true)
    {
      Serial.println("Connection to " + String(host) + " failed. Please check your setup");
    }
    errore = 2;
    // retry??
    // execOTA();
  }
  if (debug == true)
  {
    Serial.println("Got " + contentType + " payload.");
    Serial.println("Got " + String(contentLength) + " bytes from server");
    // Check what is the contentLength and if content type is `application/octet-stream`
    Serial.println("contentLength : " + String(contentLength) + ", isValidContentType : " + String(isValidContentType));
    Serial.println("");
  }
  // check contentLength and content type
  if (contentLength && isValidContentType) {
    // Check if there is enough to OTA Update
    bool canBegin = Update.begin(contentLength);
    // If yes, begin
    if (canBegin) {
      errore = 3;
     // digitalWrite(ledPin, LOW);
      if (debug == true)
      {
        Serial.println("Begin OTA. This may take 2 - 5 mins to complete. Things might be quite for a while.. Patience!");
      }

      size_t written = Update.writeStream(client);

      if (written == contentLength) {
        if (debug == true)
        {
          Serial.println("Written : " + String(written) + " successfully");
        }

      } else {
        if (debug == true)
        {
          Serial.println("Written only : " + String(written) + "/" + String(contentLength) + ". Retry?" );

        }
       
        delay (1000);
        esp_restart();
        // retry??
        // execOTA();
      }
      Serial.println("test6");
      if (Update.end()) {
        if (debug == true)
        {
          Serial.println("OTA done!");
        }

        if (Update.isFinished()) {
          errore = 0;
          if (debug == true)
          {
            Serial.println("Update successfully completed. Rebooting.");
            Serial.println("Scrittura errore : ok");
          }
          delay (1000);
          esp_restart();
        } else {
          if (debug == true)
          {
            Serial.println("Update not finished? Something went wrong!");
          }
          errore = 3;
          esp_restart();
        }
      } else {
        if (debug == true)
        {
          Serial.println("Error Occurred. Error #: " + String(Update.getError()));
        }

      }
    } else {
      // not enough space to begin OTA
      // Understand the partitions and
      // space availability
      if (debug == true)
      {
        Serial.println("Not enough space to begin OTA");
      }
      errore = 4;
      client.flush();
    }
  } else {

    if (debug == true)
    {
      Serial.println("There was no content in the response");
    }

    firm_tent ++;
    errore = 6;
    delay (300);

    switch (firm_tent) {

      case 1:
        ledflash (2, 50);
        delay (500);
        execOTA(bin2, mail_to);
        client.flush();
        break;
      case 2:
        ledflash (2, 50);
        delay (500);
        execOTA(bin3, mail_to);
        client.flush();
        break;
      case 3:
        ledflash (2, 50);
        delay (500);
        execOTA(bin4, mail_to);
        client.flush();
        break;
      case 4:
        ledflash (2, 50);
        delay (500);
        execOTA(bin4, mail_to);
        client.flush();
        break;
      default:
        esp_restart();
        break;

    }

  }
}

Any idea?
Thank You
Regards

[kolban]

I don't believe you can change the partition map using ordinary OTA functions. The partition map is a description of where in flash things are layed out. Within the partition map we can identify a number of flash areas that can be used to serve the purpose of OTA. These are the areas that are overlayed with newly downloaded binary when pulled over the Internet. On boot, one of these partitions is then chosen as the one to load the executable from. While I can see some value in being able to change the partition layout post-production, this would be a dangerous area. The nature of the partition map is an index of where things are in flash storage. If we were to push a new partition map that was itself broken, the device would be bricked (from an OTA perspective).

This makes me think that there is likely a good rule of thumb for a production system which is to think very, very carefully about partition sizes and anticipate into the future how the partition sizes will be used. Also realize that the partitions are only that ... a mapping. They allow us to reference areas of memory by name. It is optional to use partition maps. For example, if you have 4MB of flash and have only mapped out the first 3MB, that does not mean that the last MB is unavailable to you. It can still be read and written by your apps but it is your responsibility to decide how that storage is used.

Would you be able to post the back story describing what it is you are trying to achieve by changing the partition map?

[TMSio73]

Hi Kolban, sorry for the late but i m back tooday.
I made my project with default.csv partition, but after insert into the project NMEA2000 library :
#include <NMEA2000_CAN.h>
#include <N2kMessages.h>
The sketch is now 1348766 byte (102%)
I use Ble, eeprom, external FRAM memory (RAM_MB85RC_I2C.h) and Wifi.
In the sketch there is a website and too much "client.print"
I wold you like to change partition size via OTA with a litle restore sketch and after update it via OTA with the new big sketch.
I use arduino ota but im study vscode and platformio.
There is a way to do this?
Or i have to change partition size via usb ftdi?
Thank You.
Regards
Emiliano

[monkey]

Hi @kolban,

Stumbled across this when scratching my chin! I have a production device which was originally produced on a 4MB device. However, in the intervening year, the 8MB is now available and it's what I'm using. However, the production is still using the partition map for 4MB. Given that I have now got potentially 4MB of data sitting doing nothing, do you think there is a way of accessing it? It's pretty easy to determine the Flash size of the device. From there on, how would you go about writing and reading safely to the memory outside the partition? It could prove to be very useful! Other option is to write off the 4MB devices, and update the partition map asap to get a hold of the issue from here on out!

Many thanks,

Al.

[a2800276]

If you have 4MB partition maps and 8MB flash, you can manually manage the free flash space beyond the 4MB, i.e. create your own partition table for that space. It would be easier to flash two variants with native idf partition tables, though.

[monkey]

Thanks for the info. In my case, I have sent them out into the wild already, so too late to update the partition table used at build. However, if I could access additional memory via an OTA software update, that would be really useful! Don't suppose you could point to a link or something explaining how to do what you describe?

Cheers!

If you have 4MB partition maps and 8MB flash, you can manually manage the free flash space beyond the 4MB, i.e. create your own partition table for that space. It would be easier to flash two variants with native idf partition tables, though.

[a2800276]

However, if I could access additional memory via an OTA software update, that would be really useful! Don't suppose you could point to a link or something explaining how to do what you describe?

I believe if you poke around the forum you'll find an example somewhere of someone overwriting the partition table. This is clearly a recipe for disaster and you should avoid it for anything even remotely close to "production". What I suggest otoh falls squarely into the "shit that definitely voids your warranty", so -> two thumbs up.

Can't really imagine that there would be any examples projects demoing how this works (on account of the warranty thing), but this is how I would go about it:

• Find whatever systems API that will tell you how large the physical flash is, to make sure you have an 8MB device.

• Use the partition API to find the last partition, and go to it's end. Beyond that you're in "here be dragons" territory and all that flash belongs to you! Of course, if you know for sure that all existing devices have the same layout, you can skip this part, because you already know where the unused flash region begins.

• You can probably just skip the very first step, i.e. making sure you have an 8MB device, as well, because all the following steps will just fail.

• Now that you have an unused flash region, you can just use the low-level flash command and go wild. What I would do is use the existing partition table code (just spitballing it, but I assume it's located here) to write my own partition table. I assume you'll need to fiddle with this code at least a little bit, because it looks like the idf expects there to be precisely one partition table...

• Finally, you can create partitions and format them and use them as you would normally. (Based on my wildly optimistic assumption that this all would "just work™")

• Of course if you want to skip fiddling around with your own partition table, you can just access the empty flash directly, but you lose all the stdlib FILE/stream stuff or have to write your own hooks into newlib ... If you're happy with just having a big blob of flash to use, that is definitely the quickest route.

Anyway, keep us updated on your progress! Sound like a fun little project!

-tim

[boarchuz]

if I could access additional memory via an OTA software update, that would be really useful

What exactly do you want to do with it? That determines how difficult it is to do so safely. Particularly, if the bootloader needs to be aware of the change (eg. you want to move/add/resize app partitions) then that will be much more challenging than if you only want to, for example, add a SPIFFS partition for newer apps to access.

For newer apps, you could define a 'virtual' partition table describing use of the upper 4MB. There is no need to actually alter the partition table in flash. Most APIs will support this, perhaps with a little tweaking. eg. If you wanted to add another NVS partition, it would be initialised like so:

Code: Select all

const esp_partition_t my_new_nvs_partition = {
    .type = ESP_PARTITION_TYPE_DATA,
    .subtype = ESP_PARTITION_SUBTYPE_DATA_NVS,
    .address = 0x500000,
    .size = 0x10000,
    .label = "nvs2",
};
nvs_flash_init_partition_ptr(&my_new_nvs_partition);

If you want to alter app partitions, I think the only safe way would be to flash an app that effectively acts as a 3rd stage bootloader. This would be flashed to the factory partition (if it exists and is safe to do so) else first OTA partition, and the OTA data partition permanently set so that the original bootloader can only ever boot this partition. In your new 3rd stage bootloader you would do normal 2nd stage bootloader stuff, except using your new 8MB partition table instead of reading from the one in flash. Pretty complicated but I don't see any reason why it shouldn't work.

[a2800276]

Urgh, that's evil! I like it!

I think you may still have problems, because the high level partition table routines (e.g. `esp_find_partition`) will still try to use the "default" partition tables, so at least look out for that...