Code: Select all
#include "compression.h"
#include "assert.h"
#include "zlib.h"
#include <string.h>
#include "stdio.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "stdlib.h"
#include "esp_log.h"
const char * TAG = "Chaze-Compression";
/**
* @brief When recording sensor data, multiple tasks need to acquire a mutex in order to read from the bus and
* write to the buffer. The buffer is a struct that holds a counter (number of bytes written) and a malloced
* char array. If the number of bytes written equals to the BUFFER SIZE, this function is called and the
* data is compressed and saved to the flash.
* @param Buffer number to read from and compress. Either 0 or 1.
*/
void compress_and_save(uint8_t buff_num)
{
ESP_LOGI(TAG, "Using buffer number %d", buff_num);
//Get the pointer to the current buffer
buffer_t * current_buffer = buffers[buff_num];
assert(current_buffer->counter == BUFFER_SIZE); //The buffer must be filled completely.
//Data written to flash.
char * out = (char *) malloc(BUFFER_SIZE * sizeof(char));
if(out == NULL){
ESP_LOGE(TAG, "Could not allocate enough space");
//Need to write the raw, uncompressed data to the flash instead
return;
} else{
ESP_LOGI(TAG, "Start compressing.");
//Writes 'written' many chars to the out buffer previously allocated
uint32_t written = def(current_buffer->data, out, DEFAULT_COMPRESSION_LEVEL);
//Write data to flash
write_data_to_flash(out, written);
free(out);
}
}
/**
* @brief Deflates the char array given by source and writes to the char array dest.
* @param source
* @param dest
* @param level Level of compression.
* @return Number of written bytes to dest
*/
uint32_t def(char * source, char * dest, uint8_t level)
{
int ret, flush;
unsigned have;
z_stream strm;
unsigned char * in = (unsigned char *) malloc(CHUNK * sizeof(char));
unsigned char * out = (unsigned char *) malloc(CHUNK * sizeof(char));
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
ret = deflateInit2(&strm, level,Z_DEFLATED, WINDOW_SIZE ,MEM_LEVEL,Z_DEFAULT_STRATEGY);
zerr(ret);
uint32_t dest_offset = 0;
do {
memcpy(in, source, CHUNK);
strm.avail_in = CHUNK; /*We assert that the buffer is full and since chunk size = buffer size, we can read that many bytes*/
flush = Z_FINISH; //Immediately flush since we only compress one chunk.
strm.next_in = in;
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = deflate(&strm, flush);
assert(ret != Z_STREAM_ERROR);
zerr(ret);
have = CHUNK - strm.avail_out;
ESP_LOGI(TAG, "Have: %d", have);
for(int i=0;i<have;i++)
printf("%c", out[i]);
memcpy(dest+dest_offset, out, have);
dest_offset += have;
} while (strm.avail_out == 0);
assert(strm.avail_in == 0);
} while (flush != Z_FINISH);
assert(ret == Z_STREAM_END);
zerr(deflateEnd(&strm));
free(in);
free(out);
return dest_offset;
}
//For now just inflates data to check if was deflated correctly
void write_data_to_flash(char * data, uint32_t n)
{
int ret;
unsigned have;
z_stream strm;
unsigned char * in = (unsigned char *) malloc(CHUNK * sizeof(char));
unsigned char * out = (unsigned char *) malloc(CHUNK * sizeof(char));
char * to_print = (char *) malloc(CHUNK*sizeof(char));
if(to_print == NULL || in == NULL || out == NULL)
{
ESP_LOGE(TAG, "Could not allocate enough space");
return;
}
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
ret = inflateInit2(&strm,0); //Use window size in compressed stream
zerr(ret);
uint32_t to_print_offset = 0;
do {
memcpy(in, data, n);
strm.avail_in = n;
if (strm.avail_in == 0)
break;
strm.next_in = in;
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR);
zerr(ret);
have = CHUNK - strm.avail_out;
memcpy(to_print+to_print_offset, out, have);
to_print_offset+=have;
} while (strm.avail_out == 0);
} while (ret != Z_STREAM_END);
(void)inflateEnd(&strm);
free(in);
free(out);
for(int i=0;i<CHUNK;i++)
printf("%c", to_print[i]);
}
void zerr(int ret)
{
switch (ret) {
case Z_ERRNO:
ESP_LOGE(TAG, "Z err no");
break;
case Z_STREAM_ERROR:
ESP_LOGE(TAG, "Invalid compression level");
break;
case Z_DATA_ERROR:
ESP_LOGE(TAG, "Data error");
break;
case Z_MEM_ERROR:
ESP_LOGE(TAG, "Mem error");
break;
case Z_VERSION_ERROR:
ESP_LOGE(TAG, "Version error");
}
}
Code: Select all
#ifndef COMPRESSION_H
#define COMPRESSION_H
#include "stdint.h"
#define BUFFER_SIZE /*16384*/ 8192
#define CHUNK 8192 /*Must be same size as BUFFER_SIZE*/
#define DEFAULT_COMPRESSION_LEVEL 6
#define WINDOW_SIZE 9
#define MEM_LEVEL 1
#define GZIP_ENCODING 16
typedef struct {
int32_t counter;
char * data;
}buffer_t;
buffer_t * buffers[2];
void compress_and_save(uint8_t);
void write_data_to_flash(char *, uint32_t);
uint32_t def(char *, char *, uint8_t);
void zerr(int32_t);
#endif
//Working main.cpp file.
//Creates task simulating sensor readings with mutex, compresses if buffer full, switches buffer and writes to next one
/*
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "stdlib.h"
#include "stdio.h"
#include <stdio.h>
#include <string.h>
extern "C" {
#include "compression.h"
}
#include "freertos/semphr.h"
#include "esp_log.h"
SemaphoreHandle_t xSemaphore = NULL;
volatile uint8_t buff_idx = 0;
static const char * TAG = "Chaze-Compression-Main";
void sample_hr(void * pvParams)
{
char dummy[] = "Sorem ipsum lorem ich habe hier einen schönen\nAusblich wäre ich hier irgendwas rein schreibe. Ich sitze an meinem Schreibtisch\nund drehe ein bisschen Daeumchen und so weiter und so fort.\n";
for(;;){
//Try to aquire the mutex for a given amount of time then back-off
//For real life, need priorites in low sample rate sensors, random back-off time etc.
buffer_t * curr_buff = buffers[buff_idx];
if(xSemaphore != NULL)
{
if(xSemaphoreTake(xSemaphore,(TickType_t) 100) == pdTRUE)
{
ESP_LOGI(TAG, "Acquired the lock");
if(BUFFER_SIZE-curr_buff->counter >= strlen(dummy)){
ESP_LOGI(TAG, "Enough space to write");
//Enough space, we can write
memcpy(curr_buff->data+curr_buff->counter, dummy, strlen(dummy));
curr_buff->counter += strlen(dummy);
} else{
//Fill up the buffer with 'f' and set the buff_idx to buff_idx XOR 1
ESP_LOGI(TAG, "Buffer almost full, fill up");
for(int i=curr_buff->counter;i<BUFFER_SIZE-curr_buff->counter;i++)
curr_buff->data[i] = 'f';
curr_buff->counter += BUFFER_SIZE-curr_buff->counter;
ESP_LOGI(TAG, "Counter is %d", curr_buff->counter);
if(buff_idx==0)
buff_idx=1;
else buff_idx=0;
ESP_LOGI(TAG, "Call compress");
//Call compress. Switch the bit back since we want to compress the buffer that is full
compress_and_save(!buff_idx);
curr_buff->counter = 0; //Reset the compressed buffer
}
ESP_LOGI(TAG, "Release mutex");
//Release mutex
xSemaphoreGive(xSemaphore);
}
vTaskDelay(100 / portTICK_PERIOD_MS); //Back off a little longer
}
}
}
extern "C" void app_main()
{
xSemaphore = xSemaphoreCreateMutex();
//Initialize buffers 1 and 2
buffer_t buffer0 = {};
buffer0.data = (char *) malloc(BUFFER_SIZE);
buffer0.counter = 0;
buffer_t buffer1 = {};
buffer1.data = (char *) malloc(BUFFER_SIZE);
buffer1.counter = 0;
buffers[0] = &buffer0;
buffers[1] = &buffer1;
if (xTaskCreate(&sample_hr, "sample_hr", 1024 * 8, NULL, 5, NULL) != pdPASS )
{
printf("Synch task create failed.\n");
} else ESP_LOGI(TAG, "Created task");
while(1){
vTaskDelay(10000);
}
}
/*
extern "C" void app_main()
{
esp_vfs_fat_mount_config_t mount_config = {};
mount_config.max_files = 4;
mount_config.format_if_mount_failed = true;
mount_config.allocation_unit_size = CONFIG_WL_SECTOR_SIZE;
esp_err_t err = esp_vfs_fat_spiflash_mount(base_path, "storage", &mount_config, &s_wl_handle);
if (err != ESP_OK) {
ESP_LOGE(TAG, "Failed to mount FATFS (%s)", esp_err_to_name(err));
return;
}
print_dir();
remove_folder();
print_dir();
ESP_LOGI(TAG, "Opening file");
foo = fopen("/spiflash/foo.bin", "wb");
if (foo == NULL) {
ESP_LOGE(TAG, "Failed to open file for writing");
return;
}
//int ret = fprintf(foo, "written using ESP-IDF\n");
const char * tmp = esp_get_idf_version();
int ret = fwrite(tmp,1,strlen(tmp),foo);
if(ret < strlen(tmp)){
ESP_LOGE(TAG, "Error writing file");
}
if(fclose(foo) == EOF){
ESP_LOGE(TAG, "Error closing the file.");
}
else ESP_LOGI(TAG, "File written");
comp = fopen("/spiflash/c.bin", "wb");
if (comp == NULL) {
ESP_LOGE(TAG, "Failed to open file for writing");
return;
}
foo = fopen("/spiflash/foo.bin", "rb");
if(foo == NULL){
ESP_LOGE(TAG, "Error opening the file.");
}
print_dir();
uint8_t level = 6;
zerr(def(foo, comp, level));
ESP_LOGI(TAG, "Compressed.");
if(fclose(foo) == EOF){
ESP_LOGE(TAG, "Error closing file");
}
else ESP_LOGI(TAG, "Closed foo.");
if(fclose(comp) == EOF){
ESP_LOGE(TAG, "Error closing file: %s", strerror(errno));
}
else ESP_LOGI(TAG, "Closed comp.");
//Read comp
char line[128];
char *pos;
// Open file for reading
ESP_LOGI(TAG, "Reading file");
comp = fopen("/spiflash/c.bin", "rb");
if (comp == NULL) {
ESP_LOGE(TAG, "Failed to open file for reading");
}
fgets(line, sizeof(line), foo);
fclose(comp);
// strip newline
pos = strchr(line, '\n');
if (pos) {
*pos = '\0';
}
ESP_LOGI(TAG, "Read from file: '%s'", line);
//Inflate
recon = fopen("/spiflash/rec.bin", "wb");
comp = fopen("/spiflash/c.bin", "rb");
if(recon == NULL || comp == NULL){
ESP_LOGE(TAG, "Error opening file before inflating.");
}
ESP_LOGI(TAG, "Start inflating");
zerr(inf(comp, recon));
ESP_LOGI(TAG, "Stopped inflating");
if(fclose(comp) == EOF){
ESP_LOGE(TAG, "Error closing file");
}
ESP_LOGI(TAG, "Closed comp");
if(fclose(recon) == EOF){
ESP_LOGE(TAG, "Error closing file");
}
ESP_LOGI(TAG, "Closed recon");
// Open file for reading
ESP_LOGI(TAG, "Reading file");
recon = fopen("/spiflash/rec.bin", "rb");
if (recon == NULL) {
ESP_LOGE(TAG, "Failed to open file for reading");
}
fgets(line, sizeof(line), recon);
fclose(recon);
// strip newline
pos = strchr(line, '\n');
if (pos) {
*pos = '\0';
}
ESP_LOGI(TAG, "Read from file: '%s'", line);
// Open file for reading
ESP_LOGI(TAG, "Reading file");
foo = fopen("/spiflash/foo.bin", "rb");
if (foo == NULL) {
ESP_LOGE(TAG, "Failed to open file for reading");
return;
}
fgets(line, sizeof(line), foo);
fclose(foo);
// strip newline
pos = strchr(line, '\n');
if (pos) {
*pos = '\0';
}
ESP_LOGI(TAG, "Read from file: '%s'", line);
// Unmount FATFS
ESP_LOGI(TAG, "Unmounting FAT filesystem");
ESP_ERROR_CHECK( esp_vfs_fat_spiflash_unmount(base_path, s_wl_handle));
ESP_LOGI(TAG, "Done");
while(1){
vTaskDelay(1000);
}
}
*/
*/
This is working for me. I am using zlib.
I remember there were some errors when initially compiling the code, but they could be resolved easily by simply reading the error messages.
Also note that compression.h has an example main.cpp file that works at the end.
Best,
Julian
