ESP32 Forum

Code: Select all

tdefl_compressor

Code: Select all

#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "rom/miniz.h"
#include "unity.h" //testing
#include "stdlib.h"
#include "stdio.h"


#define DATASIZE (1024*1)

extern "C"{
#include "wifi_synch.h"
}

extern "C" void app_main()
{


	int x;
	char b;
	char *inbuf, *outbuf;
	tdefl_compressor *comp;
	tinfl_decompressor *decomp;
	tdefl_status status;
	size_t inbytes = 0, outbytes = 0, inpos = 0, outpos = 0, compsz;

	printf("Allocating compressor & outbuf (%d bytes)\n", sizeof(tdefl_compressor));
	comp = (tdefl_compressor *) malloc(sizeof(tdefl_compressor));
	TEST_ASSERT(comp != NULL);


	printf("Allocating data buffer and filling it with semi-random data\n");
	inbuf = (char *) malloc(DATASIZE*sizeof(char));
	TEST_ASSERT(inbuf != NULL);
	srand(0);
	for (x = 0; x < DATASIZE; x++) {
		inbuf[x] = (x & 1) ? rand() & 0xff : 0;
	}

	int free_space = heap_caps_get_free_size(MALLOC_CAP_8BIT);
	printf("Free space is %d\n", free_space);

	outbuf = (char *) malloc(DATASIZE);
	TEST_ASSERT(outbuf != NULL);
	printf("Compressing...\n");
	status = tdefl_init(comp, NULL, NULL, TDEFL_WRITE_ZLIB_HEADER | 1500);
	TEST_ASSERT(status == TDEFL_STATUS_OKAY);
	while (inbytes != DATASIZE) {
		outbytes = DATASIZE - outpos;
		inbytes = DATASIZE - inpos;
		tdefl_compress(comp, &inbuf[inpos], &inbytes, &outbuf[outpos], &outbytes, TDEFL_FINISH);
		printf("...Compressed %d into %d bytes\n", inbytes, outbytes);
		inpos += inbytes; outpos += outbytes;
	}
	compsz = outpos;
	free(comp);
	//Kill inbuffer
	for (x = 0; x < DATASIZE; x++) {
		inbuf[x] = 0;
	}
	free(inbuf);

	inbuf = outbuf;
	outbuf = (char *) malloc(DATASIZE);
	TEST_ASSERT(outbuf != NULL);
	printf("Reinflating...\n");
	decomp = (tinfl_decompressor *) malloc(sizeof(tinfl_decompressor));
	TEST_ASSERT(decomp != NULL);
	tinfl_init(decomp);
	inpos = 0; outpos = 0;
	while (inbytes != compsz) {
		outbytes = DATASIZE - outpos;
		inbytes = compsz - inpos;
		tinfl_decompress(decomp, (const mz_uint8 *)&inbuf[inpos], &inbytes, (uint8_t *)outbuf, (mz_uint8 *)&outbuf[outpos], &outbytes, TINFL_FLAG_PARSE_ZLIB_HEADER);
		printf("...Decompressed %d into %d bytes\n", inbytes, outbytes);
		inpos += inbytes; outpos += outbytes;
	}
	printf("Checking if same...\n");
	srand(0);
	for (x = 0; x < DATASIZE; x++) {
		b = (x & 1) ? rand() & 0xff : 0;
		if (outbuf[x] != b) {
			printf("Pos %x: %hhx!=%hhx\n", x, outbuf[x], b);
			TEST_ASSERT(0);
		}
	}
	printf("Great Success!\n");
	free(inbuf);
	free(outbuf);
	free(decomp);


	while(1){
		vTaskDelay(100);
	}
}

Code: Select all

#include "compression.h"
#include "assert.h"
#include "zlib.h"
#include <string.h>
#include "stdio.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "stdlib.h"
#include "esp_log.h"

const char * TAG = "Chaze-Compression";

/**
 * @brief When recording sensor data, multiple tasks need to acquire a mutex in order to read from the bus and
 * write to the buffer. The buffer is a struct that holds a counter (number of bytes written) and a malloced
 * char array. If the number of bytes written equals to the BUFFER SIZE, this function is called and the
 * data is compressed and saved to the flash.
 * @param Buffer number to read from and compress. Either 0 or 1.
 */
void compress_and_save(uint8_t buff_num)
{
	ESP_LOGI(TAG, "Using buffer number %d", buff_num);
	//Get the pointer to the current buffer
	buffer_t * current_buffer = buffers[buff_num];
	assert(current_buffer->counter == BUFFER_SIZE); //The buffer must be filled completely.

	//Data written to flash.
	char * out = (char *) malloc(BUFFER_SIZE * sizeof(char));

	if(out == NULL){
		ESP_LOGE(TAG, "Could not allocate enough space");
		//Need to write the raw, uncompressed data to the flash instead
		return;
	} else{

		ESP_LOGI(TAG, "Start compressing.");
		//Writes 'written' many chars to the out buffer previously allocated
		uint32_t written = def(current_buffer->data, out, DEFAULT_COMPRESSION_LEVEL);

		//Write data to flash
		write_data_to_flash(out, written);
		free(out);
	}
}

/**
 * @brief Deflates the char array given by source and writes to the char array dest.
 * @param source
 * @param dest
 * @param level Level of compression.
 * @return Number of written bytes to dest
 */
uint32_t def(char * source, char * dest, uint8_t level)
{
	int ret, flush;
	unsigned have;
	z_stream strm;

	unsigned char * in = (unsigned char *) malloc(CHUNK * sizeof(char));
	unsigned char * out = (unsigned char *) malloc(CHUNK * sizeof(char));

	strm.zalloc = Z_NULL;
	strm.zfree = Z_NULL;
	strm.opaque = Z_NULL;
	ret = deflateInit2(&strm, level,Z_DEFLATED, WINDOW_SIZE ,MEM_LEVEL,Z_DEFAULT_STRATEGY);
	zerr(ret);

	uint32_t dest_offset = 0;
    do {
        memcpy(in, source, CHUNK);
        strm.avail_in = CHUNK; /*We assert that the buffer is full and since chunk size = buffer size, we can read that many bytes*/
        flush = Z_FINISH; //Immediately flush since we only compress one chunk.
        strm.next_in = in;

        do {
            strm.avail_out = CHUNK;
            strm.next_out = out;
            ret = deflate(&strm, flush);
            assert(ret != Z_STREAM_ERROR);
            zerr(ret);
            have = CHUNK - strm.avail_out;
            ESP_LOGI(TAG, "Have: %d", have);
            for(int i=0;i<have;i++)
            	printf("%c", out[i]);

            memcpy(dest+dest_offset, out, have);
            dest_offset += have;
        } while (strm.avail_out == 0);
        assert(strm.avail_in == 0);

    } while (flush != Z_FINISH);
    assert(ret == Z_STREAM_END);

    zerr(deflateEnd(&strm));
    free(in);
    free(out);
    return dest_offset;
}

//For now just inflates data to check if was deflated correctly
void write_data_to_flash(char * data, uint32_t n)
{
	int ret;
	unsigned have;
	z_stream strm;

	unsigned char * in = (unsigned char *) malloc(CHUNK * sizeof(char));
	unsigned char * out = (unsigned char *) malloc(CHUNK * sizeof(char));


	char * to_print = (char *) malloc(CHUNK*sizeof(char));

	if(to_print == NULL || in == NULL || out == NULL)
	{
		ESP_LOGE(TAG, "Could not allocate enough space");
		return;
	}

	strm.zalloc = Z_NULL;
	strm.zfree = Z_NULL;
	strm.opaque = Z_NULL;
	strm.avail_in = 0;
	strm.next_in = Z_NULL;

	ret = inflateInit2(&strm,0); //Use window size in compressed stream
	zerr(ret);
	uint32_t to_print_offset = 0;

	do {
			memcpy(in, data, n);
			strm.avail_in = n;

			if (strm.avail_in == 0)
				break;
			strm.next_in = in;

			do {
				strm.avail_out = CHUNK;
				strm.next_out = out;

				ret = inflate(&strm, Z_NO_FLUSH);
				assert(ret != Z_STREAM_ERROR);
				zerr(ret);

				have = CHUNK - strm.avail_out;
				memcpy(to_print+to_print_offset, out, have);
				to_print_offset+=have;

			} while (strm.avail_out == 0);
		} while (ret != Z_STREAM_END);

	(void)inflateEnd(&strm);
	free(in);
	free(out);

	for(int i=0;i<CHUNK;i++)
		printf("%c", to_print[i]);
}


void zerr(int ret)
{
    switch (ret) {
    case Z_ERRNO:
        ESP_LOGE(TAG, "Z err no");
        break;
    case Z_STREAM_ERROR:
    	ESP_LOGE(TAG, "Invalid compression level");
        break;
    case Z_DATA_ERROR:
    	ESP_LOGE(TAG, "Data error");
        break;
    case Z_MEM_ERROR:
    	ESP_LOGE(TAG, "Mem error");
        break;
    case Z_VERSION_ERROR:
    	ESP_LOGE(TAG, "Version error");
    }
}

Code: Select all

#ifndef COMPRESSION_H
#define COMPRESSION_H

#include "stdint.h"


#define BUFFER_SIZE /*16384*/ 8192
#define CHUNK 8192 /*Must be same size as BUFFER_SIZE*/
#define DEFAULT_COMPRESSION_LEVEL 6
#define WINDOW_SIZE 9
#define MEM_LEVEL 1
#define GZIP_ENCODING 16


typedef struct {
	int32_t counter;
	char * data;
}buffer_t;

buffer_t * buffers[2];

void compress_and_save(uint8_t);
void write_data_to_flash(char *, uint32_t);
uint32_t def(char *, char *, uint8_t);
void zerr(int32_t);


#endif


//Working main.cpp file.
//Creates task simulating sensor readings with mutex, compresses if buffer full, switches buffer and writes to next one
/*
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "stdlib.h"
#include "stdio.h"
#include <stdio.h>
#include <string.h>
extern "C" {
#include "compression.h"
}
#include "freertos/semphr.h"
#include "esp_log.h"
SemaphoreHandle_t xSemaphore = NULL;
volatile uint8_t buff_idx = 0;
static const char * TAG = "Chaze-Compression-Main";
void sample_hr(void * pvParams)
{
	char dummy[] = "Sorem ipsum lorem ich habe hier einen schönen\nAusblich wäre ich hier irgendwas rein schreibe. Ich sitze an meinem Schreibtisch\nund drehe ein bisschen Daeumchen und so weiter und so fort.\n";
	for(;;){
		//Try to aquire the mutex for a given amount of time then back-off
		//For real life, need priorites in low sample rate sensors, random back-off time etc.
		buffer_t * curr_buff = buffers[buff_idx];
		if(xSemaphore != NULL)
		{
			if(xSemaphoreTake(xSemaphore,(TickType_t) 100) == pdTRUE)
			{
				ESP_LOGI(TAG, "Acquired the lock");
				if(BUFFER_SIZE-curr_buff->counter >= strlen(dummy)){
					ESP_LOGI(TAG, "Enough space to write");
					//Enough space, we can write
					memcpy(curr_buff->data+curr_buff->counter, dummy, strlen(dummy));
					curr_buff->counter += strlen(dummy);
				} else{
					//Fill up the buffer with 'f' and set the buff_idx to buff_idx XOR 1
					ESP_LOGI(TAG, "Buffer almost full, fill up");
					for(int i=curr_buff->counter;i<BUFFER_SIZE-curr_buff->counter;i++)
						curr_buff->data[i] = 'f';
					curr_buff->counter += BUFFER_SIZE-curr_buff->counter;
					ESP_LOGI(TAG, "Counter is %d", curr_buff->counter);
					if(buff_idx==0)
						buff_idx=1;
					else buff_idx=0;
					ESP_LOGI(TAG, "Call compress");
					//Call compress. Switch the bit back since we want to compress the buffer that is full
					compress_and_save(!buff_idx);
					curr_buff->counter = 0; //Reset the compressed buffer
				}
				ESP_LOGI(TAG, "Release mutex");
				//Release mutex
				xSemaphoreGive(xSemaphore);
			}
			vTaskDelay(100 / portTICK_PERIOD_MS); //Back off a little longer
		}
	}
}
extern "C" void app_main()
{
	 xSemaphore = xSemaphoreCreateMutex();
	//Initialize buffers 1 and 2
	buffer_t buffer0 = {};
	buffer0.data = (char *) malloc(BUFFER_SIZE);
	buffer0.counter = 0;
	buffer_t buffer1 = {};
	buffer1.data = (char *) malloc(BUFFER_SIZE);
	buffer1.counter = 0;
	buffers[0] = &buffer0;
	buffers[1] = &buffer1;
	if (xTaskCreate(&sample_hr, "sample_hr", 1024 * 8, NULL, 5, NULL) != pdPASS )
	{
		printf("Synch task create failed.\n");
	} else ESP_LOGI(TAG, "Created task");
	while(1){
		vTaskDelay(10000);
	}
}
/*
extern "C" void app_main()
{
	esp_vfs_fat_mount_config_t mount_config = {};
	mount_config.max_files = 4;
	mount_config.format_if_mount_failed = true;
	mount_config.allocation_unit_size = CONFIG_WL_SECTOR_SIZE;
	esp_err_t err = esp_vfs_fat_spiflash_mount(base_path, "storage", &mount_config, &s_wl_handle);
	if (err != ESP_OK) {
		ESP_LOGE(TAG, "Failed to mount FATFS (%s)", esp_err_to_name(err));
		return;
	}
	print_dir();
	remove_folder();
	print_dir();
	ESP_LOGI(TAG, "Opening file");
	foo = fopen("/spiflash/foo.bin", "wb");
	if (foo == NULL) {
		ESP_LOGE(TAG, "Failed to open file for writing");
		return;
	}
	//int ret = fprintf(foo, "written using ESP-IDF\n");
	const char * tmp = esp_get_idf_version();
	int ret = fwrite(tmp,1,strlen(tmp),foo);
	if(ret < strlen(tmp)){
		ESP_LOGE(TAG, "Error writing file");
	}
	if(fclose(foo) == EOF){
		ESP_LOGE(TAG, "Error closing the file.");
	}
	else ESP_LOGI(TAG, "File written");
	comp = fopen("/spiflash/c.bin", "wb");
	if (comp == NULL) {
		ESP_LOGE(TAG, "Failed to open file for writing");
		return;
	}
	foo = fopen("/spiflash/foo.bin", "rb");
	if(foo == NULL){
		ESP_LOGE(TAG, "Error opening the file.");
	}
	print_dir();
	uint8_t level = 6;
	zerr(def(foo, comp, level));
	ESP_LOGI(TAG, "Compressed.");
	if(fclose(foo) == EOF){
		ESP_LOGE(TAG, "Error closing file");
	}
	else ESP_LOGI(TAG, "Closed foo.");
	if(fclose(comp) == EOF){
		ESP_LOGE(TAG, "Error closing file: %s", strerror(errno));
	}
	else ESP_LOGI(TAG, "Closed comp.");
	//Read comp
	char line[128];
	char *pos;
	// Open file for reading
	ESP_LOGI(TAG, "Reading file");
	comp = fopen("/spiflash/c.bin", "rb");
	if (comp == NULL) {
		ESP_LOGE(TAG, "Failed to open file for reading");
	}
	fgets(line, sizeof(line), foo);
	fclose(comp);
	// strip newline
	pos = strchr(line, '\n');
	if (pos) {
		*pos = '\0';
	}
	ESP_LOGI(TAG, "Read from file: '%s'", line);
	//Inflate
	recon = fopen("/spiflash/rec.bin", "wb");
	comp = fopen("/spiflash/c.bin", "rb");
	if(recon == NULL || comp == NULL){
		ESP_LOGE(TAG, "Error opening file before inflating.");
	}
	ESP_LOGI(TAG, "Start inflating");
	zerr(inf(comp, recon));
	ESP_LOGI(TAG, "Stopped inflating");
	if(fclose(comp) == EOF){
		ESP_LOGE(TAG, "Error closing file");
	}
	ESP_LOGI(TAG, "Closed comp");
	if(fclose(recon) == EOF){
		ESP_LOGE(TAG, "Error closing file");
	}
	ESP_LOGI(TAG, "Closed recon");
	// Open file for reading
	ESP_LOGI(TAG, "Reading file");
	recon = fopen("/spiflash/rec.bin", "rb");
	if (recon == NULL) {
		ESP_LOGE(TAG, "Failed to open file for reading");
	}
	fgets(line, sizeof(line), recon);
	fclose(recon);
	// strip newline
	pos = strchr(line, '\n');
	if (pos) {
		*pos = '\0';
	}
	ESP_LOGI(TAG, "Read from file: '%s'", line);
	// Open file for reading
	ESP_LOGI(TAG, "Reading file");
	foo = fopen("/spiflash/foo.bin", "rb");
	if (foo == NULL) {
		ESP_LOGE(TAG, "Failed to open file for reading");
		return;
	}
	fgets(line, sizeof(line), foo);
	fclose(foo);
	// strip newline
	pos = strchr(line, '\n');
	if (pos) {
		*pos = '\0';
	}
	ESP_LOGI(TAG, "Read from file: '%s'", line);
	// Unmount FATFS
	ESP_LOGI(TAG, "Unmounting FAT filesystem");
	ESP_ERROR_CHECK( esp_vfs_fat_spiflash_unmount(base_path, s_wl_handle));
	ESP_LOGI(TAG, "Done");
	while(1){
		vTaskDelay(1000);
	}
}
*/

 */

Code: Select all

    if(!mz_zip_writer_init_file(&zip_archive, tempNameWithDir, 0))
    {
        ESP_LOGE(TAG, "Error while initializing ZIP file archive\n");
        return false;
    }

    mz_bool rc;
    ESP_LOGD(TAG, "mz_zip_writer_add_file");
    ESP_LOGI(TAG, "Max alloc size = %d", heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM));
    printf("sizeof(tdefl_compressor) = %d\n", sizeof(tdefl_compressor));
    uint32_t millis = esp_log_timestamp();
    rc = mz_zip_writer_add_file(&zip_archive, fileName, fileNameWithDir, NULL, 0, MZ_DEFAULT_COMPRESSION);
    if (rc == MZ_FALSE)
    {
        ESP_LOGE(TAG, "Could not add file: %s", mz_zip_get_error_string(mz_zip_get_last_error(&zip_archive)));
        return false;
    }

    ESP_LOGD(TAG, "mz_zip_writer_finalize_archive");
    rc = mz_zip_writer_finalize_archive( &zip_archive );
    if ( !rc )
        return false;

    ESP_LOGD(TAG, "mz_zip_writer_end");
    rc = mz_zip_writer_end( &zip_archive );
    if ( !rc )
        return false;