Add support for compressed NPZ file

modules/core/include/visp3/core/vpIoTools.h

@@ -62,13 +62,23 @@ namespace visp
 {
 #ifndef DOXYGEN_SHOULD_SKIP_THIS
 // https://github.com/BinomialLLC/basis_universal/blob/ad9386a4a1cf2a248f7bbd45f543a7448db15267/encoder/basisu_miniz.h#L665
-static inline unsigned long vp_mz_crc32(unsigned long crc, const unsigned char *ptr, size_t buf_len)
+// Karl Malbrain's compact CRC-32. See: "A compact CCITT crc16 and crc32 C implementation that balances processor
+// cache usage against speed": http://www.geocities.com/malbrain/
+// https://www.geocities.ws/malbrain/crc_c.html
+//
+// Other code:
+// stb crc32 implementation:
+// https://github.com/nothings/stb/blob/f1c79c02822848a9bed4315b12c8c8f3761e1296/stb_image_write.h#L1024-L1071
+//
+// https://github.com/cesanta/mongoose-os/blob/3d8bf2c1445389a710158353bf23be943ca3034d/src/common/cs_crc32.c#L20-L41
+static inline uint32_t vp_mz_crc32(uint32_t crc, const unsigned char *ptr, size_t buf_len)
 {
-  static const unsigned int s_crc32[16] = { 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
+  static const uint32_t s_crc32[16] = { 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c,
     0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c };
-  unsigned int crcu32 = static_cast<unsigned int>(crc);
-  if (!ptr) return 0;
-  crcu32 = ~crcu32;
+  if (!ptr) {
+    return 0;
+  }
+  uint32_t crcu32 = ~crc;
   while (buf_len--) {
     unsigned char b = *ptr++;
     crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b & 0xF)];
@@ -185,8 +195,10 @@ VISP_EXPORT void parse_zip_footer(FILE *fp, uint16_t &nrecs, size_t &global_head
 VISP_EXPORT NpyArray npz_load(const std::string &fname, const std::string &varname);
 VISP_EXPORT NpyArray npy_load(const std::string &fname);
 // Dedicated functions for saving std::string data
-VISP_EXPORT void npz_save(const std::string &zipname, std::string fname, const std::vector<std::string> &data_vec, const std::vector<size_t> &shape, const std::string &mode = "w");
-VISP_EXPORT void npz_save(const std::string &zipname, const std::string &fname, const std::string &data, const std::string &mode = "w");
+VISP_EXPORT void npz_save_str(const std::string &zipname, std::string fname, const std::vector<std::string> &data_vec,
+  const std::vector<size_t> &shape, const std::string &mode = "w", bool compress_data = false);
+VISP_EXPORT void npz_save_str(const std::string &zipname, const std::string &fname, const std::string &data,
+  const std::string &mode = "w", bool compress_data = false);
 
 template<typename T> std::vector<char> &operator+=(std::vector<char> &lhs, const T rhs)
 {
@@ -275,6 +287,9 @@ template<typename T> void npy_save(const std::string &fname, const T *data, cons
   fclose(fp);
 }
 
+VISP_EXPORT void compressData(size_t nbytes_uncompressed, std::vector<uint8_t> &uncompressed,
+  std::vector<uint8_t> &buffer_compressed, size_t &nbytes_on_disk, FILE *fp);
+
 /*!
   Save the specified \p fname array of data (\p data) into the \p zipname npz file. This function is similar to the
   <a href="https://numpy.org/doc/stable/reference/generated/numpy.savez.html">numpy.savez</a> function.
@@ -283,12 +298,14 @@ template<typename T> void npy_save(const std::string &fname, const T *data, cons
   \param[in] data : Pointer to an array of basic datatype (int, float, double, std::complex<double>, ...).
   \param[in] shape : Shape of the array, e.g. Nz x Ny x Nx.
   \param[in] mode : Writing mode, i.e. overwrite (w) or append (a) to the file.
+  \param[in] compress_data : Flag to indicate if the data should be compressed or not.
   \warning This function should also work on big-endian platform, without guarantee since it has not been tested extensively.
   \note Original library: <a href="https://github.com/rogersce/cnpy">cnpy</a> with MIT license.
 
   \sa To see how to use it, you may have a look at \ref tutorial-npz
  */
-template<typename T> void npz_save(const std::string &zipname, std::string fname, const T *data, const std::vector<size_t> &shape, const std::string &mode = "w")
+template<typename T> void npz_save(const std::string &zipname, std::string fname, const T *data, const std::vector<size_t> &shape,
+    const std::string &mode = "w", bool compress_data = false)
 {
   //first, append a .npy to the fname
   fname += ".npy";
@@ -324,12 +341,34 @@ template<typename T> void npz_save(const std::string &zipname, std::string fname
 
   // https://github.com/rogersce/cnpy/pull/58/files
   size_t nels = std::accumulate(shape.begin(), shape.end(), static_cast<size_t>(1), std::multiplies<size_t>());
-  size_t nbytes = nels*sizeof(T) + npy_header.size();
+  size_t nbytes_uncompressed = nels*sizeof(T) + npy_header.size();
+
+  // Prepare data and compression parameters
+  std::vector<uint8_t> buffer_compressed;
+  size_t nbytes_on_disk;
+  uint16_t compression_method;
+  uint32_t crc;
+
+  if (compress_data && nels > 0) {
+    // Create uncompressed buffer (header + data)
+    std::vector<uint8_t> uncompressed(nbytes_uncompressed);
+    memcpy(&uncompressed[0], &npy_header[0], npy_header.size());
+    memcpy(&uncompressed[npy_header.size()], data, nels*sizeof(T));
 
-  //get the CRC of the data to be added
-  uint32_t crc = vp_mz_crc32(0L, (uint8_t *)&npy_header[0], npy_header.size());
-  if (nels > 0) {
-    crc = vp_mz_crc32(crc, (uint8_t *)data, nels*sizeof(T));
+    // Get CRC of uncompressed data
+    crc = vp_mz_crc32(0, &uncompressed[0], nbytes_uncompressed);
+
+    compressData(nbytes_uncompressed, uncompressed, buffer_compressed, nbytes_on_disk, fp);
+    compression_method = 8; // deflate
+  }
+  else {
+    // No compression - CRC computed in two parts
+    crc = vp_mz_crc32(0, (uint8_t *)&npy_header[0], npy_header.size());
+    if (nels > 0) {
+      crc = vp_mz_crc32(crc, (uint8_t *)data, nels*sizeof(T));
+    }
+    nbytes_on_disk = nbytes_uncompressed;
+    compression_method = 0; // store
   }
 
   //build the local header
@@ -339,24 +378,24 @@ template<typename T> void npz_save(const std::string &zipname, std::string fname
   local_header += vpEndian::swap16bits(static_cast<uint16_t>(0x0403)); //second part of sig
   local_header += vpEndian::swap16bits(static_cast<uint16_t>(20)); //min version to extract
   local_header += vpEndian::swap16bits(static_cast<uint16_t>(0)); //general purpose bit flag
-  local_header += vpEndian::swap16bits(static_cast<uint16_t>(0)); //compression method
+  local_header += vpEndian::swap16bits(static_cast<uint16_t>(compression_method)); //compression method
   local_header += vpEndian::swap16bits(static_cast<uint16_t>(0)); //file last mod time
   local_header += vpEndian::swap16bits(static_cast<uint16_t>(0));     //file last mod date
   local_header += vpEndian::swap32bits(static_cast<uint32_t>(crc)); //crc
-  local_header += vpEndian::swap32bits(static_cast<uint32_t>(nbytes)); //compressed size
-  local_header += vpEndian::swap32bits(static_cast<uint32_t>(nbytes)); //uncompressed size
+  local_header += vpEndian::swap32bits(static_cast<uint32_t>(nbytes_on_disk)); //compressed size
+  local_header += vpEndian::swap32bits(static_cast<uint32_t>(nbytes_uncompressed)); //uncompressed size
   local_header += vpEndian::swap16bits(static_cast<uint16_t>(fname.size())); //fname length
   local_header += vpEndian::swap16bits(static_cast<uint16_t>(0)); //extra field length
 #else
   local_header += static_cast<uint16_t>(0x0403); //second part of sig
   local_header += static_cast<uint16_t>(20); //min version to extract
   local_header += static_cast<uint16_t>(0); //general purpose bit flag
-  local_header += static_cast<uint16_t>(0); //compression method
+  local_header += static_cast<uint16_t>(compression_method); //compression method
   local_header += static_cast<uint16_t>(0); //file last mod time
   local_header += static_cast<uint16_t>(0);     //file last mod date
   local_header += static_cast<uint32_t>(crc); //crc
-  local_header += static_cast<uint32_t>(nbytes); //compressed size
-  local_header += static_cast<uint32_t>(nbytes); //uncompressed size
+  local_header += static_cast<uint32_t>(nbytes_on_disk); //compressed size
+  local_header += static_cast<uint32_t>(nbytes_uncompressed); //uncompressed size
   local_header += static_cast<uint16_t>(fname.size()); //fname length
   local_header += static_cast<uint16_t>(0); //extra field length
 #endif
@@ -395,23 +434,28 @@ template<typename T> void npz_save(const std::string &zipname, std::string fname
   footer += vpEndian::swap16bits(static_cast<uint16_t>(nrecs+1)); //number of records on this disk
   footer += vpEndian::swap16bits(static_cast<uint16_t>(nrecs+1)); //total number of records
   footer += vpEndian::swap32bits(static_cast<uint32_t>(global_header.size())); //nbytes of global headers
-  footer += vpEndian::swap32bits(static_cast<uint32_t>(global_header_offset + nbytes + local_header.size())); //offset of start of global headers, since global header now starts after newly written array
+  footer += vpEndian::swap32bits(static_cast<uint32_t>(global_header_offset + nbytes_on_disk + local_header.size())); //offset of start of global headers, since global header now starts after newly written array
 #else
   footer += static_cast<uint16_t>(0x0605); //second part of sig
   footer += static_cast<uint16_t>(0); //number of this disk
   footer += static_cast<uint16_t>(0); //disk where footer starts
   footer += static_cast<uint16_t>(nrecs+1); //number of records on this disk
   footer += static_cast<uint16_t>(nrecs+1); //total number of records
   footer += static_cast<uint32_t>(global_header.size()); //nbytes of global headers
-  footer += static_cast<uint32_t>(global_header_offset + nbytes + local_header.size()); //offset of start of global headers, since global header now starts after newly written array
+  footer += static_cast<uint32_t>(global_header_offset + nbytes_on_disk + local_header.size()); //offset of start of global headers, since global header now starts after newly written array
 #endif
   footer += static_cast<uint16_t>(0); //zip file comment length
 
   //write everything
   fwrite(&local_header[0], sizeof(char), local_header.size(), fp);
-  fwrite(&npy_header[0], sizeof(char), npy_header.size(), fp);
-  if (data != nullptr) {
-    fwrite(&data[0], sizeof(T), nels, fp);
+  if (compress_data) {
+    fwrite(&buffer_compressed[0], sizeof(char), nbytes_on_disk, fp);
+  }
+  else {
+    fwrite(&npy_header[0], sizeof(char), npy_header.size(), fp);
+    if (data != nullptr) {
+      fwrite(&data[0], sizeof(T), nels, fp);
+    }
   }
   fwrite(&global_header[0], sizeof(char), global_header.size(), fp);
   fwrite(&footer[0], sizeof(char), footer.size(), fp);
@@ -441,16 +485,18 @@ template<typename T> void npy_save(const std::string &fname, const std::vector<T
   \param[in] fname : Identifier for the corresponding array of data.
   \param[in] data : Pointer to a 1-D array of basic datatype (int, float, double, std::complex<double>, ...).
   \param[in] mode : Writing mode, i.e. overwrite (w) or append (a) to the file.
+  \param[in] compress_data : Flag to indicate if the data should be compressed or not.
   \warning This function should also work on big-endian platform, without guarantee since it has not been tested extensively.
   \note Original library: <a href="https://github.com/rogersce/cnpy">cnpy</a> with MIT license.
 
   \sa To see how to use it, you may have a look at \ref tutorial-npz
  */
-template<typename T> void npz_save(const std::string &zipname, const std::string &fname, const std::vector<T> &data, const std::string &mode = "w")
+template<typename T> void npz_save(const std::string &zipname, const std::string &fname, const std::vector<T> &data,
+    const std::string &mode = "w", bool compress_data = false)
 {
   std::vector<size_t> shape;
   shape.push_back(data.size());
-  npz_save(zipname, fname, &data[0], shape, mode);
+  npz_save(zipname, fname, &data[0], shape, mode, compress_data);
 }
 
 template<typename T> std::vector<char> create_npy_header(const std::vector<size_t> &shape)

modules/core/src/tools/file/basisu_miniz.h

@@ -507,8 +507,8 @@ size_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void
 //  Function returns a pointer to the compressed data, or NULL on failure.
 //  *pLen_out will be set to the size of the PNG image file.
 //  The caller must mz_free() the returned heap block (which will typically be larger than *pLen_out) when it's no longer needed.
-void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, mz_uint8 w, mz_uint8 h, int num_chans, size_t *pLen_out, mz_uint level, mz_bool flip);
-void *tdefl_write_image_to_png_file_in_memory(const void *pImage, mz_uint8 w, mz_uint8 h, int num_chans, size_t *pLen_out);
+void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int h, int num_chans, size_t *pLen_out, mz_uint level, mz_bool flip);
+void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, size_t *pLen_out);
 
 // Output stream interface. The compressor uses this interface to write compressed data. It'll typically be called TDEFL_OUT_BUF_SIZE at a time.
 typedef mz_bool(*tdefl_put_buf_func_ptr)(const void *pBuf, int len, void *pUser);
@@ -2385,7 +2385,7 @@ mz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int
 // Simple PNG writer function by Alex Evans, 2011. Released into the public domain: https://gist.github.com/908299, more context at
 // http://altdevblogaday.org/2011/04/06/a-smaller-jpg-encoder/.
 // This is actually a modification of Alex's original code so PNG files generated by this function pass pngcheck.
-void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, mz_uint8 w, mz_uint8 h, int num_chans, size_t *pLen_out, mz_uint level, mz_bool flip)
+void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int h, int num_chans, size_t *pLen_out, mz_uint level, mz_bool flip)
 {
   // Using a local copy of this array here in case MINIZ_NO_ZLIB_APIS was defined.
   static const mz_uint s_tdefl_png_num_probes[11] = { 0, 1, 6, 32,  16, 32, 128, 256,  512, 768, 1500 };
@@ -2414,7 +2414,7 @@ void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, mz_uint8 w,
   // compute final size of file, grab compressed data buffer and return
   *pLen_out += 57; MZ_FREE(pComp); return out_buf.m_pBuf;
 }
-void *tdefl_write_image_to_png_file_in_memory(const void *pImage, mz_uint8 w, mz_uint8 h, int num_chans, size_t *pLen_out)
+void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, size_t *pLen_out)
 {
   // Level 6 corresponds to TDEFL_DEFAULT_MAX_PROBES or MZ_DEFAULT_LEVEL (but we can't depend on MZ_DEFAULT_LEVEL being available in case the zlib API's where #defined out)
   return tdefl_write_image_to_png_file_in_memory_ex(pImage, w, h, num_chans, pLen_out, 6, MZ_FALSE);