src/serialization.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "serialization.h"
  21
  22 #include "util/serialize.h"
  23
  24 #include <zlib.h>
  25 #include <zstd.h>
  26
  27 /* report a zlib or i/o error */
  28 static void zerr(int ret)
  29 {
  30     dstream<<"zerr: ";
  31     switch (ret) {
  32     case Z_ERRNO:
  33         if (ferror(stdin))
  34             dstream<<"error reading stdin"<<std::endl;
  35         if (ferror(stdout))
  36             dstream<<"error writing stdout"<<std::endl;
  37         break;
  38     case Z_STREAM_ERROR:
  39         dstream<<"invalid compression level"<<std::endl;
  40         break;
  41     case Z_DATA_ERROR:
  42         dstream<<"invalid or incomplete deflate data"<<std::endl;
  43         break;
  44     case Z_MEM_ERROR:
  45         dstream<<"out of memory"<<std::endl;
  46         break;
  47     case Z_VERSION_ERROR:
  48         dstream<<"zlib version mismatch!"<<std::endl;
  49                 break;
  50         default:
  51                 dstream<<"return value = "<<ret<<std::endl;
  52     }
  53 }
  54
  55 // Make sure that z is deleted in case of exception
  56 template <int (*F)(z_stream*)>
  57 class ZlibAutoDeleter {
  58 public:
  59         ZlibAutoDeleter(z_stream *ptr) : ptr_(ptr) {}
  60         ~ZlibAutoDeleter() { F(ptr_); }
  61
  62 private:
  63         z_stream *ptr_;
  64 };
  65
  66 void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level)
  67 {
  68         z_stream z;
  69         const s32 bufsize = 16384;
  70         char output_buffer[bufsize];
  71         int status = 0;
  72         int ret;
  73
  74         z.zalloc = Z_NULL;
  75         z.zfree = Z_NULL;
  76         z.opaque = Z_NULL;
  77
  78         ret = deflateInit(&z, level);
  79         if(ret != Z_OK)
  80                 throw SerializationError("compressZlib: deflateInit failed");
  81
  82         ZlibAutoDeleter<deflateEnd> deleter(&z);
  83
  84         // Point zlib to our input buffer
  85         z.next_in = (Bytef*)&data[0];
  86         z.avail_in = data_size;
  87         // And get all output
  88         for(;;)
  89         {
  90                 z.next_out = (Bytef*)output_buffer;
  91                 z.avail_out = bufsize;
  92
  93                 status = deflate(&z, Z_FINISH);
  94                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
  95                                 || status == Z_MEM_ERROR)
  96                 {
  97                         zerr(status);
  98                         throw SerializationError("compressZlib: deflate failed");
  99                 }
 100                 int count = bufsize - z.avail_out;
 101                 if(count)
 102                         os.write(output_buffer, count);
 103                 // This determines zlib has given all output
 104                 if(status == Z_STREAM_END)
 105                         break;
 106         }
 107 }
 108
 109 void compressZlib(const std::string &data, std::ostream &os, int level)
 110 {
 111         compressZlib((u8*)data.c_str(), data.size(), os, level);
 112 }
 113
 114 void decompressZlib(std::istream &is, std::ostream &os, size_t limit)
 115 {
 116         z_stream z;
 117         const s32 bufsize = 16384;
 118         char input_buffer[bufsize];
 119         char output_buffer[bufsize];
 120         int status = 0;
 121         int ret;
 122         int bytes_written = 0;
 123         int input_buffer_len = 0;
 124
 125         z.zalloc = Z_NULL;
 126         z.zfree = Z_NULL;
 127         z.opaque = Z_NULL;
 128
 129         ret = inflateInit(&z);
 130         if(ret != Z_OK)
 131                 throw SerializationError("dcompressZlib: inflateInit failed");
 132
 133         ZlibAutoDeleter<inflateEnd> deleter(&z);
 134
 135         z.avail_in = 0;
 136
 137         for(;;)
 138         {
 139                 int output_size = bufsize;
 140                 z.next_out = (Bytef*)output_buffer;
 141                 z.avail_out = output_size;
 142
 143                 if (limit) {
 144                         int limit_remaining = limit - bytes_written;
 145                         if (limit_remaining <= 0) {
 146                                 // we're aborting ahead of time - throw an error?
 147                                 break;
 148                         }
 149                         if (limit_remaining < output_size) {
 150                                 z.avail_out = output_size = limit_remaining;
 151                         }
 152                 }
 153
 154                 if(z.avail_in == 0)
 155                 {
 156                         z.next_in = (Bytef*)input_buffer;
 157                         is.read(input_buffer, bufsize);
 158                         input_buffer_len = is.gcount();
 159                         z.avail_in = input_buffer_len;
 160                 }
 161                 if(z.avail_in == 0)
 162                 {
 163                         break;
 164                 }
 165
 166                 status = inflate(&z, Z_NO_FLUSH);
 167
 168                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
 169                                 || status == Z_MEM_ERROR)
 170                 {
 171                         zerr(status);
 172                         throw SerializationError("decompressZlib: inflate failed");
 173                 }
 174                 int count = output_size - z.avail_out;
 175                 if(count)
 176                         os.write(output_buffer, count);
 177                 bytes_written += count;
 178                 if(status == Z_STREAM_END)
 179                 {
 180                         // Unget all the data that inflate didn't take
 181                         is.clear(); // Just in case EOF is set
 182                         for(u32 i=0; i < z.avail_in; i++)
 183                         {
 184                                 is.unget();
 185                                 if(is.fail() || is.bad())
 186                                 {
 187                                         dstream<<"unget #"<<i<<" failed"<<std::endl;
 188                                         dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 189                                         throw SerializationError("decompressZlib: unget failed");
 190                                 }
 191                         }
 192
 193                         break;
 194                 }
 195         }
 196 }
 197
 198 struct ZSTD_Deleter {
 199         void operator() (ZSTD_CStream* cstream) {
 200                 ZSTD_freeCStream(cstream);
 201         }
 202
 203         void operator() (ZSTD_DStream* dstream) {
 204                 ZSTD_freeDStream(dstream);
 205         }
 206 };
 207
 208 void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level)
 209 {
 210         // reusing the context is recommended for performance
 211         // it will be destroyed when the thread ends
 212         thread_local std::unique_ptr<ZSTD_CStream, ZSTD_Deleter> stream(ZSTD_createCStream());
 213
 214
 215         ZSTD_initCStream(stream.get(), level);
 216
 217         const size_t bufsize = 16384;
 218         char output_buffer[bufsize];
 219
 220         ZSTD_inBuffer input = { data, data_size, 0 };
 221         ZSTD_outBuffer output = { output_buffer, bufsize, 0 };
 222
 223         while (input.pos < input.size) {
 224                 size_t ret = ZSTD_compressStream(stream.get(), &output, &input);
 225                 if (ZSTD_isError(ret)) {
 226                         dstream << ZSTD_getErrorName(ret) << std::endl;
 227                         throw SerializationError("compressZstd: failed");
 228                 }
 229                 if (output.pos) {
 230                         os.write(output_buffer, output.pos);
 231                         output.pos = 0;
 232                 }
 233         }
 234
 235         size_t ret;
 236         do {
 237                 ret = ZSTD_endStream(stream.get(), &output);
 238                 if (ZSTD_isError(ret)) {
 239                         dstream << ZSTD_getErrorName(ret) << std::endl;
 240                         throw SerializationError("compressZstd: failed");
 241                 }
 242                 if (output.pos) {
 243                         os.write(output_buffer, output.pos);
 244                         output.pos = 0;
 245                 }
 246         } while (ret != 0);
 247
 248 }
 249
 250 void compressZstd(const std::string &data, std::ostream &os, int level)
 251 {
 252         compressZstd((u8*)data.c_str(), data.size(), os, level);
 253 }
 254
 255 void decompressZstd(std::istream &is, std::ostream &os)
 256 {
 257         // reusing the context is recommended for performance
 258         // it will be destroyed when the thread ends
 259         thread_local std::unique_ptr<ZSTD_DStream, ZSTD_Deleter> stream(ZSTD_createDStream());
 260
 261         ZSTD_initDStream(stream.get());
 262
 263         const size_t bufsize = 16384;
 264         char output_buffer[bufsize];
 265         char input_buffer[bufsize];
 266
 267         ZSTD_outBuffer output = { output_buffer, bufsize, 0 };
 268         ZSTD_inBuffer input = { input_buffer, 0, 0 };
 269         size_t ret;
 270         do
 271         {
 272                 if (input.size == input.pos) {
 273                         is.read(input_buffer, bufsize);
 274                         input.size = is.gcount();
 275                         input.pos = 0;
 276                 }
 277
 278                 ret = ZSTD_decompressStream(stream.get(), &output, &input);
 279                 if (ZSTD_isError(ret)) {
 280                         dstream << ZSTD_getErrorName(ret) << std::endl;
 281                         throw SerializationError("decompressZstd: failed");
 282                 }
 283                 if (output.pos) {
 284                         os.write(output_buffer, output.pos);
 285                         output.pos = 0;
 286                 }
 287         } while (ret != 0);
 288
 289         // Unget all the data that ZSTD_decompressStream didn't take
 290         is.clear(); // Just in case EOF is set
 291         for (u32 i = 0; i < input.size - input.pos; i++) {
 292                 is.unget();
 293                 if (is.fail() || is.bad())
 294                         throw SerializationError("decompressZstd: unget failed");
 295         }
 296 }
 297
 298 void compress(u8 *data, u32 size, std::ostream &os, u8 version, int level)
 299 {
 300         if(version >= 29)
 301         {
 302                 // map the zlib levels [0,9] to [1,10]. -1 becomes 0 which indicates the default (currently 3)
 303                 compressZstd(data, size, os, level + 1);
 304                 return;
 305         }
 306
 307         if(version >= 11)
 308         {
 309                 compressZlib(data, size, os, level);
 310                 return;
 311         }
 312
 313         if(size == 0)
 314                 return;
 315
 316         // Write length (u32)
 317
 318         u8 tmp[4];
 319         writeU32(tmp, size);
 320         os.write((char*)tmp, 4);
 321
 322         // We will be writing 8-bit pairs of more_count and byte
 323         u8 more_count = 0;
 324         u8 current_byte = data[0];
 325         for(u32 i=1; i<size; i++)
 326         {
 327                 if(
 328                         data[i] != current_byte
 329                         || more_count == 255
 330                 )
 331                 {
 332                         // write count and byte
 333                         os.write((char*)&more_count, 1);
 334                         os.write((char*)&current_byte, 1);
 335                         more_count = 0;
 336                         current_byte = data[i];
 337                 }
 338                 else
 339                 {
 340                         more_count++;
 341                 }
 342         }
 343         // write count and byte
 344         os.write((char*)&more_count, 1);
 345         os.write((char*)&current_byte, 1);
 346 }
 347
 348 void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version, int level)
 349 {
 350         compress(*data, data.getSize(), os, version, level);
 351 }
 352
 353 void compress(const std::string &data, std::ostream &os, u8 version, int level)
 354 {
 355         compress((u8*)data.c_str(), data.size(), os, version, level);
 356 }
 357
 358 void decompress(std::istream &is, std::ostream &os, u8 version)
 359 {
 360         if(version >= 29)
 361         {
 362                 decompressZstd(is, os);
 363                 return;
 364         }
 365
 366         if(version >= 11)
 367         {
 368                 decompressZlib(is, os);
 369                 return;
 370         }
 371
 372         // Read length (u32)
 373
 374         u8 tmp[4];
 375         is.read((char*)tmp, 4);
 376         u32 len = readU32(tmp);
 377
 378         // We will be reading 8-bit pairs of more_count and byte
 379         u32 count = 0;
 380         for(;;)
 381         {
 382                 u8 more_count=0;
 383                 u8 byte=0;
 384
 385                 is.read((char*)&more_count, 1);
 386
 387                 is.read((char*)&byte, 1);
 388
 389                 if(is.eof())
 390                         throw SerializationError("decompress: stream ended halfway");
 391
 392                 for(s32 i=0; i<(u16)more_count+1; i++)
 393                         os.write((char*)&byte, 1);
 394
 395                 count += (u16)more_count+1;
 396
 397                 if(count == len)
 398                         break;
 399         }
 400 }
 401
 402