src/serialization.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "serialization.h"
  21
  22 #include "util/serialize.h"
  23
  24 #include <zlib.h>
  25 #include <zstd.h>
  26
  27 /* report a zlib or i/o error */
  28 void zerr(int ret)
  29 {
  30     dstream<<"zerr: ";
  31     switch (ret) {
  32     case Z_ERRNO:
  33         if (ferror(stdin))
  34             dstream<<"error reading stdin"<<std::endl;
  35         if (ferror(stdout))
  36             dstream<<"error writing stdout"<<std::endl;
  37         break;
  38     case Z_STREAM_ERROR:
  39         dstream<<"invalid compression level"<<std::endl;
  40         break;
  41     case Z_DATA_ERROR:
  42         dstream<<"invalid or incomplete deflate data"<<std::endl;
  43         break;
  44     case Z_MEM_ERROR:
  45         dstream<<"out of memory"<<std::endl;
  46         break;
  47     case Z_VERSION_ERROR:
  48         dstream<<"zlib version mismatch!"<<std::endl;
  49                 break;
  50         default:
  51                 dstream<<"return value = "<<ret<<std::endl;
  52     }
  53 }
  54
  55 void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level)
  56 {
  57         z_stream z;
  58         const s32 bufsize = 16384;
  59         char output_buffer[bufsize];
  60         int status = 0;
  61         int ret;
  62
  63         z.zalloc = Z_NULL;
  64         z.zfree = Z_NULL;
  65         z.opaque = Z_NULL;
  66
  67         ret = deflateInit(&z, level);
  68         if(ret != Z_OK)
  69                 throw SerializationError("compressZlib: deflateInit failed");
  70
  71         // Point zlib to our input buffer
  72         z.next_in = (Bytef*)&data[0];
  73         z.avail_in = data_size;
  74         // And get all output
  75         for(;;)
  76         {
  77                 z.next_out = (Bytef*)output_buffer;
  78                 z.avail_out = bufsize;
  79
  80                 status = deflate(&z, Z_FINISH);
  81                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
  82                                 || status == Z_MEM_ERROR)
  83                 {
  84                         zerr(status);
  85                         throw SerializationError("compressZlib: deflate failed");
  86                 }
  87                 int count = bufsize - z.avail_out;
  88                 if(count)
  89                         os.write(output_buffer, count);
  90                 // This determines zlib has given all output
  91                 if(status == Z_STREAM_END)
  92                         break;
  93         }
  94
  95         deflateEnd(&z);
  96 }
  97
  98 void compressZlib(const std::string &data, std::ostream &os, int level)
  99 {
 100         compressZlib((u8*)data.c_str(), data.size(), os, level);
 101 }
 102
 103 void decompressZlib(std::istream &is, std::ostream &os, size_t limit)
 104 {
 105         z_stream z;
 106         const s32 bufsize = 16384;
 107         char input_buffer[bufsize];
 108         char output_buffer[bufsize];
 109         int status = 0;
 110         int ret;
 111         int bytes_read = 0;
 112         int bytes_written = 0;
 113         int input_buffer_len = 0;
 114
 115         z.zalloc = Z_NULL;
 116         z.zfree = Z_NULL;
 117         z.opaque = Z_NULL;
 118
 119         ret = inflateInit(&z);
 120         if(ret != Z_OK)
 121                 throw SerializationError("dcompressZlib: inflateInit failed");
 122
 123         z.avail_in = 0;
 124
 125         //dstream<<"initial fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 126
 127         for(;;)
 128         {
 129                 int output_size = bufsize;
 130                 z.next_out = (Bytef*)output_buffer;
 131                 z.avail_out = output_size;
 132
 133                 if (limit) {
 134                         int limit_remaining = limit - bytes_written;
 135                         if (limit_remaining <= 0) {
 136                                 // we're aborting ahead of time - throw an error?
 137                                 break;
 138                         }
 139                         if (limit_remaining < output_size) {
 140                                 z.avail_out = output_size = limit_remaining;
 141                         }
 142                 }
 143
 144                 if(z.avail_in == 0)
 145                 {
 146                         z.next_in = (Bytef*)input_buffer;
 147                         is.read(input_buffer, bufsize);
 148                         input_buffer_len = is.gcount();
 149                         z.avail_in = input_buffer_len;
 150                         //dstream<<"read fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 151                 }
 152                 if(z.avail_in == 0)
 153                 {
 154                         //dstream<<"z.avail_in == 0"<<std::endl;
 155                         break;
 156                 }
 157
 158                 //dstream<<"1 z.avail_in="<<z.avail_in<<std::endl;
 159                 status = inflate(&z, Z_NO_FLUSH);
 160                 //dstream<<"2 z.avail_in="<<z.avail_in<<std::endl;
 161                 bytes_read += is.gcount() - z.avail_in;
 162                 //dstream<<"bytes_read="<<bytes_read<<std::endl;
 163
 164                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
 165                                 || status == Z_MEM_ERROR)
 166                 {
 167                         zerr(status);
 168                         throw SerializationError("decompressZlib: inflate failed");
 169                 }
 170                 int count = output_size - z.avail_out;
 171                 //dstream<<"count="<<count<<std::endl;
 172                 if(count)
 173                         os.write(output_buffer, count);
 174                 bytes_written += count;
 175                 if(status == Z_STREAM_END)
 176                 {
 177                         //dstream<<"Z_STREAM_END"<<std::endl;
 178
 179                         //dstream<<"z.avail_in="<<z.avail_in<<std::endl;
 180                         //dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 181                         // Unget all the data that inflate didn't take
 182                         is.clear(); // Just in case EOF is set
 183                         for(u32 i=0; i < z.avail_in; i++)
 184                         {
 185                                 is.unget();
 186                                 if(is.fail() || is.bad())
 187                                 {
 188                                         dstream<<"unget #"<<i<<" failed"<<std::endl;
 189                                         dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 190                                         throw SerializationError("decompressZlib: unget failed");
 191                                 }
 192                         }
 193
 194                         break;
 195                 }
 196         }
 197
 198         inflateEnd(&z);
 199 }
 200
 201 struct ZSTD_Deleter {
 202         void operator() (ZSTD_CStream* cstream) {
 203                 ZSTD_freeCStream(cstream);
 204         }
 205
 206         void operator() (ZSTD_DStream* dstream) {
 207                 ZSTD_freeDStream(dstream);
 208         }
 209 };
 210
 211 void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level)
 212 {
 213         // reusing the context is recommended for performance
 214         // it will be destroyed when the thread ends
 215         thread_local std::unique_ptr<ZSTD_CStream, ZSTD_Deleter> stream(ZSTD_createCStream());
 216
 217
 218         ZSTD_initCStream(stream.get(), level);
 219
 220         const size_t bufsize = 16384;
 221         char output_buffer[bufsize];
 222
 223         ZSTD_inBuffer input = { data, data_size, 0 };
 224         ZSTD_outBuffer output = { output_buffer, bufsize, 0 };
 225
 226         while (input.pos < input.size) {
 227                 size_t ret = ZSTD_compressStream(stream.get(), &output, &input);
 228                 if (ZSTD_isError(ret)) {
 229                         dstream << ZSTD_getErrorName(ret) << std::endl;
 230                         throw SerializationError("compressZstd: failed");
 231                 }
 232                 if (output.pos) {
 233                         os.write(output_buffer, output.pos);
 234                         output.pos = 0;
 235                 }
 236         }
 237
 238         size_t ret;
 239         do {
 240                 ret = ZSTD_endStream(stream.get(), &output);
 241                 if (ZSTD_isError(ret)) {
 242                         dstream << ZSTD_getErrorName(ret) << std::endl;
 243                         throw SerializationError("compressZstd: failed");
 244                 }
 245                 if (output.pos) {
 246                         os.write(output_buffer, output.pos);
 247                         output.pos = 0;
 248                 }
 249         } while (ret != 0);
 250
 251 }
 252
 253 void compressZstd(const std::string &data, std::ostream &os, int level)
 254 {
 255         compressZstd((u8*)data.c_str(), data.size(), os, level);
 256 }
 257
 258 void decompressZstd(std::istream &is, std::ostream &os)
 259 {
 260         // reusing the context is recommended for performance
 261         // it will be destroyed when the thread ends
 262         thread_local std::unique_ptr<ZSTD_DStream, ZSTD_Deleter> stream(ZSTD_createDStream());
 263
 264         ZSTD_initDStream(stream.get());
 265
 266         const size_t bufsize = 16384;
 267         char output_buffer[bufsize];
 268         char input_buffer[bufsize];
 269
 270         ZSTD_outBuffer output = { output_buffer, bufsize, 0 };
 271         ZSTD_inBuffer input = { input_buffer, 0, 0 };
 272         size_t ret;
 273         do
 274         {
 275                 if (input.size == input.pos) {
 276                         is.read(input_buffer, bufsize);
 277                         input.size = is.gcount();
 278                         input.pos = 0;
 279                 }
 280
 281                 ret = ZSTD_decompressStream(stream.get(), &output, &input);
 282                 if (ZSTD_isError(ret)) {
 283                         dstream << ZSTD_getErrorName(ret) << std::endl;
 284                         throw SerializationError("decompressZstd: failed");
 285                 }
 286                 if (output.pos) {
 287                         os.write(output_buffer, output.pos);
 288                         output.pos = 0;
 289                 }
 290         } while (ret != 0);
 291
 292         // Unget all the data that ZSTD_decompressStream didn't take
 293         is.clear(); // Just in case EOF is set
 294         for (u32 i = 0; i < input.size - input.pos; i++) {
 295                 is.unget();
 296                 if (is.fail() || is.bad())
 297                         throw SerializationError("decompressZstd: unget failed");
 298         }
 299 }
 300
 301 void compress(u8 *data, u32 size, std::ostream &os, u8 version, int level)
 302 {
 303         if(version >= 29)
 304         {
 305                 // map the zlib levels [0,9] to [1,10]. -1 becomes 0 which indicates the default (currently 3)
 306                 compressZstd(data, size, os, level + 1);
 307                 return;
 308         }
 309
 310         if(version >= 11)
 311         {
 312                 compressZlib(data, size, os, level);
 313                 return;
 314         }
 315
 316         if(size == 0)
 317                 return;
 318
 319         // Write length (u32)
 320
 321         u8 tmp[4];
 322         writeU32(tmp, size);
 323         os.write((char*)tmp, 4);
 324
 325         // We will be writing 8-bit pairs of more_count and byte
 326         u8 more_count = 0;
 327         u8 current_byte = data[0];
 328         for(u32 i=1; i<size; i++)
 329         {
 330                 if(
 331                         data[i] != current_byte
 332                         || more_count == 255
 333                 )
 334                 {
 335                         // write count and byte
 336                         os.write((char*)&more_count, 1);
 337                         os.write((char*)&current_byte, 1);
 338                         more_count = 0;
 339                         current_byte = data[i];
 340                 }
 341                 else
 342                 {
 343                         more_count++;
 344                 }
 345         }
 346         // write count and byte
 347         os.write((char*)&more_count, 1);
 348         os.write((char*)&current_byte, 1);
 349 }
 350
 351 void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version, int level)
 352 {
 353         compress(*data, data.getSize(), os, version, level);
 354 }
 355
 356 void compress(const std::string &data, std::ostream &os, u8 version, int level)
 357 {
 358         compress((u8*)data.c_str(), data.size(), os, version, level);
 359 }
 360
 361 void decompress(std::istream &is, std::ostream &os, u8 version)
 362 {
 363         if(version >= 29)
 364         {
 365                 decompressZstd(is, os);
 366                 return;
 367         }
 368
 369         if(version >= 11)
 370         {
 371                 decompressZlib(is, os);
 372                 return;
 373         }
 374
 375         // Read length (u32)
 376
 377         u8 tmp[4];
 378         is.read((char*)tmp, 4);
 379         u32 len = readU32(tmp);
 380
 381         // We will be reading 8-bit pairs of more_count and byte
 382         u32 count = 0;
 383         for(;;)
 384         {
 385                 u8 more_count=0;
 386                 u8 byte=0;
 387
 388                 is.read((char*)&more_count, 1);
 389
 390                 is.read((char*)&byte, 1);
 391
 392                 if(is.eof())
 393                         throw SerializationError("decompress: stream ended halfway");
 394
 395                 for(s32 i=0; i<(u16)more_count+1; i++)
 396                         os.write((char*)&byte, 1);
 397
 398                 count += (u16)more_count+1;
 399
 400                 if(count == len)
 401                         break;
 402         }
 403 }
 404
 405