src/serialization.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "serialization.h"
  21
  22 #include "util/serialize.h"
  23
  24 #include <zlib.h>
  25 #include <zstd.h>
  26
  27 /* report a zlib or i/o error */
  28 void zerr(int ret)
  29 {
  30     dstream<<"zerr: ";
  31     switch (ret) {
  32     case Z_ERRNO:
  33         if (ferror(stdin))
  34             dstream<<"error reading stdin"<<std::endl;
  35         if (ferror(stdout))
  36             dstream<<"error writing stdout"<<std::endl;
  37         break;
  38     case Z_STREAM_ERROR:
  39         dstream<<"invalid compression level"<<std::endl;
  40         break;
  41     case Z_DATA_ERROR:
  42         dstream<<"invalid or incomplete deflate data"<<std::endl;
  43         break;
  44     case Z_MEM_ERROR:
  45         dstream<<"out of memory"<<std::endl;
  46         break;
  47     case Z_VERSION_ERROR:
  48         dstream<<"zlib version mismatch!"<<std::endl;
  49                 break;
  50         default:
  51                 dstream<<"return value = "<<ret<<std::endl;
  52     }
  53 }
  54
  55 void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level)
  56 {
  57         z_stream z;
  58         const s32 bufsize = 16384;
  59         char output_buffer[bufsize];
  60         int status = 0;
  61         int ret;
  62
  63         z.zalloc = Z_NULL;
  64         z.zfree = Z_NULL;
  65         z.opaque = Z_NULL;
  66
  67         ret = deflateInit(&z, level);
  68         if(ret != Z_OK)
  69                 throw SerializationError("compressZlib: deflateInit failed");
  70
  71         // Point zlib to our input buffer
  72         z.next_in = (Bytef*)&data[0];
  73         z.avail_in = data_size;
  74         // And get all output
  75         for(;;)
  76         {
  77                 z.next_out = (Bytef*)output_buffer;
  78                 z.avail_out = bufsize;
  79
  80                 status = deflate(&z, Z_FINISH);
  81                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
  82                                 || status == Z_MEM_ERROR)
  83                 {
  84                         zerr(status);
  85                         throw SerializationError("compressZlib: deflate failed");
  86                 }
  87                 int count = bufsize - z.avail_out;
  88                 if(count)
  89                         os.write(output_buffer, count);
  90                 // This determines zlib has given all output
  91                 if(status == Z_STREAM_END)
  92                         break;
  93         }
  94
  95         deflateEnd(&z);
  96 }
  97
  98 void compressZlib(const std::string &data, std::ostream &os, int level)
  99 {
 100         compressZlib((u8*)data.c_str(), data.size(), os, level);
 101 }
 102
 103 void decompressZlib(std::istream &is, std::ostream &os, size_t limit)
 104 {
 105         z_stream z;
 106         const s32 bufsize = 16384;
 107         char input_buffer[bufsize];
 108         char output_buffer[bufsize];
 109         int status = 0;
 110         int ret;
 111         int bytes_written = 0;
 112         int input_buffer_len = 0;
 113
 114         z.zalloc = Z_NULL;
 115         z.zfree = Z_NULL;
 116         z.opaque = Z_NULL;
 117
 118         ret = inflateInit(&z);
 119         if(ret != Z_OK)
 120                 throw SerializationError("dcompressZlib: inflateInit failed");
 121
 122         z.avail_in = 0;
 123
 124         for(;;)
 125         {
 126                 int output_size = bufsize;
 127                 z.next_out = (Bytef*)output_buffer;
 128                 z.avail_out = output_size;
 129
 130                 if (limit) {
 131                         int limit_remaining = limit - bytes_written;
 132                         if (limit_remaining <= 0) {
 133                                 // we're aborting ahead of time - throw an error?
 134                                 break;
 135                         }
 136                         if (limit_remaining < output_size) {
 137                                 z.avail_out = output_size = limit_remaining;
 138                         }
 139                 }
 140
 141                 if(z.avail_in == 0)
 142                 {
 143                         z.next_in = (Bytef*)input_buffer;
 144                         is.read(input_buffer, bufsize);
 145                         input_buffer_len = is.gcount();
 146                         z.avail_in = input_buffer_len;
 147                 }
 148                 if(z.avail_in == 0)
 149                 {
 150                         break;
 151                 }
 152
 153                 status = inflate(&z, Z_NO_FLUSH);
 154
 155                 if(status == Z_NEED_DICT || status == Z_DATA_ERROR
 156                                 || status == Z_MEM_ERROR)
 157                 {
 158                         zerr(status);
 159                         throw SerializationError("decompressZlib: inflate failed");
 160                 }
 161                 int count = output_size - z.avail_out;
 162                 if(count)
 163                         os.write(output_buffer, count);
 164                 bytes_written += count;
 165                 if(status == Z_STREAM_END)
 166                 {
 167                         // Unget all the data that inflate didn't take
 168                         is.clear(); // Just in case EOF is set
 169                         for(u32 i=0; i < z.avail_in; i++)
 170                         {
 171                                 is.unget();
 172                                 if(is.fail() || is.bad())
 173                                 {
 174                                         dstream<<"unget #"<<i<<" failed"<<std::endl;
 175                                         dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 176                                         throw SerializationError("decompressZlib: unget failed");
 177                                 }
 178                         }
 179
 180                         break;
 181                 }
 182         }
 183
 184         inflateEnd(&z);
 185 }
 186
 187 struct ZSTD_Deleter {
 188         void operator() (ZSTD_CStream* cstream) {
 189                 ZSTD_freeCStream(cstream);
 190         }
 191
 192         void operator() (ZSTD_DStream* dstream) {
 193                 ZSTD_freeDStream(dstream);
 194         }
 195 };
 196
 197 void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level)
 198 {
 199         // reusing the context is recommended for performance
 200         // it will be destroyed when the thread ends
 201         thread_local std::unique_ptr<ZSTD_CStream, ZSTD_Deleter> stream(ZSTD_createCStream());
 202
 203
 204         ZSTD_initCStream(stream.get(), level);
 205
 206         const size_t bufsize = 16384;
 207         char output_buffer[bufsize];
 208
 209         ZSTD_inBuffer input = { data, data_size, 0 };
 210         ZSTD_outBuffer output = { output_buffer, bufsize, 0 };
 211
 212         while (input.pos < input.size) {
 213                 size_t ret = ZSTD_compressStream(stream.get(), &output, &input);
 214                 if (ZSTD_isError(ret)) {
 215                         dstream << ZSTD_getErrorName(ret) << std::endl;
 216                         throw SerializationError("compressZstd: failed");
 217                 }
 218                 if (output.pos) {
 219                         os.write(output_buffer, output.pos);
 220                         output.pos = 0;
 221                 }
 222         }
 223
 224         size_t ret;
 225         do {
 226                 ret = ZSTD_endStream(stream.get(), &output);
 227                 if (ZSTD_isError(ret)) {
 228                         dstream << ZSTD_getErrorName(ret) << std::endl;
 229                         throw SerializationError("compressZstd: failed");
 230                 }
 231                 if (output.pos) {
 232                         os.write(output_buffer, output.pos);
 233                         output.pos = 0;
 234                 }
 235         } while (ret != 0);
 236
 237 }
 238
 239 void compressZstd(const std::string &data, std::ostream &os, int level)
 240 {
 241         compressZstd((u8*)data.c_str(), data.size(), os, level);
 242 }
 243
 244 void decompressZstd(std::istream &is, std::ostream &os)
 245 {
 246         // reusing the context is recommended for performance
 247         // it will be destroyed when the thread ends
 248         thread_local std::unique_ptr<ZSTD_DStream, ZSTD_Deleter> stream(ZSTD_createDStream());
 249
 250         ZSTD_initDStream(stream.get());
 251
 252         const size_t bufsize = 16384;
 253         char output_buffer[bufsize];
 254         char input_buffer[bufsize];
 255
 256         ZSTD_outBuffer output = { output_buffer, bufsize, 0 };
 257         ZSTD_inBuffer input = { input_buffer, 0, 0 };
 258         size_t ret;
 259         do
 260         {
 261                 if (input.size == input.pos) {
 262                         is.read(input_buffer, bufsize);
 263                         input.size = is.gcount();
 264                         input.pos = 0;
 265                 }
 266
 267                 ret = ZSTD_decompressStream(stream.get(), &output, &input);
 268                 if (ZSTD_isError(ret)) {
 269                         dstream << ZSTD_getErrorName(ret) << std::endl;
 270                         throw SerializationError("decompressZstd: failed");
 271                 }
 272                 if (output.pos) {
 273                         os.write(output_buffer, output.pos);
 274                         output.pos = 0;
 275                 }
 276         } while (ret != 0);
 277
 278         // Unget all the data that ZSTD_decompressStream didn't take
 279         is.clear(); // Just in case EOF is set
 280         for (u32 i = 0; i < input.size - input.pos; i++) {
 281                 is.unget();
 282                 if (is.fail() || is.bad())
 283                         throw SerializationError("decompressZstd: unget failed");
 284         }
 285 }
 286
 287 void compress(u8 *data, u32 size, std::ostream &os, u8 version, int level)
 288 {
 289         if(version >= 29)
 290         {
 291                 // map the zlib levels [0,9] to [1,10]. -1 becomes 0 which indicates the default (currently 3)
 292                 compressZstd(data, size, os, level + 1);
 293                 return;
 294         }
 295
 296         if(version >= 11)
 297         {
 298                 compressZlib(data, size, os, level);
 299                 return;
 300         }
 301
 302         if(size == 0)
 303                 return;
 304
 305         // Write length (u32)
 306
 307         u8 tmp[4];
 308         writeU32(tmp, size);
 309         os.write((char*)tmp, 4);
 310
 311         // We will be writing 8-bit pairs of more_count and byte
 312         u8 more_count = 0;
 313         u8 current_byte = data[0];
 314         for(u32 i=1; i<size; i++)
 315         {
 316                 if(
 317                         data[i] != current_byte
 318                         || more_count == 255
 319                 )
 320                 {
 321                         // write count and byte
 322                         os.write((char*)&more_count, 1);
 323                         os.write((char*)&current_byte, 1);
 324                         more_count = 0;
 325                         current_byte = data[i];
 326                 }
 327                 else
 328                 {
 329                         more_count++;
 330                 }
 331         }
 332         // write count and byte
 333         os.write((char*)&more_count, 1);
 334         os.write((char*)&current_byte, 1);
 335 }
 336
 337 void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version, int level)
 338 {
 339         compress(*data, data.getSize(), os, version, level);
 340 }
 341
 342 void compress(const std::string &data, std::ostream &os, u8 version, int level)
 343 {
 344         compress((u8*)data.c_str(), data.size(), os, version, level);
 345 }
 346
 347 void decompress(std::istream &is, std::ostream &os, u8 version)
 348 {
 349         if(version >= 29)
 350         {
 351                 decompressZstd(is, os);
 352                 return;
 353         }
 354
 355         if(version >= 11)
 356         {
 357                 decompressZlib(is, os);
 358                 return;
 359         }
 360
 361         // Read length (u32)
 362
 363         u8 tmp[4];
 364         is.read((char*)tmp, 4);
 365         u32 len = readU32(tmp);
 366
 367         // We will be reading 8-bit pairs of more_count and byte
 368         u32 count = 0;
 369         for(;;)
 370         {
 371                 u8 more_count=0;
 372                 u8 byte=0;
 373
 374                 is.read((char*)&more_count, 1);
 375
 376                 is.read((char*)&byte, 1);
 377
 378                 if(is.eof())
 379                         throw SerializationError("decompress: stream ended halfway");
 380
 381                 for(s32 i=0; i<(u16)more_count+1; i++)
 382                         os.write((char*)&byte, 1);
 383
 384                 count += (u16)more_count+1;
 385
 386                 if(count == len)
 387                         break;
 388         }
 389 }
 390
 391