src/util/serialize.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2010-2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "serialize.h"
  21 #include "pointer.h"
  22 #include "porting.h"
  23 #include "util/string.h"
  24 #include "exceptions.h"
  25 #include "irrlichttypes.h"
  26
  27 #include <sstream>
  28 #include <iomanip>
  29 #include <vector>
  30
  31 FloatType g_serialize_f32_type = FLOATTYPE_UNKNOWN;
  32
  33 ////
  34 //// BufReader
  35 ////
  36
  37 bool BufReader::getStringNoEx(std::string *val)
  38 {
  39         u16 num_chars;
  40         if (!getU16NoEx(&num_chars))
  41                 return false;
  42
  43         if (pos + num_chars > size) {
  44                 pos -= sizeof(num_chars);
  45                 return false;
  46         }
  47
  48         val->assign((const char *)data + pos, num_chars);
  49         pos += num_chars;
  50
  51         return true;
  52 }
  53
  54 bool BufReader::getWideStringNoEx(std::wstring *val)
  55 {
  56         u16 num_chars;
  57         if (!getU16NoEx(&num_chars))
  58                 return false;
  59
  60         if (pos + num_chars * 2 > size) {
  61                 pos -= sizeof(num_chars);
  62                 return false;
  63         }
  64
  65         for (size_t i = 0; i != num_chars; i++) {
  66                 val->push_back(readU16(data + pos));
  67                 pos += 2;
  68         }
  69
  70         return true;
  71 }
  72
  73 bool BufReader::getLongStringNoEx(std::string *val)
  74 {
  75         u32 num_chars;
  76         if (!getU32NoEx(&num_chars))
  77                 return false;
  78
  79         if (pos + num_chars > size) {
  80                 pos -= sizeof(num_chars);
  81                 return false;
  82         }
  83
  84         val->assign((const char *)data + pos, num_chars);
  85         pos += num_chars;
  86
  87         return true;
  88 }
  89
  90 bool BufReader::getRawDataNoEx(void *val, size_t len)
  91 {
  92         if (pos + len > size)
  93                 return false;
  94
  95         memcpy(val, data + pos, len);
  96         pos += len;
  97
  98         return true;
  99 }
 100
 101
 102 ////
 103 //// String
 104 ////
 105
 106 std::string serializeString(const std::string &plain)
 107 {
 108         std::string s;
 109         char buf[2];
 110
 111         if (plain.size() > STRING_MAX_LEN)
 112                 throw SerializationError("String too long for serializeString");
 113
 114         writeU16((u8 *)&buf[0], plain.size());
 115         s.append(buf, 2);
 116
 117         s.append(plain);
 118         return s;
 119 }
 120
 121 std::string deSerializeString(std::istream &is)
 122 {
 123         std::string s;
 124         char buf[2];
 125
 126         is.read(buf, 2);
 127         if (is.gcount() != 2)
 128                 throw SerializationError("deSerializeString: size not read");
 129
 130         u16 s_size = readU16((u8 *)buf);
 131         if (s_size == 0)
 132                 return s;
 133
 134         Buffer<char> buf2(s_size);
 135         is.read(&buf2[0], s_size);
 136         if (is.gcount() != s_size)
 137                 throw SerializationError("deSerializeString: couldn't read all chars");
 138
 139         s.reserve(s_size);
 140         s.append(&buf2[0], s_size);
 141         return s;
 142 }
 143
 144 ////
 145 //// Wide String
 146 ////
 147
 148 std::string serializeWideString(const std::wstring &plain)
 149 {
 150         std::string s;
 151         char buf[2];
 152
 153         if (plain.size() > WIDE_STRING_MAX_LEN)
 154                 throw SerializationError("String too long for serializeWideString");
 155
 156         writeU16((u8 *)buf, plain.size());
 157         s.append(buf, 2);
 158
 159         for (wchar_t i : plain) {
 160                 writeU16((u8 *)buf, i);
 161                 s.append(buf, 2);
 162         }
 163         return s;
 164 }
 165
 166 std::wstring deSerializeWideString(std::istream &is)
 167 {
 168         std::wstring s;
 169         char buf[2];
 170
 171         is.read(buf, 2);
 172         if (is.gcount() != 2)
 173                 throw SerializationError("deSerializeWideString: size not read");
 174
 175         u16 s_size = readU16((u8 *)buf);
 176         if (s_size == 0)
 177                 return s;
 178
 179         s.reserve(s_size);
 180         for (u32 i = 0; i < s_size; i++) {
 181                 is.read(&buf[0], 2);
 182                 if (is.gcount() != 2) {
 183                         throw SerializationError(
 184                                 "deSerializeWideString: couldn't read all chars");
 185                 }
 186
 187                 wchar_t c16 = readU16((u8 *)buf);
 188                 s.append(&c16, 1);
 189         }
 190         return s;
 191 }
 192
 193 ////
 194 //// Long String
 195 ////
 196
 197 std::string serializeLongString(const std::string &plain)
 198 {
 199         char buf[4];
 200
 201         if (plain.size() > LONG_STRING_MAX_LEN)
 202                 throw SerializationError("String too long for serializeLongString");
 203
 204         writeU32((u8*)&buf[0], plain.size());
 205         std::string s;
 206         s.append(buf, 4);
 207         s.append(plain);
 208         return s;
 209 }
 210
 211 std::string deSerializeLongString(std::istream &is)
 212 {
 213         std::string s;
 214         char buf[4];
 215
 216         is.read(buf, 4);
 217         if (is.gcount() != 4)
 218                 throw SerializationError("deSerializeLongString: size not read");
 219
 220         u32 s_size = readU32((u8 *)buf);
 221         if (s_size == 0)
 222                 return s;
 223
 224         // We don't really want a remote attacker to force us to allocate 4GB...
 225         if (s_size > LONG_STRING_MAX_LEN) {
 226                 throw SerializationError("deSerializeLongString: "
 227                         "string too long: " + itos(s_size) + " bytes");
 228         }
 229
 230         Buffer<char> buf2(s_size);
 231         is.read(&buf2[0], s_size);
 232         if ((u32)is.gcount() != s_size)
 233                 throw SerializationError("deSerializeLongString: couldn't read all chars");
 234
 235         s.reserve(s_size);
 236         s.append(&buf2[0], s_size);
 237         return s;
 238 }
 239
 240 ////
 241 //// JSON
 242 ////
 243
 244 std::string serializeJsonString(const std::string &plain)
 245 {
 246         std::ostringstream os(std::ios::binary);
 247         os << "\"";
 248
 249         for (char c : plain) {
 250                 switch (c) {
 251                         case '"':
 252                                 os << "\\\"";
 253                                 break;
 254                         case '\\':
 255                                 os << "\\\\";
 256                                 break;
 257                         case '/':
 258                                 os << "\\/";
 259                                 break;
 260                         case '\b':
 261                                 os << "\\b";
 262                                 break;
 263                         case '\f':
 264                                 os << "\\f";
 265                                 break;
 266                         case '\n':
 267                                 os << "\\n";
 268                                 break;
 269                         case '\r':
 270                                 os << "\\r";
 271                                 break;
 272                         case '\t':
 273                                 os << "\\t";
 274                                 break;
 275                         default: {
 276                                 if (c >= 32 && c <= 126) {
 277                                         os << c;
 278                                 } else {
 279                                         u32 cnum = (u8)c;
 280                                         os << "\\u" << std::hex << std::setw(4)
 281                                                 << std::setfill('0') << cnum;
 282                                 }
 283                                 break;
 284                         }
 285                 }
 286         }
 287
 288         os << "\"";
 289         return os.str();
 290 }
 291
 292 std::string deSerializeJsonString(std::istream &is)
 293 {
 294         std::ostringstream os(std::ios::binary);
 295         char c, c2;
 296
 297         // Parse initial doublequote
 298         is >> c;
 299         if (c != '"')
 300                 throw SerializationError("JSON string must start with doublequote");
 301
 302         // Parse characters
 303         for (;;) {
 304                 c = is.get();
 305                 if (is.eof())
 306                         throw SerializationError("JSON string ended prematurely");
 307
 308                 if (c == '"') {
 309                         return os.str();
 310                 }
 311
 312                 if (c == '\\') {
 313                         c2 = is.get();
 314                         if (is.eof())
 315                                 throw SerializationError("JSON string ended prematurely");
 316                         switch (c2) {
 317                                 case 'b':
 318                                         os << '\b';
 319                                         break;
 320                                 case 'f':
 321                                         os << '\f';
 322                                         break;
 323                                 case 'n':
 324                                         os << '\n';
 325                                         break;
 326                                 case 'r':
 327                                         os << '\r';
 328                                         break;
 329                                 case 't':
 330                                         os << '\t';
 331                                         break;
 332                                 case 'u': {
 333                                         int hexnumber;
 334                                         char hexdigits[4 + 1];
 335
 336                                         is.read(hexdigits, 4);
 337                                         if (is.eof())
 338                                                 throw SerializationError("JSON string ended prematurely");
 339                                         hexdigits[4] = 0;
 340
 341                                         std::istringstream tmp_is(hexdigits, std::ios::binary);
 342                                         tmp_is >> std::hex >> hexnumber;
 343                                         os << (char)hexnumber;
 344                                         break;
 345                                 }
 346                                 default:
 347                                         os << c2;
 348                                         break;
 349                         }
 350                 } else {
 351                         os << c;
 352                 }
 353         }
 354
 355         return os.str();
 356 }
 357
 358 std::string serializeJsonStringIfNeeded(const std::string &s)
 359 {
 360         for (size_t i = 0; i < s.size(); ++i) {
 361                 if (s[i] <= 0x1f || s[i] >= 0x7f || s[i] == ' ' || s[i] == '\"')
 362                         return serializeJsonString(s);
 363         }
 364         return s;
 365 }
 366
 367 std::string deSerializeJsonStringIfNeeded(std::istream &is)
 368 {
 369         std::ostringstream tmp_os;
 370         bool expect_initial_quote = true;
 371         bool is_json = false;
 372         bool was_backslash = false;
 373         for (;;) {
 374                 char c = is.get();
 375                 if (is.eof())
 376                         break;
 377
 378                 if (expect_initial_quote && c == '"') {
 379                         tmp_os << c;
 380                         is_json = true;
 381                 } else if(is_json) {
 382                         tmp_os << c;
 383                         if (was_backslash)
 384                                 was_backslash = false;
 385                         else if (c == '\\')
 386                                 was_backslash = true;
 387                         else if (c == '"')
 388                                 break; // Found end of string
 389                 } else {
 390                         if (c == ' ') {
 391                                 // Found end of word
 392                                 is.unget();
 393                                 break;
 394                         }
 395
 396                         tmp_os << c;
 397                 }
 398                 expect_initial_quote = false;
 399         }
 400         if (is_json) {
 401                 std::istringstream tmp_is(tmp_os.str(), std::ios::binary);
 402                 return deSerializeJsonString(tmp_is);
 403         }
 404
 405         return tmp_os.str();
 406 }
 407
 408 ////
 409 //// String/Struct conversions
 410 ////
 411
 412 bool deSerializeStringToStruct(std::string valstr,
 413         std::string format, void *out, size_t olen)
 414 {
 415         size_t len = olen;
 416         std::vector<std::string *> strs_alloced;
 417         std::string *str;
 418         char *f, *snext;
 419         size_t pos;
 420
 421         char *s = &valstr[0];
 422         char *buf = new char[len];
 423         char *bufpos = buf;
 424
 425         char *fmtpos, *fmt = &format[0];
 426         while ((f = strtok_r(fmt, ",", &fmtpos)) && s) {
 427                 fmt = nullptr;
 428
 429                 bool is_unsigned = false;
 430                 int width = 0;
 431                 char valtype = *f;
 432
 433                 width = (int)strtol(f + 1, &f, 10);
 434                 if (width && valtype == 's')
 435                         valtype = 'i';
 436
 437                 switch (valtype) {
 438                         case 'u':
 439                                 is_unsigned = true;
 440                                 /* FALLTHROUGH */
 441                         case 'i':
 442                                 if (width == 16) {
 443                                         bufpos += PADDING(bufpos, u16);
 444                                         if ((bufpos - buf) + sizeof(u16) <= len) {
 445                                                 if (is_unsigned)
 446                                                         *(u16 *)bufpos = (u16)strtoul(s, &s, 10);
 447                                                 else
 448                                                         *(s16 *)bufpos = (s16)strtol(s, &s, 10);
 449                                         }
 450                                         bufpos += sizeof(u16);
 451                                 } else if (width == 32) {
 452                                         bufpos += PADDING(bufpos, u32);
 453                                         if ((bufpos - buf) + sizeof(u32) <= len) {
 454                                                 if (is_unsigned)
 455                                                         *(u32 *)bufpos = (u32)strtoul(s, &s, 10);
 456                                                 else
 457                                                         *(s32 *)bufpos = (s32)strtol(s, &s, 10);
 458                                         }
 459                                         bufpos += sizeof(u32);
 460                                 } else if (width == 64) {
 461                                         bufpos += PADDING(bufpos, u64);
 462                                         if ((bufpos - buf) + sizeof(u64) <= len) {
 463                                                 if (is_unsigned)
 464                                                         *(u64 *)bufpos = (u64)strtoull(s, &s, 10);
 465                                                 else
 466                                                         *(s64 *)bufpos = (s64)strtoll(s, &s, 10);
 467                                         }
 468                                         bufpos += sizeof(u64);
 469                                 }
 470                                 s = strchr(s, ',');
 471                                 break;
 472                         case 'b':
 473                                 snext = strchr(s, ',');
 474                                 if (snext)
 475                                         *snext++ = 0;
 476
 477                                 bufpos += PADDING(bufpos, bool);
 478                                 if ((bufpos - buf) + sizeof(bool) <= len)
 479                                         *(bool *)bufpos = is_yes(std::string(s));
 480                                 bufpos += sizeof(bool);
 481
 482                                 s = snext;
 483                                 break;
 484                         case 'f':
 485                                 bufpos += PADDING(bufpos, float);
 486                                 if ((bufpos - buf) + sizeof(float) <= len)
 487                                         *(float *)bufpos = strtof(s, &s);
 488                                 bufpos += sizeof(float);
 489
 490                                 s = strchr(s, ',');
 491                                 break;
 492                         case 's':
 493                                 while (*s == ' ' || *s == '\t')
 494                                         s++;
 495                                 if (*s++ != '"') //error, expected string
 496                                         goto fail;
 497                                 snext = s;
 498
 499                                 while (snext[0] && !(snext[-1] != '\\' && snext[0] == '"'))
 500                                         snext++;
 501                                 *snext++ = 0;
 502
 503                                 bufpos += PADDING(bufpos, std::string *);
 504
 505                                 str = new std::string(s);
 506                                 pos = 0;
 507                                 while ((pos = str->find("\\\"", pos)) != std::string::npos)
 508                                         str->erase(pos, 1);
 509
 510                                 if ((bufpos - buf) + sizeof(std::string *) <= len)
 511                                         *(std::string **)bufpos = str;
 512                                 bufpos += sizeof(std::string *);
 513                                 strs_alloced.push_back(str);
 514
 515                                 s = *snext ? snext + 1 : nullptr;
 516                                 break;
 517                         case 'v':
 518                                 while (*s == ' ' || *s == '\t')
 519                                         s++;
 520                                 if (*s++ != '(') //error, expected vector
 521                                         goto fail;
 522
 523                                 if (width == 2) {
 524                                         bufpos += PADDING(bufpos, v2f);
 525
 526                                         if ((bufpos - buf) + sizeof(v2f) <= len) {
 527                                         v2f *v = (v2f *)bufpos;
 528                                                 v->X = strtof(s, &s);
 529                                                 s++;
 530                                                 v->Y = strtof(s, &s);
 531                                         }
 532
 533                                         bufpos += sizeof(v2f);
 534                                 } else if (width == 3) {
 535                                         bufpos += PADDING(bufpos, v3f);
 536                                         if ((bufpos - buf) + sizeof(v3f) <= len) {
 537                                                 v3f *v = (v3f *)bufpos;
 538                                                 v->X = strtof(s, &s);
 539                                                 s++;
 540                                                 v->Y = strtof(s, &s);
 541                                                 s++;
 542                                                 v->Z = strtof(s, &s);
 543                                         }
 544
 545                                         bufpos += sizeof(v3f);
 546                                 }
 547                                 s = strchr(s, ',');
 548                                 break;
 549                         default: //error, invalid format specifier
 550                                 goto fail;
 551                 }
 552
 553                 if (s && *s == ',')
 554                         s++;
 555
 556                 if ((size_t)(bufpos - buf) > len) //error, buffer too small
 557                         goto fail;
 558         }
 559
 560         if (f && *f) { //error, mismatched number of fields and values
 561 fail:
 562                 for (size_t i = 0; i != strs_alloced.size(); i++)
 563                         delete strs_alloced[i];
 564                 delete[] buf;
 565                 return false;
 566         }
 567
 568         memcpy(out, buf, olen);
 569         delete[] buf;
 570         return true;
 571 }
 572
 573 // Casts *buf to a signed or unsigned fixed-width integer of 'w' width
 574 #define SIGN_CAST(w, buf) (is_unsigned ? *((u##w *) buf) : *((s##w *) buf))
 575
 576 bool serializeStructToString(std::string *out,
 577         std::string format, void *value)
 578 {
 579         std::ostringstream os;
 580         std::string str;
 581         char *f;
 582         size_t strpos;
 583
 584         char *bufpos = (char *) value;
 585         char *fmtpos, *fmt = &format[0];
 586         while ((f = strtok_r(fmt, ",", &fmtpos))) {
 587                 fmt = nullptr;
 588                 bool is_unsigned = false;
 589                 int width = 0;
 590                 char valtype = *f;
 591
 592                 width = (int)strtol(f + 1, &f, 10);
 593                 if (width && valtype == 's')
 594                         valtype = 'i';
 595
 596                 switch (valtype) {
 597                         case 'u':
 598                                 is_unsigned = true;
 599                                 /* FALLTHROUGH */
 600                         case 'i':
 601                                 if (width == 16) {
 602                                         bufpos += PADDING(bufpos, u16);
 603                                         os << SIGN_CAST(16, bufpos);
 604                                         bufpos += sizeof(u16);
 605                                 } else if (width == 32) {
 606                                         bufpos += PADDING(bufpos, u32);
 607                                         os << SIGN_CAST(32, bufpos);
 608                                         bufpos += sizeof(u32);
 609                                 } else if (width == 64) {
 610                                         bufpos += PADDING(bufpos, u64);
 611                                         os << SIGN_CAST(64, bufpos);
 612                                         bufpos += sizeof(u64);
 613                                 }
 614                                 break;
 615                         case 'b':
 616                                 bufpos += PADDING(bufpos, bool);
 617                                 os << std::boolalpha << *((bool *) bufpos);
 618                                 bufpos += sizeof(bool);
 619                                 break;
 620                         case 'f':
 621                                 bufpos += PADDING(bufpos, float);
 622                                 os << *((float *) bufpos);
 623                                 bufpos += sizeof(float);
 624                                 break;
 625                         case 's':
 626                                 bufpos += PADDING(bufpos, std::string *);
 627                                 str = **((std::string **) bufpos);
 628
 629                                 strpos = 0;
 630                                 while ((strpos = str.find('"', strpos)) != std::string::npos) {
 631                                         str.insert(strpos, 1, '\\');
 632                                         strpos += 2;
 633                                 }
 634
 635                                 os << str;
 636                                 bufpos += sizeof(std::string *);
 637                                 break;
 638                         case 'v':
 639                                 if (width == 2) {
 640                                         bufpos += PADDING(bufpos, v2f);
 641                                         v2f *v = (v2f *) bufpos;
 642                                         os << '(' << v->X << ", " << v->Y << ')';
 643                                         bufpos += sizeof(v2f);
 644                                 } else {
 645                                         bufpos += PADDING(bufpos, v3f);
 646                                         v3f *v = (v3f *) bufpos;
 647                                         os << '(' << v->X << ", " << v->Y << ", " << v->Z << ')';
 648                                         bufpos += sizeof(v3f);
 649                                 }
 650                                 break;
 651                         default:
 652                                 return false;
 653                 }
 654                 os << ", ";
 655         }
 656         *out = os.str();
 657
 658         // Trim off the trailing comma and space
 659         if (out->size() >= 2)
 660                 out->resize(out->size() - 2);
 661
 662         return true;
 663 }
 664
 665 #undef SIGN_CAST
 666
 667 ////
 668 //// Other
 669 ////
 670
 671 std::string serializeHexString(const std::string &data, bool insert_spaces)
 672 {
 673         std::string result;
 674         result.reserve(data.size() * (2 + insert_spaces));
 675
 676         static const char hex_chars[] = "0123456789abcdef";
 677
 678         const size_t len = data.size();
 679         for (size_t i = 0; i != len; i++) {
 680                 u8 byte = data[i];
 681                 result.push_back(hex_chars[(byte >> 4) & 0x0F]);
 682                 result.push_back(hex_chars[(byte >> 0) & 0x0F]);
 683                 if (insert_spaces && i != len - 1)
 684                         result.push_back(' ');
 685         }
 686
 687         return result;
 688 }