util/sectors2sqlite.py

   1 #!/usr/bin/python3
   2
   3 # Loads block files from sectors folders into map.sqlite database.
   4 # The sectors folder should be safe to remove after this prints "Finished."
   5
   6 import time, os
   7
   8 try:
   9         import sqlite3
  10 except:
  11         exit('You need to have the Python sqlite3 module.')
  12
  13 path = "../world/"
  14
  15 paths = []
  16
  17 # sectors2 gets to try first
  18 if os.path.isdir(path + 'sectors2/'):
  19         paths.append('sectors2')
  20 if os.path.isdir(path + 'sectors/'):
  21         paths.append('sectors')
  22
  23 if not paths:
  24         exit('Could not find sectors folder at ' + path + 'sectors2/ or ' + path + 'sectors/')
  25
  26 def uint(u):
  27         u = int('0x'+u, 16)
  28         return (u if u < 2**11 else u - 2**12)
  29
  30 def int64(u):
  31         while u >= 2**63:
  32                 u -= 2**64
  33         while u <= -2**63:
  34                 u += 2**64
  35         return u
  36
  37 # Convert sector folder(s) to integer
  38 def getSectorPos(dirname):
  39         if len(dirname) == 8:
  40                 # Old layout
  41                 x = uint(dirname[:4])
  42                 z = uint(dirname[4:])
  43         elif len(dirname) == 7:
  44                 # New layout
  45                 x = uint(dirname[:3])
  46                 z = uint(dirname[4:])
  47         else:
  48                 print('Terrible sector at ' + dirname)
  49                 return
  50
  51         return x, z
  52
  53 # Convert block file to integer position
  54 def getBlockPos(sectordir, blockfile):
  55         p2d = getSectorPos(sectordir)
  56
  57         if not p2d:
  58                 return
  59
  60         if len(blockfile) != 4:
  61                 print("Invalid block filename: " + blockfile)
  62
  63         y = uint(blockfile)
  64
  65         return p2d[0], y, p2d[1]
  66
  67 # Convert location to integer
  68 def getBlockAsInteger(p):
  69         return int64(p[2]*16777216 + p[1]*4096 + p[0])
  70
  71 # Init
  72
  73 create = False
  74 if not os.path.isfile(path + 'map.sqlite'):
  75         create = True
  76
  77 conn = sqlite3.connect(path + 'map.sqlite')
  78
  79 if not conn:
  80         exit('Could not open database.')
  81
  82 cur = conn.cursor()
  83
  84 if create:
  85         cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (`pos` INT NOT NULL PRIMARY KEY, `data` BLOB);")
  86         conn.commit()
  87         print('Created database at ' + path + 'map.sqlite')
  88
  89 # Crawl the folders
  90
  91 count = 0
  92 t = time.time()
  93 for base in paths:
  94         v = 0
  95         if base == 'sectors':
  96                 v = 1
  97         elif base == 'sectors2':
  98                 v= 2
  99         else:
 100                 print('Ignoring base ' + base)
 101                 continue
 102
 103         for root, dirs, files in os.walk(path + base):
 104                 if files:
 105                         for block in files:
 106                                 pos = getBlockAsInteger(getBlockPos(root[(-8 if v == 1 else -7 if v == 2 else 0):], block))
 107
 108                                 if pos is None:
 109                                         print('Ignoring broken path ' + root + '/' + block)
 110                                         continue
 111
 112                                 f = open(root+'/'+block, 'rb')
 113                                 cur.execute('INSERT OR IGNORE INTO `blocks` VALUES(?, ?)', (pos, f.read()))
 114                                 f.close()
 115                                 count += 1
 116
 117                 if(time.time() - t > 3):
 118                         t = time.time()
 119                         print(str(count)+' blocks processed...')
 120
 121 conn.commit()
 122
 123 print('Finished. (' + str(count) + ' blocks)')