1 /++ 2 Provides LZMA (aka .xz) and .tar file read-only support. 3 Combine to read .tar.xz files, or use in conjunction with 4 other files to read other types of .tar files. 5 6 Also has a custom archive called arcz read and write support. 7 It is designed to efficiently pack and randomly access large 8 numbers of similar files. Unlike .zip files, it will do 9 cross-file compression (meaning it can significantly shrink 10 archives with several small but similar files), and unlike 11 tar.gz files, it supports random access without decompressing 12 the whole archive to get an individual file. It is designed 13 for large numbers of small, similar files. 14 +/ 15 module arsd.archive; 16 17 version(WithoutLzmaDecoder) {} else 18 version=WithLzmaDecoder; 19 20 version(WithoutArczCode) {} else 21 version=WithArczCode; 22 23 /+ 24 /++ 25 Reads a tar file and passes the chunks to your handler. Use it like: 26 27 TarFile f = TarFile("filename.tar"); 28 foreach(part; f) { 29 if(part.isNewFile) { 30 31 } 32 } 33 34 FIXME not implemented 35 +/ 36 struct TarFile { 37 this(string filename) { 38 39 } 40 } 41 +/ 42 43 inout(char)[] upToZero(inout(char)[] a) { 44 int i = 0; 45 while(i < a.length && a[i]) i++; 46 return a[0 .. i]; 47 } 48 49 50 /++ 51 A header of a file in the archive. This represents the 52 binary format of the header block. 53 +/ 54 align(512) 55 struct TarFileHeader { 56 align(1): 57 char[100] fileName_ = 0; 58 char[8] fileMode_ = 0; 59 char[8] ownerUid_ = 0; 60 char[8] ownerGid_ = 0; 61 char[12] size_ = 0; // in octal 62 char[12] mtime_ = 0; // octal unix timestamp 63 char[8] checksum_ = 0; // right????? 64 char[1] fileType_ = 0; // hard link, soft link, etc 65 char[100] linkFileName_ = 0; 66 char[6] ustarMagic_ = 0; // if "ustar\0", remaining fields are set 67 char[2] ustarVersion_ = 0; 68 char[32] ownerName_ = 0; 69 char[32] groupName_ = 0; 70 char[8] deviceMajorNumber_ = 0; 71 char[8] deviceMinorNumber_ = 0; 72 char[155] filenamePrefix_ = 0; 73 74 /// Returns the filename. You should cache the return value as long as TarFileHeader is in scope (it returns a slice after calling strlen) 75 const(char)[] filename() { 76 import core.stdc.string; 77 if(filenamePrefix_[0]) 78 return upToZero(filenamePrefix_[]) ~ upToZero(fileName_[]); 79 return upToZero(fileName_[]); 80 } 81 82 /// 83 ulong size() { 84 import core.stdc.stdlib; 85 return strtoul(size_.ptr, null, 8); 86 } 87 88 /// 89 TarFileType type() { 90 if(fileType_[0] == 0) 91 return TarFileType.normal; 92 else 93 return cast(TarFileType) (fileType_[0] - '0'); 94 } 95 } 96 97 /// There's other types but this is all I care about. You can still detect the char by `((cast(char) type) + '0')` 98 enum TarFileType { 99 normal = 0, /// 100 hardLink = 1, /// 101 symLink = 2, /// 102 characterSpecial = 3, /// 103 blockSpecial = 4, /// 104 directory = 5, /// 105 fifo = 6 /// 106 } 107 108 109 110 111 /++ 112 Low level tar file processor. You must pass it a 113 TarFileHeader buffer as well as a size_t for context. 114 Both must be initialized to all zeroes on first call, 115 then not modified in between calls. 116 117 Each call must populate the dataBuffer with 512 bytes. 118 119 returns true if still work to do. 120 +/ 121 bool processTar( 122 TarFileHeader* header, 123 long* bytesRemainingOnCurrentFile, 124 ubyte[] dataBuffer, 125 scope void delegate(TarFileHeader* header, bool isNewFile, bool fileFinished, ubyte[] data) handleData 126 ) 127 { 128 assert(dataBuffer.length == 512); 129 assert(bytesRemainingOnCurrentFile !is null); 130 assert(header !is null); 131 132 if(*bytesRemainingOnCurrentFile) { 133 bool isNew = *bytesRemainingOnCurrentFile == header.size(); 134 if(*bytesRemainingOnCurrentFile <= 512) { 135 handleData(header, isNew, true, dataBuffer[0 .. cast(size_t) *bytesRemainingOnCurrentFile]); 136 *bytesRemainingOnCurrentFile = 0; 137 } else { 138 handleData(header, isNew, false, dataBuffer[]); 139 *bytesRemainingOnCurrentFile -= 512; 140 } 141 } else { 142 *header = *(cast(TarFileHeader*) dataBuffer.ptr); 143 auto s = header.size(); 144 *bytesRemainingOnCurrentFile = s; 145 if(header.type() == TarFileType.directory) 146 handleData(header, true, false, null); 147 if(s == 0 && header.type == TarFileType.normal) 148 return false; 149 } 150 151 return true; 152 } 153 154 /// 155 unittest { 156 void main() { 157 TarFileHeader tfh; 158 long size; 159 160 import std.stdio; 161 ubyte[512] buffer; 162 foreach(chunk; File("/home/me/test/pl.tar", "r").byChunk(buffer[])) { 163 processTar(&tfh, &size, buffer[], 164 (header, isNewFile, fileFinished, data) { 165 if(isNewFile) 166 writeln("**** " , header.filename, " ", header.size); 167 write(cast(string) data); 168 if(fileFinished) 169 writeln("+++++++++++++++"); 170 171 }); 172 } 173 } 174 175 main(); 176 } 177 178 179 ulong readVla(ref ubyte[] data) { 180 ulong n; 181 182 n = data[0] & 0x7f; 183 if(!(data[0] & 0x80)) 184 data = data[1 .. $]; 185 186 int i = 0; 187 while(data[0] & 0x80) { 188 i++; 189 data = data[1 .. $]; 190 191 ubyte b = data[0]; 192 if(b == 0) return 0; 193 194 195 n |= cast(ulong) (b & 0x7F) << (i * 7); 196 } 197 return n; 198 } 199 200 /++ 201 A simple .xz file decoder. 202 203 FIXME: it doesn't implement very many checks, instead 204 assuming things are what it expects. Don't use this without 205 assertions enabled! 206 207 Use it by feeding it xz file data chunks. It will give you 208 back decompressed data chunks; 209 210 BEWARE OF REUSED BUFFERS. See the example. 211 +/ 212 version(WithLzmaDecoder) 213 struct XzDecoder { 214 /++ 215 Start decoding by feeding it some initial data. You must 216 send it at least enough bytes for the header (> 16 bytes prolly); 217 try to send it a reasonably sized chunk. 218 +/ 219 this(ubyte[] initialData) { 220 221 ubyte[6] magic; 222 223 magic[] = initialData[0 .. magic.length]; 224 initialData = initialData[magic.length .. $]; 225 226 if(cast(string) magic != "\xFD7zXZ\0") 227 throw new Exception("not an xz file"); 228 229 ubyte[2] streamFlags = initialData[0 .. 2]; 230 initialData = initialData[2 .. $]; 231 232 // size of the check at the end in the footer. im just ignoring tbh 233 checkSize = streamFlags[1] == 0 ? 0 : (4 << ((streamFlags[1]-1) / 3)); 234 235 //uint crc32 = initialData[0 .. 4]; // FIXME just cast it. this is the crc of the flags. 236 initialData = initialData[4 .. $]; 237 238 239 // now we are into an xz block... 240 241 int blockHeaderSize = (initialData[0] + 1) * 4; 242 243 auto srcPostHeader = initialData[blockHeaderSize .. $]; 244 245 initialData = initialData[1 .. $]; 246 247 ubyte blockFlags = initialData[0]; 248 initialData = initialData[1 .. $]; 249 250 if(blockFlags & 0x40) { 251 compressedSize = readVla(initialData); 252 } 253 254 if(blockFlags & 0x80) { 255 uncompressedSize = readVla(initialData); 256 } 257 258 auto filterCount = (blockFlags & 0b11) + 1; 259 260 ubyte props; 261 262 foreach(f; 0 .. filterCount) { 263 auto fid = readVla(initialData); 264 auto sz = readVla(initialData); 265 266 assert(fid == 0x21); 267 assert(sz == 1); 268 269 props = initialData[0]; 270 initialData = initialData[1 .. $]; 271 } 272 273 //writeln(src.ptr); 274 //writeln(srcPostHeader.ptr); 275 276 // there should be some padding to a multiple of 4... 277 // three bytes of zeroes given the assumptions here 278 279 initialData = initialData[3 .. $]; 280 281 // and then a header crc 282 283 initialData = initialData[4 .. $]; // skip header crc 284 285 assert(initialData.ptr is srcPostHeader.ptr); 286 287 // skip unknown header bytes 288 while(initialData.ptr < srcPostHeader.ptr) { 289 initialData = initialData[1 .. $]; 290 } 291 292 // should finally be at compressed data... 293 294 //writeln(compressedSize); 295 //writeln(uncompressedSize); 296 297 if(Lzma2Dec_Allocate(&lzmaDecoder, props) != SRes.OK) { 298 assert(0); 299 } 300 301 Lzma2Dec_Init(&lzmaDecoder); 302 303 unprocessed = initialData; 304 } 305 306 ~this() { 307 LzmaDec_FreeProbs(&lzmaDecoder.decoder); 308 } 309 310 /++ 311 You tell it where you want the data. 312 313 You must pass it the existing unprocessed data 314 315 Returns slice of dest that is actually filled up so far. 316 +/ 317 ubyte[] processData(ubyte[] dest, ubyte[] src) { 318 319 size_t destLen = dest.length; 320 size_t srcLen = src.length; 321 322 ELzmaStatus status; 323 324 auto res = Lzma2Dec_DecodeToBuf( 325 &lzmaDecoder, 326 dest.ptr, 327 &destLen, 328 src.ptr, 329 &srcLen, 330 LZMA_FINISH_ANY, 331 &status 332 ); 333 334 if(res != 0) { 335 import std.conv; 336 throw new Exception(to!string(res)); 337 } 338 339 /+ 340 import std.stdio; 341 writeln(res, " ", status); 342 writeln(srcLen); 343 writeln(destLen, ": ", cast(string) dest[0 .. destLen]); 344 +/ 345 346 if(status == LZMA_STATUS_NEEDS_MORE_INPUT) { 347 unprocessed = src[srcLen .. $]; 348 finished_ = false; 349 needsMoreData_ = true; 350 } else if(status == LZMA_STATUS_FINISHED_WITH_MARK || status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) { 351 unprocessed = null; 352 finished_ = true; 353 needsMoreData_ = false; 354 } else if(status == LZMA_STATUS_NOT_FINISHED) { 355 unprocessed = src[srcLen .. $]; 356 finished_ = false; 357 needsMoreData_ = false; 358 } else { 359 // wtf 360 import std.conv; 361 assert(0, to!string(status)); 362 } 363 364 return dest[0 .. destLen]; 365 } 366 367 /// 368 bool finished() { 369 return finished_; 370 } 371 372 /// 373 bool needsMoreData() { 374 return needsMoreData_; 375 } 376 377 bool finished_; 378 bool needsMoreData_; 379 380 CLzma2Dec lzmaDecoder; 381 int checkSize; 382 ulong compressedSize; /// 383 ulong uncompressedSize; /// 384 385 ubyte[] unprocessed; /// 386 } 387 388 /// 389 version(WithLzmaDecoder) 390 unittest { 391 392 void main() { 393 ubyte[512] dest; // into tar size chunks! 394 ubyte[1024] src; 395 396 import std.stdio; 397 398 //auto file = File("/home/me/test/amazing.txt.xz", "rb"); 399 auto file = File("/home/me/Android/ldcdl/test.tar.xz", "rb"); 400 auto bfr = file.rawRead(src[]); 401 402 XzDecoder xzd = XzDecoder(bfr); 403 404 // not necessarily set, don't rely on them 405 writeln(xzd.compressedSize, " / ", xzd.uncompressedSize); 406 407 // for tar 408 TarFileHeader tfh; 409 long size; 410 411 long sum = 0; 412 while(!xzd.finished) { 413 // as long as your are not finished, there is more work to do. But it doesn't 414 // necessarily need more data, so that is a separate check. 415 if(xzd.needsMoreData) { 416 // if it needs more data, append new stuff to the end of the buffer, after 417 // the existing unprocessed stuff. If your buffer is too small, you may be 418 // forced to grow it here, but anything >= 1 KB seems OK in my tests. 419 bfr = file.rawRead(src[bfr.length - xzd.unprocessed.length .. $]); 420 } else { 421 // otherwise, you want to continue working with existing unprocessed data 422 bfr = xzd.unprocessed; 423 } 424 //write(cast(string) xzd.processData(dest[], bfr)); 425 426 auto buffer = xzd.processData(dest[], bfr); 427 428 // if the buffer is empty we are probably done 429 // or need more data, so continue the loop to evaluate. 430 if(buffer.length == 0) 431 continue; 432 433 // our tar code requires specifically 512 byte pieces 434 while(!xzd.finished && buffer.length != 512) { 435 // need more data hopefully 436 assert(xzd.needsMoreData); 437 // using the existing buffer... 438 bfr = file.rawRead(src[bfr.length - xzd.unprocessed.length .. $]); 439 auto nbuffer = xzd.processData(dest[buffer.length .. $], bfr); 440 buffer = dest[0 .. buffer.length + nbuffer.length]; 441 } 442 443 sum += buffer.length; 444 445 // process the buffer through the tar file handler 446 processTar(&tfh, &size, buffer[], 447 (header, isNewFile, fileFinished, data) { 448 if(isNewFile) 449 writeln("**** " , header.filename, " ", header.size); 450 //write(cast(string) data); 451 if(fileFinished) 452 writeln("+++++++++++++++"); 453 454 }); 455 } 456 457 writeln(sum); 458 } 459 460 main(); 461 } 462 463 version(WithArczCode) { 464 /* The code in this section was originally written by Ketmar Dark for his arcz.d module. I modified it afterward. */ 465 466 /** ARZ chunked archive format processor. 467 * 468 * This module provides `std.stdio.File`-like interface to ARZ archives. 469 * 470 * Copyright: Copyright Ketmar Dark, 2016 471 * 472 * License: Boost License 1.0 473 */ 474 // module iv.arcz; 475 476 // use Balz compressor if available 477 static if (__traits(compiles, { import iv.balz; })) enum arcz_has_balz = true; else enum arcz_has_balz = false; 478 static if (__traits(compiles, { import iv.zopfli; })) enum arcz_has_zopfli = true; else enum arcz_has_zopfli = false; 479 static if (arcz_has_balz) import iv.balz; 480 static if (arcz_has_zopfli) import iv.zopfli; 481 482 // comment this to free pakced chunk buffer right after using 483 // i.e. `AZFile` will allocate new block for each new chunk 484 //version = arcz_use_more_memory; 485 486 public import core.stdc.stdio : SEEK_SET, SEEK_CUR, SEEK_END; 487 488 489 // ////////////////////////////////////////////////////////////////////////// // 490 /// ARZ archive accessor. Use this to open ARZ archives, and open packed files from ARZ archives. 491 public struct ArzArchive { 492 private: 493 static assert(size_t.sizeof >= (void*).sizeof); 494 private import core.stdc.stdio : FILE, fopen, fclose, fread, fseek; 495 private import etc.c.zlib; 496 497 static struct ChunkInfo { 498 uint ofs; // offset in file 499 uint pksize; // packed chunk size (same as chunk size: chunk is unpacked) 500 } 501 502 static struct FileInfo { 503 string name; 504 uint chunk; 505 uint chunkofs; // offset of first file byte in unpacked chunk 506 uint size; // unpacked file size 507 } 508 509 static struct Nfo { 510 uint rc = 1; // refcounter 511 ChunkInfo[] chunks; 512 FileInfo[string] files; 513 uint chunkSize; 514 uint lastChunkSize; 515 bool useBalz; 516 FILE* afl; // archive file, we'll keep it opened 517 518 @disable this (this); // no copies! 519 520 static void decRef (size_t me) { 521 if (me) { 522 auto nfo = cast(Nfo*)me; 523 assert(nfo.rc); 524 if (--nfo.rc == 0) { 525 import core.memory : GC; 526 import core.stdc.stdlib : free; 527 if (nfo.afl !is null) fclose(nfo.afl); 528 nfo.chunks.destroy; 529 nfo.files.destroy; 530 nfo.afl = null; 531 GC.removeRange(cast(void*)nfo/*, Nfo.sizeof*/); 532 free(nfo); 533 debug(arcz_rc) { import core.stdc.stdio : printf; printf("Nfo %p freed\n", nfo); } 534 } 535 } 536 } 537 } 538 539 size_t nfop; // hide it from GC 540 541 private @property Nfo* nfo () { pragma(inline, true); return cast(Nfo*)nfop; } 542 void decRef () { pragma(inline, true); Nfo.decRef(nfop); nfop = 0; } 543 544 static uint readUint (FILE* fl) { 545 if (fl is null) throw new Exception("cannot read from closed file"); 546 uint v; 547 if (fread(&v, 1, v.sizeof, fl) != v.sizeof) throw new Exception("file reading error"); 548 version(BigEndian) { 549 import core.bitop : bswap; 550 v = bswap(v); 551 } else version(LittleEndian) { 552 // nothing to do 553 } else { 554 static assert(0, "wtf?!"); 555 } 556 return v; 557 } 558 559 static uint readUbyte (FILE* fl) { 560 if (fl is null) throw new Exception("cannot read from closed file"); 561 ubyte v; 562 if (fread(&v, 1, v.sizeof, fl) != v.sizeof) throw new Exception("file reading error"); 563 return v; 564 } 565 566 static void readBuf (FILE* fl, void[] buf) { 567 if (buf.length > 0) { 568 if (fl is null) throw new Exception("cannot read from closed file"); 569 if (fread(buf.ptr, 1, buf.length, fl) != buf.length) throw new Exception("file reading error"); 570 } 571 } 572 573 static T* xalloc(T, bool clear=true) (uint mem) if (T.sizeof > 0) { 574 import core.exception : onOutOfMemoryError; 575 assert(mem != 0); 576 static if (clear) { 577 import core.stdc.stdlib : calloc; 578 auto res = calloc(mem, T.sizeof); 579 if (res is null) onOutOfMemoryError(); 580 static if (is(T == struct)) { 581 import core.stdc.string : memcpy; 582 static immutable T i = T.init; 583 foreach (immutable idx; 0..mem) memcpy(res+idx, &i, T.sizeof); 584 } 585 debug(arcz_alloc) { import core.stdc.stdio : printf; printf("allocated %u bytes at %p\n", cast(uint)(mem*T.sizeof), res); } 586 return cast(T*)res; 587 } else { 588 import core.stdc.stdlib : malloc; 589 auto res = malloc(mem*T.sizeof); 590 if (res is null) onOutOfMemoryError(); 591 static if (is(T == struct)) { 592 import core.stdc.string : memcpy; 593 static immutable T i = T.init; 594 foreach (immutable idx; 0..mem) memcpy(res+idx, &i, T.sizeof); 595 } 596 debug(arcz_alloc) { import core.stdc.stdio : printf; printf("allocated %u bytes at %p\n", cast(uint)(mem*T.sizeof), res); } 597 return cast(T*)res; 598 } 599 } 600 601 static void xfree(T) (T* ptr) { 602 if (ptr !is null) { 603 import core.stdc.stdlib : free; 604 debug(arcz_alloc) { import core.stdc.stdio : printf; printf("freing at %p\n", ptr); } 605 free(ptr); 606 } 607 } 608 609 static if (arcz_has_balz) static ubyte balzDictSize (uint blockSize) { 610 foreach (ubyte bits; Balz.MinDictBits..Balz.MaxDictBits+1) { 611 if ((1U<<bits) >= blockSize) return bits; 612 } 613 return Balz.MaxDictBits; 614 } 615 616 // unpack exactly `destlen` bytes 617 static if (arcz_has_balz) static void unpackBlockBalz (void* dest, uint destlen, const(void)* src, uint srclen, uint blocksize) { 618 Unbalz bz; 619 bz.reinit(balzDictSize(blocksize)); 620 int ipos, opos; 621 auto dc = bz.decompress( 622 // reader 623 (buf) { 624 import core.stdc.string : memcpy; 625 if (ipos >= srclen) return 0; 626 uint rd = destlen-ipos; 627 if (rd > buf.length) rd = cast(uint)buf.length; 628 memcpy(buf.ptr, src+ipos, rd); 629 ipos += rd; 630 return rd; 631 }, 632 // writer 633 (buf) { 634 //if (opos+buf.length > destlen) throw new Exception("error unpacking archive"); 635 uint wr = destlen-opos; 636 if (wr > buf.length) wr = cast(uint)buf.length; 637 if (wr > 0) { 638 import core.stdc.string : memcpy; 639 memcpy(dest+opos, buf.ptr, wr); 640 opos += wr; 641 } 642 }, 643 // unpack length 644 destlen 645 ); 646 if (opos != destlen) throw new Exception("error unpacking archive"); 647 } 648 649 static void unpackBlockZLib (void* dest, uint destlen, const(void)* src, uint srclen, uint blocksize) { 650 z_stream zs; 651 zs.avail_in = 0; 652 zs.avail_out = 0; 653 // initialize unpacker 654 if (inflateInit2(&zs, 15) != Z_OK) throw new Exception("can't initialize zlib"); 655 scope(exit) inflateEnd(&zs); 656 zs.next_in = cast(typeof(zs.next_in))src; 657 zs.avail_in = srclen; 658 zs.next_out = cast(typeof(zs.next_out))dest; 659 zs.avail_out = destlen; 660 while (zs.avail_out > 0) { 661 auto err = inflate(&zs, Z_SYNC_FLUSH); 662 if (err != Z_STREAM_END && err != Z_OK) throw new Exception("error unpacking archive"); 663 if (err == Z_STREAM_END) break; 664 } 665 if (zs.avail_out != 0) throw new Exception("error unpacking archive"); 666 } 667 668 static void unpackBlock (void* dest, uint destlen, const(void)* src, uint srclen, uint blocksize, bool useBalz) { 669 if (useBalz) { 670 static if (arcz_has_balz) { 671 unpackBlockBalz(dest, destlen, src, srclen, blocksize); 672 } else { 673 throw new Exception("no Balz support was compiled in ArcZ"); 674 } 675 } else { 676 unpackBlockZLib(dest, destlen, src, srclen, blocksize); 677 } 678 } 679 680 public: 681 this (in ArzArchive arc) { 682 assert(nfop == 0); 683 nfop = arc.nfop; 684 if (nfop) ++nfo.rc; 685 } 686 687 this (this) { 688 if (nfop) ++nfo.rc; 689 } 690 691 ~this () { close(); } 692 693 void opAssign (in ArzArchive arc) { 694 if (arc.nfop) { 695 auto n = cast(Nfo*)arc.nfop; 696 ++n.rc; 697 } 698 decRef(); 699 nfop = arc.nfop; 700 } 701 702 void close () { decRef(); } 703 704 @property FileInfo[string] files () { return (nfop ? nfo.files : null); } 705 706 void openArchive (const(char)[] filename) { 707 debug/*(arcz)*/ import core.stdc.stdio : printf; 708 FILE* fl = null; 709 scope(exit) if (fl !is null) fclose(fl); 710 close(); 711 if (filename.length == 0) throw new Exception("cannot open unnamed archive file"); 712 if (false && filename.length < 2048) { // FIXME the alloca fails on win64 for some reason 713 import core.stdc.stdlib : alloca; 714 auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1]; 715 tfn[0..filename.length] = filename[]; 716 tfn[filename.length] = 0; 717 fl = fopen(tfn.ptr, "rb"); 718 } else { 719 import core.stdc.stdlib : malloc, free; 720 auto tfn = (cast(char*)malloc(filename.length+1))[0..filename.length+1]; 721 if (tfn !is null) { 722 tfn[0 .. filename.length] = filename[]; 723 tfn[filename.length] = 0; 724 scope(exit) free(tfn.ptr); 725 fl = fopen(tfn.ptr, "rb"); 726 } 727 } 728 if (fl is null) throw new Exception("cannot open archive file '"~filename.idup~"'"); 729 char[4] sign; 730 bool useBalz; 731 readBuf(fl, sign[]); 732 if (sign != "CZA2") throw new Exception("invalid archive file '"~filename.idup~"'"); 733 switch (readUbyte(fl)) { 734 case 0: useBalz = false; break; 735 case 1: useBalz = true; break; 736 default: throw new Exception("invalid version of archive file '"~filename.idup~"'"); 737 } 738 uint indexofs = readUint(fl); // index offset in file 739 uint pkidxsize = readUint(fl); // packed index size 740 uint idxsize = readUint(fl); // unpacked index size 741 if (pkidxsize == 0 || idxsize == 0 || indexofs == 0) throw new Exception("invalid archive file '"~filename.idup~"'"); 742 // now read index 743 ubyte* idxbuf = null; 744 scope(exit) xfree(idxbuf); 745 { 746 auto pib = xalloc!ubyte(pkidxsize); 747 scope(exit) xfree(pib); 748 if (fseek(fl, indexofs, 0) < 0) throw new Exception("seek error in archive file '"~filename.idup~"'"); 749 readBuf(fl, pib[0..pkidxsize]); 750 idxbuf = xalloc!ubyte(idxsize); 751 unpackBlock(idxbuf, idxsize, pib, pkidxsize, idxsize, useBalz); 752 } 753 754 // parse index and build structures 755 uint idxbufpos = 0; 756 757 ubyte getUbyte () { 758 if (idxsize-idxbufpos < ubyte.sizeof) throw new Exception("invalid index for archive file '"~filename.idup~"'"); 759 return idxbuf[idxbufpos++]; 760 } 761 762 uint getUint () { 763 if (idxsize-idxbufpos < uint.sizeof) throw new Exception("invalid index for archive file '"~filename.idup~"'"); 764 version(BigEndian) { 765 import core.bitop : bswap; 766 uint v = *cast(uint*)(idxbuf+idxbufpos); 767 idxbufpos += 4; 768 return bswap(v); 769 } else version(LittleEndian) { 770 uint v = *cast(uint*)(idxbuf+idxbufpos); 771 idxbufpos += 4; 772 return v; 773 } else { 774 static assert(0, "wtf?!"); 775 } 776 } 777 778 void getBuf (void[] buf) { 779 if (buf.length > 0) { 780 import core.stdc.string : memcpy; 781 if (idxsize-idxbufpos < buf.length) throw new Exception("invalid index for archive file '"~filename.idup~"'"); 782 memcpy(buf.ptr, idxbuf+idxbufpos, buf.length); 783 idxbufpos += buf.length; 784 } 785 } 786 787 // allocate shared info struct 788 Nfo* nfo = xalloc!Nfo(1); 789 assert(nfo.rc == 1); 790 debug(arcz_rc) { import core.stdc.stdio : printf; printf("Nfo %p allocated\n", nfo); } 791 scope(failure) decRef(); 792 nfop = cast(size_t)nfo; 793 { 794 import core.memory : GC; 795 GC.addRange(nfo, Nfo.sizeof); 796 } 797 798 // read chunk info and data 799 nfo.useBalz = useBalz; 800 nfo.chunkSize = getUint; 801 auto ccount = getUint; // chunk count 802 nfo.lastChunkSize = getUint; 803 debug(arcz_dirread) printf("chunk size: %u\nchunk count: %u\nlast chunk size:%u\n", nfo.chunkSize, ccount, nfo.lastChunkSize); 804 if (ccount == 0 || nfo.chunkSize < 1 || nfo.lastChunkSize < 1 || nfo.lastChunkSize > nfo.chunkSize) throw new Exception("invalid archive file '"~filename.idup~"'"); 805 nfo.chunks.length = ccount; 806 // chunk offsets and sizes 807 foreach (ref ci; nfo.chunks) { 808 ci.ofs = getUint; 809 ci.pksize = getUint; 810 } 811 // read file count and info 812 auto fcount = getUint; 813 if (fcount == 0) throw new Exception("empty archive file '"~filename.idup~"'"); 814 // calc name buffer position and size 815 //immutable uint nbofs = idxbufpos+fcount*(5*4); 816 //if (nbofs >= idxsize) throw new Exception("invalid index in archive file '"~filename.idup~"'"); 817 //immutable uint nbsize = idxsize-nbofs; 818 debug(arcz_dirread) printf("file count: %u\n", fcount); 819 foreach (immutable _; 0..fcount) { 820 uint nameofs = getUint; 821 uint namelen = getUint; 822 if (namelen == 0) { 823 // skip unnamed file 824 //throw new Exception("invalid archive file '"~filename.idup~"'"); 825 getUint; // chunk number 826 getUint; // offset in chunk 827 getUint; // unpacked size 828 debug(arcz_dirread) printf("skipped empty file\n"); 829 } else { 830 //if (nameofs >= nbsize || namelen > nbsize || nameofs+namelen > nbsize) throw new Exception("invalid index in archive file '"~filename.idup~"'"); 831 if (nameofs >= idxsize || namelen > idxsize || nameofs+namelen > idxsize) throw new Exception("invalid index in archive file '"~filename.idup~"'"); 832 FileInfo fi; 833 auto nb = new char[](namelen); 834 nb[0..namelen] = (cast(char*)idxbuf)[nameofs..nameofs+namelen]; 835 fi.name = cast(string)(nb); // it is safe here 836 fi.chunk = getUint; // chunk number 837 fi.chunkofs = getUint; // offset in chunk 838 fi.size = getUint; // unpacked size 839 debug(arcz_dirread) printf("file size: %u\nfile chunk: %u\noffset in chunk:%u; name: [%.*s]\n", fi.size, fi.chunk, fi.chunkofs, cast(uint)fi.name.length, fi.name.ptr); 840 nfo.files[fi.name] = fi; 841 } 842 } 843 // transfer achive file ownership 844 nfo.afl = fl; 845 fl = null; 846 } 847 848 bool exists (const(char)[] name) { if (nfop) return ((name in nfo.files) !is null); else return false; } 849 850 AZFile open (const(char)[] name) { 851 if (!nfop) throw new Exception("can't open file from non-opened archive"); 852 if (auto fi = name in nfo.files) { 853 auto zl = xalloc!LowLevelPackedRO(1); 854 scope(failure) xfree(zl); 855 debug(arcz_rc) { import core.stdc.stdio : printf; printf("Zl %p allocated\n", zl); } 856 zl.setup(nfo, fi.chunk, fi.chunkofs, fi.size); 857 AZFile fl; 858 fl.zlp = cast(size_t)zl; 859 return fl; 860 } 861 throw new Exception("can't open file '"~name.idup~"' from archive"); 862 } 863 864 private: 865 static struct LowLevelPackedRO { 866 private import etc.c.zlib; 867 868 uint rc = 1; 869 size_t nfop; // hide it from GC 870 871 private @property inout(Nfo*) nfo () inout pure nothrow @trusted @nogc { pragma(inline, true); return cast(typeof(return))nfop; } 872 static void decRef (size_t me) { 873 if (me) { 874 auto zl = cast(LowLevelPackedRO*)me; 875 assert(zl.rc); 876 if (--zl.rc == 0) { 877 import core.stdc.stdlib : free; 878 if (zl.chunkData !is null) free(zl.chunkData); 879 version(arcz_use_more_memory) if (zl.pkdata !is null) free(zl.pkdata); 880 Nfo.decRef(zl.nfop); 881 free(zl); 882 debug(arcz_rc) { import core.stdc.stdio : printf; printf("Zl %p freed\n", zl); } 883 } else { 884 //debug(arcz_rc) { import core.stdc.stdio : printf; printf("Zl %p; rc after decRef is %u\n", zl, zl.rc); } 885 } 886 } 887 } 888 889 uint nextchunk; // next chunk to read 890 uint curcpos; // position in current chunk 891 uint curcsize; // number of valid bytes in `chunkData` 892 uint stchunk; // starting chunk 893 uint stofs; // offset in starting chunk 894 uint totalsize; // total file size 895 uint pos; // current file position 896 uint lastrdpos; // last actual read position 897 z_stream zs; 898 ubyte* chunkData; // can be null 899 version(arcz_use_more_memory) { 900 ubyte* pkdata; 901 uint pkdatasize; 902 } 903 904 @disable this (this); 905 906 void setup (Nfo* anfo, uint astchunk, uint astofs, uint asize) { 907 assert(anfo !is null); 908 assert(rc == 1); 909 nfop = cast(size_t)anfo; 910 ++anfo.rc; 911 nextchunk = stchunk = astchunk; 912 //curcpos = 0; 913 stofs = astofs; 914 totalsize = asize; 915 } 916 917 @property bool eof () { pragma(inline, true); return (pos >= totalsize); } 918 919 // return less than chunk size if our file fits in one non-full chunk completely 920 uint justEnoughMemory () pure const nothrow @safe @nogc { 921 pragma(inline, true); 922 version(none) { 923 return nfo.chunkSize; 924 } else { 925 return (totalsize < nfo.chunkSize && stofs+totalsize < nfo.chunkSize ? stofs+totalsize : nfo.chunkSize); 926 } 927 } 928 929 void unpackNextChunk () { 930 if (nfop == 0) assert(0, "wtf?!"); 931 //scope(failure) if (chunkData !is null) { xfree(chunkData); chunkData = null; } 932 debug(arcz_unp) { import core.stdc.stdio : printf; printf("unpacking chunk %u\n", nextchunk); } 933 // allocate buffer for unpacked data 934 if (chunkData is null) { 935 // optimize things a little: if our file fits in less then one chunk, allocate "just enough" memory 936 chunkData = xalloc!(ubyte, false)(justEnoughMemory); 937 } 938 auto chunk = &nfo.chunks[nextchunk]; 939 if (chunk.pksize == nfo.chunkSize) { 940 // unpacked chunk, just read it 941 debug(arcz_unp) { import core.stdc.stdio : printf; printf(" chunk is not packed\n"); } 942 if (fseek(nfo.afl, chunk.ofs, 0) < 0) throw new Exception("ARCZ reading error"); 943 if (fread(chunkData, 1, nfo.chunkSize, nfo.afl) != nfo.chunkSize) throw new Exception("ARCZ reading error"); 944 curcsize = nfo.chunkSize; 945 } else { 946 // packed chunk, unpack it 947 // allocate buffer for packed data 948 version(arcz_use_more_memory) { 949 import core.stdc.stdlib : realloc; 950 if (pkdatasize < chunk.pksize) { 951 import core.exception : onOutOfMemoryError; 952 auto newpk = realloc(pkdata, chunk.pksize); 953 if (newpk is null) onOutOfMemoryError(); 954 debug(arcz_alloc) { import core.stdc.stdio : printf; printf("reallocated from %u to %u bytes; %p -> %p\n", cast(uint)pkdatasize, cast(uint)chunk.pksize, pkdata, newpk); } 955 pkdata = cast(ubyte*)newpk; 956 pkdatasize = chunk.pksize; 957 } 958 alias pkd = pkdata; 959 } else { 960 auto pkd = xalloc!(ubyte, false)(chunk.pksize); 961 scope(exit) xfree(pkd); 962 } 963 if (fseek(nfo.afl, chunk.ofs, 0) < 0) throw new Exception("ARCZ reading error"); 964 if (fread(pkd, 1, chunk.pksize, nfo.afl) != chunk.pksize) throw new Exception("ARCZ reading error"); 965 uint upsize = (nextchunk == nfo.chunks.length-1 ? nfo.lastChunkSize : nfo.chunkSize); // unpacked chunk size 966 immutable uint cksz = upsize; 967 immutable uint jem = justEnoughMemory; 968 if (upsize > jem) upsize = jem; 969 debug(arcz_unp) { import core.stdc.stdio : printf; printf(" unpacking %u bytes to %u bytes\n", chunk.pksize, upsize); } 970 ArzArchive.unpackBlock(chunkData, upsize, pkd, chunk.pksize, cksz, nfo.useBalz); 971 curcsize = upsize; 972 } 973 curcpos = 0; 974 // fix first chunk offset if necessary 975 if (nextchunk == stchunk && stofs > 0) { 976 // it's easier to just memmove it 977 import core.stdc.string : memmove; 978 assert(stofs < curcsize); 979 memmove(chunkData, chunkData+stofs, curcsize-stofs); 980 curcsize -= stofs; 981 } 982 ++nextchunk; // advance to next chunk 983 } 984 985 void syncReadPos () { 986 if (pos >= totalsize || pos == lastrdpos) return; 987 immutable uint fcdata = nfo.chunkSize-stofs; // number of our bytes in the first chunk 988 // does our pos lie in the first chunk? 989 if (pos < fcdata) { 990 // yep, just read it 991 if (nextchunk != stchunk+1) { 992 nextchunk = stchunk; 993 unpackNextChunk(); // we'll need it anyway 994 } else { 995 // just rewind 996 curcpos = 0; 997 } 998 curcpos += pos; 999 lastrdpos = pos; 1000 return; 1001 } 1002 // find the chunk we want 1003 uint npos = pos-fcdata; 1004 uint xblock = stchunk+1+npos/nfo.chunkSize; 1005 uint curcstart = (xblock-(stchunk+1))*nfo.chunkSize+fcdata; 1006 if (xblock != nextchunk-1) { 1007 // read and unpack this chunk 1008 nextchunk = xblock; 1009 unpackNextChunk(); 1010 } else { 1011 // just rewind 1012 curcpos = 0; 1013 } 1014 assert(pos >= curcstart && pos < curcstart+nfo.chunkSize); 1015 uint skip = pos-curcstart; 1016 lastrdpos = pos; 1017 curcpos += skip; 1018 } 1019 1020 int read (void* buf, uint count) { 1021 if (buf is null) return -1; 1022 if (count == 0 || totalsize == 0) return 0; 1023 if (totalsize >= 0 && pos >= totalsize) return 0; // EOF 1024 syncReadPos(); 1025 assert(lastrdpos == pos); 1026 if (cast(long)pos+count > totalsize) count = totalsize-pos; 1027 auto res = count; 1028 while (count > 0) { 1029 debug(arcz_read) { import core.stdc.stdio : printf; printf("reading %u bytes; pos=%u; lastrdpos=%u; curcpos=%u; curcsize=%u\n", count, pos, lastrdpos, curcpos, curcsize); } 1030 import core.stdc.string : memcpy; 1031 if (curcpos >= curcsize) { 1032 unpackNextChunk(); // we want next chunk! 1033 debug(arcz_read) { import core.stdc.stdio : printf; printf(" *reading %u bytes; pos=%u; lastrdpos=%u; curcpos=%u; curcsize=%u\n", count, pos, lastrdpos, curcpos, curcsize); } 1034 } 1035 assert(curcpos < curcsize && curcsize != 0); 1036 int rd = (curcsize-curcpos >= count ? count : curcsize-curcpos); 1037 assert(rd > 0); 1038 memcpy(buf, chunkData+curcpos, rd); 1039 curcpos += rd; 1040 pos += rd; 1041 lastrdpos += rd; 1042 buf += rd; 1043 count -= rd; 1044 } 1045 assert(pos == lastrdpos); 1046 return res; 1047 } 1048 1049 long lseek (long ofs, int origin) { 1050 //TODO: overflow checks 1051 switch (origin) { 1052 case SEEK_SET: break; 1053 case SEEK_CUR: ofs += pos; break; 1054 case SEEK_END: 1055 if (ofs > 0) ofs = 0; 1056 if (-ofs > totalsize) ofs = -cast(long)totalsize; 1057 ofs += totalsize; 1058 break; 1059 default: 1060 return -1; 1061 } 1062 if (ofs < 0) return -1; 1063 if (totalsize >= 0 && ofs > totalsize) ofs = totalsize; 1064 pos = cast(uint)ofs; 1065 return pos; 1066 } 1067 } 1068 } 1069 1070 1071 // ////////////////////////////////////////////////////////////////////////// // 1072 /// Opened file. 1073 public struct AZFile { 1074 private: 1075 size_t zlp; 1076 1077 private @property inout(ArzArchive.LowLevelPackedRO)* zl () inout pure nothrow @trusted @nogc { pragma(inline, true); return cast(typeof(return))zlp; } 1078 private void decRef () { pragma(inline, true); ArzArchive.LowLevelPackedRO.decRef(zlp); zlp = 0; } 1079 1080 public: 1081 this (in AZFile afl) { 1082 assert(zlp == 0); 1083 zlp = afl.zlp; 1084 if (zlp) ++zl.rc; 1085 } 1086 1087 this (this) { 1088 if (zlp) ++zl.rc; 1089 } 1090 1091 ~this () { close(); } 1092 1093 void opAssign (in AZFile afl) { 1094 if (afl.zlp) { 1095 auto n = cast(ArzArchive.LowLevelPackedRO*)afl.zlp; 1096 ++n.rc; 1097 } 1098 decRef(); 1099 zlp = afl.zlp; 1100 } 1101 1102 void close () { decRef(); } 1103 1104 @property bool isOpen () const pure nothrow @safe @nogc { pragma(inline, true); return (zlp != 0); } 1105 @property uint size () const pure nothrow @safe @nogc { pragma(inline, true); return (zlp ? zl.totalsize : 0); } 1106 @property uint tell () const pure nothrow @safe @nogc { pragma(inline, true); return (zlp ? zl.pos : 0); } 1107 1108 void seek (long ofs, int origin=SEEK_SET) { 1109 if (!zlp) throw new Exception("can't seek in closed file"); 1110 auto res = zl.lseek(ofs, origin); 1111 if (res < 0) throw new Exception("seek error"); 1112 } 1113 1114 private import std.traits : isMutable; 1115 1116 //TODO: overflow check 1117 T[] rawRead(T) (T[] buf) if (isMutable!T) { 1118 if (!zlp) throw new Exception("can't read from closed file"); 1119 if (buf.length > 0) { 1120 auto res = zl.read(buf.ptr, cast(int) (buf.length*T.sizeof)); 1121 if (res == -1 || res%T.sizeof != 0) throw new Exception("read error"); 1122 return buf[0..res/T.sizeof]; 1123 } else { 1124 return buf[0..0]; 1125 } 1126 } 1127 } 1128 1129 1130 // ////////////////////////////////////////////////////////////////////////// // 1131 /** this class can be used to create archive file. 1132 * 1133 * Example: 1134 * -------------------- 1135 * import std.file, std.path, std.stdio : File; 1136 * 1137 * enum ArcName = "z00.arz"; 1138 * enum DirName = "experimental-docs"; 1139 * 1140 * ubyte[] rdbuf; 1141 * rdbuf.length = 65536; 1142 * 1143 * auto arcz = new ArzCreator(ArcName); 1144 * long total = 0; 1145 * foreach (DirEntry e; dirEntries(DirName, SpanMode.breadth)) { 1146 * if (e.isFile) { 1147 * assert(e.size < uint.max); 1148 * //writeln(e.name); 1149 * total += e.size; 1150 * string fname = e.name[DirName.length+1..$]; 1151 * arcz.newFile(fname, cast(uint)e.size); 1152 * auto fi = File(e.name); 1153 * for (;;) { 1154 * auto rd = fi.rawRead(rdbuf[]); 1155 * if (rd.length == 0) break; 1156 * arcz.rawWrite(rd[]); 1157 * } 1158 * } 1159 * } 1160 * arcz.close(); 1161 * writeln(total, " bytes packed to ", getSize(ArcName), " (", arcz.chunksWritten, " chunks, ", arcz.filesWritten, " files)"); 1162 * -------------------- 1163 */ 1164 final class ArzCreator { 1165 private import etc.c.zlib; 1166 private import core.stdc.stdio : FILE, fopen, fclose, ftell, fseek, fwrite; 1167 1168 public: 1169 //WARNING! don't change the order! 1170 enum Compressor { 1171 ZLib, // default 1172 Balz, 1173 BalzMax, // Balz, maximum compression 1174 Zopfli, // this will fallback to zlib if no zopfli support was compiled in 1175 } 1176 1177 private: 1178 static struct ChunkInfo { 1179 uint ofs; // offset in file 1180 uint pksize; // packed chunk size 1181 } 1182 1183 static struct FileInfo { 1184 string name; 1185 uint chunk; 1186 uint chunkofs; // offset of first file byte in unpacked chunk 1187 uint size; // unpacked file size 1188 } 1189 1190 private: 1191 ubyte[] chunkdata; 1192 uint cdpos; 1193 FILE* arcfl; 1194 ChunkInfo[] chunks; 1195 FileInfo[] files; 1196 uint lastChunkSize; 1197 uint statChunks, statFiles; 1198 Compressor cpr = Compressor.ZLib; 1199 1200 private: 1201 void writeUint (uint v) { 1202 if (arcfl is null) throw new Exception("write error"); 1203 version(BigEndian) { 1204 import core.bitop : bswap; 1205 v = bswap(v); 1206 } else version(LittleEndian) { 1207 // nothing to do 1208 } else { 1209 static assert(0, "wtf?!"); 1210 } 1211 if (fwrite(&v, 1, v.sizeof, arcfl) != v.sizeof) throw new Exception("write error"); // signature 1212 } 1213 1214 void writeUbyte (ubyte v) { 1215 if (arcfl is null) throw new Exception("write error"); 1216 if (fwrite(&v, 1, v.sizeof, arcfl) != v.sizeof) throw new Exception("write error"); // signature 1217 } 1218 1219 void writeBuf (const(void)[] buf) { 1220 if (buf.length > 0) { 1221 if (arcfl is null) throw new Exception("write error"); 1222 if (fwrite(buf.ptr, 1, buf.length, arcfl) != buf.length) throw new Exception("write error"); // signature 1223 } 1224 } 1225 1226 static if (arcz_has_balz) long writePackedBalz (const(void)[] upbuf) { 1227 assert(upbuf.length > 0 && upbuf.length < int.max); 1228 long res = 0; 1229 Balz bz; 1230 int ipos, opos; 1231 bz.reinit(ArzArchive.balzDictSize(cast(uint)upbuf.length)); 1232 bz.compress( 1233 // reader 1234 (buf) { 1235 import core.stdc.string : memcpy; 1236 if (ipos >= upbuf.length) return 0; 1237 uint rd = cast(uint)upbuf.length-ipos; 1238 if (rd > buf.length) rd = cast(uint)buf.length; 1239 memcpy(buf.ptr, upbuf.ptr+ipos, rd); 1240 ipos += rd; 1241 return rd; 1242 }, 1243 // writer 1244 (buf) { 1245 res += buf.length; 1246 writeBuf(buf[]); 1247 }, 1248 // max mode 1249 (cpr == Compressor.BalzMax) 1250 ); 1251 return res; 1252 } 1253 1254 static if (arcz_has_zopfli) long writePackedZopfli (const(void)[] upbuf) { 1255 ubyte[] indata; 1256 void* odata; 1257 size_t osize; 1258 ZopfliOptions opts; 1259 ZopfliCompress(opts, ZOPFLI_FORMAT_ZLIB, upbuf.ptr, upbuf.length, &odata, &osize); 1260 writeBuf(odata[0..osize]); 1261 ZopfliFree(odata); 1262 return cast(long)osize; 1263 } 1264 1265 long writePackedZLib (const(void)[] upbuf) { 1266 assert(upbuf.length > 0 && upbuf.length < int.max); 1267 long res = 0; 1268 z_stream zs; 1269 ubyte[2048] obuf; 1270 zs.next_out = obuf.ptr; 1271 zs.avail_out = cast(uint)obuf.length; 1272 zs.next_in = null; 1273 zs.avail_in = 0; 1274 // initialize packer 1275 if (deflateInit2(&zs, Z_BEST_COMPRESSION, Z_DEFLATED, 15, 9, 0) != Z_OK) throw new Exception("can't write packed data"); 1276 scope(exit) deflateEnd(&zs); 1277 zs.next_in = cast(typeof(zs.next_in))upbuf.ptr; 1278 zs.avail_in = cast(uint)upbuf.length; 1279 while (zs.avail_in > 0) { 1280 if (zs.avail_out == 0) { 1281 res += cast(uint)obuf.length; 1282 writeBuf(obuf[]); 1283 zs.next_out = obuf.ptr; 1284 zs.avail_out = cast(uint)obuf.length; 1285 } 1286 auto err = deflate(&zs, Z_NO_FLUSH); 1287 if (err != Z_OK) throw new Exception("zlib compression error"); 1288 } 1289 while (zs.avail_out != obuf.length) { 1290 res += cast(uint)obuf.length-zs.avail_out; 1291 writeBuf(obuf[0..$-zs.avail_out]); 1292 zs.next_out = obuf.ptr; 1293 zs.avail_out = cast(uint)obuf.length; 1294 auto err = deflate(&zs, Z_FINISH); 1295 if (err != Z_OK && err != Z_STREAM_END) throw new Exception("zlib compression error"); 1296 // succesfully flushed? 1297 //if (err != Z_STREAM_END) throw new VFSException("zlib compression error"); 1298 } 1299 return res; 1300 } 1301 1302 // return size of packed data written 1303 uint writePackedBuf (const(void)[] upbuf) { 1304 assert(upbuf.length > 0 && upbuf.length < int.max); 1305 long res = 0; 1306 final switch (cpr) { 1307 case Compressor.ZLib: 1308 res = writePackedZLib(upbuf); 1309 break; 1310 case Compressor.Balz: 1311 case Compressor.BalzMax: 1312 static if (arcz_has_balz) { 1313 res = writePackedBalz(upbuf); 1314 break; 1315 } else { 1316 throw new Exception("no Balz support was compiled in ArcZ"); 1317 } 1318 case Compressor.Zopfli: 1319 static if (arcz_has_zopfli) { 1320 res = writePackedZopfli(upbuf); 1321 //break; 1322 } else { 1323 //new Exception("no Zopfli support was compiled in ArcZ"); 1324 res = writePackedZLib(upbuf); 1325 } 1326 break; 1327 } 1328 if (res > uint.max) throw new Exception("output archive too big"); 1329 return cast(uint)res; 1330 } 1331 1332 void flushData () { 1333 if (cdpos > 0) { 1334 ChunkInfo ci; 1335 auto pos = ftell(arcfl); 1336 if (pos < 0 || pos >= uint.max) throw new Exception("output archive too big"); 1337 ci.ofs = cast(uint)pos; 1338 auto wlen = writePackedBuf(chunkdata[0..cdpos]); 1339 ci.pksize = wlen; 1340 if (cdpos == chunkdata.length && ci.pksize >= chunkdata.length) { 1341 // wow, this chunk is unpackable 1342 //{ import std.stdio; writeln("unpackable chunk found!"); } 1343 if (fseek(arcfl, pos, 0) < 0) throw new Exception("can't seek in output file"); 1344 writeBuf(chunkdata[0..cdpos]); 1345 version(Posix) { 1346 import core.stdc.stdio : fileno; 1347 import core.sys.posix.unistd : ftruncate; 1348 pos = ftell(arcfl); 1349 if (pos < 0 || pos >= uint.max) throw new Exception("output archive too big"); 1350 if (ftruncate(fileno(arcfl), cast(uint)pos) < 0) throw new Exception("error truncating output file"); 1351 } 1352 ci.pksize = cdpos; 1353 } 1354 if (cdpos < chunkdata.length) lastChunkSize = cast(uint)cdpos; 1355 cdpos = 0; 1356 chunks ~= ci; 1357 } else { 1358 lastChunkSize = cast(uint)chunkdata.length; 1359 } 1360 } 1361 1362 void closeArc () { 1363 flushData(); 1364 // write index 1365 //assert(ftell(arcfl) > 0 && ftell(arcfl) < uint.max); 1366 assert(chunkdata.length < uint.max); 1367 assert(chunks.length < uint.max); 1368 assert(files.length < uint.max); 1369 // create index in memory 1370 ubyte[] index; 1371 1372 void putUint (uint v) { 1373 index ~= v&0xff; 1374 index ~= (v>>8)&0xff; 1375 index ~= (v>>16)&0xff; 1376 index ~= (v>>24)&0xff; 1377 } 1378 1379 void putUbyte (ubyte v) { 1380 index ~= v; 1381 } 1382 1383 void putBuf (const(void)[] buf) { 1384 assert(buf.length > 0); 1385 index ~= (cast(const(ubyte)[])buf)[]; 1386 } 1387 1388 // create index in memory 1389 { 1390 // chunk size 1391 putUint(cast(uint)chunkdata.length); 1392 // chunk count 1393 putUint(cast(uint)chunks.length); 1394 // last chunk size 1395 putUint(lastChunkSize); // 0: last chunk is full 1396 // chunk offsets and sizes 1397 foreach (ref ci; chunks) { 1398 putUint(ci.ofs); 1399 putUint(ci.pksize); 1400 } 1401 // file count 1402 putUint(cast(uint)files.length); 1403 uint nbofs = cast(uint)index.length+cast(uint)files.length*(5*4); 1404 //uint nbofs = 0; 1405 // files 1406 foreach (ref fi; files) { 1407 // name: length(byte), chars 1408 assert(fi.name.length > 0 && fi.name.length <= 16384); 1409 putUint(nbofs); 1410 putUint(cast(uint)fi.name.length); 1411 nbofs += cast(uint)fi.name.length+1; // put zero byte there to ease C interfacing 1412 //putBuf(fi.name[]); 1413 // chunk number 1414 putUint(fi.chunk); 1415 // offset in unpacked chunk 1416 putUint(fi.chunkofs); 1417 // unpacked size 1418 putUint(fi.size); 1419 } 1420 // names 1421 foreach (ref fi; files) { 1422 putBuf(fi.name[]); 1423 putUbyte(0); // this means nothing, it is here just for convenience (hello, C!) 1424 } 1425 assert(index.length < uint.max); 1426 } 1427 auto cpos = ftell(arcfl); 1428 if (cpos < 0 || cpos > uint.max) throw new Exception("output archive too big"); 1429 // write packed index 1430 debug(arcz_writer) { import core.stdc.stdio : pinrtf; printf("index size: %u\n", cast(uint)index.length); } 1431 auto pkisz = writePackedBuf(index[]); 1432 debug(arcz_writer) { import core.stdc.stdio : pinrtf; printf("packed index size: %u\n", cast(uint)pkisz); } 1433 // write index info 1434 if (fseek(arcfl, 5, 0) < 0) throw new Exception("seek error"); 1435 // index offset in file 1436 writeUint(cast(uint) cpos); 1437 // packed index size 1438 writeUint(pkisz); 1439 // unpacked index size 1440 writeUint(cast(uint)index.length); 1441 // done 1442 statChunks = cast(uint)chunks.length; 1443 statFiles = cast(uint)files.length; 1444 } 1445 1446 public: 1447 this (const(char)[] fname, uint chunkSize=256*1024, Compressor acpr=Compressor.ZLib) { 1448 import std.internal.cstring; 1449 assert(chunkSize > 0 && chunkSize < 32*1024*1024); // arbitrary limit 1450 static if (!arcz_has_balz) { 1451 if (acpr == Compressor.Balz || acpr == Compressor.BalzMax) throw new Exception("no Balz support was compiled in ArcZ"); 1452 } 1453 static if (!arcz_has_zopfli) { 1454 //if (acpr == Compressor.Zopfli) throw new Exception("no Zopfli support was compiled in ArcZ"); 1455 } 1456 cpr = acpr; 1457 arcfl = fopen(fname.tempCString, "wb"); 1458 if (arcfl is null) throw new Exception("can't create output file '"~fname.idup~"'"); 1459 cdpos = 0; 1460 chunkdata.length = chunkSize; 1461 scope(failure) { fclose(arcfl); arcfl = null; } 1462 writeBuf("CZA2"); // signature 1463 if (cpr == Compressor.Balz || cpr == Compressor.BalzMax) { 1464 writeUbyte(1); // version 1465 } else { 1466 writeUbyte(0); // version 1467 } 1468 writeUint(0); // offset to index 1469 writeUint(0); // packed index size 1470 writeUint(0); // unpacked index size 1471 } 1472 1473 ~this () { close(); } 1474 1475 void close () { 1476 if (arcfl !is null) { 1477 scope(exit) { fclose(arcfl); arcfl = null; } 1478 closeArc(); 1479 } 1480 chunkdata = null; 1481 chunks = null; 1482 files = null; 1483 lastChunkSize = 0; 1484 cdpos = 0; 1485 } 1486 1487 // valid after closing 1488 @property uint chunksWritten () const pure nothrow @safe @nogc { pragma(inline, true); return statChunks; } 1489 @property uint filesWritten () const pure nothrow @safe @nogc { pragma(inline, true); return statFiles; } 1490 1491 void newFile (string name, uint size) { 1492 FileInfo fi; 1493 assert(name.length <= 255); 1494 fi.name = name; 1495 fi.chunk = cast(uint)chunks.length; 1496 fi.chunkofs = cast(uint)cdpos; 1497 fi.size = size; 1498 files ~= fi; 1499 } 1500 1501 void rawWrite(T) (const(T)[] buffer) { 1502 if (buffer.length > 0) { 1503 auto src = cast(const(ubyte)*)buffer.ptr; 1504 auto len = buffer.length*T.sizeof; 1505 while (len > 0) { 1506 if (cdpos == chunkdata.length) flushData(); 1507 if (cdpos < chunkdata.length) { 1508 auto wr = chunkdata.length-cdpos; 1509 if (wr > len) wr = len; 1510 chunkdata[cdpos..cdpos+wr] = src[0..wr]; 1511 cdpos += wr; 1512 len -= wr; 1513 src += wr; 1514 } 1515 } 1516 } 1517 } 1518 } 1519 1520 1521 // ////////////////////////////////////////////////////////////////////////// // 1522 /* arcz file format: 1523 header 1524 ====== 1525 db 'CZA2' ; signature 1526 db version ; 0: zlib; 1: balz 1527 dd indexofs ; offset to packed index 1528 dd pkindexsz ; size of packed index 1529 dd upindexsz ; size of unpacked index 1530 1531 1532 index 1533 ===== 1534 dd chunksize ; unpacked chunk size in bytes 1535 dd chunkcount ; number of chunks in file 1536 dd lastchunksz ; size of last chunk (it may be incomplete); 0: last chunk is completely used (all `chunksize` bytes) 1537 1538 then chunk offsets and sizes follows: 1539 dd chunkofs ; from file start 1540 dd pkchunksz ; size of (possibly packed) chunk data; if it equals to `chunksize`, this chunk is not packed 1541 1542 then file list follows: 1543 dd filecount ; number of files in archive 1544 1545 then file info follows: 1546 dd nameofs ; (in index) 1547 dd namelen ; length of name (can't be 0) 1548 dd firstchunk ; chunk where file starts 1549 dd firstofs ; offset in first chunk (unpacked) where file starts 1550 dd filesize ; unpacked file size 1551 1552 then name buffer follows -- just bytes 1553 */ 1554 1555 } 1556 1557 version(WithLzmaDecoder) { 1558 1559 /* *************************************************** */ 1560 /* The rest of the file is copy/paste of external code */ 1561 /* *************************************************** */ 1562 1563 /* LzmaDec.h -- LZMA Decoder 1564 2017-04-03 : Igor Pavlov : Public domain */ 1565 // also by Lasse Collin 1566 /* ported to D by ketmar */ 1567 private nothrow @trusted @nogc: 1568 1569 //version = _LZMA_PROB32; 1570 /* _LZMA_PROB32 can increase the speed on some CPUs, 1571 but memory usage for CLzmaDec::probs will be doubled in that case */ 1572 1573 //version = _LZMA_SIZE_OPT; 1574 1575 alias Byte = ubyte; 1576 alias UInt16 = ushort; 1577 alias UInt32 = uint; 1578 alias SizeT = size_t; 1579 1580 version(_LZMA_PROB32) { 1581 alias CLzmaProb = UInt32; 1582 } else { 1583 alias CLzmaProb = UInt16; 1584 } 1585 1586 public enum SRes { 1587 OK, 1588 ERROR_UNSUPPORTED, 1589 ERROR_MEM, 1590 ERROR_DATA, 1591 ERROR_INPUT_EOF, 1592 ERROR_FAIL, 1593 } 1594 1595 /* ---------- LZMA Properties ---------- */ 1596 1597 public enum LZMA_PROPS_SIZE = 5; 1598 1599 public struct CLzmaProps { 1600 uint lc, lp, pb; 1601 UInt32 dicSize; 1602 } 1603 1604 /* LzmaProps_Decode - decodes properties 1605 Returns: 1606 SRes.OK 1607 SRes.ERROR_UNSUPPORTED - Unsupported properties 1608 */ 1609 1610 //!SRes LzmaProps_Decode(CLzmaProps *p, const(Byte)* data, uint size); 1611 1612 1613 /* ---------- LZMA Decoder state ---------- */ 1614 1615 /* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case. 1616 Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */ 1617 1618 enum LZMA_REQUIRED_INPUT_MAX = 20; 1619 1620 public struct CLzmaDec { 1621 private: 1622 CLzmaProps prop; 1623 CLzmaProb* probs; 1624 public Byte* dic; 1625 const(Byte)* buf; 1626 UInt32 range, code; 1627 public SizeT dicPos; 1628 public SizeT dicBufSize; 1629 UInt32 processedPos; 1630 UInt32 checkDicSize; 1631 uint state; 1632 UInt32[4] reps; 1633 uint remainLen; 1634 int needFlush; 1635 int needInitState; 1636 UInt32 numProbs; 1637 uint tempBufSize; 1638 Byte[LZMA_REQUIRED_INPUT_MAX] tempBuf; 1639 } 1640 1641 //#define LzmaDec_Construct(p) { (p).dic = 0; (p).probs = 0; } 1642 1643 //void LzmaDec_Init(CLzmaDec *p); 1644 1645 /* There are two types of LZMA streams: 1646 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. 1647 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ 1648 1649 public alias ELzmaFinishMode = int; 1650 public enum /*ELzmaFinishMode*/ { 1651 LZMA_FINISH_ANY, /* finish at any point */ 1652 LZMA_FINISH_END /* block must be finished at the end */ 1653 } 1654 1655 /* ELzmaFinishMode has meaning only if the decoding reaches output limit !!! 1656 1657 You must use LZMA_FINISH_END, when you know that current output buffer 1658 covers last bytes of block. In other cases you must use LZMA_FINISH_ANY. 1659 1660 If LZMA decoder sees end marker before reaching output limit, it returns SRes.OK, 1661 and output value of destLen will be less than output buffer size limit. 1662 You can check status result also. 1663 1664 You can use multiple checks to test data integrity after full decompression: 1665 1) Check Result and "status" variable. 1666 2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize. 1667 3) Check that output(srcLen) = compressedSize, if you know real compressedSize. 1668 You must use correct finish mode in that case. */ 1669 1670 public alias ELzmaStatus = int; 1671 public enum /*ELzmaStatus*/ { 1672 LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */ 1673 LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ 1674 LZMA_STATUS_NOT_FINISHED, /* stream was not finished */ 1675 LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */ 1676 LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */ 1677 } 1678 1679 /* ELzmaStatus is used only as output value for function call */ 1680 1681 1682 /* ---------- Interfaces ---------- */ 1683 1684 /* There are 3 levels of interfaces: 1685 1) Dictionary Interface 1686 2) Buffer Interface 1687 3) One Call Interface 1688 You can select any of these interfaces, but don't mix functions from different 1689 groups for same object. */ 1690 1691 1692 /* There are two variants to allocate state for Dictionary Interface: 1693 1) LzmaDec_Allocate / LzmaDec_Free 1694 2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs 1695 You can use variant 2, if you set dictionary buffer manually. 1696 For Buffer Interface you must always use variant 1. 1697 1698 LzmaDec_Allocate* can return: 1699 SRes.OK 1700 SRes.ERROR_MEM - Memory allocation error 1701 SRes.ERROR_UNSUPPORTED - Unsupported properties 1702 */ 1703 1704 /* 1705 SRes LzmaDec_AllocateProbs(CLzmaDec *p, const(Byte)* props, uint propsSize); 1706 void LzmaDec_FreeProbs(CLzmaDec *p); 1707 1708 SRes LzmaDec_Allocate(CLzmaDec *state, const(Byte)* prop, uint propsSize); 1709 void LzmaDec_Free(CLzmaDec *state); 1710 */ 1711 1712 /* ---------- Dictionary Interface ---------- */ 1713 1714 /* You can use it, if you want to eliminate the overhead for data copying from 1715 dictionary to some other external buffer. 1716 You must work with CLzmaDec variables directly in this interface. 1717 1718 STEPS: 1719 LzmaDec_Constr() 1720 LzmaDec_Allocate() 1721 for (each new stream) 1722 { 1723 LzmaDec_Init() 1724 while (it needs more decompression) 1725 { 1726 LzmaDec_DecodeToDic() 1727 use data from CLzmaDec::dic and update CLzmaDec::dicPos 1728 } 1729 } 1730 LzmaDec_Free() 1731 */ 1732 1733 /* LzmaDec_DecodeToDic 1734 1735 The decoding to internal dictionary buffer (CLzmaDec::dic). 1736 You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!! 1737 1738 finishMode: 1739 It has meaning only if the decoding reaches output limit (dicLimit). 1740 LZMA_FINISH_ANY - Decode just dicLimit bytes. 1741 LZMA_FINISH_END - Stream must be finished after dicLimit. 1742 1743 Returns: 1744 SRes.OK 1745 status: 1746 LZMA_STATUS_FINISHED_WITH_MARK 1747 LZMA_STATUS_NOT_FINISHED 1748 LZMA_STATUS_NEEDS_MORE_INPUT 1749 LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK 1750 SRes.ERROR_DATA - Data error 1751 */ 1752 1753 //SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const(Byte)* src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); 1754 1755 1756 /* ---------- Buffer Interface ---------- */ 1757 1758 /* It's zlib-like interface. 1759 See LzmaDec_DecodeToDic description for information about STEPS and return results, 1760 but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need 1761 to work with CLzmaDec variables manually. 1762 1763 finishMode: 1764 It has meaning only if the decoding reaches output limit (*destLen). 1765 LZMA_FINISH_ANY - Decode just destLen bytes. 1766 LZMA_FINISH_END - Stream must be finished after (*destLen). 1767 */ 1768 1769 //SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const(Byte)* src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); 1770 1771 1772 /* ---------- One Call Interface ---------- */ 1773 1774 /* LzmaDecode 1775 1776 finishMode: 1777 It has meaning only if the decoding reaches output limit (*destLen). 1778 LZMA_FINISH_ANY - Decode just destLen bytes. 1779 LZMA_FINISH_END - Stream must be finished after (*destLen). 1780 1781 Returns: 1782 SRes.OK 1783 status: 1784 LZMA_STATUS_FINISHED_WITH_MARK 1785 LZMA_STATUS_NOT_FINISHED 1786 LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK 1787 SRes.ERROR_DATA - Data error 1788 SRes.ERROR_MEM - Memory allocation error 1789 SRes.ERROR_UNSUPPORTED - Unsupported properties 1790 SRes.ERROR_INPUT_EOF - It needs more bytes in input buffer (src). 1791 */ 1792 1793 /* 1794 SRes LzmaDecode(Byte *dest, SizeT *destLen, const(Byte)* src, SizeT *srcLen, 1795 const(Byte)* propData, uint propSize, ELzmaFinishMode finishMode, 1796 ELzmaStatus *status, ISzAllocPtr alloc); 1797 */ 1798 1799 // ////////////////////////////////////////////////////////////////////////// // 1800 private: 1801 1802 enum kNumTopBits = 24; 1803 enum kTopValue = 1U<<kNumTopBits; 1804 1805 enum kNumBitModelTotalBits = 11; 1806 enum kBitModelTotal = 1<<kNumBitModelTotalBits; 1807 enum kNumMoveBits = 5; 1808 1809 enum RC_INIT_SIZE = 5; 1810 1811 //enum NORMALIZE = "if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); }"; 1812 1813 //#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 1814 //#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); 1815 //#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); 1816 enum GET_BIT2(string p, string i, string A0, string A1) = 1817 "ttt = *("~p~"); if (range < kTopValue) { range <<= 8; code = (code<<8)|(*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound)\n"~ 1818 "{ range = bound; *("~p~") = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); "~i~" = ("~i~"+"~i~"); "~A0~" } else\n"~ 1819 "{ range -= bound; code -= bound; *("~p~") = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); "~i~" = ("~i~"+"~i~")+1; "~A1~" }"; 1820 //#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) 1821 enum GET_BIT(string p, string i) = GET_BIT2!(p, i, "", ""); 1822 1823 enum TREE_GET_BIT(string probs, string i) = "{"~GET_BIT!("("~probs~"+"~i~")", i)~"}"; 1824 1825 enum TREE_DECODE(string probs, string limit, string i) = 1826 "{ "~i~" = 1; do { "~TREE_GET_BIT!(probs, i)~" } while ("~i~" < "~limit~"); "~i~" -= "~limit~"; }"; 1827 1828 1829 version(_LZMA_SIZE_OPT) { 1830 enum TREE_6_DECODE(string probs, string i) = TREE_DECODE!(probs, "(1<<6)", i); 1831 } else { 1832 enum TREE_6_DECODE(string probs, string i) = 1833 "{ "~i~" = 1;\n"~ 1834 TREE_GET_BIT!(probs, i)~ 1835 TREE_GET_BIT!(probs, i)~ 1836 TREE_GET_BIT!(probs, i)~ 1837 TREE_GET_BIT!(probs, i)~ 1838 TREE_GET_BIT!(probs, i)~ 1839 TREE_GET_BIT!(probs, i)~ 1840 i~" -= 0x40; }"; 1841 } 1842 1843 enum NORMAL_LITER_DEC = GET_BIT!("prob + symbol", "symbol"); 1844 enum MATCHED_LITER_DEC = 1845 "matchByte <<= 1;\n"~ 1846 "bit = (matchByte & offs);\n"~ 1847 "probLit = prob + offs + bit + symbol;\n"~ 1848 GET_BIT2!("probLit", "symbol", "offs &= ~bit;", "offs &= bit;"); 1849 1850 enum NORMALIZE_CHECK = "if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); }"; 1851 1852 //#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 1853 //#define UPDATE_0_CHECK range = bound; 1854 //#define UPDATE_1_CHECK range -= bound; code -= bound; 1855 enum GET_BIT2_CHECK(string p, string i, string A0, string A1) = 1856 "ttt = *("~p~"); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound)\n"~ 1857 "{ range = bound; "~i~" = ("~i~" + "~i~"); "~A0~" } else\n"~ 1858 "{ range -= bound; code -= bound; "~i~" = ("~i~" + "~i~") + 1; "~A1~" }"; 1859 enum GET_BIT_CHECK(string p, string i) = GET_BIT2_CHECK!(p, i, "{}", "{}"); 1860 enum TREE_DECODE_CHECK(string probs, string limit, string i) = 1861 "{ "~i~" = 1; do { "~GET_BIT_CHECK!(probs~"+"~i, i)~" } while ("~i~" < "~limit~"); "~i~" -= "~limit~"; }"; 1862 1863 1864 enum kNumPosBitsMax = 4; 1865 enum kNumPosStatesMax = (1 << kNumPosBitsMax); 1866 1867 enum kLenNumLowBits = 3; 1868 enum kLenNumLowSymbols = (1 << kLenNumLowBits); 1869 enum kLenNumMidBits = 3; 1870 enum kLenNumMidSymbols = (1 << kLenNumMidBits); 1871 enum kLenNumHighBits = 8; 1872 enum kLenNumHighSymbols = (1 << kLenNumHighBits); 1873 1874 enum LenChoice = 0; 1875 enum LenChoice2 = (LenChoice + 1); 1876 enum LenLow = (LenChoice2 + 1); 1877 enum LenMid = (LenLow + (kNumPosStatesMax << kLenNumLowBits)); 1878 enum LenHigh = (LenMid + (kNumPosStatesMax << kLenNumMidBits)); 1879 enum kNumLenProbs = (LenHigh + kLenNumHighSymbols); 1880 1881 1882 enum kNumStates = 12; 1883 enum kNumLitStates = 7; 1884 1885 enum kStartPosModelIndex = 4; 1886 enum kEndPosModelIndex = 14; 1887 enum kNumFullDistances = (1 << (kEndPosModelIndex >> 1)); 1888 1889 enum kNumPosSlotBits = 6; 1890 enum kNumLenToPosStates = 4; 1891 1892 enum kNumAlignBits = 4; 1893 enum kAlignTableSize = (1 << kNumAlignBits); 1894 1895 enum kMatchMinLen = 2; 1896 enum kMatchSpecLenStart = (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols); 1897 1898 enum IsMatch = 0; 1899 enum IsRep = (IsMatch + (kNumStates << kNumPosBitsMax)); 1900 enum IsRepG0 = (IsRep + kNumStates); 1901 enum IsRepG1 = (IsRepG0 + kNumStates); 1902 enum IsRepG2 = (IsRepG1 + kNumStates); 1903 enum IsRep0Long = (IsRepG2 + kNumStates); 1904 enum PosSlot = (IsRep0Long + (kNumStates << kNumPosBitsMax)); 1905 enum SpecPos = (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)); 1906 enum Align = (SpecPos + kNumFullDistances - kEndPosModelIndex); 1907 enum LenCoder = (Align + kAlignTableSize); 1908 enum RepLenCoder = (LenCoder + kNumLenProbs); 1909 enum Literal = (RepLenCoder + kNumLenProbs); 1910 1911 enum LZMA_BASE_SIZE = 1846; 1912 enum LZMA_LIT_SIZE = 0x300; 1913 1914 static assert(Literal == LZMA_BASE_SIZE); 1915 1916 //#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p).lc + (p).lp))) 1917 1918 enum LZMA_DIC_MIN = (1 << 12); 1919 1920 /* First LZMA-symbol is always decoded. 1921 And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization 1922 Out: 1923 Result: 1924 SRes.OK - OK 1925 SRes.ERROR_DATA - Error 1926 p->remainLen: 1927 < kMatchSpecLenStart : normal remain 1928 = kMatchSpecLenStart : finished 1929 = kMatchSpecLenStart + 1 : Flush marker (unused now) 1930 = kMatchSpecLenStart + 2 : State Init Marker (unused now) 1931 */ 1932 1933 private SRes LzmaDec_DecodeReal (CLzmaDec* p, SizeT limit, const(Byte)* bufLimit) { 1934 CLzmaProb* probs = p.probs; 1935 1936 uint state = p.state; 1937 UInt32 rep0 = p.reps.ptr[0], rep1 = p.reps.ptr[1], rep2 = p.reps.ptr[2], rep3 = p.reps.ptr[3]; 1938 uint pbMask = (1U<<(p.prop.pb))-1; 1939 uint lpMask = (1U<<(p.prop.lp))-1; 1940 uint lc = p.prop.lc; 1941 1942 Byte* dic = p.dic; 1943 SizeT dicBufSize = p.dicBufSize; 1944 SizeT dicPos = p.dicPos; 1945 1946 UInt32 processedPos = p.processedPos; 1947 UInt32 checkDicSize = p.checkDicSize; 1948 uint len = 0; 1949 1950 const(Byte)* buf = p.buf; 1951 UInt32 range = p.range; 1952 UInt32 code = p.code; 1953 1954 do { 1955 CLzmaProb *prob; 1956 UInt32 bound; 1957 uint ttt; 1958 uint posState = processedPos & pbMask; 1959 1960 prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; 1961 ttt = *(prob); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 1962 { 1963 uint symbol; 1964 range = bound; *(prob) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 1965 prob = probs + Literal; 1966 if (processedPos != 0 || checkDicSize != 0) 1967 prob += (cast(UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + 1968 (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); 1969 processedPos++; 1970 1971 if (state < kNumLitStates) 1972 { 1973 state -= (state < 4) ? state : 3; 1974 symbol = 1; 1975 version(_LZMA_SIZE_OPT) { 1976 do { mixin(NORMAL_LITER_DEC); } while (symbol < 0x100); 1977 } else { 1978 mixin(NORMAL_LITER_DEC); 1979 mixin(NORMAL_LITER_DEC); 1980 mixin(NORMAL_LITER_DEC); 1981 mixin(NORMAL_LITER_DEC); 1982 mixin(NORMAL_LITER_DEC); 1983 mixin(NORMAL_LITER_DEC); 1984 mixin(NORMAL_LITER_DEC); 1985 mixin(NORMAL_LITER_DEC); 1986 } 1987 } 1988 else 1989 { 1990 uint matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; 1991 uint offs = 0x100; 1992 state -= (state < 10) ? 3 : 6; 1993 symbol = 1; 1994 version(_LZMA_SIZE_OPT) { 1995 do 1996 { 1997 uint bit; 1998 CLzmaProb *probLit; 1999 mixin(MATCHED_LITER_DEC); 2000 } 2001 while (symbol < 0x100); 2002 } else { 2003 { 2004 uint bit; 2005 CLzmaProb *probLit; 2006 mixin(MATCHED_LITER_DEC); 2007 mixin(MATCHED_LITER_DEC); 2008 mixin(MATCHED_LITER_DEC); 2009 mixin(MATCHED_LITER_DEC); 2010 mixin(MATCHED_LITER_DEC); 2011 mixin(MATCHED_LITER_DEC); 2012 mixin(MATCHED_LITER_DEC); 2013 mixin(MATCHED_LITER_DEC); 2014 } 2015 } 2016 } 2017 2018 dic[dicPos++] = cast(Byte)symbol; 2019 continue; 2020 } 2021 2022 { 2023 range -= bound; code -= bound; *(prob) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2024 prob = probs + IsRep + state; 2025 ttt = *(prob); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2026 { 2027 range = bound; *(prob) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2028 state += kNumStates; 2029 prob = probs + LenCoder; 2030 } 2031 else 2032 { 2033 range -= bound; code -= bound; *(prob) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2034 if (checkDicSize == 0 && processedPos == 0) 2035 return SRes.ERROR_DATA; 2036 prob = probs + IsRepG0 + state; 2037 ttt = *(prob); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2038 { 2039 range = bound; *(prob) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2040 prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; 2041 ttt = *(prob); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2042 { 2043 range = bound; *(prob) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2044 dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; 2045 dicPos++; 2046 processedPos++; 2047 state = state < kNumLitStates ? 9 : 11; 2048 continue; 2049 } 2050 range -= bound; code -= bound; *(prob) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2051 } 2052 else 2053 { 2054 UInt32 distance; 2055 range -= bound; code -= bound; *(prob) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2056 prob = probs + IsRepG1 + state; 2057 ttt = *(prob); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2058 { 2059 range = bound; *(prob) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2060 distance = rep1; 2061 } 2062 else 2063 { 2064 range -= bound; code -= bound; *(prob) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2065 prob = probs + IsRepG2 + state; 2066 ttt = *(prob); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2067 { 2068 range = bound; *(prob) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2069 distance = rep2; 2070 } 2071 else 2072 { 2073 range -= bound; code -= bound; *(prob) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2074 distance = rep3; 2075 rep3 = rep2; 2076 } 2077 rep2 = rep1; 2078 } 2079 rep1 = rep0; 2080 rep0 = distance; 2081 } 2082 state = state < kNumLitStates ? 8 : 11; 2083 prob = probs + RepLenCoder; 2084 } 2085 2086 version(_LZMA_SIZE_OPT) { 2087 { 2088 uint lim, offset; 2089 CLzmaProb *probLen = prob + LenChoice; 2090 ttt = *(probLen); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2091 { 2092 range = bound; *(probLen) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2093 probLen = prob + LenLow + (posState << kLenNumLowBits); 2094 offset = 0; 2095 lim = (1 << kLenNumLowBits); 2096 } 2097 else 2098 { 2099 range -= bound; code -= bound; *(probLen) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2100 probLen = prob + LenChoice2; 2101 ttt = *(probLen); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2102 { 2103 range = bound; *(probLen) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2104 probLen = prob + LenMid + (posState << kLenNumMidBits); 2105 offset = kLenNumLowSymbols; 2106 lim = (1 << kLenNumMidBits); 2107 } 2108 else 2109 { 2110 range -= bound; code -= bound; *(probLen) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2111 probLen = prob + LenHigh; 2112 offset = kLenNumLowSymbols + kLenNumMidSymbols; 2113 lim = (1 << kLenNumHighBits); 2114 } 2115 } 2116 mixin(TREE_DECODE!("probLen", "lim", "len")); 2117 len += offset; 2118 } 2119 } else { 2120 { 2121 CLzmaProb *probLen = prob + LenChoice; 2122 ttt = *(probLen); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2123 { 2124 range = bound; *(probLen) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2125 probLen = prob + LenLow + (posState << kLenNumLowBits); 2126 len = 1; 2127 mixin(TREE_GET_BIT!("probLen", "len")); 2128 mixin(TREE_GET_BIT!("probLen", "len")); 2129 mixin(TREE_GET_BIT!("probLen", "len")); 2130 len -= 8; 2131 } 2132 else 2133 { 2134 range -= bound; code -= bound; *(probLen) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2135 probLen = prob + LenChoice2; 2136 ttt = *(probLen); if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } bound = (range>>kNumBitModelTotalBits)*ttt; if (code < bound) 2137 { 2138 range = bound; *(probLen) = cast(CLzmaProb)(ttt+((kBitModelTotal-ttt)>>kNumMoveBits)); 2139 probLen = prob + LenMid + (posState << kLenNumMidBits); 2140 len = 1; 2141 mixin(TREE_GET_BIT!("probLen", "len")); 2142 mixin(TREE_GET_BIT!("probLen", "len")); 2143 mixin(TREE_GET_BIT!("probLen", "len")); 2144 } 2145 else 2146 { 2147 range -= bound; code -= bound; *(probLen) = cast(CLzmaProb)(ttt-(ttt>>kNumMoveBits)); 2148 probLen = prob + LenHigh; 2149 mixin(TREE_DECODE!("probLen", "(1 << kLenNumHighBits)", "len")); 2150 len += kLenNumLowSymbols + kLenNumMidSymbols; 2151 } 2152 } 2153 } 2154 } 2155 2156 if (state >= kNumStates) 2157 { 2158 UInt32 distance; 2159 prob = probs + PosSlot + 2160 ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); 2161 mixin(TREE_6_DECODE!("prob", "distance")); 2162 if (distance >= kStartPosModelIndex) 2163 { 2164 uint posSlot = cast(uint)distance; 2165 uint numDirectBits = cast(uint)(((distance >> 1) - 1)); 2166 distance = (2 | (distance & 1)); 2167 if (posSlot < kEndPosModelIndex) 2168 { 2169 distance <<= numDirectBits; 2170 prob = probs + SpecPos + distance - posSlot - 1; 2171 { 2172 UInt32 mask = 1; 2173 uint i = 1; 2174 do 2175 { 2176 mixin(GET_BIT2!("prob + i", "i", "{}" , "distance |= mask;")); 2177 mask <<= 1; 2178 } 2179 while (--numDirectBits != 0); 2180 } 2181 } 2182 else 2183 { 2184 numDirectBits -= kNumAlignBits; 2185 do 2186 { 2187 if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } 2188 range >>= 1; 2189 2190 { 2191 UInt32 t; 2192 code -= range; 2193 t = (0 - (cast(UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ 2194 distance = (distance << 1) + (t + 1); 2195 code += range & t; 2196 } 2197 /* 2198 distance <<= 1; 2199 if (code >= range) 2200 { 2201 code -= range; 2202 distance |= 1; 2203 } 2204 */ 2205 } 2206 while (--numDirectBits != 0); 2207 prob = probs + Align; 2208 distance <<= kNumAlignBits; 2209 { 2210 uint i = 1; 2211 mixin(GET_BIT2!("prob + i", "i", "", "distance |= 1;")); 2212 mixin(GET_BIT2!("prob + i", "i", "", "distance |= 2;")); 2213 mixin(GET_BIT2!("prob + i", "i", "", "distance |= 4;")); 2214 mixin(GET_BIT2!("prob + i", "i", "", "distance |= 8;")); 2215 } 2216 if (distance == cast(UInt32)0xFFFFFFFF) 2217 { 2218 len += kMatchSpecLenStart; 2219 state -= kNumStates; 2220 break; 2221 } 2222 } 2223 } 2224 2225 rep3 = rep2; 2226 rep2 = rep1; 2227 rep1 = rep0; 2228 rep0 = distance + 1; 2229 if (checkDicSize == 0) 2230 { 2231 if (distance >= processedPos) 2232 { 2233 p.dicPos = dicPos; 2234 return SRes.ERROR_DATA; 2235 } 2236 } 2237 else if (distance >= checkDicSize) 2238 { 2239 p.dicPos = dicPos; 2240 return SRes.ERROR_DATA; 2241 } 2242 state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; 2243 } 2244 2245 len += kMatchMinLen; 2246 2247 { 2248 SizeT rem; 2249 uint curLen; 2250 SizeT pos; 2251 2252 if ((rem = limit - dicPos) == 0) 2253 { 2254 p.dicPos = dicPos; 2255 return SRes.ERROR_DATA; 2256 } 2257 2258 curLen = ((rem < len) ? cast(uint)rem : len); 2259 pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); 2260 2261 processedPos += curLen; 2262 2263 len -= curLen; 2264 if (curLen <= dicBufSize - pos) 2265 { 2266 Byte *dest = dic + dicPos; 2267 ptrdiff_t src = cast(ptrdiff_t)pos - cast(ptrdiff_t)dicPos; 2268 const(Byte)* lim = dest + curLen; 2269 dicPos += curLen; 2270 do 2271 *(dest) = cast(Byte)*(dest + src); 2272 while (++dest != lim); 2273 } 2274 else 2275 { 2276 do 2277 { 2278 dic[dicPos++] = dic[pos]; 2279 if (++pos == dicBufSize) 2280 pos = 0; 2281 } 2282 while (--curLen != 0); 2283 } 2284 } 2285 } 2286 } 2287 while (dicPos < limit && buf < bufLimit); 2288 2289 if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } 2290 2291 p.buf = buf; 2292 p.range = range; 2293 p.code = code; 2294 p.remainLen = len; 2295 p.dicPos = dicPos; 2296 p.processedPos = processedPos; 2297 p.reps.ptr[0] = rep0; 2298 p.reps.ptr[1] = rep1; 2299 p.reps.ptr[2] = rep2; 2300 p.reps.ptr[3] = rep3; 2301 p.state = state; 2302 2303 return SRes.OK; 2304 } 2305 2306 private void LzmaDec_WriteRem (CLzmaDec* p, SizeT limit) { 2307 if (p.remainLen != 0 && p.remainLen < kMatchSpecLenStart) 2308 { 2309 Byte *dic = p.dic; 2310 SizeT dicPos = p.dicPos; 2311 SizeT dicBufSize = p.dicBufSize; 2312 uint len = p.remainLen; 2313 SizeT rep0 = p.reps.ptr[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ 2314 SizeT rem = limit - dicPos; 2315 if (rem < len) 2316 len = cast(uint)(rem); 2317 2318 if (p.checkDicSize == 0 && p.prop.dicSize - p.processedPos <= len) 2319 p.checkDicSize = p.prop.dicSize; 2320 2321 p.processedPos += len; 2322 p.remainLen -= len; 2323 while (len != 0) 2324 { 2325 len--; 2326 dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; 2327 dicPos++; 2328 } 2329 p.dicPos = dicPos; 2330 } 2331 } 2332 2333 private SRes LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const(Byte)* bufLimit) 2334 { 2335 do 2336 { 2337 SizeT limit2 = limit; 2338 if (p.checkDicSize == 0) 2339 { 2340 UInt32 rem = p.prop.dicSize - p.processedPos; 2341 if (limit - p.dicPos > rem) 2342 limit2 = p.dicPos + rem; 2343 } 2344 2345 if (auto sres = LzmaDec_DecodeReal(p, limit2, bufLimit)) return sres; 2346 2347 if (p.checkDicSize == 0 && p.processedPos >= p.prop.dicSize) 2348 p.checkDicSize = p.prop.dicSize; 2349 2350 LzmaDec_WriteRem(p, limit); 2351 } 2352 while (p.dicPos < limit && p.buf < bufLimit && p.remainLen < kMatchSpecLenStart); 2353 2354 if (p.remainLen > kMatchSpecLenStart) 2355 p.remainLen = kMatchSpecLenStart; 2356 2357 return SRes.OK; 2358 } 2359 2360 alias ELzmaDummy = int; 2361 enum /*ELzmaDummy*/ { 2362 DUMMY_ERROR, /* unexpected end of input stream */ 2363 DUMMY_LIT, 2364 DUMMY_MATCH, 2365 DUMMY_REP 2366 } 2367 2368 private ELzmaDummy LzmaDec_TryDummy(const(CLzmaDec)* p, const(Byte)* buf, SizeT inSize) 2369 { 2370 UInt32 range = p.range; 2371 UInt32 code = p.code; 2372 const(Byte)* bufLimit = buf + inSize; 2373 const(CLzmaProb)* probs = p.probs; 2374 uint state = p.state; 2375 ELzmaDummy res; 2376 2377 { 2378 const(CLzmaProb)* prob; 2379 UInt32 bound; 2380 uint ttt; 2381 uint posState = (p.processedPos) & ((1 << p.prop.pb) - 1); 2382 2383 prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; 2384 ttt = *(prob); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2385 { 2386 range = bound; 2387 2388 /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ 2389 2390 prob = probs + Literal; 2391 if (p.checkDicSize != 0 || p.processedPos != 0) 2392 prob += (cast(UInt32)LZMA_LIT_SIZE * 2393 ((((p.processedPos) & ((1 << (p.prop.lp)) - 1)) << p.prop.lc) + 2394 (p.dic[(p.dicPos == 0 ? p.dicBufSize : p.dicPos) - 1] >> (8 - p.prop.lc)))); 2395 2396 if (state < kNumLitStates) 2397 { 2398 uint symbol = 1; 2399 do { mixin(GET_BIT_CHECK!("prob + symbol", "symbol")); } while (symbol < 0x100); 2400 } 2401 else 2402 { 2403 uint matchByte = p.dic[p.dicPos - p.reps.ptr[0] + 2404 (p.dicPos < p.reps.ptr[0] ? p.dicBufSize : 0)]; 2405 uint offs = 0x100; 2406 uint symbol = 1; 2407 do 2408 { 2409 uint bit; 2410 const(CLzmaProb)* probLit; 2411 matchByte <<= 1; 2412 bit = (matchByte & offs); 2413 probLit = prob + offs + bit + symbol; 2414 mixin(GET_BIT2_CHECK!("probLit", "symbol", "offs &= ~bit;", "offs &= bit;")); 2415 } 2416 while (symbol < 0x100); 2417 } 2418 res = DUMMY_LIT; 2419 } 2420 else 2421 { 2422 uint len; 2423 range -= bound; code -= bound; 2424 2425 prob = probs + IsRep + state; 2426 ttt = *(prob); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2427 { 2428 range = bound; 2429 state = 0; 2430 prob = probs + LenCoder; 2431 res = DUMMY_MATCH; 2432 } 2433 else 2434 { 2435 range -= bound; code -= bound; 2436 res = DUMMY_REP; 2437 prob = probs + IsRepG0 + state; 2438 ttt = *(prob); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2439 { 2440 range = bound; 2441 prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; 2442 ttt = *(prob); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2443 { 2444 range = bound; 2445 if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } 2446 return DUMMY_REP; 2447 } 2448 else 2449 { 2450 range -= bound; code -= bound; 2451 } 2452 } 2453 else 2454 { 2455 range -= bound; code -= bound; 2456 prob = probs + IsRepG1 + state; 2457 ttt = *(prob); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2458 { 2459 range = bound; 2460 } 2461 else 2462 { 2463 range -= bound; code -= bound; 2464 prob = probs + IsRepG2 + state; 2465 ttt = *(prob); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2466 { 2467 range = bound; 2468 } 2469 else 2470 { 2471 range -= bound; code -= bound; 2472 } 2473 } 2474 } 2475 state = kNumStates; 2476 prob = probs + RepLenCoder; 2477 } 2478 { 2479 uint limit, offset; 2480 const(CLzmaProb)* probLen = prob + LenChoice; 2481 ttt = *(probLen); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2482 { 2483 range = bound; 2484 probLen = prob + LenLow + (posState << kLenNumLowBits); 2485 offset = 0; 2486 limit = 1 << kLenNumLowBits; 2487 } 2488 else 2489 { 2490 range -= bound; code -= bound; 2491 probLen = prob + LenChoice2; 2492 ttt = *(probLen); if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) 2493 { 2494 range = bound; 2495 probLen = prob + LenMid + (posState << kLenNumMidBits); 2496 offset = kLenNumLowSymbols; 2497 limit = 1 << kLenNumMidBits; 2498 } 2499 else 2500 { 2501 range -= bound; code -= bound; 2502 probLen = prob + LenHigh; 2503 offset = kLenNumLowSymbols + kLenNumMidSymbols; 2504 limit = 1 << kLenNumHighBits; 2505 } 2506 } 2507 mixin(TREE_DECODE_CHECK!("probLen", "limit", "len")); 2508 len += offset; 2509 } 2510 2511 if (state < 4) 2512 { 2513 uint posSlot; 2514 prob = probs + PosSlot + 2515 ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << 2516 kNumPosSlotBits); 2517 mixin(TREE_DECODE_CHECK!("prob", "1 << kNumPosSlotBits", "posSlot")); 2518 if (posSlot >= kStartPosModelIndex) 2519 { 2520 uint numDirectBits = ((posSlot >> 1) - 1); 2521 2522 /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ 2523 2524 if (posSlot < kEndPosModelIndex) 2525 { 2526 prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; 2527 } 2528 else 2529 { 2530 numDirectBits -= kNumAlignBits; 2531 do 2532 { 2533 if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } 2534 range >>= 1; 2535 code -= range & (((code - range) >> 31) - 1); 2536 /* if (code >= range) code -= range; */ 2537 } 2538 while (--numDirectBits != 0); 2539 prob = probs + Align; 2540 numDirectBits = kNumAlignBits; 2541 } 2542 { 2543 uint i = 1; 2544 do 2545 { 2546 mixin(GET_BIT_CHECK!("prob + i", "i")); 2547 } 2548 while (--numDirectBits != 0); 2549 } 2550 } 2551 } 2552 } 2553 } 2554 if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } 2555 return res; 2556 } 2557 2558 2559 void LzmaDec_InitDicAndState(CLzmaDec *p, bool initDic, bool initState) 2560 { 2561 p.needFlush = 1; 2562 p.remainLen = 0; 2563 p.tempBufSize = 0; 2564 2565 if (initDic) 2566 { 2567 p.processedPos = 0; 2568 p.checkDicSize = 0; 2569 p.needInitState = 1; 2570 } 2571 if (initState) 2572 p.needInitState = 1; 2573 } 2574 2575 public void LzmaDec_Init(CLzmaDec *p) 2576 { 2577 p.dicPos = 0; 2578 LzmaDec_InitDicAndState(p, true, true); 2579 } 2580 2581 private void LzmaDec_InitStateReal(CLzmaDec *p) 2582 { 2583 SizeT numProbs = (Literal+(cast(UInt32)LZMA_LIT_SIZE<<((&p.prop).lc+(&p.prop).lp))); 2584 SizeT i; 2585 CLzmaProb *probs = p.probs; 2586 for (i = 0; i < numProbs; i++) 2587 probs[i] = kBitModelTotal >> 1; 2588 p.reps.ptr[0] = p.reps.ptr[1] = p.reps.ptr[2] = p.reps.ptr[3] = 1; 2589 p.state = 0; 2590 p.needInitState = 0; 2591 } 2592 2593 public SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const(Byte)* src, SizeT *srcLen, 2594 ELzmaFinishMode finishMode, ELzmaStatus *status) 2595 { 2596 SizeT inSize = *srcLen; 2597 (*srcLen) = 0; 2598 LzmaDec_WriteRem(p, dicLimit); 2599 2600 *status = LZMA_STATUS_NOT_SPECIFIED; 2601 2602 while (p.remainLen != kMatchSpecLenStart) 2603 { 2604 int checkEndMarkNow; 2605 2606 if (p.needFlush) 2607 { 2608 for (; inSize > 0 && p.tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) 2609 p.tempBuf.ptr[p.tempBufSize++] = *src++; 2610 if (p.tempBufSize < RC_INIT_SIZE) 2611 { 2612 *status = LZMA_STATUS_NEEDS_MORE_INPUT; 2613 return SRes.OK; 2614 } 2615 if (p.tempBuf.ptr[0] != 0) 2616 return SRes.ERROR_DATA; 2617 p.code = 2618 (cast(UInt32)p.tempBuf.ptr[1] << 24) 2619 | (cast(UInt32)p.tempBuf.ptr[2] << 16) 2620 | (cast(UInt32)p.tempBuf.ptr[3] << 8) 2621 | (cast(UInt32)p.tempBuf.ptr[4]); 2622 p.range = 0xFFFFFFFF; 2623 p.needFlush = 0; 2624 p.tempBufSize = 0; 2625 } 2626 2627 checkEndMarkNow = 0; 2628 if (p.dicPos >= dicLimit) 2629 { 2630 if (p.remainLen == 0 && p.code == 0) 2631 { 2632 *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; 2633 return SRes.OK; 2634 } 2635 if (finishMode == LZMA_FINISH_ANY) 2636 { 2637 *status = LZMA_STATUS_NOT_FINISHED; 2638 return SRes.OK; 2639 } 2640 if (p.remainLen != 0) 2641 { 2642 *status = LZMA_STATUS_NOT_FINISHED; 2643 return SRes.ERROR_DATA; 2644 } 2645 checkEndMarkNow = 1; 2646 } 2647 2648 if (p.needInitState) 2649 LzmaDec_InitStateReal(p); 2650 2651 if (p.tempBufSize == 0) 2652 { 2653 SizeT processed; 2654 const(Byte)* bufLimit; 2655 if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) 2656 { 2657 int dummyRes = LzmaDec_TryDummy(p, src, inSize); 2658 if (dummyRes == DUMMY_ERROR) 2659 { 2660 import core.stdc.string : memcpy; 2661 memcpy(p.tempBuf.ptr, src, inSize); 2662 p.tempBufSize = cast(uint)inSize; 2663 (*srcLen) += inSize; 2664 *status = LZMA_STATUS_NEEDS_MORE_INPUT; 2665 return SRes.OK; 2666 } 2667 if (checkEndMarkNow && dummyRes != DUMMY_MATCH) 2668 { 2669 *status = LZMA_STATUS_NOT_FINISHED; 2670 return SRes.ERROR_DATA; 2671 } 2672 bufLimit = src; 2673 } 2674 else 2675 bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; 2676 p.buf = src; 2677 if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) 2678 return SRes.ERROR_DATA; 2679 processed = cast(SizeT)(p.buf - src); 2680 (*srcLen) += processed; 2681 src += processed; 2682 inSize -= processed; 2683 } 2684 else 2685 { 2686 uint rem = p.tempBufSize, lookAhead = 0; 2687 while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) 2688 p.tempBuf.ptr[rem++] = src[lookAhead++]; 2689 p.tempBufSize = rem; 2690 if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) 2691 { 2692 int dummyRes = LzmaDec_TryDummy(p, p.tempBuf.ptr, rem); 2693 if (dummyRes == DUMMY_ERROR) 2694 { 2695 (*srcLen) += lookAhead; 2696 *status = LZMA_STATUS_NEEDS_MORE_INPUT; 2697 return SRes.OK; 2698 } 2699 if (checkEndMarkNow && dummyRes != DUMMY_MATCH) 2700 { 2701 *status = LZMA_STATUS_NOT_FINISHED; 2702 return SRes.ERROR_DATA; 2703 } 2704 } 2705 p.buf = p.tempBuf.ptr; 2706 if (LzmaDec_DecodeReal2(p, dicLimit, p.buf) != 0) 2707 return SRes.ERROR_DATA; 2708 2709 { 2710 uint kkk = cast(uint)(p.buf - p.tempBuf.ptr); 2711 if (rem < kkk) 2712 return SRes.ERROR_FAIL; /* some internal error */ 2713 rem -= kkk; 2714 if (lookAhead < rem) 2715 return SRes.ERROR_FAIL; /* some internal error */ 2716 lookAhead -= rem; 2717 } 2718 (*srcLen) += lookAhead; 2719 src += lookAhead; 2720 inSize -= lookAhead; 2721 p.tempBufSize = 0; 2722 } 2723 } 2724 if (p.code == 0) 2725 *status = LZMA_STATUS_FINISHED_WITH_MARK; 2726 return (p.code == 0) ? SRes.OK : SRes.ERROR_DATA; 2727 } 2728 2729 public SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const(Byte)* src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) 2730 { 2731 import core.stdc.string : memcpy; 2732 SizeT outSize = *destLen; 2733 SizeT inSize = *srcLen; 2734 *srcLen = *destLen = 0; 2735 for (;;) 2736 { 2737 SizeT inSizeCur = inSize, outSizeCur, dicPos; 2738 ELzmaFinishMode curFinishMode; 2739 SRes res; 2740 if (p.dicPos == p.dicBufSize) 2741 p.dicPos = 0; 2742 dicPos = p.dicPos; 2743 if (outSize > p.dicBufSize - dicPos) 2744 { 2745 outSizeCur = p.dicBufSize; 2746 curFinishMode = LZMA_FINISH_ANY; 2747 } 2748 else 2749 { 2750 outSizeCur = dicPos + outSize; 2751 curFinishMode = finishMode; 2752 } 2753 2754 res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); 2755 src += inSizeCur; 2756 inSize -= inSizeCur; 2757 *srcLen += inSizeCur; 2758 outSizeCur = p.dicPos - dicPos; 2759 memcpy(dest, p.dic + dicPos, outSizeCur); 2760 dest += outSizeCur; 2761 outSize -= outSizeCur; 2762 *destLen += outSizeCur; 2763 if (res != 0) 2764 return res; 2765 if (outSizeCur == 0 || outSize == 0) 2766 return SRes.OK; 2767 } 2768 } 2769 2770 public void LzmaDec_FreeProbs(CLzmaDec *p) { 2771 import core.stdc.stdlib : free; 2772 if (p.probs !is null) free(p.probs); 2773 p.probs = null; 2774 } 2775 2776 private void LzmaDec_FreeDict(CLzmaDec *p) { 2777 import core.stdc.stdlib : free; 2778 if (p.dic !is null) free(p.dic); 2779 p.dic = null; 2780 } 2781 2782 public void LzmaDec_Free(CLzmaDec *p) { 2783 LzmaDec_FreeProbs(p); 2784 LzmaDec_FreeDict(p); 2785 } 2786 2787 public SRes LzmaProps_Decode(CLzmaProps *p, const(Byte)*data, uint size) 2788 { 2789 UInt32 dicSize; 2790 Byte d; 2791 2792 if (size < LZMA_PROPS_SIZE) 2793 return SRes.ERROR_UNSUPPORTED; 2794 else 2795 dicSize = data[1] | (data[2] << 8) | (data[3] << 16) | (data[4] << 24); 2796 2797 if (dicSize < LZMA_DIC_MIN) 2798 dicSize = LZMA_DIC_MIN; 2799 p.dicSize = dicSize; 2800 2801 d = data[0]; 2802 if (d >= (9 * 5 * 5)) 2803 return SRes.ERROR_UNSUPPORTED; 2804 2805 p.lc = d % 9; 2806 d /= 9; 2807 p.pb = d / 5; 2808 p.lp = d % 5; 2809 2810 return SRes.OK; 2811 } 2812 2813 private SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const(CLzmaProps)* propNew) { 2814 import core.stdc.stdlib : malloc; 2815 UInt32 numProbs = (Literal+(cast(UInt32)LZMA_LIT_SIZE<<((propNew).lc+(propNew).lp))); 2816 if (!p.probs || numProbs != p.numProbs) 2817 { 2818 LzmaDec_FreeProbs(p); 2819 p.probs = cast(CLzmaProb *)malloc(numProbs * CLzmaProb.sizeof); 2820 p.numProbs = numProbs; 2821 if (!p.probs) 2822 return SRes.ERROR_MEM; 2823 } 2824 return SRes.OK; 2825 } 2826 2827 public SRes LzmaDec_AllocateProbs(CLzmaDec *p, const(Byte)* props, uint propsSize) 2828 { 2829 CLzmaProps propNew; 2830 if (auto sres = LzmaProps_Decode(&propNew, props, propsSize)) return sres; 2831 if (auto sres = LzmaDec_AllocateProbs2(p, &propNew)) return sres; 2832 p.prop = propNew; 2833 return SRes.OK; 2834 } 2835 2836 public SRes LzmaDec_Allocate(CLzmaDec *p, const(Byte)*props, uint propsSize) 2837 { 2838 import core.stdc.stdlib : malloc; 2839 CLzmaProps propNew; 2840 SizeT dicBufSize; 2841 if (auto sres = LzmaProps_Decode(&propNew, props, propsSize)) return sres; 2842 if (auto sres = LzmaDec_AllocateProbs2(p, &propNew)) return sres; 2843 2844 { 2845 UInt32 dictSize = propNew.dicSize; 2846 SizeT mask = (1U << 12) - 1; 2847 if (dictSize >= (1U << 30)) mask = (1U << 22) - 1; 2848 else if (dictSize >= (1U << 22)) mask = (1U << 20) - 1; 2849 dicBufSize = (cast(SizeT)dictSize + mask) & ~mask; 2850 if (dicBufSize < dictSize) 2851 dicBufSize = dictSize; 2852 } 2853 2854 if (!p.dic || dicBufSize != p.dicBufSize) 2855 { 2856 LzmaDec_FreeDict(p); 2857 p.dic = cast(Byte *)malloc(dicBufSize); 2858 if (!p.dic) 2859 { 2860 LzmaDec_FreeProbs(p); 2861 return SRes.ERROR_MEM; 2862 } 2863 } 2864 p.dicBufSize = dicBufSize; 2865 p.prop = propNew; 2866 return SRes.OK; 2867 } 2868 2869 public SRes LzmaDecode(Byte *dest, SizeT *destLen, const(Byte)* src, SizeT *srcLen, 2870 const(Byte)* propData, uint propSize, ELzmaFinishMode finishMode, 2871 ELzmaStatus *status) 2872 { 2873 CLzmaDec p; 2874 SRes res; 2875 SizeT outSize = *destLen, inSize = *srcLen; 2876 *destLen = *srcLen = 0; 2877 *status = LZMA_STATUS_NOT_SPECIFIED; 2878 if (inSize < RC_INIT_SIZE) 2879 return SRes.ERROR_INPUT_EOF; 2880 //LzmaDec_Construct(&p); 2881 p.dic = null; p.probs = null; 2882 if (auto sres = LzmaDec_AllocateProbs(&p, propData, propSize)) return sres; 2883 p.dic = dest; 2884 p.dicBufSize = outSize; 2885 LzmaDec_Init(&p); 2886 *srcLen = inSize; 2887 res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); 2888 *destLen = p.dicPos; 2889 if (res == SRes.OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) 2890 res = SRes.ERROR_INPUT_EOF; 2891 LzmaDec_FreeProbs(&p); 2892 return res; 2893 } 2894 2895 2896 2897 /* Lzma2Dec.c -- LZMA2 Decoder 2898 2009-05-03 : Igor Pavlov : Public domain */ 2899 // also by Lasse Collin 2900 // ported to D by adr. 2901 2902 /* 2903 00000000 - EOS 2904 00000001 U U - Uncompressed Reset Dic 2905 00000010 U U - Uncompressed No Reset 2906 100uuuuu U U P P - LZMA no reset 2907 101uuuuu U U P P - LZMA reset state 2908 110uuuuu U U P P S - LZMA reset state + new prop 2909 111uuuuu U U P P S - LZMA reset state + new prop + reset dic 2910 2911 u, U - Unpack Size 2912 P - Pack Size 2913 S - Props 2914 */ 2915 2916 struct CLzma2Dec 2917 { 2918 CLzmaDec decoder; 2919 UInt32 packSize; 2920 UInt32 unpackSize; 2921 int state; 2922 Byte control; 2923 bool needInitDic; 2924 bool needInitState; 2925 bool needInitProp; 2926 } 2927 2928 enum LZMA2_CONTROL_LZMA = (1 << 7); 2929 enum LZMA2_CONTROL_COPY_NO_RESET = 2; 2930 enum LZMA2_CONTROL_COPY_RESET_DIC = 1; 2931 enum LZMA2_CONTROL_EOF = 0; 2932 2933 auto LZMA2_IS_UNCOMPRESSED_STATE(P)(P p) { return (((p).control & LZMA2_CONTROL_LZMA) == 0); } 2934 2935 auto LZMA2_GET_LZMA_MODE(P)(P p) { return (((p).control >> 5) & 3); } 2936 auto LZMA2_IS_THERE_PROP(P)(P mode) { return ((mode) >= 2); } 2937 2938 enum LZMA2_LCLP_MAX = 4; 2939 auto LZMA2_DIC_SIZE_FROM_PROP(P)(P p) { return ((cast(UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)); } 2940 2941 enum ELzma2State 2942 { 2943 LZMA2_STATE_CONTROL, 2944 LZMA2_STATE_UNPACK0, 2945 LZMA2_STATE_UNPACK1, 2946 LZMA2_STATE_PACK0, 2947 LZMA2_STATE_PACK1, 2948 LZMA2_STATE_PROP, 2949 LZMA2_STATE_DATA, 2950 LZMA2_STATE_DATA_CONT, 2951 LZMA2_STATE_FINISHED, 2952 LZMA2_STATE_ERROR 2953 } 2954 2955 static SRes Lzma2Dec_GetOldProps(Byte prop, Byte *props) 2956 { 2957 UInt32 dicSize; 2958 if (prop > 40) 2959 return SRes.ERROR_UNSUPPORTED; 2960 dicSize = (prop == 40) ? 0xFFFFFFFF : LZMA2_DIC_SIZE_FROM_PROP(prop); 2961 props[0] = cast(Byte)LZMA2_LCLP_MAX; 2962 props[1] = cast(Byte)(dicSize); 2963 props[2] = cast(Byte)(dicSize >> 8); 2964 props[3] = cast(Byte)(dicSize >> 16); 2965 props[4] = cast(Byte)(dicSize >> 24); 2966 return SRes.OK; 2967 } 2968 2969 SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop) 2970 { 2971 Byte[LZMA_PROPS_SIZE] props; 2972 auto wtf = Lzma2Dec_GetOldProps(prop, props.ptr); 2973 if(wtf != 0) return wtf; 2974 return LzmaDec_AllocateProbs(&p.decoder, props.ptr, LZMA_PROPS_SIZE); 2975 } 2976 2977 SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop) 2978 { 2979 Byte[LZMA_PROPS_SIZE] props; 2980 auto wtf = Lzma2Dec_GetOldProps(prop, props.ptr); 2981 if(wtf != 0) return wtf; 2982 return LzmaDec_Allocate(&p.decoder, props.ptr, LZMA_PROPS_SIZE); 2983 } 2984 2985 void Lzma2Dec_Init(CLzma2Dec *p) 2986 { 2987 p.state = ELzma2State.LZMA2_STATE_CONTROL; 2988 p.needInitDic = true; 2989 p.needInitState = true; 2990 p.needInitProp = true; 2991 LzmaDec_Init(&p.decoder); 2992 } 2993 2994 static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) 2995 { 2996 switch(p.state) 2997 { 2998 default: return ELzma2State.LZMA2_STATE_ERROR; 2999 case ELzma2State.LZMA2_STATE_CONTROL: 3000 p.control = b; 3001 if (p.control == 0) 3002 return ELzma2State.LZMA2_STATE_FINISHED; 3003 if (LZMA2_IS_UNCOMPRESSED_STATE(p)) 3004 { 3005 if ((p.control & 0x7F) > 2) 3006 return ELzma2State.LZMA2_STATE_ERROR; 3007 p.unpackSize = 0; 3008 } 3009 else 3010 p.unpackSize = cast(UInt32)(p.control & 0x1F) << 16; 3011 return ELzma2State.LZMA2_STATE_UNPACK0; 3012 3013 case ELzma2State.LZMA2_STATE_UNPACK0: 3014 p.unpackSize |= cast(UInt32)b << 8; 3015 return ELzma2State.LZMA2_STATE_UNPACK1; 3016 3017 case ELzma2State.LZMA2_STATE_UNPACK1: 3018 p.unpackSize |= cast(UInt32)b; 3019 p.unpackSize++; 3020 return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? ELzma2State.LZMA2_STATE_DATA : ELzma2State.LZMA2_STATE_PACK0; 3021 3022 case ELzma2State.LZMA2_STATE_PACK0: 3023 p.packSize = cast(UInt32)b << 8; 3024 return ELzma2State.LZMA2_STATE_PACK1; 3025 3026 case ELzma2State.LZMA2_STATE_PACK1: 3027 p.packSize |= cast(UInt32)b; 3028 p.packSize++; 3029 return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? ELzma2State.LZMA2_STATE_PROP: 3030 (p.needInitProp ? ELzma2State.LZMA2_STATE_ERROR : ELzma2State.LZMA2_STATE_DATA); 3031 3032 case ELzma2State.LZMA2_STATE_PROP: 3033 { 3034 int lc, lp; 3035 if (b >= (9 * 5 * 5)) 3036 return ELzma2State.LZMA2_STATE_ERROR; 3037 lc = b % 9; 3038 b /= 9; 3039 p.decoder.prop.pb = b / 5; 3040 lp = b % 5; 3041 if (lc + lp > LZMA2_LCLP_MAX) 3042 return ELzma2State.LZMA2_STATE_ERROR; 3043 p.decoder.prop.lc = lc; 3044 p.decoder.prop.lp = lp; 3045 p.needInitProp = false; 3046 return ELzma2State.LZMA2_STATE_DATA; 3047 } 3048 } 3049 } 3050 3051 static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, Byte *src, SizeT size) 3052 { 3053 import core.stdc.string; 3054 memcpy(p.dic + p.dicPos, src, size); 3055 p.dicPos += size; 3056 if (p.checkDicSize == 0 && p.prop.dicSize - p.processedPos <= size) 3057 p.checkDicSize = p.prop.dicSize; 3058 p.processedPos += cast(UInt32)size; 3059 } 3060 3061 SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, 3062 Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) 3063 { 3064 SizeT inSize = *srcLen; 3065 *srcLen = 0; 3066 *status = LZMA_STATUS_NOT_SPECIFIED; 3067 3068 while (p.state != ELzma2State.LZMA2_STATE_FINISHED) 3069 { 3070 SizeT dicPos = p.decoder.dicPos; 3071 if (p.state == ELzma2State.LZMA2_STATE_ERROR) 3072 return SRes.ERROR_DATA; 3073 if (dicPos == dicLimit && finishMode == LZMA_FINISH_ANY) 3074 { 3075 *status = LZMA_STATUS_NOT_FINISHED; 3076 return SRes.OK; 3077 } 3078 if (p.state != ELzma2State.LZMA2_STATE_DATA && p.state != ELzma2State.LZMA2_STATE_DATA_CONT) 3079 { 3080 if (*srcLen == inSize) 3081 { 3082 *status = LZMA_STATUS_NEEDS_MORE_INPUT; 3083 return SRes.OK; 3084 } 3085 (*srcLen)++; 3086 p.state = Lzma2Dec_UpdateState(p, *src++); 3087 continue; 3088 } 3089 { 3090 SizeT destSizeCur = dicLimit - dicPos; 3091 SizeT srcSizeCur = inSize - *srcLen; 3092 ELzmaFinishMode curFinishMode = LZMA_FINISH_ANY; 3093 3094 if (p.unpackSize <= destSizeCur) 3095 { 3096 destSizeCur = cast(SizeT)p.unpackSize; 3097 curFinishMode = LZMA_FINISH_END; 3098 } 3099 3100 if (LZMA2_IS_UNCOMPRESSED_STATE(p)) 3101 { 3102 if (*srcLen == inSize) 3103 { 3104 *status = LZMA_STATUS_NEEDS_MORE_INPUT; 3105 return SRes.OK; 3106 } 3107 3108 if (p.state == ELzma2State.LZMA2_STATE_DATA) 3109 { 3110 bool initDic = (p.control == LZMA2_CONTROL_COPY_RESET_DIC); 3111 if (initDic) 3112 p.needInitProp = p.needInitState = true; 3113 else if (p.needInitDic) 3114 return SRes.ERROR_DATA; 3115 p.needInitDic = false; 3116 LzmaDec_InitDicAndState(&p.decoder, initDic, false); 3117 } 3118 3119 if (srcSizeCur > destSizeCur) 3120 srcSizeCur = destSizeCur; 3121 3122 if (srcSizeCur == 0) 3123 return SRes.ERROR_DATA; 3124 3125 LzmaDec_UpdateWithUncompressed(&p.decoder, src, srcSizeCur); 3126 3127 src += srcSizeCur; 3128 *srcLen += srcSizeCur; 3129 p.unpackSize -= cast(UInt32)srcSizeCur; 3130 p.state = (p.unpackSize == 0) ? ELzma2State.LZMA2_STATE_CONTROL : ELzma2State.LZMA2_STATE_DATA_CONT; 3131 } 3132 else 3133 { 3134 SizeT outSizeProcessed; 3135 SRes res; 3136 3137 if (p.state == ELzma2State.LZMA2_STATE_DATA) 3138 { 3139 int mode = LZMA2_GET_LZMA_MODE(p); 3140 bool initDic = (mode == 3); 3141 bool initState = (mode > 0); 3142 if ((!initDic && p.needInitDic) || (!initState && p.needInitState)) 3143 return SRes.ERROR_DATA; 3144 3145 LzmaDec_InitDicAndState(&p.decoder, initDic, initState); 3146 p.needInitDic = false; 3147 p.needInitState = false; 3148 p.state = ELzma2State.LZMA2_STATE_DATA_CONT; 3149 } 3150 if (srcSizeCur > p.packSize) 3151 srcSizeCur = cast(SizeT)p.packSize; 3152 3153 res = LzmaDec_DecodeToDic(&p.decoder, dicPos + destSizeCur, src, &srcSizeCur, curFinishMode, status); 3154 3155 src += srcSizeCur; 3156 *srcLen += srcSizeCur; 3157 p.packSize -= cast(UInt32)srcSizeCur; 3158 3159 outSizeProcessed = p.decoder.dicPos - dicPos; 3160 p.unpackSize -= cast(UInt32)outSizeProcessed; 3161 3162 if(res != 0) return res; 3163 if (*status == LZMA_STATUS_NEEDS_MORE_INPUT) 3164 return res; 3165 3166 if (srcSizeCur == 0 && outSizeProcessed == 0) 3167 { 3168 if (*status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK || 3169 p.unpackSize != 0 || p.packSize != 0) 3170 return SRes.ERROR_DATA; 3171 p.state = ELzma2State.LZMA2_STATE_CONTROL; 3172 } 3173 if (*status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) 3174 *status = LZMA_STATUS_NOT_FINISHED; 3175 } 3176 } 3177 } 3178 *status = LZMA_STATUS_FINISHED_WITH_MARK; 3179 return SRes.OK; 3180 } 3181 3182 SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) 3183 { 3184 import core.stdc.string; 3185 SizeT outSize = *destLen, inSize = *srcLen; 3186 *srcLen = *destLen = 0; 3187 for (;;) 3188 { 3189 SizeT srcSizeCur = inSize, outSizeCur, dicPos; 3190 ELzmaFinishMode curFinishMode; 3191 SRes res; 3192 if (p.decoder.dicPos == p.decoder.dicBufSize) 3193 p.decoder.dicPos = 0; 3194 dicPos = p.decoder.dicPos; 3195 if (outSize > p.decoder.dicBufSize - dicPos) 3196 { 3197 outSizeCur = p.decoder.dicBufSize; 3198 curFinishMode = LZMA_FINISH_ANY; 3199 } 3200 else 3201 { 3202 outSizeCur = dicPos + outSize; 3203 curFinishMode = finishMode; 3204 } 3205 3206 res = Lzma2Dec_DecodeToDic(p, outSizeCur, src, &srcSizeCur, curFinishMode, status); 3207 src += srcSizeCur; 3208 inSize -= srcSizeCur; 3209 *srcLen += srcSizeCur; 3210 outSizeCur = p.decoder.dicPos - dicPos; 3211 memcpy(dest, p.decoder.dic + dicPos, outSizeCur); 3212 dest += outSizeCur; 3213 outSize -= outSizeCur; 3214 *destLen += outSizeCur; 3215 if (res != 0) 3216 return res; 3217 if (outSizeCur == 0 || outSize == 0) 3218 return SRes.OK; 3219 } 3220 } 3221 3222 SRes Lzma2Decode(Byte *dest, SizeT *destLen, Byte *src, SizeT *srcLen, 3223 Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status) 3224 { 3225 CLzma2Dec decoder; 3226 SRes res; 3227 SizeT outSize = *destLen, inSize = *srcLen; 3228 Byte[LZMA_PROPS_SIZE] props; 3229 3230 //Lzma2Dec_Construct(&decoder); 3231 3232 *destLen = *srcLen = 0; 3233 *status = LZMA_STATUS_NOT_SPECIFIED; 3234 decoder.decoder.dic = dest; 3235 decoder.decoder.dicBufSize = outSize; 3236 3237 auto wtf = Lzma2Dec_GetOldProps(prop, props.ptr); 3238 if(wtf != 0) return wtf; 3239 wtf = LzmaDec_AllocateProbs(&decoder.decoder, props.ptr, LZMA_PROPS_SIZE); 3240 if(wtf != 0) return wtf; 3241 3242 *srcLen = inSize; 3243 res = Lzma2Dec_DecodeToDic(&decoder, outSize, src, srcLen, finishMode, status); 3244 *destLen = decoder.decoder.dicPos; 3245 if (res == SRes.OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) 3246 res = SRes.ERROR_INPUT_EOF; 3247 3248 LzmaDec_FreeProbs(&decoder.decoder); 3249 return res; 3250 } 3251 3252 }