diff --git a/src/SharpCompress/Common/Zip/ZipFilePart.cs b/src/SharpCompress/Common/Zip/ZipFilePart.cs index 77dc4abba..63d2a39fa 100644 --- a/src/SharpCompress/Common/Zip/ZipFilePart.cs +++ b/src/SharpCompress/Common/Zip/ZipFilePart.cs @@ -4,7 +4,7 @@ using System.Linq; using SharpCompress.Common.Zip.Headers; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT.InputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.Deflate64; using SharpCompress.Compressors.Explode; @@ -122,7 +122,7 @@ protected Stream CreateDecompressionStream(Stream stream, ZipCompressionMethod m } case ZipCompressionMethod.BZip2: { - return new BZip2Stream(stream, CompressionMode.Decompress, false); + return new BZip2ParallelInputStream(stream); } case ZipCompressionMethod.LZMA: { diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs new file mode 100644 index 000000000..d8e703f44 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs @@ -0,0 +1,270 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// Compresses and writes a single BZip2 block + /// + /// Block encoding consists of the following stages: + /// 1. Run-Length Encoding[1] - write() + /// 2. Burrows Wheeler Transform - close() (through BZip2DivSufSort) + /// 3. Write block header - close() + /// 4. Move To Front Transform - close() (through BZip2HuffmanStageEncoder) + /// 5. Run-Length Encoding[2] - close() (through BZip2HuffmanStageEncoder) + /// 6. Create and write Huffman tables - close() (through BZip2HuffmanStageEncoder) + /// 7. Huffman encode and write data - close() (through BZip2HuffmanStageEncoder) + /// + internal class BZip2BlockCompressor + { + // The stream to which compressed BZip2 data is written + private readonly IBZip2BitOutputStream bitOutputStream; + + // CRC builder for the block + private readonly CRC32 crc = new CRC32(); + + // The RLE'd block data + private readonly byte[] block; + + // Current length of the data within the block array + private int blockLength; + + // A limit beyond which new data will not be accepted into the block + private readonly int blockLengthLimit; + + // The values that are present within the RLE'd block data. For each index, true if that + // value is present within the data, otherwise false + private readonly bool[] blockValuesPresent = new bool[256]; + + // The Burrows Wheeler Transformed block data + private readonly int[] bwtBlock; + + // The current RLE value being accumulated (undefined when rleLength is 0) + private int rleCurrentValue = -1; + + // The repeat count of the current RLE value + private int rleLength; + + /// + /// Determines if any bytes have been written to the block. + /// True if one or more bytes has been written to the block, otherwise false. + /// + public bool IsEmpty + { + get { return ((this.blockLength == 0) && (this.rleLength == 0)); } + } + + /// + /// Gets the CRC of the completed block. Only valid after calling Close(). + /// + public uint CRC + { + get { return this.crc.CRC; } + } + + /// + /// Public constructor + /// + /// The BZip2BitOutputStream to which compressed BZip2 data is written + /// The declared block size in bytes. Up to this many bytes will be accepted + /// into the block after Run-Length Encoding is applied + public BZip2BlockCompressor(IBZip2BitOutputStream bitOutputStream, int blockSize) + { + this.bitOutputStream = bitOutputStream; + + // One extra byte is added to allow for the block wrap applied in close() + this.block = new byte[blockSize + 1]; + this.bwtBlock = new int[blockSize + 1]; + this.blockLengthLimit = blockSize - 6; // 5 bytes for one RLE run plus one byte - see Write(int) + } + + /// + /// Writes a byte to the block, accumulating to an RLE run where possible + /// + /// The byte to write + /// True if the byte was written, or false if the block is already full + public bool Write(int value) + { + if (this.blockLength > this.blockLengthLimit) + return false; + + if (this.rleLength == 0) + { + this.rleCurrentValue = value; + this.rleLength = 1; + } + else if (this.rleCurrentValue != value) + { + // This path commits us to write 6 bytes - one RLE run (5 bytes) plus one extra + this.WriteRun(this.rleCurrentValue & 0xff, this.rleLength); + this.rleCurrentValue = value; + this.rleLength = 1; + } + else + { + if (this.rleLength == 254) + { + this.WriteRun(this.rleCurrentValue & 0xff, 255); + this.rleLength = 0; + } + else + { + this.rleLength++; + } + } + + return true; + } + + /// + /// Writes an array to the block + /// + /// The array to write + /// The offset within the input data to write from + /// The number of bytes of input data to write + /// The actual number of input bytes written. May be less than the number requested, or + /// zero if the block is already full + public int Write(byte[] data, int offset, int length) + { + var written = 0; + + while (length-- > 0) + { + if (!this.Write(data[offset++])) + break; + written++; + } + + return written; + } + + /// Compresses and writes out the block. + /// Exception on any I/O error writing the data + public void CloseBlock() + { + // If an RLE run is in progress, write it out + if (this.rleLength > 0) + this.WriteRun(this.rleCurrentValue & 0xff, this.rleLength); + + // Apply a one byte block wrap required by the BWT implementation + this.block[this.blockLength] = this.block[0]; + + // Perform the Burrows Wheeler Transform + var divSufSort = new BZip2DivSufSort(this.block, this.bwtBlock, this.blockLength); + var bwtStartPointer = divSufSort.BWT(); + + // Write out the block header + this.bitOutputStream.WriteBits(24, BZip2Constants.BLOCK_HEADER_MARKER_1); + this.bitOutputStream.WriteBits(24, BZip2Constants.BLOCK_HEADER_MARKER_2); + this.bitOutputStream.WriteInteger(this.crc.CRC); + this.bitOutputStream.WriteBoolean(false); // Randomised block flag. We never create randomised blocks + this.bitOutputStream.WriteBits(24, (uint)bwtStartPointer); + + // Write out the symbol map + this.WriteSymbolMap(); + + // Perform the Move To Front Transform and Run-Length Encoding[2] stages + var mtfEncoder = new BZip2MTFAndRLE2StageEncoder( + this.bwtBlock, + this.blockLength, + this.blockValuesPresent + ); + mtfEncoder.Encode(); + + // Perform the Huffman Encoding stage and write out the encoded data + var huffmanEncoder = new BZip2HuffmanStageEncoder( + this.bitOutputStream, + mtfEncoder.MtfBlock, + mtfEncoder.MtfLength, + mtfEncoder.MtfAlphabetSize, + mtfEncoder.MtfSymbolFrequencies + ); + huffmanEncoder.Encode(); + } + + /// + /// Write the Huffman symbol to output byte map + /// + /// on any I/O error writing the data + private void WriteSymbolMap() + { + var condensedInUse = new bool[16]; + + for (var i = 0; i < 16; i++) + { + for (int j = 0, k = i << 4; j < 16; j++, k++) + { + if (this.blockValuesPresent[k]) + { + condensedInUse[i] = true; + } + } + } + + for (var i = 0; i < 16; i++) + { + this.bitOutputStream.WriteBoolean(condensedInUse[i]); + } + + for (var i = 0; i < 16; i++) + { + if (condensedInUse[i]) + { + for (int j = 0, k = i * 16; j < 16; j++, k++) + { + this.bitOutputStream.WriteBoolean(this.blockValuesPresent[k]); + } + } + } + } + + /// + /// Writes an RLE run to the block array, updating the block CRC and present values array as required + /// + /// The value to write + /// The run length of the value to write + private void WriteRun(int value, int runLength) + { + this.blockValuesPresent[value] = true; + this.crc.UpdateCrc(value, runLength); + + var byteValue = (byte)value; + switch (runLength) + { + case 1: + this.block[this.blockLength] = byteValue; + this.blockLength = this.blockLength + 1; + break; + + case 2: + this.block[this.blockLength] = byteValue; + this.block[this.blockLength + 1] = byteValue; + this.blockLength = this.blockLength + 2; + break; + + case 3: + this.block[this.blockLength] = byteValue; + this.block[this.blockLength + 1] = byteValue; + this.block[this.blockLength + 2] = byteValue; + this.blockLength = this.blockLength + 3; + break; + + default: + runLength -= 4; + this.blockValuesPresent[runLength] = true; + this.block[this.blockLength] = byteValue; + this.block[this.blockLength + 1] = byteValue; + this.block[this.blockLength + 2] = byteValue; + this.block[this.blockLength + 3] = byteValue; + this.block[this.blockLength + 4] = (byte)runLength; + this.blockLength = this.blockLength + 5; + break; + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs new file mode 100644 index 000000000..b1c4bb2c4 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs @@ -0,0 +1,980 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// Reads and decompresses a single BZip2 block + /// + /// Block decoding consists of the following stages: + /// 1. Read block header - BZip2BlockDecompressor() + /// 2. Read Huffman tables - readHuffmanTables() + /// 3. Read and decode Huffman encoded data - decodeHuffmanData() + /// 4. Run-Length Decoding[2] - decodeHuffmanData() + /// 5. Inverse Move To Front Transform - decodeHuffmanData() + /// 6. Inverse Burrows Wheeler Transform - initialiseInverseBWT() + /// 7. Run-Length Decoding[1] - read() + /// 8. Optional Block De-Randomisation - read() (through decodeNextBWTByte()) + /// + internal class BZip2BlockDecompressor + { + /// + /// The BZip2 specification originally included the optional addition of a slight pseudo-random + /// perturbation to the input data, in order to work around the block sorting algorithm's non- + /// optimal performance on some types of input. The current mainline bzip2 does not require this + /// and will not create randomised blocks, but compatibility is still required for old data (and + /// third party compressors that haven't caught up). When decompressing a randomised block, for + /// each value N in this array, a 1 will be XOR'd onto the output of the Burrows-Wheeler + /// transform stage after N bytes, then the next N taken from the following entry. + /// + private static readonly int[] RNUMS = + { + 619, + 720, + 127, + 481, + 931, + 816, + 813, + 233, + 566, + 247, + 985, + 724, + 205, + 454, + 863, + 491, + 741, + 242, + 949, + 214, + 733, + 859, + 335, + 708, + 621, + 574, + 73, + 654, + 730, + 472, + 419, + 436, + 278, + 496, + 867, + 210, + 399, + 680, + 480, + 51, + 878, + 465, + 811, + 169, + 869, + 675, + 611, + 697, + 867, + 561, + 862, + 687, + 507, + 283, + 482, + 129, + 807, + 591, + 733, + 623, + 150, + 238, + 59, + 379, + 684, + 877, + 625, + 169, + 643, + 105, + 170, + 607, + 520, + 932, + 727, + 476, + 693, + 425, + 174, + 647, + 73, + 122, + 335, + 530, + 442, + 853, + 695, + 249, + 445, + 515, + 909, + 545, + 703, + 919, + 874, + 474, + 882, + 500, + 594, + 612, + 641, + 801, + 220, + 162, + 819, + 984, + 589, + 513, + 495, + 799, + 161, + 604, + 958, + 533, + 221, + 400, + 386, + 867, + 600, + 782, + 382, + 596, + 414, + 171, + 516, + 375, + 682, + 485, + 911, + 276, + 98, + 553, + 163, + 354, + 666, + 933, + 424, + 341, + 533, + 870, + 227, + 730, + 475, + 186, + 263, + 647, + 537, + 686, + 600, + 224, + 469, + 68, + 770, + 919, + 190, + 373, + 294, + 822, + 808, + 206, + 184, + 943, + 795, + 384, + 383, + 461, + 404, + 758, + 839, + 887, + 715, + 67, + 618, + 276, + 204, + 918, + 873, + 777, + 604, + 560, + 951, + 160, + 578, + 722, + 79, + 804, + 96, + 409, + 713, + 940, + 652, + 934, + 970, + 447, + 318, + 353, + 859, + 672, + 112, + 785, + 645, + 863, + 803, + 350, + 139, + 93, + 354, + 99, + 820, + 908, + 609, + 772, + 154, + 274, + 580, + 184, + 79, + 626, + 630, + 742, + 653, + 282, + 762, + 623, + 680, + 81, + 927, + 626, + 789, + 125, + 411, + 521, + 938, + 300, + 821, + 78, + 343, + 175, + 128, + 250, + 170, + 774, + 972, + 275, + 999, + 639, + 495, + 78, + 352, + 126, + 857, + 956, + 358, + 619, + 580, + 124, + 737, + 594, + 701, + 612, + 669, + 112, + 134, + 694, + 363, + 992, + 809, + 743, + 168, + 974, + 944, + 375, + 748, + 52, + 600, + 747, + 642, + 182, + 862, + 81, + 344, + 805, + 988, + 739, + 511, + 655, + 814, + 334, + 249, + 515, + 897, + 955, + 664, + 981, + 649, + 113, + 974, + 459, + 893, + 228, + 433, + 837, + 553, + 268, + 926, + 240, + 102, + 654, + 459, + 51, + 686, + 754, + 806, + 760, + 493, + 403, + 415, + 394, + 687, + 700, + 946, + 670, + 656, + 610, + 738, + 392, + 760, + 799, + 887, + 653, + 978, + 321, + 576, + 617, + 626, + 502, + 894, + 679, + 243, + 440, + 680, + 879, + 194, + 572, + 640, + 724, + 926, + 56, + 204, + 700, + 707, + 151, + 457, + 449, + 797, + 195, + 791, + 558, + 945, + 679, + 297, + 59, + 87, + 824, + 713, + 663, + 412, + 693, + 342, + 606, + 134, + 108, + 571, + 364, + 631, + 212, + 174, + 643, + 304, + 329, + 343, + 97, + 430, + 751, + 497, + 314, + 983, + 374, + 822, + 928, + 140, + 206, + 73, + 263, + 980, + 736, + 876, + 478, + 430, + 305, + 170, + 514, + 364, + 692, + 829, + 82, + 855, + 953, + 676, + 246, + 369, + 970, + 294, + 750, + 807, + 827, + 150, + 790, + 288, + 923, + 804, + 378, + 215, + 828, + 592, + 281, + 565, + 555, + 710, + 82, + 896, + 831, + 547, + 261, + 524, + 462, + 293, + 465, + 502, + 56, + 661, + 821, + 976, + 991, + 658, + 869, + 905, + 758, + 745, + 193, + 768, + 550, + 608, + 933, + 378, + 286, + 215, + 979, + 792, + 961, + 61, + 688, + 793, + 644, + 986, + 403, + 106, + 366, + 905, + 644, + 372, + 567, + 466, + 434, + 645, + 210, + 389, + 550, + 919, + 135, + 780, + 773, + 635, + 389, + 707, + 100, + 626, + 958, + 165, + 504, + 920, + 176, + 193, + 713, + 857, + 265, + 203, + 50, + 668, + 108, + 645, + 990, + 626, + 197, + 510, + 357, + 358, + 850, + 858, + 364, + 936, + 638, + }; + + // Maximum possible number of Huffman table selectors + private const int HUFFMAN_MAXIMUM_SELECTORS = + (900000 / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH) + 1; + + // Provides bits of input to decode + private readonly IBZip2BitInputStream bitInputStream; + + // Calculates the block CRC from the fully decoded bytes of the block + private readonly CRC32 crc = new CRC32(); + + // The CRC of the current block as read from the block header + private readonly uint blockCRC; + + // true if the current block is randomised, otherwise false + private readonly bool blockRandomised; + + // + // Huffman Decoding stage + // + + // The end-of-block Huffman symbol. Decoding of the block ends when this is encountered + private int huffmanEndOfBlockSymbol; + + /// + /// A map from Huffman symbol index to output character. Some types of data (e.g. ASCII text) + /// may contain only a limited number of byte values; Huffman symbols are only allocated to + /// those values that actually occur in the uncompressed data. + /// + private readonly byte[] huffmanSymbolMap = new byte[256]; + + // + // Move To Front stage + // + + /// + /// Counts of each byte value within the bwtTransformedArray data. Collected at the Move + /// To Front stage, consumed by the Inverse Burrows Wheeler Transform stage + /// + private readonly int[] bwtByteCounts = new int[256]; + + /// + /// The Burrows-Wheeler Transform processed data. Read at the Move To Front stage, consumed by the + /// Inverse Burrows Wheeler Transform stage + /// + private byte[] bwtBlock; + + // + // Inverse Burrows-Wheeler Transform stage + // + + /// + /// At each position contains the union of :- + /// An output character (8 bits) + /// A pointer from each position to its successor (24 bits, left shifted 8 bits) + /// As the pointer cannot exceed the maximum block size of 900k, 24 bits is more than enough to + /// hold it; Folding the character data into the spare bits while performing the inverse BWT, + /// when both pieces of information are available, saves a large number of memory accesses in + /// the final decoding stages. + /// + private int[] bwtMergedPointers = Array.Empty(); + + // The current merged pointer into the Burrow-Wheeler Transform array + private int bwtCurrentMergedPointer; + + /// + /// The actual length in bytes of the current block at the Inverse Burrows Wheeler Transform + /// stage (before final Run-Length Decoding) + /// + private int bwtBlockLength; + + // The number of output bytes that have been decoded up to the Inverse Burrows Wheeler Transform stage + private int bwtBytesDecoded; + + // + // Run-Length Encoding and Random Perturbation stage + // + + // The most recently RLE decoded byte + private int rleLastDecodedByte = -1; + + /// + /// The number of previous identical output bytes decoded. After 4 identical bytes, the next byte + /// decoded is an RLE repeat count + /// + private int rleAccumulator; + + // The RLE repeat count of the current decoded byte. When this reaches zero, a new byte is decoded + private int rleRepeat; + + // If the current block is randomised, the position within the RNUMS randomisation array + private int randomIndex; + + // If the current block is randomised, the remaining count at the current RNUMS position + private int randomCount = BZip2BlockDecompressor.RNUMS[0] - 1; + + // Minimum number of alternative Huffman tables + public const int HUFFMAN_MINIMUM_TABLES = 2; + + // Maximum number of alternative Huffman tables + public const int HUFFMAN_MAXIMUM_TABLES = 6; + + /// + /// Read and decode the block's Huffman tables + /// @return A decoder for the Huffman stage that uses the decoded tables + /// + /// if the input stream reaches EOF before all table data has been read + private BZip2HuffmanStageDecoder ReadHuffmanTables() + { + var tableCodeLengths = new byte[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE + ]; + + // Read Huffman symbol to output byte map + uint huffmanUsedRanges = this.bitInputStream.ReadBits(16); + int huffmanSymbolCount = 0; + + for (int i = 0; i < 16; i++) + { + if ((huffmanUsedRanges & ((1 << 15) >> i)) == 0) + continue; + for (int j = 0, k = i << 4; j < 16; j++, k++) + { + if (this.bitInputStream.ReadBoolean()) + { + this.huffmanSymbolMap[huffmanSymbolCount++] = (byte)k; + } + } + } + var endOfBlockSymbol = huffmanSymbolCount + 1; + this.huffmanEndOfBlockSymbol = endOfBlockSymbol; + + // Read total number of tables and selectors + uint totalTables = this.bitInputStream.ReadBits(3); + uint totalSelectors = this.bitInputStream.ReadBits(15); + if ( + (totalTables < BZip2BlockDecompressor.HUFFMAN_MINIMUM_TABLES) + || (totalTables > BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES) + || (totalSelectors < 1) + || (totalSelectors > BZip2BlockDecompressor.HUFFMAN_MAXIMUM_SELECTORS) + ) + { + throw new IOException("BZip2 block Huffman tables invalid"); + } + + // Read and decode MTFed Huffman selector list + var tableMTF = new MoveToFront(); + var selectors = new byte[totalSelectors]; + for (var selector = 0; selector < totalSelectors; selector++) + { + selectors[selector] = tableMTF.IndexToFront((int)this.bitInputStream.ReadUnary()); + } + + // Read the Canonical Huffman code lengths for each table + for (var table = 0; table < totalTables; table++) + { + int currentLength = (int)this.bitInputStream.ReadBits(5); + for (var i = 0; i <= endOfBlockSymbol; i++) + { + while (this.bitInputStream.ReadBoolean()) + { + currentLength += this.bitInputStream.ReadBoolean() ? -1 : 1; + } + tableCodeLengths[table, i] = (byte)currentLength; + } + } + + return new BZip2HuffmanStageDecoder( + this.bitInputStream, + endOfBlockSymbol + 1, + tableCodeLengths, + selectors + ); + } + + /// + /// Reads the Huffman encoded data from the input stream, performs Run-Length Decoding and + /// applies the Move To Front transform to reconstruct the Burrows-Wheeler Transform array + /// + /// The Huffman decoder through which symbols are read + /// if an end-of-block symbol was not decoded within the declared block size + private void DecodeHuffmanData(BZip2HuffmanStageDecoder huffmanDecoder) + { + var symbolMTF = new MoveToFront(); + int _bwtBlockLength = 0; + int repeatCount = 0; + int repeatIncrement = 1; + int mtfValue = 0; + + while (true) + { + var nextSymbol = huffmanDecoder.NextSymbol(); + + if (nextSymbol == BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA) + { + repeatCount += repeatIncrement; + repeatIncrement <<= 1; + } + else if (nextSymbol == BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB) + { + repeatCount += repeatIncrement << 1; + repeatIncrement <<= 1; + } + else + { + byte nextByte; + if (repeatCount > 0) + { + if (_bwtBlockLength + repeatCount > this.bwtBlock.Length) + throw new IOException("BZip2 block exceeds declared block size"); + + nextByte = this.huffmanSymbolMap[mtfValue]; + this.bwtByteCounts[nextByte & 0xff] += repeatCount; + while (--repeatCount >= 0) + { + this.bwtBlock[_bwtBlockLength++] = nextByte; + } + + repeatCount = 0; + repeatIncrement = 1; + } + + if (nextSymbol == this.huffmanEndOfBlockSymbol) + break; + + if (_bwtBlockLength >= this.bwtBlock.Length) + throw new IOException("BZip2 block exceeds declared block size"); + + mtfValue = symbolMTF.IndexToFront(nextSymbol - 1) & 0xff; + + nextByte = this.huffmanSymbolMap[mtfValue]; + this.bwtByteCounts[nextByte & 0xff]++; + this.bwtBlock[_bwtBlockLength++] = nextByte; + } + } + + this.bwtBlockLength = _bwtBlockLength; + } + + /// + /// Set up the Inverse Burrows-Wheeler Transform merged pointer array + /// + /// The start pointer into the BWT array + /// if the given start pointer is invalid + private void InitialiseInverseBWT(uint bwtStartPointer) + { + var _bwtMergedPointers = new int[this.bwtBlockLength]; + var characterBase = new int[256]; + + if ((bwtStartPointer < 0) || (bwtStartPointer >= this.bwtBlockLength)) + throw new IOException("BZip2 start pointer invalid"); + + // Cumulatise character counts + Array.ConstrainedCopy(this.bwtByteCounts, 0, characterBase, 1, 255); + for (int i = 2; i <= 255; i++) + { + characterBase[i] += characterBase[i - 1]; + } + + // Merged-Array Inverse Burrows-Wheeler Transform + // Combining the output characters and forward pointers into a single array here, where we + // have already read both of the corresponding values, cuts down on memory accesses in the + // final walk through the array + for (int i = 0; i < this.bwtBlockLength; i++) + { + int value = this.bwtBlock[i] & 0xff; + _bwtMergedPointers[characterBase[value]++] = (i << 8) + value; + } + + //this.bwtBlock = null; + this.bwtMergedPointers = _bwtMergedPointers; + this.bwtCurrentMergedPointer = _bwtMergedPointers[bwtStartPointer]; + } + + /// + /// Decodes a byte from the Burrows-Wheeler Transform stage. If the block has randomisation + /// applied, reverses the randomisation + /// + /// The decoded byte + private int decodeNextBWTByte() + { + int nextDecodedByte = this.bwtCurrentMergedPointer & 0xff; + this.bwtCurrentMergedPointer = this.bwtMergedPointers[ + this.bwtCurrentMergedPointer >> 8 + ]; + + if (this.blockRandomised) + { + if (--this.randomCount == 0) + { + nextDecodedByte ^= 1; + this.randomIndex = (this.randomIndex + 1) % 512; + this.randomCount = BZip2BlockDecompressor.RNUMS[this.randomIndex]; + } + } + + this.bwtBytesDecoded++; + + return nextDecodedByte; + } + + /// + /// Public constructor + /// + /// The BZip2BitInputStream to read from + /// The maximum decoded size of the block + /// If the block could not be decoded + public BZip2BlockDecompressor(IBZip2BitInputStream bitInputStream, uint blockSize) + { + this.bitInputStream = bitInputStream; + this.bwtBlock = new byte[blockSize]; + + // Read block header + this.blockCRC = this.bitInputStream.ReadInteger(); //.ReadBits(32); + this.blockRandomised = this.bitInputStream.ReadBoolean(); + uint bwtStartPointer = this.bitInputStream.ReadBits(24); + + // Read block data and decode through to the Inverse Burrows Wheeler Transform stage + var huffmanDecoder = this.ReadHuffmanTables(); + this.DecodeHuffmanData(huffmanDecoder); + this.InitialiseInverseBWT(bwtStartPointer); + } + + /// + /// Decodes a byte from the final Run-Length Encoding stage, pulling a new byte from the + /// Burrows-Wheeler Transform stage when required + /// + /// The decoded byte, or -1 if there are no more bytes + public int Read() + { + while (this.rleRepeat < 1) + { + if (this.bwtBytesDecoded == this.bwtBlockLength) + return -1; + + int nextByte = this.decodeNextBWTByte(); + if (nextByte != this.rleLastDecodedByte) + { + // New byte, restart accumulation + this.rleLastDecodedByte = nextByte; + this.rleRepeat = 1; + this.rleAccumulator = 1; + this.crc.UpdateCrc(nextByte); + } + else + { + if (++this.rleAccumulator == 4) + { + // Accumulation complete, start repetition + int _rleRepeat = this.decodeNextBWTByte() + 1; + this.rleRepeat = _rleRepeat; + this.rleAccumulator = 0; + this.crc.UpdateCrc(nextByte, _rleRepeat); + } + else + { + this.rleRepeat = 1; + this.crc.UpdateCrc(nextByte); + } + } + } + + this.rleRepeat--; + + return this.rleLastDecodedByte; + } + + /// + /// Decodes multiple bytes from the final Run-Length Encoding stage, pulling new bytes from the + /// Burrows-Wheeler Transform stage when required + /// + /// The array to write to + /// The starting position within the array + /// The number of bytes to read + /// The number of bytes actually read, or -1 if there are no bytes left in the block + public int Read(byte[] destination, int offset, int length) + { + int i; + for (i = 0; i < length; i++, offset++) + { + var decoded = this.Read(); + if (decoded == -1) + return (i == 0) ? -1 : i; + + destination[offset] = (byte)decoded; + } + return i; + } + + public int Read(Stream x, int maxLength) + { + int i; + for (i = 0; i < maxLength; i++) + { + var decoded = this.Read(); + if (decoded == -1) + return (i == 0) ? -1 : i; + + x.WriteByte((byte)decoded); + } + return i; + } + + public int ReadAll(Stream x) + { + int count = 0; + while (true) + { + var decoded = this.Read(); + if (decoded == -1) + return (count == 0) ? -1 : count; + count++; + x.WriteByte((byte)decoded); + } + } + + /// + /// Verify and return the block CRC. This method may only be called after all of the block's bytes have been read + /// + /// The block CRC + /// if the CRC verification failed + public uint CheckCrc() + { + if (this.blockCRC != this.crc.CRC) + throw new IOException("BZip2 block CRC error"); + + return this.crc.CRC; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs new file mode 100644 index 000000000..6d52a1ab6 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs @@ -0,0 +1,2981 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// DivSufSort suffix array generator + /// Based on libdivsufsort 1.2.3 patched to support BZip2 + /// + /// + /// This is a simple conversion of the original C with two minor bugfixes applied(see "BUGFIX" + /// comments within the class). Documentation within the class is largely absent. + /// + internal class BZip2DivSufSort + { + private class StackEntry + { + public readonly int a; + public readonly int b; + public readonly int c; + public readonly int d; + + public StackEntry(int a, int b, int c, int d) + { + this.a = a; + this.b = b; + this.c = c; + this.d = d; + } + } + + private class PartitionResult + { + public readonly int first; + public readonly int last; + + public PartitionResult(int first, int last) + { + this.first = first; + this.last = last; + } + } + + private class TRBudget + { + private int budget; + public int chance; + + public bool update(int size, int n) + { + this.budget -= n; + if (this.budget <= 0) + { + if (--this.chance == 0) + { + return false; + } + this.budget += size; + } + + return true; + } + + public TRBudget(int budget, int chance) + { + this.budget = budget; + this.chance = chance; + } + } + + private const int STACK_SIZE = 64; + private const int BUCKET_A_SIZE = 256; + private const int BUCKET_B_SIZE = 65536; + private const int SS_BLOCKSIZE = 1024; + private const int INSERTIONSORT_THRESHOLD = 8; + + private static readonly int[] log2table = + { + -1, + 0, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + }; + + private readonly int[] SA; + private readonly byte[] T; + private readonly int n; + + /// + /// Public constructor + /// + /// T The input array + /// SA The output array + /// The length of the input data + public BZip2DivSufSort(byte[] T, int[] SA, int n) + { + this.T = T; + this.SA = SA; + this.n = n; + } + + /// + /// Performs a Burrows Wheeler Transform on the input array + /// + /// the index of the first character of the input array within the output array + public int BWT() + { + if (this.n == 0) + return 0; + + if (this.n == 1) + { + this.SA[0] = this.T[0]; + return 0; + } + + var bucketA = new int[BZip2DivSufSort.BUCKET_A_SIZE]; + var bucketB = new int[BZip2DivSufSort.BUCKET_B_SIZE]; + + int m = this.sortTypeBstar(bucketA, bucketB); + return 0 < m ? this.constructBWT(bucketA, bucketB) : 0; + } + + // ReSharper disable LoopVariableIsNeverChangedInsideLoop + private static void swapElements(int[] array1, int index1, int[] array2, int index2) + { + var temp = array1[index1]; + array1[index1] = array2[index2]; + array2[index2] = temp; + } + + private int ssCompare(int p1, int p2, int depth) + { + int U1n, + U2n; // pointers within T + int U1, + U2; + + for ( + U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = this.SA[p1 + 1] + 2, U2n = + this.SA[p2 + 1] + 2; + (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); + ++U1, ++U2 + ) { } + + return U1 < U1n + ? (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) + : (U2 < U2n ? -1 : 0); + } + + private int ssCompareLast(int PA, int p1, int p2, int depth, int size) + { + int U1, + U2, + U1n, + U2n; + + for ( + U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = size, U2n = + this.SA[(p2 + 1)] + 2; + (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); + ++U1, ++U2 + ) { } + + if (U1 < U1n) + return (U2 < U2n) ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1; + + if (U2 == U2n) + return 1; + + for ( + U1 = U1 % size, U1n = this.SA[PA] + 2; + (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); + ++U1, ++U2 + ) { } + + return U1 < U1n + ? (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) + : (U2 < U2n ? -1 : 0); + } + + private void ssInsertionSort(int PA, int first, int last, int depth) + { + int i; // pointer within SA + + for (i = last - 2; first <= i; --i) + { + int j; // pointer within SA + int t; + int r; + for ( + t = this.SA[i], j = i + 1; + 0 < (r = this.ssCompare(PA + t, PA + this.SA[j], depth)); + + ) + { + do + { + this.SA[j - 1] = this.SA[j]; + } while ((++j < last) && (this.SA[j] < 0)); + if (last <= j) + { + break; + } + } + if (r == 0) + { + this.SA[j] = ~this.SA[j]; + } + this.SA[j - 1] = t; + } + } + + private void ssFixdown(int Td, int PA, int sa, int i, int size) + { + int j, + k; + int v; + int c; + + for ( + v = this.SA[sa + i], c = (this.T[Td + this.SA[PA + v]]) & 0xff; + (j = 2 * i + 1) < size; + this.SA[sa + i] = this.SA[sa + k], i = k + ) + { + int d = this.T[Td + this.SA[PA + this.SA[sa + (k = j++)]]] & 0xff; + int e; + if (d < (e = this.T[Td + this.SA[PA + this.SA[sa + j]]] & 0xff)) + { + k = j; + d = e; + } + if (d <= c) + break; + } + this.SA[sa + i] = v; + } + + private void ssHeapSort(int Td, int PA, int sa, int size) + { + int i; + + int m = size; + if ((size % 2) == 0) + { + m--; + if ( + (this.T[Td + this.SA[PA + this.SA[sa + (m / 2)]]] & 0xff) + < (this.T[Td + this.SA[PA + this.SA[sa + m]]] & 0xff) + ) + { + swapElements(this.SA, sa + m, this.SA, sa + (m / 2)); + } + } + + for (i = m / 2 - 1; 0 <= i; --i) + { + this.ssFixdown(Td, PA, sa, i, m); + } + + if ((size % 2) == 0) + { + swapElements(this.SA, sa, this.SA, sa + m); + this.ssFixdown(Td, PA, sa, 0, m); + } + + for (i = m - 1; 0 < i; --i) + { + int t = this.SA[sa]; + this.SA[sa] = this.SA[sa + i]; + this.ssFixdown(Td, PA, sa, 0, i); + this.SA[sa + i] = t; + } + } + + private int ssMedian3(int Td, int PA, int v1, int v2, int v3) + { + var T_v1 = this.T[Td + this.SA[PA + this.SA[v1]]] & 0xff; + var T_v2 = this.T[Td + this.SA[PA + this.SA[v2]]] & 0xff; + var T_v3 = this.T[Td + this.SA[PA + this.SA[v3]]] & 0xff; + + if (T_v1 > T_v2) + { + var temp = v1; + v1 = v2; + v2 = temp; + var T_vtemp = T_v1; + T_v1 = T_v2; + T_v2 = T_vtemp; + } + if (T_v2 > T_v3) + { + return T_v1 > T_v3 ? v1 : v3; + } + return v2; + } + + private int ssMedian5(int Td, int PA, int v1, int v2, int v3, int v4, int v5) + { + var T_v1 = this.T[Td + this.SA[PA + this.SA[v1]]] & 0xff; + var T_v2 = this.T[Td + this.SA[PA + this.SA[v2]]] & 0xff; + var T_v3 = this.T[Td + this.SA[PA + this.SA[v3]]] & 0xff; + var T_v4 = this.T[Td + this.SA[PA + this.SA[v4]]] & 0xff; + var T_v5 = this.T[Td + this.SA[PA + this.SA[v5]]] & 0xff; + int temp; + int T_vtemp; + + // ReSharper disable RedundantAssignment + if (T_v2 > T_v3) + { + temp = v2; + v2 = v3; + v3 = temp; + T_vtemp = T_v2; + T_v2 = T_v3; + T_v3 = T_vtemp; + } + if (T_v4 > T_v5) + { + temp = v4; + v4 = v5; + v5 = temp; + T_vtemp = T_v4; + T_v4 = T_v5; + T_v5 = T_vtemp; + } + if (T_v2 > T_v4) + { + temp = v2; + v2 = v4; + v4 = temp; + T_vtemp = T_v2; + T_v2 = T_v4; + T_v4 = T_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + T_vtemp = T_v3; + T_v3 = T_v5; + T_v5 = T_vtemp; + } + if (T_v1 > T_v3) + { + temp = v1; + v1 = v3; + v3 = temp; + T_vtemp = T_v1; + T_v1 = T_v3; + T_v3 = T_vtemp; + } + if (T_v1 > T_v4) + { + temp = v1; + v1 = v4; + v4 = temp; + T_vtemp = T_v1; + T_v1 = T_v4; + T_v4 = T_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + T_vtemp = T_v3; + T_v3 = T_v5; + T_v5 = T_vtemp; + } + // ReSharper restore RedundantAssignment + + return T_v3 > T_v4 ? v4 : v3; + } + + private int ssPivot(int Td, int PA, int first, int last) + { + int t = last - first; + int middle = first + t / 2; + + if (t <= 512) + { + if (t <= 32) + { + return this.ssMedian3(Td, PA, first, middle, last - 1); + } + t >>= 2; + return this.ssMedian5(Td, PA, first, first + t, middle, last - 1 - t, last - 1); + } + t >>= 3; + return this.ssMedian3( + Td, + PA, + this.ssMedian3(Td, PA, first, first + t, first + (t << 1)), + this.ssMedian3(Td, PA, middle - t, middle, middle + t), + this.ssMedian3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1) + ); + } + + private static int ssLog(int x) + { + return ((x & 0xff00) != 0) + ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] + : BZip2DivSufSort.log2table[x & 0xff]; + } + + private int ssSubstringPartition(int PA, int first, int last, int depth) + { + int a, + b; + + for (a = first - 1, b = last; ; ) + { + for ( + ; + (++a < b) + && ( + (this.SA[PA + this.SA[a]] + depth) >= (this.SA[PA + this.SA[a] + 1] + 1) + ); + + ) + { + this.SA[a] = ~this.SA[a]; + } + for ( + ; + (a < --b) + && ( + (this.SA[PA + this.SA[b]] + depth) < (this.SA[PA + this.SA[b] + 1] + 1) + ); + + ) { } + if (b <= a) + { + break; + } + int t = ~this.SA[b]; + this.SA[b] = this.SA[a]; + this.SA[a] = t; + } + if (first < a) + this.SA[first] = ~this.SA[first]; + + return a; + } + + private void ssMultiKeyIntroSort(int PA, int first, int last, int depth) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int ssize; + int limit; + int x = 0; + + for (ssize = 0, limit = ssLog(last - first); ; ) + { + if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) + { + if (1 < (last - first)) + { + this.ssInsertionSort(PA, first, last, depth); + } + if (ssize == 0) + return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + depth = entry.c; + limit = entry.d; + continue; + } + + int Td = depth; + if (limit-- == 0) + { + this.ssHeapSort(Td, PA, first, last - first); + } + int a; + int v; + if (limit < 0) + { + for ( + a = first + 1, v = this.T[Td + this.SA[PA + this.SA[first]]] & 0xff; + a < last; + ++a + ) + { + if ((x = (this.T[Td + this.SA[PA + this.SA[a]]] & 0xff)) != v) + { + if (1 < (a - first)) + { + break; + } + v = x; + first = a; + } + } + if ((this.T[Td + this.SA[PA + this.SA[first]] - 1] & 0xff) < v) + { + first = this.ssSubstringPartition(PA, first, a, depth); + } + if ((a - first) <= (last - a)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry(a, last, depth, -1); + last = a; + depth += 1; + limit = ssLog(a - first); + } + else + { + first = a; + limit = -1; + } + } + else + { + if (1 < (last - a)) + { + stack[ssize++] = new StackEntry(first, a, depth + 1, ssLog(a - first)); + first = a; + limit = -1; + } + else + { + last = a; + depth += 1; + limit = ssLog(a - first); + } + } + continue; + } + + a = this.ssPivot(Td, PA, first, last); + v = this.T[Td + this.SA[PA + this.SA[a]]] & 0xff; + swapElements(this.SA, first, this.SA, a); + + int b; + for ( + b = first; + (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) == v); + + ) { } + if (((a = b) < last) && (x < v)) + { + for ( + ; + (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v); + + ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + } + int c; + for ( + c = last; + (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) == v); + + ) { } + int d; + if ((b < (d = c)) && (x > v)) + { + for ( + ; + (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v); + + ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c; ) + { + swapElements(this.SA, b, this.SA, c); + for ( + ; + (++b < c) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v); + + ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + for ( + ; + (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v); + + ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + + int s; + int t; + if ((s = a - first) > (t = b - a)) + { + s = t; + } + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + + a = first + (b - a); + c = last - (d - c); + b = + (v <= (this.T[Td + this.SA[PA + this.SA[a]] - 1] & 0xff)) + ? a + : this.ssSubstringPartition(PA, a, c, depth); + + if ((a - first) <= (last - c)) + { + if ((last - c) <= (c - b)) + { + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); + stack[ssize++] = new StackEntry(c, last, depth, limit); + last = a; + } + else if ((a - first) <= (c - b)) + { + stack[ssize++] = new StackEntry(c, last, depth, limit); + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); + last = a; + } + else + { + stack[ssize++] = new StackEntry(c, last, depth, limit); + stack[ssize++] = new StackEntry(first, a, depth, limit); + first = b; + last = c; + depth += 1; + limit = ssLog(c - b); + } + } + else + { + if ((a - first) <= (c - b)) + { + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); + stack[ssize++] = new StackEntry(first, a, depth, limit); + first = c; + } + else if ((last - c) <= (c - b)) + { + stack[ssize++] = new StackEntry(first, a, depth, limit); + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); + first = c; + } + else + { + stack[ssize++] = new StackEntry(first, a, depth, limit); + stack[ssize++] = new StackEntry(c, last, depth, limit); + first = b; + last = c; + depth += 1; + limit = ssLog(c - b); + } + } + } + else + { + limit += 1; + if ((this.T[Td + this.SA[PA + this.SA[first]] - 1] & 0xff) < v) + { + first = this.ssSubstringPartition(PA, first, last, depth); + limit = ssLog(last - first); + } + depth += 1; + } + } + } + + private static void ssBlockSwap( + int[] array1, + int first1, + int[] array2, + int first2, + int size + ) + { + int a, + b; + int i; + for (i = size, a = first1, b = first2; 0 < i; --i, ++a, ++b) + { + swapElements(array1, a, array2, b); + } + } + + private void ssMergeForward( + int PA, + int[] buf, + int bufoffset, + int first, + int middle, + int last, + int depth + ) + { + int i, + j, + k; + int t; + + int bufend = bufoffset + (middle - first) - 1; + ssBlockSwap(buf, bufoffset, this.SA, first, middle - first); + + for (t = this.SA[first], i = first, j = bufoffset, k = middle; ; ) + { + int r = this.ssCompare(PA + buf[j], PA + this.SA[k], depth); + if (r < 0) + { + do + { + this.SA[i++] = buf[j]; + if (bufend <= j) + { + buf[j] = t; + return; + } + buf[j++] = this.SA[i]; + } while (buf[j] < 0); + } + else if (r > 0) + { + do + { + this.SA[i++] = this.SA[k]; + this.SA[k++] = this.SA[i]; + if (last <= k) + { + while (j < bufend) + { + this.SA[i++] = buf[j]; + buf[j++] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + } while (this.SA[k] < 0); + } + else + { + this.SA[k] = ~this.SA[k]; + do + { + this.SA[i++] = buf[j]; + if (bufend <= j) + { + buf[j] = t; + return; + } + buf[j++] = this.SA[i]; + } while (buf[j] < 0); + + do + { + this.SA[i++] = this.SA[k]; + this.SA[k++] = this.SA[i]; + if (last <= k) + { + while (j < bufend) + { + this.SA[i++] = buf[j]; + buf[j++] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + } while (this.SA[k] < 0); + } + } + } + + private void ssMergeBackward( + int PA, + int[] buf, + int bufoffset, + int first, + int middle, + int last, + int depth + ) + { + int p1, + p2; + int i, + j, + k; + int t; + + int bufend = bufoffset + (last - middle); + ssBlockSwap(buf, bufoffset, this.SA, middle, last - middle); + + int x = 0; + if (buf[bufend - 1] < 0) + { + x |= 1; + p1 = PA + ~buf[bufend - 1]; + } + else + { + p1 = PA + buf[bufend - 1]; + } + if (this.SA[middle - 1] < 0) + { + x |= 2; + p2 = PA + ~this.SA[middle - 1]; + } + else + { + p2 = PA + this.SA[middle - 1]; + } + for (t = this.SA[last - 1], i = last - 1, j = bufend - 1, k = middle - 1; ; ) + { + int r = this.ssCompare(p1, p2, depth); + if (r > 0) + { + if ((x & 1) != 0) + { + do + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } while (buf[j] < 0); + x ^= 1; + } + this.SA[i--] = buf[j]; + if (j <= bufoffset) + { + buf[j] = t; + return; + } + buf[j--] = this.SA[i]; + + if (buf[j] < 0) + { + x |= 1; + p1 = PA + ~buf[j]; + } + else + { + p1 = PA + buf[j]; + } + } + else if (r < 0) + { + if ((x & 2) != 0) + { + do + { + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + } while (this.SA[k] < 0); + x ^= 2; + } + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + if (k < first) + { + while (bufoffset < j) + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + + if (this.SA[k] < 0) + { + x |= 2; + p2 = PA + ~this.SA[k]; + } + else + { + p2 = PA + this.SA[k]; + } + } + else + { + if ((x & 1) != 0) + { + do + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } while (buf[j] < 0); + x ^= 1; + } + this.SA[i--] = ~buf[j]; + if (j <= bufoffset) + { + buf[j] = t; + return; + } + buf[j--] = this.SA[i]; + + if ((x & 2) != 0) + { + do + { + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + } while (this.SA[k] < 0); + x ^= 2; + } + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + if (k < first) + { + while (bufoffset < j) + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + + if (buf[j] < 0) + { + x |= 1; + p1 = PA + ~buf[j]; + } + else + { + p1 = PA + buf[j]; + } + if (this.SA[k] < 0) + { + x |= 2; + p2 = PA + ~this.SA[k]; + } + else + { + p2 = PA + this.SA[k]; + } + } + } + } + + private static int getIDX(int a) + { + return (0 <= a) ? a : ~a; + } + + private void ssMergeCheckEqual(int PA, int depth, int a) + { + if ( + (0 <= this.SA[a]) + && (this.ssCompare(PA + getIDX(this.SA[a - 1]), PA + this.SA[a], depth) == 0) + ) + { + this.SA[a] = ~this.SA[a]; + } + } + + private void ssMerge( + int PA, + int first, + int middle, + int last, + int[] buf, + int bufoffset, + int bufsize, + int depth + ) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int ssize; + int check; + + for (check = 0, ssize = 0; ; ) + { + if ((last - middle) <= bufsize) + { + if ((first < middle) && (middle < last)) + { + this.ssMergeBackward(PA, buf, bufoffset, first, middle, last, depth); + } + + if ((check & 1) != 0) + { + this.ssMergeCheckEqual(PA, depth, first); + } + if ((check & 2) != 0) + { + this.ssMergeCheckEqual(PA, depth, last); + } + if (ssize == 0) + return; + var entry = stack[--ssize]; + first = entry.a; + middle = entry.b; + last = entry.c; + check = entry.d; + continue; + } + + if ((middle - first) <= bufsize) + { + if (first < middle) + { + this.ssMergeForward(PA, buf, bufoffset, first, middle, last, depth); + } + if ((check & 1) != 0) + { + this.ssMergeCheckEqual(PA, depth, first); + } + if ((check & 2) != 0) + { + this.ssMergeCheckEqual(PA, depth, last); + } + if (ssize == 0) + return; + var entry = stack[--ssize]; + first = entry.a; + middle = entry.b; + last = entry.c; + check = entry.d; + continue; + } + + int m; + int len; + int half; + for ( + m = 0, len = Math.Min(middle - first, last - middle), half = len >> 1; + 0 < len; + len = half, half >>= 1 + ) + { + if ( + this.ssCompare( + PA + getIDX(this.SA[middle + m + half]), + PA + getIDX(this.SA[middle - m - half - 1]), + depth + ) < 0 + ) + { + m += half + 1; + half -= (len & 1) ^ 1; + } + } + + if (0 < m) + { + ssBlockSwap(this.SA, middle - m, this.SA, middle, m); + int j; + int i = j = middle; + int next = 0; + if ((middle + m) < last) + { + if (this.SA[middle + m] < 0) + { + for (; this.SA[i - 1] < 0; --i) { } + this.SA[middle + m] = ~this.SA[middle + m]; + } + for (j = middle; this.SA[j] < 0; ++j) { } + next = 1; + } + if ((i - first) <= (last - j)) + { + stack[ssize++] = new StackEntry( + j, + middle + m, + last, + (check & 2) | (next & 1) + ); + middle -= m; + last = i; + check = (check & 1); + } + else + { + if ((i == middle) && (middle == j)) + { + next <<= 1; + } + stack[ssize++] = new StackEntry( + first, + middle - m, + i, + (check & 1) | (next & 2) + ); + first = j; + middle += m; + check = (check & 2) | (next & 1); + } + } + else + { + if ((check & 1) != 0) + { + this.ssMergeCheckEqual(PA, depth, first); + } + this.ssMergeCheckEqual(PA, depth, middle); + if ((check & 2) != 0) + { + this.ssMergeCheckEqual(PA, depth, last); + } + if (ssize == 0) + return; + var entry = stack[--ssize]; + first = entry.a; + middle = entry.b; + last = entry.c; + check = entry.d; + } + } + } + + private void subStringSort( + int PA, + int first, + int last, + int[] buf, + int bufoffset, + int bufsize, + int depth, + bool lastsuffix, + int size + ) + { + int a; + int i; + int k; + + if (lastsuffix) + ++first; + + for ( + a = first, i = 0; + (a + BZip2DivSufSort.SS_BLOCKSIZE) < last; + a += BZip2DivSufSort.SS_BLOCKSIZE, ++i + ) + { + this.ssMultiKeyIntroSort(PA, a, a + BZip2DivSufSort.SS_BLOCKSIZE, depth); + int[] curbuf = this.SA; + int curbufoffset = a + BZip2DivSufSort.SS_BLOCKSIZE; + int curbufsize = last - (a + BZip2DivSufSort.SS_BLOCKSIZE); + if (curbufsize <= bufsize) + { + curbufsize = bufsize; + curbuf = buf; + curbufoffset = bufoffset; + } + int b; + int j; + for ( + b = a, k = BZip2DivSufSort.SS_BLOCKSIZE, j = i; + (j & 1) != 0; + b -= k, k <<= 1, j >>= 1 + ) + this.ssMerge(PA, b - k, b, b + k, curbuf, curbufoffset, curbufsize, depth); + } + + this.ssMultiKeyIntroSort(PA, a, last, depth); + + for (k = BZip2DivSufSort.SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) + { + if ((i & 1) != 0) + { + this.ssMerge(PA, a - k, a, last, buf, bufoffset, bufsize, depth); + a -= k; + } + } + + if (lastsuffix) + { + int r; + for ( + a = first, i = this.SA[first - 1], r = 1; + (a < last) + && ( + (this.SA[a] < 0) + || ( + 0 + < (r = this.ssCompareLast(PA, PA + i, PA + this.SA[a], depth, size)) + ) + ); + ++a + ) + { + this.SA[a - 1] = this.SA[a]; + } + if (r == 0) + { + this.SA[a] = ~this.SA[a]; + } + this.SA[a - 1] = i; + } + } + + private int trGetC(int ISA, int ISAd, int ISAn, int p) + { + return ( + ((ISAd + p) < ISAn) + ? this.SA[ISAd + p] + : this.SA[ISA + ((ISAd - ISA + p) % (ISAn - ISA))] + ); + } + + private void trFixdown(int ISA, int ISAd, int ISAn, int sa, int i, int size) + { + int j, + k; + int v; + int c; + + for ( + v = this.SA[sa + i], c = this.trGetC(ISA, ISAd, ISAn, v); + (j = 2 * i + 1) < size; + this.SA[sa + i] = this.SA[sa + k], i = k + ) + { + k = j++; + int d = this.trGetC(ISA, ISAd, ISAn, this.SA[sa + k]); + int e; + if (d < (e = this.trGetC(ISA, ISAd, ISAn, this.SA[sa + j]))) + { + k = j; + d = e; + } + if (d <= c) + { + break; + } + } + this.SA[sa + i] = v; + } + + private void trHeapSort(int ISA, int ISAd, int ISAn, int sa, int size) + { + int i; + + int m = size; + if ((size % 2) == 0) + { + m--; + if ( + this.trGetC(ISA, ISAd, ISAn, this.SA[sa + (m / 2)]) + < this.trGetC(ISA, ISAd, ISAn, this.SA[sa + m]) + ) + { + swapElements(this.SA, sa + m, this.SA, sa + (m / 2)); + } + } + + for (i = m / 2 - 1; 0 <= i; --i) + { + this.trFixdown(ISA, ISAd, ISAn, sa, i, m); + } + + if ((size % 2) == 0) + { + swapElements(this.SA, sa + 0, this.SA, sa + m); + this.trFixdown(ISA, ISAd, ISAn, sa, 0, m); + } + + for (i = m - 1; 0 < i; --i) + { + int t = this.SA[sa + 0]; + this.SA[sa + 0] = this.SA[sa + i]; + this.trFixdown(ISA, ISAd, ISAn, sa, 0, i); + this.SA[sa + i] = t; + } + } + + private void trInsertionSort(int ISA, int ISAd, int ISAn, int first, int last) + { + int a; + + for (a = first + 1; a < last; ++a) + { + int b; + int t; + int r; + for ( + t = this.SA[a], b = a - 1; + 0 + > ( + r = + this.trGetC(ISA, ISAd, ISAn, t) + - this.trGetC(ISA, ISAd, ISAn, this.SA[b]) + ); + + ) + { + do + { + this.SA[b + 1] = this.SA[b]; + } while ((first <= --b) && (this.SA[b] < 0)); + if (b < first) + { + break; + } + } + if (r == 0) + { + this.SA[b] = ~this.SA[b]; + } + this.SA[b + 1] = t; + } + } + + private static int trLog(int x) + { + return ((x & 0xffff0000) != 0) + ? ( + ((x & 0xff000000) != 0) + ? 24 + BZip2DivSufSort.log2table[(x >> 24) & 0xff] + : 16 + BZip2DivSufSort.log2table[(x >> 16) & 0xff] + ) + : ( + ((x & 0x0000ff00) != 0) + ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] + : 0 + BZip2DivSufSort.log2table[(x >> 0) & 0xff] + ); + } + + private int trMedian3(int ISA, int ISAd, int ISAn, int v1, int v2, int v3) + { + var SA_v1 = this.trGetC(ISA, ISAd, ISAn, this.SA[v1]); + var SA_v2 = this.trGetC(ISA, ISAd, ISAn, this.SA[v2]); + var SA_v3 = this.trGetC(ISA, ISAd, ISAn, this.SA[v3]); + + if (SA_v1 > SA_v2) + { + var temp = v1; + v1 = v2; + v2 = temp; + var SA_vtemp = SA_v1; + SA_v1 = SA_v2; + SA_v2 = SA_vtemp; + } + if (SA_v2 > SA_v3) + { + return SA_v1 > SA_v3 ? v1 : v3; + } + + return v2; + } + + private int trMedian5(int ISA, int ISAd, int ISAn, int v1, int v2, int v3, int v4, int v5) + { + var SA_v1 = this.trGetC(ISA, ISAd, ISAn, this.SA[v1]); + var SA_v2 = this.trGetC(ISA, ISAd, ISAn, this.SA[v2]); + var SA_v3 = this.trGetC(ISA, ISAd, ISAn, this.SA[v3]); + var SA_v4 = this.trGetC(ISA, ISAd, ISAn, this.SA[v4]); + var SA_v5 = this.trGetC(ISA, ISAd, ISAn, this.SA[v5]); + int temp; + int SA_vtemp; + + // ReSharper disable RedundantAssignment + if (SA_v2 > SA_v3) + { + temp = v2; + v2 = v3; + v3 = temp; + SA_vtemp = SA_v2; + SA_v2 = SA_v3; + SA_v3 = SA_vtemp; + } + if (SA_v4 > SA_v5) + { + temp = v4; + v4 = v5; + v5 = temp; + SA_vtemp = SA_v4; + SA_v4 = SA_v5; + SA_v5 = SA_vtemp; + } + if (SA_v2 > SA_v4) + { + temp = v2; + v2 = v4; + v4 = temp; + SA_vtemp = SA_v2; + SA_v2 = SA_v4; + SA_v4 = SA_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + SA_vtemp = SA_v3; + SA_v3 = SA_v5; + SA_v5 = SA_vtemp; + } + if (SA_v1 > SA_v3) + { + temp = v1; + v1 = v3; + v3 = temp; + SA_vtemp = SA_v1; + SA_v1 = SA_v3; + SA_v3 = SA_vtemp; + } + if (SA_v1 > SA_v4) + { + temp = v1; + v1 = v4; + v4 = temp; + SA_vtemp = SA_v1; + SA_v1 = SA_v4; + SA_v4 = SA_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + SA_vtemp = SA_v3; + SA_v3 = SA_v5; + SA_v5 = SA_vtemp; + } + // ReSharper restore RedundantAssignment + + return SA_v3 > SA_v4 ? v4 : v3; + } + + private int trPivot(int ISA, int ISAd, int ISAn, int first, int last) + { + int t = last - first; + int middle = first + t / 2; + + if (t <= 512) + { + if (t <= 32) + { + return this.trMedian3(ISA, ISAd, ISAn, first, middle, last - 1); + } + t >>= 2; + return this.trMedian5( + ISA, + ISAd, + ISAn, + first, + first + t, + middle, + last - 1 - t, + last - 1 + ); + } + t >>= 3; + return this.trMedian3( + ISA, + ISAd, + ISAn, + this.trMedian3(ISA, ISAd, ISAn, first, first + t, first + (t << 1)), + this.trMedian3(ISA, ISAd, ISAn, middle - t, middle, middle + t), + this.trMedian3(ISA, ISAd, ISAn, last - 1 - (t << 1), last - 1 - t, last - 1) + ); + } + + private void lsUpdateGroup(int ISA, int first, int last) + { + int a; + + for (a = first; a < last; ++a) + { + int b; + if (0 <= this.SA[a]) + { + b = a; + do + { + this.SA[ISA + this.SA[a]] = a; + } while ((++a < last) && (0 <= this.SA[a])); + this.SA[b] = b - a; + if (last <= a) + { + break; + } + } + b = a; + do + { + this.SA[a] = ~this.SA[a]; + } while (this.SA[++a] < 0); + int t = a; + do + { + this.SA[ISA + this.SA[b]] = t; + } while (++b <= a); + } + } + + private void lsIntroSort(int ISA, int ISAd, int ISAn, int first, int last) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int limit; + int x = 0; + int ssize; + + for (ssize = 0, limit = trLog(last - first); ; ) + { + if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) + { + if (1 < (last - first)) + { + this.trInsertionSort(ISA, ISAd, ISAn, first, last); + this.lsUpdateGroup(ISA, first, last); + } + else if ((last - first) == 1) + { + this.SA[first] = -1; + } + if (ssize == 0) + return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + limit = entry.c; + continue; + } + + int a; + int b; + if (limit-- == 0) + { + this.trHeapSort(ISA, ISAd, ISAn, first, last - first); + for (a = last - 1; first < a; a = b) + { + for ( + x = this.trGetC(ISA, ISAd, ISAn, this.SA[a]), b = a - 1; + (first <= b) && (this.trGetC(ISA, ISAd, ISAn, this.SA[b]) == x); + --b + ) + { + this.SA[b] = ~this.SA[b]; + } + } + this.lsUpdateGroup(ISA, first, last); + if (ssize == 0) + return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + limit = entry.c; + continue; + } + + a = this.trPivot(ISA, ISAd, ISAn, first, last); + swapElements(this.SA, first, this.SA, a); + int v = this.trGetC(ISA, ISAd, ISAn, this.SA[first]); + + for ( + b = first; + (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) == v); + + ) { } + if (((a = b) < last) && (x < v)) + { + for (; (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + } + int c; + for (c = last; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) == v); ) + { } + int d; + if ((b < (d = c)) && (x > v)) + { + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c; ) + { + swapElements(this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + + int s; + int t; + if ((s = a - first) > (t = b - a)) + { + s = t; + } + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + + a = first + (b - a); + b = last - (d - c); + + for (c = first, v = a - 1; c < a; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + if (b < last) + { + for (c = a, v = b - 1; c < b; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + if ((b - a) == 1) + { + this.SA[a] = -1; + } + + if ((a - first) <= (last - b)) + { + if (first < a) + { + stack[ssize++] = new StackEntry(b, last, limit, 0); + last = a; + } + else + { + first = b; + } + } + else + { + if (b < last) + { + stack[ssize++] = new StackEntry(first, a, limit, 0); + first = b; + } + else + { + last = a; + } + } + } + else + { + if (ssize == 0) + return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + limit = entry.c; + } + } + } + + private void lsSort(int ISA, int x, int depth) + { + int ISAd; + + for (ISAd = ISA + depth; -x < this.SA[0]; ISAd += (ISAd - ISA)) + { + int first = 0; + int skip = 0; + int last; + int t; + do + { + if ((t = this.SA[first]) < 0) + { + first -= t; + skip += t; + } + else + { + if (skip != 0) + { + this.SA[first + skip] = skip; + skip = 0; + } + last = this.SA[ISA + t] + 1; + this.lsIntroSort(ISA, ISAd, ISA + x, first, last); + first = last; + } + } while (first < x); + + if (skip != 0) + this.SA[first + skip] = skip; + + if (x < (ISAd - ISA)) + { + first = 0; + do + { + if ((t = this.SA[first]) < 0) + first -= t; + else + { + last = this.SA[ISA + t] + 1; + int i; + for (i = first; i < last; ++i) + this.SA[ISA + this.SA[i]] = i; + first = last; + } + } while (first < x); + break; + } + } + } + + private PartitionResult trPartition(int ISA, int ISAd, int ISAn, int first, int last, int v) + { + int a, + b, + c, + d; + var x = 0; + + for ( + b = first - 1; + (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) == v); + + ) { } + if (((a = b) < last) && (x < v)) + { + for (; (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + } + for (c = last; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) == v); ) { } + if ((b < (d = c)) && (x > v)) + { + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c; ) + { + swapElements(this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + int t; + int s; + if ((s = a - first) > (t = b - a)) + s = t; + + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + first += (b - a); + last -= (d - c); + } + + return new PartitionResult(first, last); + } + + private void trCopy(int ISA, int ISAn, int first, int a, int b, int last, int depth) + { + int c, + d, + e; + int s; + int v = b - 1; + + for (c = first, d = a - 1; c <= d; ++c) + { + if ((s = this.SA[c] - depth) < 0) + { + s += ISAn - ISA; + } + if (this.SA[ISA + s] == v) + { + this.SA[++d] = s; + this.SA[ISA + s] = d; + } + } + for (c = last - 1, e = d + 1, d = b; e < d; --c) + { + if ((s = this.SA[c] - depth) < 0) + { + s += ISAn - ISA; + } + if (this.SA[ISA + s] == v) + { + this.SA[--d] = s; + this.SA[ISA + s] = d; + } + } + } + + private void trIntroSort( + int ISA, + int ISAd, + int ISAn, + int first, + int last, + TRBudget budget, + int size + ) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int s; + int x = 0; + int limit; + int ssize; + + for (ssize = 0, limit = trLog(last - first); ; ) + { + int a; + int b; + int c; + int v; + int next; + if (limit < 0) + { + if (limit == -1) + { + if (!budget.update(size, last - first)) + break; + var result = this.trPartition(ISA, ISAd - 1, ISAn, first, last, last - 1); + a = result.first; + b = result.last; + if ((first < a) || (b < last)) + { + if (a < last) + { + for (c = first, v = a - 1; c < a; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + if (b < last) + { + for (c = a, v = b - 1; c < b; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + + stack[ssize++] = new StackEntry(0, a, b, 0); + stack[ssize++] = new StackEntry(ISAd - 1, first, last, -2); + if ((a - first) <= (last - b)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry(ISAd, b, last, trLog(last - b)); + last = a; + limit = trLog(a - first); + } + else if (1 < (last - b)) + { + first = b; + limit = trLog(last - b); + } + else + { + if (ssize == 0) + return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } + else + { + if (1 < (last - b)) + { + stack[ssize++] = new StackEntry( + ISAd, + first, + a, + trLog(a - first) + ); + first = b; + limit = trLog(last - b); + } + else if (1 < (a - first)) + { + last = a; + limit = trLog(a - first); + } + else + { + if (ssize == 0) + return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } + } + else + { + for (c = first; c < last; ++c) + { + this.SA[ISA + this.SA[c]] = c; + } + if (ssize == 0) + return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } + else if (limit == -2) + { + a = stack[--ssize].b; + b = stack[ssize].c; + this.trCopy(ISA, ISAn, first, a, b, last, ISAd - ISA); + if (ssize == 0) + return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + else + { + if (0 <= this.SA[first]) + { + a = first; + do + { + this.SA[ISA + this.SA[a]] = a; + } while ((++a < last) && (0 <= this.SA[a])); + first = a; + } + if (first < last) + { + a = first; + do + { + this.SA[a] = ~this.SA[a]; + } while (this.SA[++a] < 0); + next = + (this.SA[ISA + this.SA[a]] != this.SA[ISAd + this.SA[a]]) + ? trLog(a - first + 1) + : -1; + if (++a < last) + { + for (b = first, v = a - 1; b < a; ++b) + { + this.SA[ISA + this.SA[b]] = v; + } + } + + if ((a - first) <= (last - a)) + { + stack[ssize++] = new StackEntry(ISAd, a, last, -3); + ISAd += 1; + last = a; + limit = next; + } + else + { + if (1 < (last - a)) + { + stack[ssize++] = new StackEntry(ISAd + 1, first, a, next); + first = a; + limit = -3; + } + else + { + ISAd += 1; + last = a; + limit = next; + } + } + } + else + { + if (ssize == 0) + return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } + continue; + } + + if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) + { + if (!budget.update(size, last - first)) + break; + this.trInsertionSort(ISA, ISAd, ISAn, first, last); + limit = -3; + continue; + } + + if (limit-- == 0) + { + if (!budget.update(size, last - first)) + break; + this.trHeapSort(ISA, ISAd, ISAn, first, last - first); + for (a = last - 1; first < a; a = b) + { + for ( + x = this.trGetC(ISA, ISAd, ISAn, this.SA[a]), b = a - 1; + (first <= b) && (this.trGetC(ISA, ISAd, ISAn, this.SA[b]) == x); + --b + ) + { + this.SA[b] = ~this.SA[b]; + } + } + limit = -3; + continue; + } + + a = this.trPivot(ISA, ISAd, ISAn, first, last); + + swapElements(this.SA, first, this.SA, a); + v = this.trGetC(ISA, ISAd, ISAn, this.SA[first]); + for ( + b = first; + (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) == v); + + ) { } + if (((a = b) < last) && (x < v)) + { + for (; (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + } + for (c = last; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) == v); ) + { } + int d; + if ((b < (d = c)) && (x > v)) + { + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c; ) + { + swapElements(this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) + { + if (x == v) + { + swapElements(this.SA, b, this.SA, a); + ++a; + } + } + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) + { + if (x == v) + { + swapElements(this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + + int t; + if ((s = a - first) > (t = b - a)) + { + s = t; + } + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements(this.SA, e, this.SA, f); + } + + a = first + (b - a); + b = last - (d - c); + next = (this.SA[ISA + this.SA[a]] != v) ? trLog(b - a) : -1; + + for (c = first, v = a - 1; c < a; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + if (b < last) + { + for (c = a, v = b - 1; c < b; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + + if ((a - first) <= (last - b)) + { + if ((last - b) <= (b - a)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + last = a; + } + else if (1 < (last - b)) + { + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + first = b; + } + else if (1 < (b - a)) + { + ISAd += 1; + first = a; + last = b; + limit = next; + } + else + { + if (ssize == 0) + return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } + else if ((a - first) <= (b - a)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + last = a; + } + else if (1 < (b - a)) + { + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } + else + { + first = b; + } + } + else + { + if (1 < (b - a)) + { + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } + else + { + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + last = a; + } + } + } + else + { + if ((a - first) <= (b - a)) + { + if (1 < (last - b)) + { + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + first = b; + } + else if (1 < (a - first)) + { + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + last = a; + } + else if (1 < (b - a)) + { + ISAd += 1; + first = a; + last = b; + limit = next; + } + else + { + stack[ssize++] = new StackEntry(ISAd, first, last, limit); + } + } + else if ((last - b) <= (b - a)) + { + if (1 < (last - b)) + { + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + first = b; + } + else if (1 < (b - a)) + { + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } + else + { + last = a; + } + } + else + { + if (1 < (b - a)) + { + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } + else + { + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + first = b; + } + } + } + } + else + { + if (!budget.update(size, last - first)) + break; // BUGFIX : Added to prevent an infinite loop in the original code + limit += 1; + ISAd += 1; + } + } + + for (s = 0; s < ssize; ++s) + { + if (stack[s].d == -3) + { + this.lsUpdateGroup(ISA, stack[s].b, stack[s].c); + } + } + } + + private void trSort(int ISA, int x, int depth) + { + int first = 0; + + if (-x < this.SA[0]) + { + var budget = new TRBudget(x, trLog(x) * 2 / 3 + 1); + do + { + int t; + if ((t = this.SA[first]) < 0) + { + first -= t; + } + else + { + int last = this.SA[ISA + t] + 1; + if (1 < (last - first)) + { + this.trIntroSort(ISA, ISA + depth, ISA + x, first, last, budget, x); + if (budget.chance == 0) + { + // Switch to Larsson-Sadakane sorting algorithm. + if (0 < first) + { + this.SA[0] = -first; + } + this.lsSort(ISA, x, depth); + break; + } + } + first = last; + } + } while (first < x); + } + } + + private static int BUCKET_B(int c0, int c1) + { + return (c1 << 8) | c0; + } + + private static int BUCKET_BSTAR(int c0, int c1) + { + return (c0 << 8) | c1; + } + + private int sortTypeBstar(int[] bucketA, int[] bucketB) + { + var tempbuf = new int[256]; + + int i, + j, + k, + t; + int c0, + c1; + int flag; + + for (i = 1, flag = 1; i < this.n; ++i) + { + if (this.T[i - 1] != this.T[i]) + { + if ((this.T[i - 1] & 0xff) > (this.T[i] & 0xff)) + { + flag = 0; + } + break; + } + } + i = this.n - 1; + int m = this.n; + + int ti, + ti1, + t0; + if ( + ((ti = (this.T[i] & 0xff)) < (t0 = (this.T[0] & 0xff))) + || ((this.T[i] == this.T[0]) && (flag != 0)) + ) + { + if (flag == 0) + { + ++bucketB[BUCKET_BSTAR(ti, t0)]; + this.SA[--m] = i; + } + else + { + ++bucketB[BUCKET_B(ti, t0)]; + } + for ( + --i; + (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); + --i + ) + { + ++bucketB[BUCKET_B(ti, ti1)]; + } + } + + for (; 0 <= i; ) + { + do + { + ++bucketA[this.T[i] & 0xff]; + } while ((0 <= --i) && ((this.T[i] & 0xff) >= (this.T[i + 1] & 0xff))); + + if (0 <= i) + { + ++bucketB[BUCKET_BSTAR(this.T[i] & 0xff, this.T[i + 1] & 0xff)]; + this.SA[--m] = i; + for ( + --i; + (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); + --i + ) + { + ++bucketB[BUCKET_B(ti, ti1)]; + } + } + } + m = this.n - m; + if (m == 0) + { + for (i = 0; i < this.n; ++i) + { + this.SA[i] = i; + } + return 0; + } + + for (c0 = 0, i = -1, j = 0; c0 < 256; ++c0) + { + t = i + bucketA[c0]; + bucketA[c0] = i + j; + i = t + bucketB[BUCKET_B(c0, c0)]; + for (c1 = c0 + 1; c1 < 256; ++c1) + { + j += bucketB[BUCKET_BSTAR(c0, c1)]; + bucketB[(c0 << 8) | c1] = j; + i += bucketB[BUCKET_B(c0, c1)]; + } + } + + int PAb = this.n - m; + int ISAb = m; + for (i = m - 2; 0 <= i; --i) + { + t = this.SA[PAb + i]; + c0 = this.T[t] & 0xff; + c1 = this.T[t + 1] & 0xff; + this.SA[--bucketB[BUCKET_BSTAR(c0, c1)]] = i; + } + t = this.SA[PAb + m - 1]; + c0 = this.T[t] & 0xff; + c1 = this.T[t + 1] & 0xff; + this.SA[--bucketB[BUCKET_BSTAR(c0, c1)]] = m - 1; + + int[] buf = this.SA; + int bufoffset = m; + int bufsize = this.n - (2 * m); + if (bufsize <= 256) + { + buf = tempbuf; + bufoffset = 0; + bufsize = 256; + } + + for (c0 = 255, j = m; 0 < j; --c0) + { + for (c1 = 255; c0 < c1; j = i, --c1) + { + i = bucketB[BUCKET_BSTAR(c0, c1)]; + if (1 < (j - i)) + { + this.subStringSort( + PAb, + i, + j, + buf, + bufoffset, + bufsize, + 2, + this.SA[i] == (m - 1), + this.n + ); + } + } + } + + for (i = m - 1; 0 <= i; --i) + { + if (0 <= this.SA[i]) + { + j = i; + do + { + this.SA[ISAb + this.SA[i]] = i; + } while ((0 <= --i) && (0 <= this.SA[i])); + this.SA[i + 1] = i - j; + if (i <= 0) + { + break; + } + } + j = i; + do + { + this.SA[ISAb + (this.SA[i] = ~this.SA[i])] = j; + } while (this.SA[--i] < 0); + this.SA[ISAb + this.SA[i]] = j; + } + + this.trSort(ISAb, m, 1); + + i = this.n - 1; + j = m; + if ( + ((this.T[i] & 0xff) < (this.T[0] & 0xff)) + || ((this.T[i] == this.T[0]) && (flag != 0)) + ) + { + if (flag == 0) + { + this.SA[this.SA[ISAb + --j]] = i; + } + for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) { } + } + for (; 0 <= i; ) + { + for (--i; (0 <= i) && ((this.T[i] & 0xff) >= (this.T[i + 1] & 0xff)); --i) { } + if (0 <= i) + { + this.SA[this.SA[ISAb + --j]] = i; + for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) { } + } + } + + for (c0 = 255, i = this.n - 1, k = m - 1; 0 <= c0; --c0) + { + for (c1 = 255; c0 < c1; --c1) + { + t = i - bucketB[BUCKET_B(c0, c1)]; + bucketB[BUCKET_B(c0, c1)] = i + 1; + + for (i = t, j = bucketB[BUCKET_BSTAR(c0, c1)]; j <= k; --i, --k) + { + this.SA[i] = this.SA[k]; + } + } + t = i - bucketB[BUCKET_B(c0, c0)]; + bucketB[BUCKET_B(c0, c0)] = i + 1; + if (c0 < 255) + { + bucketB[BUCKET_BSTAR(c0, c0 + 1)] = t + 1; + } + i = bucketA[c0]; + } + + return m; + } + + private int constructBWT(int[] bucketA, int[] bucketB) + { + int i; + int t = 0; + int s, + s1; + int c0, + c1, + c2 = 0; + var orig = -1; + + for (c1 = 254; 0 <= c1; --c1) + { + int j; + for ( + i = bucketB[BUCKET_BSTAR(c1, c1 + 1)], j = bucketA[c1 + 1], t = 0, c2 = -1; + i <= j; + --j + ) + { + if (0 <= (s1 = s = this.SA[j])) + { + if (--s < 0) + s = this.n - 1; + + if ((c0 = (this.T[s] & 0xff)) <= c1) + { + this.SA[j] = ~s1; + if ((0 < s) && ((this.T[s - 1] & 0xff) > c0)) + { + s = ~s; + } + if (c2 == c0) + { + this.SA[--t] = s; + } + else + { + if (0 <= c2) + bucketB[BUCKET_B(c2, c1)] = t; + + this.SA[t = bucketB[BUCKET_B(c2 = c0, c1)] - 1] = s; + } + } + } + else + { + this.SA[j] = ~s; + } + } + } + + for (i = 0; i < this.n; ++i) + { + if (0 <= (s1 = s = this.SA[i])) + { + if (--s < 0) + s = this.n - 1; + + if ((c0 = (this.T[s] & 0xff)) >= (this.T[s + 1] & 0xff)) + { + if ((0 < s) && ((this.T[s - 1] & 0xff) < c0)) + s = ~s; + + if (c0 == c2) + { + this.SA[++t] = s; + } + else + { + if (c2 != -1) // BUGFIX: Original code can write to bucketA[-1] + bucketA[c2] = t; + this.SA[t = bucketA[c2 = c0] + 1] = s; + } + } + } + else + { + s1 = ~s1; + } + + if (s1 == 0) + { + this.SA[i] = this.T[this.n - 1]; + orig = i; + } + else + { + this.SA[i] = this.T[s1 - 1]; + } + } + + return orig; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs new file mode 100644 index 000000000..20a302799 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs @@ -0,0 +1,186 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// A decoder for the BZip2 Huffman coding stage + /// + internal class BZip2HuffmanStageDecoder + { + // The BZip2BitInputStream from which Huffman codes are read + private readonly IBZip2BitInputStream bitInputStream; + + // The longest Huffman code length accepted by the decoder + private const int HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH = 23; + + // The Huffman table number to use for each group of 50 symbols + private readonly byte[] selectors; + + // The minimum code length for each Huffman table + private readonly int[] minimumLengths = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES + ]; + + /// + /// An array of values for each Huffman table that must be subtracted from the numerical value of + /// a Huffman code of a given bit length to give its canonical code index + /// + private readonly int[,] codeBases = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2 + ]; + + /// + /// An array of values for each Huffman table that gives the highest numerical value of a Huffman + /// code of a given bit length + /// + private readonly int[,] codeLimits = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 1 + ]; + + // A mapping for each Huffman table from canonical code index to output symbol + private readonly int[,] codeSymbols = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE + ]; + + // The Huffman table for the current group + private int currentTable; + + // The index of the current group within the selectors array + private int groupIndex = -1; + + // The byte position within the current group. A new group is selected every 50 decoded bytes + private int groupPosition = -1; + + /// + /// Public constructor + /// + /// The BZip2BitInputStream from which Huffman codes are read + /// The total number of codes (uniform for each table) + /// The Canonical Huffman code lengths for each table + /// The Huffman table number to use for each group of 50 symbols + public BZip2HuffmanStageDecoder( + IBZip2BitInputStream bitInputStream, + int alphabetSize, + byte[,] tableCodeLengths, + byte[] selectors + ) + { + this.bitInputStream = bitInputStream; + this.selectors = selectors; + this.currentTable = this.selectors[0]; + this.CreateHuffmanDecodingTables(alphabetSize, tableCodeLengths); + } + + /// + /// Decodes and returns the next symbol + /// + /// The decoded symbol + /// if the end of the input stream is reached while decoding + public int NextSymbol() + { + // Move to next group selector if required + if (((++this.groupPosition % BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH) == 0)) + { + this.groupIndex++; + if (this.groupIndex == this.selectors.Length) + throw new IOException("Error decoding BZip2 block"); + + this.currentTable = this.selectors[this.groupIndex] & 0xff; + } + + var codeLength = this.minimumLengths[this.currentTable]; + + // Starting with the minimum bit length for the table, read additional bits one at a time + // until a complete code is recognised + for ( + uint codeBits = this.bitInputStream.ReadBits(codeLength); + codeLength <= BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH; + codeLength++ + ) + { + if (codeBits <= this.codeLimits[this.currentTable, codeLength]) + { + // Convert the code to a symbol index and return + return this.codeSymbols[ + this.currentTable, + codeBits - this.codeBases[this.currentTable, codeLength] + ]; + } + codeBits = (codeBits << 1) | this.bitInputStream.ReadBits(1); + } + + // A valid code was not recognised + throw new IOException("Error decoding BZip2 block"); + } + + /// + /// Constructs Huffman decoding tables from lists of Canonical Huffman code lengths + /// + /// The total number of codes (uniform for each table) + /// The Canonical Huffman code lengths for each table + private void CreateHuffmanDecodingTables(int alphabetSize, byte[,] tableCodeLengths) + { + for (int table = 0; table < tableCodeLengths.GetLength(0); table++) + { + int minimumLength = BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH; + int maximumLength = 0; + + // Find the minimum and maximum code length for the table + for (int i = 0; i < alphabetSize; i++) + { + maximumLength = Math.Max(tableCodeLengths[table, i], maximumLength); + minimumLength = Math.Min(tableCodeLengths[table, i], minimumLength); + } + this.minimumLengths[table] = minimumLength; + + // Calculate the first output symbol for each code length + for (int i = 0; i < alphabetSize; i++) + { + this.codeBases[table, tableCodeLengths[table, i] + 1]++; + } + for ( + int i = 1; + i < BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2; + i++ + ) + { + this.codeBases[table, i] += this.codeBases[table, i - 1]; + } + + // Calculate the first and last Huffman code for each code length (codes at a given length are sequential in value) + for (int i = minimumLength, code = 0; i <= maximumLength; i++) + { + int base1 = code; + code += this.codeBases[table, i + 1] - this.codeBases[table, i]; + this.codeBases[table, i] = base1 - this.codeBases[table, i]; + this.codeLimits[table, i] = code - 1; + code <<= 1; + } + + // Populate the mapping from canonical code index to output symbol + for ( + int bitLength = minimumLength, codeIndex = 0; + bitLength <= maximumLength; + bitLength++ + ) + { + for (int symbol = 0; symbol < alphabetSize; symbol++) + { + if (tableCodeLengths[table, symbol] == bitLength) + this.codeSymbols[table, codeIndex++] = symbol; + } + } + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs new file mode 100644 index 000000000..9f83d67a5 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs @@ -0,0 +1,387 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// An encoder for the BZip2 Huffman encoding stage + /// + internal class BZip2HuffmanStageEncoder + { + // Used in initial Huffman table generation + private const int HUFFMAN_HIGH_SYMBOL_COST = 15; + + // The longest Huffman code length created by the encoder + private const int HUFFMAN_ENCODE_MAXIMUM_CODE_LENGTH = 20; + + // Number of symbols decoded after which a new Huffman table is selected + public const int HUFFMAN_GROUP_RUN_LENGTH = 50; + + // The BZip2BitOutputStream to which the Huffman tables and data is written + private readonly IBZip2BitOutputStream bitOutputStream; + + // The output of the Move To Front Transform and Run Length Encoding[2] stages + private readonly ushort[] mtfBlock; + + // The actual number of values contained in the mtfBlock array + private readonly int mtfLength; + + // The number of unique values in the mtfBlock array + private readonly int mtfAlphabetSize; + + // The global frequencies of values within the mtfBlock array + private readonly int[] mtfSymbolFrequencies; + + // The Canonical Huffman code lengths for each table + private readonly int[,] huffmanCodeLengths; + + // Merged code symbols for each table. The value at each position is ((code length << 24) | code) + private readonly int[,] huffmanMergedCodeSymbols; + + // The selectors for each segment + private readonly byte[] selectors; + + /// + /// Public constructor + /// + /// The BZip2BitOutputStream to write to + /// The MTF block data + /// The actual length of the MTF block + /// The size of the MTF block's alphabet + /// The frequencies the MTF block's symbols + public BZip2HuffmanStageEncoder( + IBZip2BitOutputStream bitOutputStream, + ushort[] mtfBlock, + int mtfLength, + int mtfAlphabetSize, + int[] mtfSymbolFrequencies + ) + { + this.bitOutputStream = bitOutputStream; + this.mtfBlock = mtfBlock; + this.mtfSymbolFrequencies = mtfSymbolFrequencies; + this.mtfAlphabetSize = mtfAlphabetSize; + this.mtfLength = mtfLength; + + var totalTables = selectTableCount(mtfLength); + + this.huffmanCodeLengths = new int[totalTables, mtfAlphabetSize]; + this.huffmanMergedCodeSymbols = new int[totalTables, mtfAlphabetSize]; + this.selectors = new byte[ + (mtfLength + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH - 1) + / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH + ]; + } + + /// + /// Encodes and writes the block data + /// + /// on any I/O error writing the data + public void Encode() + { + // Create optimised selector list and Huffman tables + this.generateHuffmanOptimisationSeeds(); + for (var i = 3; i >= 0; i--) + { + this.optimiseSelectorsAndHuffmanTables(i == 0); + } + this.assignHuffmanCodeSymbols(); + + // Write out the tables and the block data encoded with them + this.writeSelectorsAndHuffmanTables(); + this.writeBlockData(); + } + + /// + /// Selects an appropriate table count for a given MTF length + /// + /// The length to select a table count for + /// The selected table count + private static int selectTableCount(int mtfLength) + { + if (mtfLength >= 2400) + return 6; + if (mtfLength >= 1200) + return 5; + if (mtfLength >= 600) + return 4; + return mtfLength >= 200 ? 3 : 2; + } + + /// + /// Generate a Huffman code length table for a given list of symbol frequencies + /// + /// The total number of symbols + /// The frequencies of the symbols + /// The array to which the generated code lengths should be written + /// + private static void generateHuffmanCodeLengths( + int alphabetSize, + int[,] symbolFrequencies, + int[,] codeLengths, + int index + ) + { + var mergedFrequenciesAndIndices = new int[alphabetSize]; + var sortedFrequencies = new int[alphabetSize]; + + // The Huffman allocator needs its input symbol frequencies to be sorted, but we need to return code lengths in the same order as the + // corresponding frequencies are passed in + + // The symbol frequency and index are merged into a single array of integers - frequency in the high 23 bits, index in the low 9 bits. + // 2^23 = 8,388,608 which is higher than the maximum possible frequency for one symbol in a block + // 2^9 = 512 which is higher than the maximum possible alphabet size (== 258) + // Sorting this array simultaneously sorts the frequencies and leaves a lookup that can be used to cheaply invert the sort + for (int i = 0; i < alphabetSize; i++) + mergedFrequenciesAndIndices[i] = (symbolFrequencies[index, i] << 9) | i; + + Array.Sort(mergedFrequenciesAndIndices); + for (int i = 0; i < alphabetSize; i++) + sortedFrequencies[i] = mergedFrequenciesAndIndices[i] >> 9; + + // Allocate code lengths - the allocation is in place, so the code lengths will be in the sortedFrequencies array afterwards + HuffmanAllocator.AllocateHuffmanCodeLengths( + sortedFrequencies, + BZip2HuffmanStageEncoder.HUFFMAN_ENCODE_MAXIMUM_CODE_LENGTH + ); + + // Reverse the sort to place the code lengths in the same order as the symbols whose frequencies were passed in + for (int i = 0; i < alphabetSize; i++) + codeLengths[index, mergedFrequenciesAndIndices[i] & 0x1ff] = sortedFrequencies[i]; + } + + /// + /// Generate initial Huffman code length tables, giving each table a different low cost section + /// of the alphabet that is roughly equal in overall cumulative frequency. Note that the initial + /// tables are invalid for actual Huffman code generation, and only serve as the seed for later + /// iterative optimisation in optimiseSelectorsAndHuffmanTables(int) + /// + private void generateHuffmanOptimisationSeeds() + { + int totalTables = this.huffmanCodeLengths.GetLength(0); + int remainingLength = this.mtfLength; + int lowCostEnd = -1; + + for (int i = 0; i < totalTables; i++) + { + int targetCumulativeFrequency = remainingLength / (totalTables - i); + int lowCostStart = lowCostEnd + 1; + int actualCumulativeFrequency = 0; + + while ( + (actualCumulativeFrequency < targetCumulativeFrequency) + && (lowCostEnd < (this.mtfAlphabetSize - 1)) + ) + { + actualCumulativeFrequency += this.mtfSymbolFrequencies[++lowCostEnd]; + } + + if ( + (lowCostEnd > lowCostStart) + && (i != 0) + && (i != (totalTables - 1)) + && (((totalTables - i) & 1) == 0) + ) + actualCumulativeFrequency -= this.mtfSymbolFrequencies[lowCostEnd--]; + + for (var j = 0; j < this.mtfAlphabetSize; j++) + { + if ((j < lowCostStart) || (j > lowCostEnd)) + this.huffmanCodeLengths[i, j] = + BZip2HuffmanStageEncoder.HUFFMAN_HIGH_SYMBOL_COST; + } + + remainingLength -= actualCumulativeFrequency; + } + } + + /// + /// Co-optimise the selector list and the alternative Huffman table code lengths. This method is + /// called repeatedly in the hope that the total encoded size of the selectors, the Huffman code + /// lengths and the block data encoded with them will converge towards a minimum.
+ /// If the data is highly incompressible, it is possible that the total encoded size will + /// instead diverge (increase) slightly.
+ /// + /// If true, write out the (final) chosen selectors + private void optimiseSelectorsAndHuffmanTables(bool storeSelectors) + { + int totalTables = this.huffmanCodeLengths.GetLength(0); + var tableFrequencies = new int[totalTables, this.mtfAlphabetSize]; + + int selectorIndex = 0; + + // Find the best table for each group of 50 block bytes based on the current Huffman code lengths + for (int groupStart = 0; groupStart < this.mtfLength; ) + { + int groupEnd = + Math.Min( + groupStart + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, + this.mtfLength + ) - 1; + + // Calculate the cost of this group when encoded by each table + var cost = new int[totalTables]; + for (int i = groupStart; i <= groupEnd; i++) + { + int value = this.mtfBlock[i]; + for (var j = 0; j < totalTables; j++) + { + cost[j] += this.huffmanCodeLengths[j, value]; + } + } + + // Find the table with the least cost for this group + byte bestTable = 0; + var bestCost = cost[0]; + for (byte i = 1; i < totalTables; i++) + { + var tableCost = cost[i]; + if (tableCost < bestCost) + { + bestCost = tableCost; + bestTable = i; + } + } + + // Accumulate symbol frequencies for the table chosen for this block + for (int i = groupStart; i <= groupEnd; i++) + { + tableFrequencies[bestTable, this.mtfBlock[i]]++; + } + + // Store a selector indicating the table chosen for this block + if (storeSelectors) + { + this.selectors[selectorIndex++] = bestTable; + } + + groupStart = groupEnd + 1; + } + + // Generate new Huffman code lengths based on the frequencies for each table accumulated in this iteration + for (int i = 0; i < totalTables; i++) + { + generateHuffmanCodeLengths( + this.mtfAlphabetSize, + tableFrequencies, + this.huffmanCodeLengths, + i + ); + } + } + + // Assigns Canonical Huffman codes based on the calculated lengths + private void assignHuffmanCodeSymbols() + { + int totalTables = this.huffmanCodeLengths.GetLength(0); + + for (int i = 0; i < totalTables; i++) + { + int minimumLength = 32; + int maximumLength = 0; + for (var j = 0; j < this.mtfAlphabetSize; j++) + { + var length = this.huffmanCodeLengths[i, j]; + if (length > maximumLength) + { + maximumLength = length; + } + if (length < minimumLength) + { + minimumLength = length; + } + } + + int code = 0; + for (int j = minimumLength; j <= maximumLength; j++) + { + for (var k = 0; k < this.mtfAlphabetSize; k++) + { + if ((this.huffmanCodeLengths[i, k] & 0xff) == j) + { + this.huffmanMergedCodeSymbols[i, k] = (j << 24) | code; + code++; + } + } + code <<= 1; + } + } + } + + /// + /// Write out the selector list and Huffman tables + /// + /// on any I/O error writing the data + private void writeSelectorsAndHuffmanTables() + { + int totalSelectors = this.selectors.Length; + int totalTables = this.huffmanCodeLengths.GetLength(0); + + this.bitOutputStream.WriteBits(3, (uint)totalTables); + this.bitOutputStream.WriteBits(15, (uint)totalSelectors); + + // Write the selectors + var selectorMTF = new MoveToFront(); + for (int i = 0; i < totalSelectors; i++) + { + this.bitOutputStream.WriteUnary(selectorMTF.ValueToFront(this.selectors[i])); + } + + // Write the Huffman tables + for (int i = 0; i < totalTables; i++) + { + var currentLength = this.huffmanCodeLengths[i, 0]; + + this.bitOutputStream.WriteBits(5, (uint)currentLength); + + for (var j = 0; j < this.mtfAlphabetSize; j++) + { + var codeLength = this.huffmanCodeLengths[i, j]; + var value = (currentLength < codeLength) ? 2u : 3u; + var delta = Math.Abs(codeLength - currentLength); + while (delta-- > 0) + { + this.bitOutputStream.WriteBits(2, value); + } + this.bitOutputStream.WriteBoolean(false); + currentLength = codeLength; + } + } + } + + /// + /// Writes out the encoded block data + /// + /// on any I/O error writing the data + private void writeBlockData() + { + int selectorIndex = 0; + + for (int mtfIndex = 0; mtfIndex < this.mtfLength; ) + { + int groupEnd = + Math.Min( + mtfIndex + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, + this.mtfLength + ) - 1; + + int index = this.selectors[selectorIndex++]; + + while (mtfIndex <= groupEnd) + { + var mergedCodeSymbol = this.huffmanMergedCodeSymbols[ + index, + this.mtfBlock[mtfIndex++] + ]; + this.bitOutputStream.WriteBits(mergedCodeSymbol >> 24, (uint)mergedCodeSymbol); + } + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs new file mode 100644 index 000000000..5985c6f4a --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs @@ -0,0 +1,168 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// An encoder for the BZip2 Move To Front Transform and Run-Length Encoding[2] stages + /// + /// An encoder for the BZip2 Move To Front Transform and Run-Length Encoding[2] stages. + /// Although conceptually these two stages are separate, it is computationally efficient to perform them in one pass. + /// + internal class BZip2MTFAndRLE2StageEncoder + { + // The Burrows-Wheeler transformed block + private readonly int[] bwtBlock; + + // Actual length of the data in the bwtBlock array + private readonly int bwtLength; + + // At each position, true if the byte value with that index is present within the block, otherwise false + private readonly bool[] bwtValuesInUse; + + // The output of the Move To Front Transform and Run-Length Encoding[2] stages + private readonly ushort[] mtfBlock; + + // The global frequencies of values within the mtfBlock array + private readonly int[] mtfSymbolFrequencies = new int[ + BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE + ]; + + // Maximum possible Huffman alphabet size + public const int HUFFMAN_MAXIMUM_ALPHABET_SIZE = 258; + + // Huffman symbol used for run-length encoding + public const ushort RLE_SYMBOL_RUNA = 0; + + // Huffman symbol used for run-length encoding + public const ushort RLE_SYMBOL_RUNB = 1; + + // Gets the encoded MTF block + public ushort[] MtfBlock + { + get { return this.mtfBlock; } + } + + // Gets The actual length of the MTF block + public int MtfLength { get; private set; } + + // Gets the size of the MTF block's alphabet + public int MtfAlphabetSize { get; private set; } + + // Gets the frequencies of the MTF block's symbols + public int[] MtfSymbolFrequencies + { + get { return this.mtfSymbolFrequencies; } + } + + /// + /// Public constructor + /// + /// The Burrows Wheeler Transformed block data + /// The actual length of the BWT data + /// The values that are present within the BWT data. For each index, + /// true if that value is present within the data, otherwise false + public BZip2MTFAndRLE2StageEncoder(int[] bwtBlock, int bwtLength, bool[] bwtValuesPresent) + { + this.bwtBlock = bwtBlock; + this.bwtLength = bwtLength; + this.bwtValuesInUse = bwtValuesPresent; + this.mtfBlock = new ushort[bwtLength + 1]; + } + + // Performs the Move To Front transform and Run Length Encoding[1] stages + public void Encode() + { + var huffmanSymbolMap = new byte[256]; + var symbolMTF = new MoveToFront(); + + int totalUniqueValues = 0; + for (var i = 0; i < 256; i++) + { + if (this.bwtValuesInUse[i]) + huffmanSymbolMap[i] = (byte)totalUniqueValues++; + } + + int endOfBlockSymbol = totalUniqueValues + 1; + int mtfIndex = 0; + int repeatCount = 0; + int totalRunAs = 0; + int totalRunBs = 0; + + for (var i = 0; i < this.bwtLength; i++) + { + // Move To Front + int mtfPosition = symbolMTF.ValueToFront(huffmanSymbolMap[this.bwtBlock[i] & 0xff]); + + // Run Length Encode + if (mtfPosition == 0) + { + repeatCount++; + } + else + { + if (repeatCount > 0) + { + repeatCount--; + while (true) + { + if ((repeatCount & 1) == 0) + { + this.mtfBlock[mtfIndex++] = + BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA; + totalRunAs++; + } + else + { + this.mtfBlock[mtfIndex++] = + BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB; + totalRunBs++; + } + + if (repeatCount <= 1) + break; + + repeatCount = (repeatCount - 2) >> 1; + } + repeatCount = 0; + } + + this.mtfBlock[mtfIndex++] = (char)(mtfPosition + 1); + this.mtfSymbolFrequencies[mtfPosition + 1]++; + } + } + + if (repeatCount > 0) + { + repeatCount--; + while (true) + { + if ((repeatCount & 1) == 0) + { + this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA; + totalRunAs++; + } + else + { + this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB; + totalRunBs++; + } + + if (repeatCount <= 1) + break; + + repeatCount = (repeatCount - 2) >> 1; + } + } + + this.mtfBlock[mtfIndex] = (ushort)endOfBlockSymbol; + this.mtfSymbolFrequencies[endOfBlockSymbol]++; + this.mtfSymbolFrequencies[BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA] += totalRunAs; + this.mtfSymbolFrequencies[BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB] += totalRunBs; + + this.MtfLength = mtfIndex + 1; + this.MtfAlphabetSize = endOfBlockSymbol + 1; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs new file mode 100644 index 000000000..31594e263 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs @@ -0,0 +1,298 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// A CRC32 calculator + /// + internal class CRC32 + { + /// The CRC lookup table + private static readonly uint[] Crc32Lookup = + { + 0x00000000, + 0x04c11db7, + 0x09823b6e, + 0x0d4326d9, + 0x130476dc, + 0x17c56b6b, + 0x1a864db2, + 0x1e475005, + 0x2608edb8, + 0x22c9f00f, + 0x2f8ad6d6, + 0x2b4bcb61, + 0x350c9b64, + 0x31cd86d3, + 0x3c8ea00a, + 0x384fbdbd, + 0x4c11db70, + 0x48d0c6c7, + 0x4593e01e, + 0x4152fda9, + 0x5f15adac, + 0x5bd4b01b, + 0x569796c2, + 0x52568b75, + 0x6a1936c8, + 0x6ed82b7f, + 0x639b0da6, + 0x675a1011, + 0x791d4014, + 0x7ddc5da3, + 0x709f7b7a, + 0x745e66cd, + 0x9823b6e0, + 0x9ce2ab57, + 0x91a18d8e, + 0x95609039, + 0x8b27c03c, + 0x8fe6dd8b, + 0x82a5fb52, + 0x8664e6e5, + 0xbe2b5b58, + 0xbaea46ef, + 0xb7a96036, + 0xb3687d81, + 0xad2f2d84, + 0xa9ee3033, + 0xa4ad16ea, + 0xa06c0b5d, + 0xd4326d90, + 0xd0f37027, + 0xddb056fe, + 0xd9714b49, + 0xc7361b4c, + 0xc3f706fb, + 0xceb42022, + 0xca753d95, + 0xf23a8028, + 0xf6fb9d9f, + 0xfbb8bb46, + 0xff79a6f1, + 0xe13ef6f4, + 0xe5ffeb43, + 0xe8bccd9a, + 0xec7dd02d, + 0x34867077, + 0x30476dc0, + 0x3d044b19, + 0x39c556ae, + 0x278206ab, + 0x23431b1c, + 0x2e003dc5, + 0x2ac12072, + 0x128e9dcf, + 0x164f8078, + 0x1b0ca6a1, + 0x1fcdbb16, + 0x018aeb13, + 0x054bf6a4, + 0x0808d07d, + 0x0cc9cdca, + 0x7897ab07, + 0x7c56b6b0, + 0x71159069, + 0x75d48dde, + 0x6b93dddb, + 0x6f52c06c, + 0x6211e6b5, + 0x66d0fb02, + 0x5e9f46bf, + 0x5a5e5b08, + 0x571d7dd1, + 0x53dc6066, + 0x4d9b3063, + 0x495a2dd4, + 0x44190b0d, + 0x40d816ba, + 0xaca5c697, + 0xa864db20, + 0xa527fdf9, + 0xa1e6e04e, + 0xbfa1b04b, + 0xbb60adfc, + 0xb6238b25, + 0xb2e29692, + 0x8aad2b2f, + 0x8e6c3698, + 0x832f1041, + 0x87ee0df6, + 0x99a95df3, + 0x9d684044, + 0x902b669d, + 0x94ea7b2a, + 0xe0b41de7, + 0xe4750050, + 0xe9362689, + 0xedf73b3e, + 0xf3b06b3b, + 0xf771768c, + 0xfa325055, + 0xfef34de2, + 0xc6bcf05f, + 0xc27dede8, + 0xcf3ecb31, + 0xcbffd686, + 0xd5b88683, + 0xd1799b34, + 0xdc3abded, + 0xd8fba05a, + 0x690ce0ee, + 0x6dcdfd59, + 0x608edb80, + 0x644fc637, + 0x7a089632, + 0x7ec98b85, + 0x738aad5c, + 0x774bb0eb, + 0x4f040d56, + 0x4bc510e1, + 0x46863638, + 0x42472b8f, + 0x5c007b8a, + 0x58c1663d, + 0x558240e4, + 0x51435d53, + 0x251d3b9e, + 0x21dc2629, + 0x2c9f00f0, + 0x285e1d47, + 0x36194d42, + 0x32d850f5, + 0x3f9b762c, + 0x3b5a6b9b, + 0x0315d626, + 0x07d4cb91, + 0x0a97ed48, + 0x0e56f0ff, + 0x1011a0fa, + 0x14d0bd4d, + 0x19939b94, + 0x1d528623, + 0xf12f560e, + 0xf5ee4bb9, + 0xf8ad6d60, + 0xfc6c70d7, + 0xe22b20d2, + 0xe6ea3d65, + 0xeba91bbc, + 0xef68060b, + 0xd727bbb6, + 0xd3e6a601, + 0xdea580d8, + 0xda649d6f, + 0xc423cd6a, + 0xc0e2d0dd, + 0xcda1f604, + 0xc960ebb3, + 0xbd3e8d7e, + 0xb9ff90c9, + 0xb4bcb610, + 0xb07daba7, + 0xae3afba2, + 0xaafbe615, + 0xa7b8c0cc, + 0xa379dd7b, + 0x9b3660c6, + 0x9ff77d71, + 0x92b45ba8, + 0x9675461f, + 0x8832161a, + 0x8cf30bad, + 0x81b02d74, + 0x857130c3, + 0x5d8a9099, + 0x594b8d2e, + 0x5408abf7, + 0x50c9b640, + 0x4e8ee645, + 0x4a4ffbf2, + 0x470cdd2b, + 0x43cdc09c, + 0x7b827d21, + 0x7f436096, + 0x7200464f, + 0x76c15bf8, + 0x68860bfd, + 0x6c47164a, + 0x61043093, + 0x65c52d24, + 0x119b4be9, + 0x155a565e, + 0x18197087, + 0x1cd86d30, + 0x029f3d35, + 0x065e2082, + 0x0b1d065b, + 0x0fdc1bec, + 0x3793a651, + 0x3352bbe6, + 0x3e119d3f, + 0x3ad08088, + 0x2497d08d, + 0x2056cd3a, + 0x2d15ebe3, + 0x29d4f654, + 0xc5a92679, + 0xc1683bce, + 0xcc2b1d17, + 0xc8ea00a0, + 0xd6ad50a5, + 0xd26c4d12, + 0xdf2f6bcb, + 0xdbee767c, + 0xe3a1cbc1, + 0xe760d676, + 0xea23f0af, + 0xeee2ed18, + 0xf0a5bd1d, + 0xf464a0aa, + 0xf9278673, + 0xfde69bc4, + 0x89b8fd09, + 0x8d79e0be, + 0x803ac667, + 0x84fbdbd0, + 0x9abc8bd5, + 0x9e7d9662, + 0x933eb0bb, + 0x97ffad0c, + 0xafb010b1, + 0xab710d06, + 0xa6322bdf, + 0xa2f33668, + 0xbcb4666d, + 0xb8757bda, + 0xb5365d03, + 0xb1f740b4, + }; + + /// The current CRC + private uint crc = 0xffffffff; + + /// Gets the current CRC + public uint CRC => ~this.crc; + + /// Updates the CRC with a single byte + /// The value to update the CRC with + public void UpdateCrc(int value) + { + this.crc = (this.crc << 8) ^ CRC32.Crc32Lookup[((this.crc >> 24) ^ value) & 0xff]; + } + + /// Update the CRC with a sequence of identical bytes + /// The value to update the CRC with + /// The number of bytes + public void UpdateCrc(int value, int count) + { + while (count-- > 0) + { + this.crc = (this.crc << 8) ^ CRC32.Crc32Lookup[((this.crc >> 24) ^ value) & 0xff]; + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs new file mode 100644 index 000000000..bc1eea761 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs @@ -0,0 +1,220 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// An in-place, length restricted Canonical Huffman code length allocator + /// + /// Based on the algorithm proposed by R.L.Milidiú, A.A.Pessoa and E.S.Laber + /// in "In-place Length-Restricted Prefix Coding" (see: http://www-di.inf.puc-rio.br/~laber/public/spire98.ps) + /// and incorporating additional ideas from the implementation of "shcodec" by Simakov Alexander + /// (see: http://webcenter.ru/~xander/) + /// + internal static class HuffmanAllocator + { + /// Allocates Canonical Huffman code lengths in place based on a sorted frequency array + /// On input, a sorted array of symbol frequencies; On output, an array of Canonical Huffman code lenghts + /// The maximum code length. Must be at least ceil(log2(array.length)) + public static void AllocateHuffmanCodeLengths(int[] array, int maximumLength) + { + switch (array.Length) + { + case 2: + array[1] = 1; + break; + case 1: + array[0] = 1; + return; + } + + // Pass 1 : Set extended parent pointers + SetExtendedParentPointers(array); + + // Pass 2 : Find number of nodes to relocate in order to achieve maximum code length + int nodesToRelocate = FindNodesToRelocate(array, maximumLength); + + // Pass 3 : Generate code lengths + if ((array[0] % array.Length) >= nodesToRelocate) + { + AllocateNodeLengths(array); + } + else + { + var insertDepth = maximumLength - SignificantBits(nodesToRelocate - 1); + AllocateNodeLengthsWithRelocation(array, nodesToRelocate, insertDepth); + } + } + + /// + /// + /// + /// The code length array + /// The input position + /// The number of internal nodes to be relocated + /// The smallest k such that nodesToMove <= k <= i and i <= (array[k] % array.length) + private static int First(int[] array, int i, int nodesToMove) + { + int length = array.Length; + int limit = i; + int k = array.Length - 2; + + while ((i >= nodesToMove) && ((array[i] % length) > limit)) + { + k = i; + i -= (limit - i + 1); + } + i = Math.Max(nodesToMove - 1, i); + + while (k > (i + 1)) + { + var temp = (i + k) >> 1; + if ((array[temp] % length) > limit) + k = temp; + else + i = temp; + } + + return k; + } + + /// + /// Fills the code array with extended parent pointers + /// + /// The code length array + private static void SetExtendedParentPointers(int[] array) + { + int length = array.Length; + + array[0] += array[1]; + + for (int headNode = 0, tailNode = 1, topNode = 2; tailNode < (length - 1); tailNode++) + { + int temp; + if ((topNode >= length) || (array[headNode] < array[topNode])) + { + temp = array[headNode]; + array[headNode++] = tailNode; + } + else + { + temp = array[topNode++]; + } + + if ( + (topNode >= length) + || ((headNode < tailNode) && (array[headNode] < array[topNode])) + ) + { + temp += array[headNode]; + array[headNode++] = tailNode + length; + } + else + { + temp += array[topNode++]; + } + + array[tailNode] = temp; + } + } + + /// + /// Finds the number of nodes to relocate in order to achieve a given code length limit + /// + /// The code length array + /// The maximum bit length for the generated codes + /// The number of nodes to relocate + private static int FindNodesToRelocate(int[] array, int maximumLength) + { + int currentNode = array.Length - 2; + + for ( + var currentDepth = 1; + (currentDepth < (maximumLength - 1)) && (currentNode > 1); + currentDepth++ + ) + currentNode = First(array, currentNode - 1, 0); + + return currentNode; + } + + /// + /// A final allocation pass with no code length limit + /// + /// The code length array + private static void AllocateNodeLengths(int[] array) + { + int firstNode = array.Length - 2; + int nextNode = array.Length - 1; + + for (int currentDepth = 1, availableNodes = 2; availableNodes > 0; currentDepth++) + { + int lastNode = firstNode; + firstNode = First(array, lastNode - 1, 0); + + for (var i = availableNodes - (lastNode - firstNode); i > 0; i--) + array[nextNode--] = currentDepth; + + availableNodes = (lastNode - firstNode) << 1; + } + } + + /// + /// A final allocation pass that relocates nodes in order to achieve a maximum code length limit + /// + /// The code length array + /// The number of internal nodes to be relocated + /// The depth at which to insert relocated nodes + private static void AllocateNodeLengthsWithRelocation( + int[] array, + int nodesToMove, + int insertDepth + ) + { + int firstNode = array.Length - 2; + int nextNode = array.Length - 1; + int currentDepth = (insertDepth == 1) ? 2 : 1; + int nodesLeftToMove = (insertDepth == 1) ? nodesToMove - 2 : nodesToMove; + + for (int availableNodes = currentDepth << 1; availableNodes > 0; currentDepth++) + { + int lastNode = firstNode; + firstNode = + (firstNode <= nodesToMove) + ? firstNode + : First(array, lastNode - 1, nodesToMove); + + int offset = 0; + if (currentDepth >= insertDepth) + { + offset = Math.Min(nodesLeftToMove, 1 << (currentDepth - insertDepth)); + } + else if (currentDepth == (insertDepth - 1)) + { + offset = 1; + if ((array[firstNode]) == lastNode) + firstNode++; + } + + for (var i = availableNodes - (lastNode - firstNode + offset); i > 0; i--) + array[nextNode--] = currentDepth; + + nodesLeftToMove -= offset; + availableNodes = (lastNode - firstNode + offset) << 1; + } + } + + private static int SignificantBits(int x) + { + int n; + for (n = 0; x > 0; n++) + { + x >>= 1; + } + return n; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs new file mode 100644 index 000000000..a9fa23251 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs @@ -0,0 +1,61 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// A 256 entry Move To Front transform + /// + internal class MoveToFront + { + /// The Move To Front list + private readonly byte[] mtf; + + /// Public constructor + public MoveToFront() + { + this.mtf = new byte[256]; + for (int i = 0; i < 256; i++) + this.mtf[i] = (byte)i; + } + + /// Moves a value to the head of the MTF list (forward Move To Front transform) + /// The value to move + /// The position the value moved from + public int ValueToFront(byte value) + { + int index = 0; + byte temp = this.mtf[0]; + + if (value == temp) + return index; + + this.mtf[0] = value; + while (temp != value) + { + index++; + byte temp2 = temp; + temp = this.mtf[index]; + this.mtf[index] = temp2; + } + + return index; + } + + /// Gets the value from a given index and moves it to the front of the MTF list (inverse Move To Front transform) + /// The index to move + /// The value at the given index + public byte IndexToFront(int index) + { + byte value = this.mtf[index]; + Array.ConstrainedCopy(this.mtf, 0, this.mtf, 1, index); + this.mtf[0] = value; + + return value; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs new file mode 100644 index 000000000..d9f9e73de --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs @@ -0,0 +1,114 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + /// + /// Implements a bit-wise input stream + /// + internal class BZip2BitInputStream : IBZip2BitInputStream + { + // The stream from which bits are read + private Stream _inputStream; + + // A buffer of bits read from the input stream that have not yet been returned + private uint _bitBuffer; + + // The number of bits currently buffered in bitBuffer + private int _bitCount; + + /// Public constructor + /// The input stream to wrap + public BZip2BitInputStream(Stream inputStream) => this._inputStream = inputStream; + + public void Dispose() + { + // do nothing, the BZip2BitInputStream is not the owner of the _inputStream so don't dispose it + } + + /// Reads a single bit from the wrapped input stream + /// true if the bit read was 1, otherwise false + /// if no more bits are available in the input stream + public bool ReadBoolean() + { + if (this._bitCount > 0) + { + this._bitCount--; + } + else + { + int byteRead = this._inputStream.ReadByte(); + + if (byteRead < 0) + throw new IOException("Insufficient data"); + + this._bitBuffer = (this._bitBuffer << 8) | (uint)byteRead; + this._bitCount += 7; + } + + return ((this._bitBuffer & (1 << this._bitCount))) != 0; + } + + /// Reads a zero-terminated unary number from the wrapped input stream + /// The unary number + /// if no more bits are available in the input stream + public uint ReadUnary() + { + for (uint unaryCount = 0; ; unaryCount++) + { + if (this._bitCount > 0) + { + this._bitCount--; + } + else + { + var byteRead = this._inputStream.ReadByte(); + + if (byteRead < 0) + throw new IOException("Insufficient data"); + + this._bitBuffer = (this._bitBuffer << 8) | (uint)byteRead; + this._bitCount += 7; + } + + if (((this._bitBuffer & (1 << this._bitCount))) == 0) + return unaryCount; + } + } + + /// Reads up to 32 bits from the wrapped input stream + /// The number of bits to read (maximum 32) + /// The bits requested, right-aligned within the integer + /// if no more bits are available in the input stream + public uint ReadBits(int count) + { + if (this._bitCount < count) + { + while (this._bitCount < count) + { + int byteRead = this._inputStream.ReadByte(); + + if (byteRead < 0) + throw new IOException("Insufficient data"); + + this._bitBuffer = (this._bitBuffer << 8) | (uint)byteRead; + this._bitCount += 8; + } + } + + this._bitCount -= count; + + return (uint)((this._bitBuffer >> this._bitCount) & ((1 << count) - 1)); + } + + /// Reads 32 bits of input as an integer + /// The integer read + /// if 32 bits are not available in the input stream + public uint ReadInteger() => (this.ReadBits(16) << 16) | (this.ReadBits(16)); + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs new file mode 100644 index 000000000..6e8c70159 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs @@ -0,0 +1,196 @@ +// Added by drone1400, December 2025 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + /// + /// Helper class for splitting a BZip2 stream into multiple Memory Streams made up of individual BZip2 blocks. + /// The spitting is done by looking for the magic block header 6 byte sequence. + /// The output MemoryStreams bits are left aligned. + /// + internal class BZip2BitInputStreamSplitter : IDisposable + { + // The stream from which bits are read + private BZip2BitInputStream? _bitInputStream; + + // internal temporary buffer + private MemoryStream? _buffer = null; + + private int _blockLevel = 0; + private int _blockSizeBytes = 0; + private uint _finalCrc = 0; + private ulong _crtVal = 0x00; + private int _bitCount = 0; + + public bool IsStreamComplete => this._bitInputStream is null; + public int BlockLevel => this._blockLevel; + public int BlockSizeBytes => this._blockSizeBytes; + public uint FinalCrc => this._finalCrc; + + public BZip2BitInputStreamSplitter( + Stream inputStream, + InputStreamHeaderCheckType inputStreamHeaderCheck = + InputStreamHeaderCheckType.FULL_HEADER, + int manualBlockLevel = 9 + ) + { + this._bitInputStream = new BZip2BitInputStream(inputStream); + this.Initialize(inputStreamHeaderCheck, manualBlockLevel); + } + + private void Initialize(InputStreamHeaderCheckType inputStreamHeaderCheck, int blockLevel) + { + if (this._bitInputStream is null) + return; + + /* Read the stream header */ + try + { + switch (inputStreamHeaderCheck) + { + case InputStreamHeaderCheckType.FULL_HEADER: + { + uint marker1 = this._bitInputStream.ReadBits(16); + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if ( + marker1 != BZip2Constants.STREAM_START_MARKER_1 + || marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZ: + { + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if ( + marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZH: + { + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + } + + this._blockLevel = blockLevel; + this._blockSizeBytes = blockLevel * 100000; + + this._crtVal = this._bitInputStream.ReadBits(24); + this._crtVal <<= 24; + this._crtVal |= this._bitInputStream.ReadBits(24); + + // expecting the first block header immediately after + if (this._crtVal != BZip2Constants.BLOCK_HEADER_MARKER) + { + throw new IOException("BZip2 stream format error"); + } + + this._buffer = new MemoryStream(this._blockSizeBytes + 100000); + } + catch (IOException) + { + // If the stream header was not valid, stop trying to read more data + this._bitInputStream = null; + this._buffer = null; + throw; + } + } + + /// + /// Finds the next BZip2 block and returns an in memory copy of it + /// + /// MemoryStream + public MemoryStream? CopyNextBlock() + { + void FlushByte() + { + this._buffer.WriteByte((byte)(this._crtVal >> 48)); + this._crtVal &= 0x0000FFFF_FFFFFFFF; + this._bitCount = 0; + } + + void FlushPartialByte() + { + byte partialByte = (byte)(this._crtVal >> 48); + partialByte <<= (8 - this._bitCount); + this._buffer.WriteByte(partialByte); + + this._crtVal &= 0x0000FFFF_FFFFFFFF; + this._bitCount = 0; + } + + if (this._bitInputStream is null || this._buffer is null) + return null; + + while (true) + { + this._crtVal <<= 1; + this._crtVal |= this._bitInputStream.ReadBits(1); + ulong testVal = this._crtVal & 0x0000FFFF_FFFFFFFF; + this._bitCount++; + if (this._bitCount == 8) + { + FlushByte(); + } + + if (testVal == BZip2Constants.STREAM_END_MARKER) + { + // we found the magic bit sequence for end of stream, that means we're done! + if (this._bitCount > 0) + { + FlushPartialByte(); + } + + // read the 32 bit CRC at the end of the file + this._finalCrc = this._bitInputStream.ReadInteger(); + + MemoryStream retBuffer = this._buffer; + + // clear buffer and input stream to mark that we are done processing blocks for good + this._buffer = null; + this._bitInputStream = null; + return retBuffer; + } + + if (testVal == BZip2Constants.BLOCK_HEADER_MARKER) + { + // we found the magic bit sequence! + if (this._bitCount > 0) + { + FlushPartialByte(); + } + MemoryStream retBuffer = this._buffer; + this._buffer = new MemoryStream(this._blockSizeBytes); + return retBuffer; + } + } + } + + public void Dispose() + { + this._bitInputStream?.Dispose(); + this._buffer?.Dispose(); + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs new file mode 100644 index 000000000..2950e925b --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs @@ -0,0 +1,293 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + /// An InputStream wrapper that decompresses BZip2 data + /// Instances of this class are not threadsafe + public class BZip2InputStream : Stream + { + // The stream from which compressed BZip2 data is read and decoded + private Stream _inputStream; + + // True if the underlying stream will be closed with the current Stream + private readonly bool _isOwner; + + // An InputStream wrapper that provides bit-level reads + private BZip2BitInputStream _bitInputStream; + + // (@code true} if the end of the compressed stream has been reached, otherwise false + private bool _streamComplete; + + /// + /// The declared block size of the stream (before final run-length decoding). The final block + /// will usually be smaller, but no block in the stream has to be exactly this large, and an + /// encoder could in theory choose to mix blocks of any size up to this value. Its function is + /// therefore as a hint to the decompressor as to how much working space is sufficient to + /// decompress blocks in a given stream + /// + private uint _streamBlockSize; + + // The merged CRC of all blocks decompressed so far + private uint _streamCrc; + + // The decompressor for the current block + private BZip2BlockDecompressor? _blockDecompressor; + + /// Public constructor + /// The InputStream to wrap + /// if true, will close the stream when done + /// + /// Used when is NoHeader + public BZip2InputStream( + Stream inputStream, + bool isOwner = true, + InputStreamHeaderCheckType inputStreamHeaderCheck = + InputStreamHeaderCheckType.FULL_HEADER, + int manualBlockLevel = 9 + ) + { + this._inputStream = inputStream; + this._bitInputStream = new BZip2BitInputStream(inputStream); + this._isOwner = isOwner; + + // initialize stream immediately + this.InitializeStream(inputStreamHeaderCheck, manualBlockLevel); + // prepare first block + this.InitializeNextBlock(); + } + + #region Implementation of abstract members of Stream + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'Flush()' method." + ); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); + + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'SetLength(long value)' method." + ); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method." + ); + + public override bool CanRead => this._inputStream.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => this._inputStream.Length; + public override long Position + { + get => this._inputStream.Position; + set => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support Set operation for property 'Position'." + ); + } + + public override int ReadByte() + { + var nextByte = this._blockDecompressor?.Read() ?? -1; + + // if current block has reached its end, prepare next block and try reading again + if (nextByte == -1) + { + if (this.InitializeNextBlock()) + { + nextByte = this._blockDecompressor?.Read() ?? -1; + } + } + + return nextByte; + } + + public override int Read(byte[] destination, int offset, int length) + { + int bytesRead = this._blockDecompressor?.Read(destination, offset, length) ?? 0; + + // if current block has reached its end, prepare next block and try reading again + if (bytesRead == -1) + { + if (this.InitializeNextBlock()) + { + bytesRead = this._blockDecompressor?.Read(destination, offset, length) ?? 0; + } + } + + if (bytesRead == -1) + bytesRead = 0; + + return bytesRead; + } + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this._streamComplete = true; + this._blockDecompressor = null; + + this._bitInputStream.Dispose(); + + if (this._isOwner) + { + //this._inputStream.Close(); + this._inputStream.Dispose(); + } + } + +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + + /// Reads the stream header and checks that the data appears to be a valid BZip2 stream + /// if the stream header is not valid + private void InitializeStream( + InputStreamHeaderCheckType inputStreamHeaderCheck, + int blockLevel + ) + { + /* If the stream has been explicitly closed, throw an exception */ + if (this._bitInputStream is null) + throw new IOException("Stream closed"); + + // If we're already at the end of the stream, do nothing + if (this._streamComplete) + return; + + // Read the stream header + try + { + switch (inputStreamHeaderCheck) + { + case InputStreamHeaderCheckType.FULL_HEADER: + { + uint marker1 = this._bitInputStream.ReadBits(16); + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if ( + marker1 != BZip2Constants.STREAM_START_MARKER_1 + || marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZ: + { + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if ( + marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZH: + { + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + } + + this._streamBlockSize = (uint)(blockLevel * 100000); + } + catch (IOException) + { + // If the stream header was not valid, stop trying to read more data + this._streamComplete = true; + throw; + } + } + + /// Prepares a new block for decompression if any remain in the stream + /// If a previous block has completed, its CRC is checked and merged into the stream CRC. + /// If the previous block was the final block in the stream, the stream CRC is validated + /// true if a block was successfully initialised, or false if the end of file marker was encountered + /// If either the block or stream CRC check failed, if the following data is + /// not a valid block-header or end-of-file marker, or if the following block could not be decoded + private bool InitializeNextBlock() + { + // If we're already at the end of the stream, do nothing + if (this._streamComplete) + return false; + + // If a block is complete, check the block CRC and integrate it into the stream CRC + if (this._blockDecompressor != null) + { + uint blockCrc = this._blockDecompressor.CheckCrc(); + this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + } + + // Read block-header or end-of-stream marker + uint marker1 = this._bitInputStream.ReadBits(24); + uint marker2 = this._bitInputStream.ReadBits(24); + + if ( + marker1 == BZip2Constants.BLOCK_HEADER_MARKER_1 + && marker2 == BZip2Constants.BLOCK_HEADER_MARKER_2 + ) + { + // Initialise a new block + try + { + this._blockDecompressor = new BZip2BlockDecompressor( + this._bitInputStream, + this._streamBlockSize + ); + } + catch (IOException) + { + // If the block could not be decoded, stop trying to read more data + this._streamComplete = true; + throw; + } + return true; + } + if ( + marker1 == BZip2Constants.STREAM_END_MARKER_1 + && marker2 == BZip2Constants.STREAM_END_MARKER_2 + ) + { + // Read and verify the end-of-stream CRC + this._streamComplete = true; + uint storedCombinedCrc = this._bitInputStream.ReadInteger(); // .ReadBits(32); + + if (storedCombinedCrc != this._streamCrc) + throw new IOException("BZip2 stream CRC error"); + + return false; + } + + // If what was read is not a valid block-header or end-of-stream marker, the stream is broken + this._streamComplete = true; + throw new IOException("BZip2 stream format error"); + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs new file mode 100644 index 000000000..27742b331 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs @@ -0,0 +1,119 @@ +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; + +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + internal class BZip2ParallelInputDataBlock : IDisposable + { + private int _blockId = 0; + private MemoryStream? _inputBlockBuffer; + private MemoryStream? _outputBuffer; + private int _outputBufferSize; + private int _blockSizeBytes; + private bool _isCrcOk = false; + private uint _crcValue; + + public long Length => this._outputBuffer?.Length ?? 0; + public long Position => this._outputBuffer?.Position ?? 0; + public bool IsDone => this.Position >= this.Length; + public int BlockId => this._blockId; + public bool IsCrcOk => this._isCrcOk; + public uint CrcValue => this._crcValue; + + public BZip2ParallelInputDataBlock( + int blockId, + MemoryStream inputBlockBuffer, + int blockSizeBytes, + int outputBufferSize + ) + { + this._blockId = blockId; + this._inputBlockBuffer = inputBlockBuffer; + this._blockSizeBytes = blockSizeBytes; + this._outputBufferSize = outputBufferSize; + } + + public void Dispose() + { + this._inputBlockBuffer?.Dispose(); + this._outputBuffer?.Dispose(); + } + + /// + /// Decompresses the full block + /// + public void Decompress() + { + if (this._inputBlockBuffer is null) + throw new IOException("Attempted to Decompress the same block twice"); + + this._outputBuffer = new MemoryStream(this._outputBufferSize); + + // note: we need to skip the first 6 magic bytes... + this._inputBlockBuffer.Position = 6; + using BZip2BitInputStream inputStream = new BZip2BitInputStream(this._inputBlockBuffer); + BZip2BlockDecompressor blockDecompressor = new BZip2BlockDecompressor( + inputStream, + (uint)this._blockSizeBytes + ); + + int readCount = blockDecompressor.ReadAll(this._outputBuffer); + + // reset buffer position + this._outputBuffer.Position = 0; + + try + { + this._crcValue = blockDecompressor.CheckCrc(); + this._isCrcOk = true; + } + catch (Exception) + { + this._crcValue = 0; + this._isCrcOk = false; + } + + this._inputBlockBuffer.Dispose(); + this._inputBlockBuffer = null; + } + + /// + /// reads a single byte from the output buffer + /// + /// + public int Read() + { + if (this._outputBuffer is null) + throw new IOException( + "Attempted to read decompressed data before decompressing block" + ); + + if (this._isCrcOk == false) + return -1; + if (this._outputBuffer.Position < this._outputBuffer.Length) + return this._outputBuffer.ReadByte(); + return -1; + } + + /// + /// Reads a number of bytes from the output buffer into the destination buffer + /// + /// destination buffer + /// starting position within destination buffer + /// maximum number of bytes to read + /// + public int Read(byte[] destination, int offset, int length) + { + if (this._outputBuffer is null) + throw new IOException( + "Attempted to read decompressed data before decompressing block" + ); + + if (this._isCrcOk == false) + return 0; + + return this._outputBuffer.Read(destination, offset, length); + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs new file mode 100644 index 000000000..d2fdd10a0 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs @@ -0,0 +1,398 @@ +// Added by drone1400, July 2025 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + /// An InputStream wrapper that decompresses BZip2 data using multiple threads + /// Instances of this class are not threadsafe + public class BZip2ParallelInputStream : Stream + { + #region Private fields + + // block size fields + private readonly int _blockLevel; + private readonly int _blockSizeBytes; + + private int _mtNextInputBlockId = 0; + private int _mtNextOutputBlockId = 0; + + // yikes! sounds bad right? + // if one of the worker threads, this is set to true and any further attempt to + // write/flush/close the stream will throw an exception + private bool _unsafeFatalException = false; + private Exception? _lastWorkerException = null; + + // dictionary of processed blocks + private readonly Dictionary _mtDecodedBlocks = + new Dictionary(); + + private int _mtPendingBlocks = 0; + private readonly int _mtMaxPendingBlocks; + + private readonly int _mtWorkerBufferSize; + + private readonly object _syncRootProcesing = new object(); + + // The stream from which compressed BZip2 data is read and decoded + private Stream _inputStream; + + // True if the underlying stream will be closed with the current Stream + private readonly bool _isOwner; + + // An InputStream wrapper that provides bit-level reads + private BZip2BitInputStreamSplitter _inputStreamSplitter; + + // The merged CRC of all blocks decompressed so far + private uint _streamCrc; + + #endregion + + #region Public methods + + /// Public constructor + /// The InputStream to wrap + /// True if the underlying stream will be closed with the current Stream + /// + /// Used when is NoHeader + /// Maximum number of blocks to read/decompress at the same time + /// Minimum buffer size for each worker thread during decompression, in bytes. + public BZip2ParallelInputStream( + Stream inputStream, + bool isOwner = true, + InputStreamHeaderCheckType inputStreamHeaderCheck = + InputStreamHeaderCheckType.FULL_HEADER, + int manualBlockLevel = 9, + int maxPendingBlocks = 0, + int workerBufferSize = 12582912 + ) + { + if (maxPendingBlocks == 0) + { + maxPendingBlocks = Environment.ProcessorCount; + } + this._mtMaxPendingBlocks = maxPendingBlocks; + this._mtWorkerBufferSize = workerBufferSize; + + // initialize stream + this._inputStream = inputStream; + this._isOwner = isOwner; + this._inputStreamSplitter = new BZip2BitInputStreamSplitter( + inputStream, + inputStreamHeaderCheck, + manualBlockLevel + ); + + this._blockLevel = this._inputStreamSplitter.BlockLevel; + this._blockSizeBytes = this._inputStreamSplitter.BlockSizeBytes; + } + + #endregion + + /// + /// Decompress a block of data + /// + /// + /// if compressing the block somehow fails... + private void MultiThreadWorkerAction(object? blockData) + { + try + { + if (blockData is BZip2ParallelInputDataBlock dataBlock) + { + // do the work + dataBlock.Decompress(); + + // queue up the finished decompressed block data + lock (this._syncRootProcesing) + { + this._mtDecodedBlocks.Add(dataBlock.BlockId, dataBlock); + } + } + } + catch (Exception ex) + { + // set this without any locks... + this._lastWorkerException = ex; + this._unsafeFatalException = true; + + throw new IOException( + "BZip2 Processing thread somehow crashed... See inner exception for details!", + ex + ); + } + finally + { + lock (this._syncRootProcesing) + { + this._mtPendingBlocks--; + } + } + } + + /// + /// If there are not too many pending blocks being processed, reads another block from the input stream and adds it to the work queue. + /// + /// True if the data block was queued up, false otherwise + /// if one of the worker threads or writing to the final bit stream experienced an exception + /// Should only be called from within or + private bool TryToQueueUpAnotherBlock() + { + if (this._unsafeFatalException) + { + throw new IOException( + "One of the decompression threads somehow failed... This should never happen.", + this._lastWorkerException + ); + } + + if (this._inputStreamSplitter.IsStreamComplete) + { + return false; + } + + lock (this._syncRootProcesing) + { + if (this._mtPendingBlocks >= this._mtMaxPendingBlocks) + return false; + } + + // extract the next block... + MemoryStream? ms = this._inputStreamSplitter.CopyNextBlock(); + + // this should be impossible if this._inputStreamSplitter.IsStreamComplete is not true + if (ms is null) + return false; + + lock (this._syncRootProcesing) + { + this._mtPendingBlocks++; + BZip2ParallelInputDataBlock data = new BZip2ParallelInputDataBlock( + this._mtNextInputBlockId++, + ms, + this._blockSizeBytes, + this._mtWorkerBufferSize + ); + ThreadPool.QueueUserWorkItem(this.MultiThreadWorkerAction, data); + } + + return true; + } + + /// + /// Checks if we are done processing the stream. This is done by checing if there are any pending blocks to be decoded or read. + /// Also by checking if the input stream splitter helper has reached the end of stream markers. + /// + /// True if we are done, false otherwise + /// If there's a CRC error at end of the stream. + /// /// Should only be called from within or + private bool CheckIfCompletelyDone() + { + if (this._unsafeFatalException) + { + throw new IOException( + "One of the decompression threads somehow failed... This should never happen.", + this._lastWorkerException + ); + } + + lock (this._syncRootProcesing) + { + if ( + this._mtPendingBlocks == 0 + && this._mtDecodedBlocks.Count == 0 + && this._inputStreamSplitter.IsStreamComplete + ) + { + if (this._inputStreamSplitter.FinalCrc != this._streamCrc) + { + throw new IOException("BZip2 stream CRC error"); + } + // we are compeltely done, can no longer do anything! + return true; + } + } + + return false; + } + + #region Implementation of abstract members of Stream + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'Flush()' method." + ); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); + + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'SetLength(long value)' method." + ); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method." + ); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => this._inputStream.Length; + public override long Position + { + get => this._inputStream.Position; + set => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support Set operation for property 'Position'." + ); + } + + public override int ReadByte() + { + while (true) + { + // check if the next output block is done + BZip2ParallelInputDataBlock? data = null; + lock (this._syncRootProcesing) + { + if (this._mtDecodedBlocks.ContainsKey(this._mtNextOutputBlockId)) + { + data = this._mtDecodedBlocks[this._mtNextOutputBlockId]; + } + } + + if (data != null) + { + if (data.IsCrcOk == false) + { + this._unsafeFatalException = true; + throw new IOException("BZip2 block decompression CRC error..."); + } + + if (data.IsDone == false) + { + // can read next byte + return data.Read(); + } + + // decompressor is done! + lock (this._syncRootProcesing) + { + uint blockCrc = data.CrcValue; + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + + this._mtDecodedBlocks.Remove(this._mtNextOutputBlockId); + this._mtNextOutputBlockId++; + + // continue to processing next block that is already decoded, skip queueing up more blocks... + continue; + } + } + + // queue up the next block + bool queueSuccessful = this.TryToQueueUpAnotherBlock(); + + // check if we are completely done + if (this.CheckIfCompletelyDone()) + return -1; + + if (!queueSuccessful) + { + Thread.Sleep(1); + } + } + } + + public override int Read(byte[] destination, int offset, int length) + { + int totalRead = 0; + int remaining = length; + + while (true) + { + // check if the next output block is done + BZip2ParallelInputDataBlock? data = null; + lock (this._syncRootProcesing) + { + if (this._mtDecodedBlocks.ContainsKey(this._mtNextOutputBlockId)) + { + data = this._mtDecodedBlocks[this._mtNextOutputBlockId]; + } + } + + if (data != null) + { + if (data.IsCrcOk == false) + { + this._unsafeFatalException = true; + throw new IOException("BZip2 block decompression CRC error..."); + } + + int readCount = data.Read(destination, offset, remaining); + remaining -= readCount; + offset += readCount; + totalRead += readCount; + + if (totalRead >= length) + return totalRead; + + if (data.IsDone) + { + // decompressor is done! + lock (this._syncRootProcesing) + { + uint blockCrc = data.CrcValue; + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + + this._mtDecodedBlocks.Remove(this._mtNextOutputBlockId); + this._mtNextOutputBlockId++; + + // continue to processing next block that is already decoded, skip queueing up more blocks... + continue; + } + } + } + + // queue up the next block + bool queueSuccessful = this.TryToQueueUpAnotherBlock(); + + // check if we are completely done + if (this.CheckIfCompletelyDone()) + return totalRead; + + if (!queueSuccessful) + { + Thread.Sleep(1); + } + } + } + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this._mtDecodedBlocks.Clear(); + + if (this._isOwner) + { + //this._inputStream.Close(); + this._inputStream.Dispose(); + } + } + +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs new file mode 100644 index 000000000..3a6d24067 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs @@ -0,0 +1,45 @@ +// Added by drone1400, December 2025 +// Location: https://github.com/drone1400/bzip2 + +namespace SharpCompress.Compressors.BZip2MT.Interface +{ + internal static class BZip2Constants + { + public const uint STREAM_START_MARKER = 0x425A68; + public const ulong STREAM_END_MARKER = 0x177245385090; + public const ulong BLOCK_HEADER_MARKER = 0x314159265359; + + /// The first 2 bytes of a Bzip2 marker, BZ + public const uint STREAM_START_MARKER_1 = 0x425a; + + /// The 'h' that distinguishes BZip from BZip2 + public const uint STREAM_START_MARKER_2 = 0x68; + + /// First three bytes of the end of stream marker + public const uint STREAM_END_MARKER_1 = 0x177245; + + /// Last three bytes of the end of stream marker + public const uint STREAM_END_MARKER_2 = 0x385090; + + // First three bytes of the block header marker + public const uint BLOCK_HEADER_MARKER_1 = 0x314159; + + // Last three bytes of the block header marker + public const uint BLOCK_HEADER_MARKER_2 = 0x265359; + } + + public enum InputStreamHeaderCheckType + { + // check for full header (ex: BZh9 ) + FULL_HEADER, + + // skips BZ part of header + NO_BZ, + + // skips BZh part of header + NO_BZH, + + // skips BZh and block level, aka the whole header + NO_HEADER, + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs new file mode 100644 index 000000000..1da856cbe --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs @@ -0,0 +1,29 @@ +using System; +using System.IO; + +namespace SharpCompress.Compressors.BZip2MT.Interface +{ + internal interface IBZip2BitInputStream : IDisposable + { + /// Reads a single bit from the wrapped input stream + /// true if the bit read was 1, otherwise false + /// if no more bits are available in the input stream + public bool ReadBoolean(); + + /// Reads a zero-terminated unary number from the wrapped input stream + /// The unary number + /// if no more bits are available in the input stream + public uint ReadUnary(); + + /// Reads up to 32 bits from the wrapped input stream + /// The number of bits to read (maximum 32) + /// The bits requested, right-aligned within the integer + /// if no more bits are available in the input stream + public uint ReadBits(int count); + + /// Reads 32 bits of input as an integer + /// The integer read + /// if 32 bits are not available in the input stream + public uint ReadInteger(); + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs new file mode 100644 index 000000000..6aa5651da --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs @@ -0,0 +1,48 @@ +// Added by drone1400, July 2022 +// Location: https://github.com/drone1400/bzip2 + +using System; + +namespace SharpCompress.Compressors.BZip2MT.Interface +{ + /// + /// Interface for a stream wrapper that implements bit-wise write operations + /// + internal interface IBZip2BitOutputStream : IDisposable + { + /// + /// Writes a single bit to the wrapped output stream + /// + /// The bit to write + /// if an error occurs writing to the stream + public void WriteBoolean(bool value); + + /// + /// Writes a zero-terminated unary number to the wrapped output stream + /// + /// The number to write (must be non-negative) + /// if an error occurs writing to the stream + public void WriteUnary(int value); + + /// + /// Writes up to 24 bits to the wrapped output stream + /// + /// The number of bits to write (maximum 24) + /// The bits to write + /// if an error occurs writing to the stream + public void WriteBits(int count, uint value); + + /// + /// Writes an integer as 32 bits of output + /// + /// The integer to write + /// if an error occurs writing to the stream + public void WriteInteger(uint value); + + /// + /// Writes any remaining bits to the output stream, zero padding to a whole byte as required + /// + /// if an error occurs writing to the stream + public void Flush(); + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs new file mode 100644 index 000000000..3e6fccb3f --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs @@ -0,0 +1,93 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + /// Implements a bit-wise output stream + /// + /// Allows the writing of single bit booleans, unary numbers, bit + /// strings of arbitrary length(up to 24 bits), and bit aligned 32-bit integers.A single byte at a + /// time is written to the wrapped stream when sufficient bits have been accumulated + /// + internal class BZip2BitOutputStream : IBZip2BitOutputStream + { + // The stream to which bits are written + private readonly Stream _outputStream; + + // A buffer of bits waiting to be written to the output stream + private uint _bitBuffer; + + // The number of bits currently buffered in bitBuffer + private int _bitCount; + + /// + /// Public constructor + /// + /// The OutputStream to wrap + public BZip2BitOutputStream(Stream outputStream) => this._outputStream = outputStream; + + public void Dispose() + { + // do nothing, the BZip2BitOutputStream is not the owner of the _outputStream so don't dispose it + } + + #region IBZip2BitOutputStream implementation + + public void WriteBoolean(bool value) + { + this._bitCount++; + this._bitBuffer |= ((value ? 1u : 0u) << (32 - this._bitCount)); + + if (this._bitCount == 8) + { + this._outputStream.WriteByte((byte)(this._bitBuffer >> 24)); + this._bitBuffer = 0; + this._bitCount = 0; + } + } + + public void WriteUnary(int value) + { + while (value-- > 0) + { + this.WriteBoolean(true); + } + this.WriteBoolean(false); + } + + public void WriteBits(int count, uint value) + { + this._bitBuffer |= ((value << (32 - count)) >> this._bitCount); + this._bitCount += count; + + while (this._bitCount >= 8) + { + this._outputStream.WriteByte((byte)(this._bitBuffer >> 24)); + this._bitBuffer <<= 8; + this._bitCount -= 8; + } + } + + public void WriteInteger(uint value) + { + this.WriteBits(16, (value >> 16) & 0xffff); + this.WriteBits(16, value & 0xffff); + } + + public void Flush() + { + if (this._bitCount > 0) + { + this.WriteBits(8 - this._bitCount, 0); + } + } + + #endregion + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs new file mode 100644 index 000000000..4288ce2a6 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs @@ -0,0 +1,205 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + /// An OutputStream wrapper that compresses BZip2 data + /// Instances of this class are not threadsafe + public class BZip2OutputStream : Stream + { + // The stream to which compressed BZip2 data is written + private Stream _outputStream; + + // An OutputStream wrapper that provides bit-level writes + private readonly BZip2BitOutputStream _bitOutputStream; + + // (@code true} if the compressed stream has been finished, otherwise false + private bool _streamFinished; + + // The declared maximum block size of the stream (before final run-length decoding) + private readonly int _streamBlockSize; + + // The merged CRC of all blocks compressed so far + private uint _streamCrc; + + // The compressor for the current block + private BZip2BlockCompressor? _blockCompressor; + + // True if the underlying stream will be closed with the current Stream + private bool _isOwner; + + /// Public constructor + /// The output stream to write to + /// The BZip2 block size as a multiple of 100,000 bytes (minimum 1, maximum 9) + /// True if the underlying stream will be closed with the current Stream + /// On any I/O error writing to the output stream + /// Larger block sizes require more memory for both compression and decompression, + /// but give better compression ratios. 9 will usually be the best value to use + public BZip2OutputStream( + Stream outputStream, + bool isOwner = true, + int blockSizeMultiplier = 9 + ) + { + if (outputStream is null) + throw new ArgumentException("Null output stream"); + + if ((blockSizeMultiplier < 1) || (blockSizeMultiplier > 9)) + throw new ArgumentException("Invalid BZip2 block size" + blockSizeMultiplier); + + this._streamBlockSize = blockSizeMultiplier * 100000; + this._outputStream = outputStream; + this._bitOutputStream = new BZip2BitOutputStream(this._outputStream); + this._isOwner = isOwner; + + this._bitOutputStream.WriteBits(16, BZip2Constants.STREAM_START_MARKER_1); + this._bitOutputStream.WriteBits(8, BZip2Constants.STREAM_START_MARKER_2); + this._bitOutputStream.WriteBits(8, (uint)('0' + blockSizeMultiplier)); + + this.InitialiseNextBlock(); + } + + #region Implementation of abstract members of Stream + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'Flush()' method." + ); + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method." + ); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); + + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'SetLength(long value)' method." + ); + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => this._outputStream.CanWrite; + + public override long Length => this._outputStream.Length; + + public override long Position + { + get => this._outputStream.Position; + set => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support Set operation for property 'Position'." + ); + } + + public override void WriteByte(byte value) + { + if (this._outputStream is null) + throw new IOException("Stream closed"); + + if (this._blockCompressor is null || this._streamFinished) + throw new IOException("Write beyond end of stream"); + + if (!this._blockCompressor.Write(value & 0xff)) + { + this.CloseBlock(); + this.InitialiseNextBlock(); + this._blockCompressor.Write(value & 0xff); + } + } + + public override void Write(byte[] data, int offset, int length) + { + if (this._outputStream is null) + throw new IOException("Stream closed"); + + if (this._blockCompressor is null || this._streamFinished) + throw new IOException("Write beyond end of stream"); + + while (length > 0) + { + int bytesWritten; + if ((bytesWritten = this._blockCompressor.Write(data, offset, length)) < length) + { + this.CloseBlock(); + this.InitialiseNextBlock(); + } + offset += bytesWritten; + length -= bytesWritten; + } + } + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this.Finish(); + if (this._isOwner) + { + this._outputStream.Dispose(); + } + } + +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + + /// Initialises a new block for compression + private void InitialiseNextBlock() => + this._blockCompressor = new BZip2BlockCompressor( + this._bitOutputStream, + this._streamBlockSize + ); + + /// Compress and write out the block currently in progress + /// If no bytes have been written to the block, it is discarded + /// On any I/O error writing to the output stream + private void CloseBlock() + { + if (this._blockCompressor is null || this._blockCompressor.IsEmpty) + return; + + this._blockCompressor.CloseBlock(); + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ this._blockCompressor.CRC; + } + + /// Compresses and writes out any as yet unwritten data, then writes the end of the BZip2 stream + /// The underlying OutputStream is not closed + /// On any I/O error writing to the output stream + private void Finish() + { + if (!this._streamFinished) + { + this._streamFinished = true; + try + { + this.CloseBlock(); + this._bitOutputStream.WriteBits(24, BZip2Constants.STREAM_END_MARKER_1); + this._bitOutputStream.WriteBits(24, BZip2Constants.STREAM_END_MARKER_2); + this._bitOutputStream.WriteInteger(this._streamCrc); + this._bitOutputStream.Flush(); + this._outputStream.Flush(); + } + finally + { + this._blockCompressor = null; + } + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs new file mode 100644 index 000000000..58de1843d --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs @@ -0,0 +1,176 @@ +// Added by drone1400, July 2022 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.Interface; + +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + /// A collection of bit output data + /// + /// Allows the writing of single bit booleans, unary numbers, bit + /// strings of arbitrary length(up to 24 bits), and bit aligned 32-bit integers.A single byte at a + /// time is written to a list of structures that serves as a buffer for use in parallelized + /// execution of block compression + /// + internal class BZip2ParallelOutputDataBlock : IBZip2BitOutputStream + { + private BZip2BitOutputStream _internalBitStream; + private MemoryStream _buffer; + private long _bitCount = 0; + + /// + /// Compressed block CRC to be stored here when block is finished + /// + public uint BlockCrc => this._blockCrc; + private uint _blockCrc = 0; + + /// + /// Indicates that the compression block is full + /// + public bool IsFull => this._isFull; + private bool _isFull = false; + + /// + /// Block numeric id for distinguishing blocks + /// + public int BlockId => this._blockId; + private int _blockId; + + /// + /// Number of bytes loaded into the block compressor + /// + public int LoadedBytes => this._loadedBytes; + private int _loadedBytes = 0; + private readonly BZip2BlockCompressor _compressor; + + /// + /// Public constructor + /// + /// block size in bytes, also initial internal buffer list capacity + /// Block number id, used to distinguish blocks in multithreadding + public BZip2ParallelOutputDataBlock(int blockSizeBytes, int blockId) + { + this._buffer = new MemoryStream(blockSizeBytes + 100000); + this._internalBitStream = new BZip2BitOutputStream(this._buffer); + this._blockId = blockId; + this._compressor = new BZip2BlockCompressor(this, blockSizeBytes); + } + + public void Dispose() + { + this._internalBitStream.Dispose(); + this._buffer.Dispose(); + } + + /// + /// Loads a byte into the 's first RLE stage + /// + /// Byte + /// True if byte was loaded, false if byte could not be loaded because block compressor is full + public bool LoadByte(byte value) + { + if (this._compressor.Write(value)) + { + this._loadedBytes++; + return true; + } + + // could not load the byte, means block is full + this._isFull = true; + + return false; + } + + /// + /// Loads bytes from a buffer into the 's first RLE stage + /// + /// Byte buffer + /// Byte buffer offset + /// Number of bytes to load + /// Number of bytes actually loaded + public int LoadBytes(byte[] buff, int offset, int length) + { + int count = this._compressor.Write(buff, offset, length); + this._loadedBytes += count; + if (count < length) + { + // could not load all the bytes, means block is full + this._isFull = true; + } + return count; + } + + /// + /// Starts the actual compression + /// + public void CompressBytes() + { + this._compressor.CloseBlock(); + this._blockCrc = this._compressor.CRC; + } + + /// + /// Writes all the buffer data to the real + /// + /// The real bit output stream + /// if an error occurs writing to the stream + public void WriteToRealOutputStream(BZip2BitOutputStream stream) + { + this._internalBitStream.Flush(); + + this._buffer.Position = 0; + while (this._bitCount >= 8) + { + int b = this._buffer.ReadByte(); + stream.WriteBits(8, (uint)b); + this._bitCount -= 8; + } + + if (this._bitCount > 0) + { + int b = this._buffer.ReadByte(); + b = (b >> (8 - (int)this._bitCount)); + stream.WriteBits((int)this._bitCount, (uint)b); + this._bitCount = 0; + } + } + + #region IBZip2BitOutputStream implementation + + public void WriteBoolean(bool value) + { + this._bitCount++; + this._internalBitStream.WriteBoolean(value); + } + + public void WriteUnary(int value) + { + while (value-- > 0) + { + this._bitCount++; + this._internalBitStream.WriteBoolean(true); + } + this._bitCount++; + this._internalBitStream.WriteBoolean(false); + } + + public void WriteBits(int count, uint value) + { + this._bitCount += count; + this._internalBitStream.WriteBits(count, value); + } + + public void WriteInteger(uint value) + { + this._bitCount += 32; + this._internalBitStream.WriteInteger(value); + } + + public void Flush() => this._internalBitStream.Flush(); + + #endregion + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs new file mode 100644 index 000000000..9e370b800 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs @@ -0,0 +1,425 @@ +// Added by drone1400, July 2022 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; + +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + /// An OutputStream wrapper that compresses BZip2 data using multiple threads + /// Instances of this class are not threadsafe + public class BZip2ParallelOutputStream : Stream + { + // block size fields + private readonly int _compressBlockSize; + private readonly int _blockLevel; + + private int _mtNextInputBlockId = 0; + private int _mtNextOutputBlockId = 0; + + // flag indicating input stream is finished + private bool _mtStreamIsFinished = false; + + // yikes! sounds bad right? + // if one of the worker threads, this is set to true and any furhter attempt to + // write/flush/close the stream will throw an exception + private bool _unsafeFatalException = false; + private Exception? _lastWorkerException = null; + + // dictionary of processed blocks + private readonly Dictionary _mtProcessedBlocks = + new Dictionary(); + + private int _mtPendingBlocks = 0; + + private readonly object _syncRootProcesing = new object(); + private readonly object _syncRootOutputStream = new object(); + + // The output stream + private readonly Stream _outputStream; + + // The bit output stream + private readonly BZip2BitOutputStream _bitStream; + + // The merged CRC of all blocks compressed so far + private uint _streamCrc = 0; + + // + private BZip2ParallelOutputDataBlock _currentBlockBuffer; + + // True if the underlying stream will be closed with the current Stream + private readonly bool _isOwner; + + // for debug purposes... + // private int _debugMaxMetaBufferDataPairCount = 0; + + /// + /// Public constructor + /// + /// The output stream in which to write the compressed data + /// True if the underlying stream will be closed with the current Stream + /// The BZip2 block size as a multiple of 100,000 bytes (minimum 1, maximum 9) + /// For best performance, compressor thread number should be equal to CPU core count, but results may vary + public BZip2ParallelOutputStream(Stream output, bool isOwner = true, int blockLevel = 9) + { + this._outputStream = output; + this._bitStream = new BZip2BitOutputStream(output); + + if (blockLevel < 1) + blockLevel = 1; + if (blockLevel > 9) + blockLevel = 9; + this._blockLevel = blockLevel; + + // evaluate thread count + // if (compressorThreads < 1) compressorThreads = 1; + // if (compressorThreads > ABSOLUTE_MAX_THREADS) compressorThreads = ABSOLUTE_MAX_THREADS; + // this._mtCompressorThreads = compressorThreads; + + // supposedly a block can only expand 1.25x, so 0.8 of normal block size should always be safe... + this._compressBlockSize = 100000 * this._blockLevel; + + // initialize initial meta buffer + this._currentBlockBuffer = new BZip2ParallelOutputDataBlock( + this._compressBlockSize, + this._mtNextInputBlockId++ + ); + + this._isOwner = isOwner; + + // write the bz2 header... + this.WriteBz2Header(); + } + + /// + /// Writes the next pending Output data block to the output bit stream + /// + /// True if a block was written, false if didn't find any pending blocks + /// + private bool TryWriteOutputBlockAndIncrementId() + { + if (this._unsafeFatalException) + { + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); + } + + try + { + BZip2ParallelOutputDataBlock? currentOutput = null; + + lock (this._syncRootProcesing) + { + // check if the next output block can be extracted + if (this._mtProcessedBlocks.ContainsKey(this._mtNextOutputBlockId)) + { + currentOutput = this._mtProcessedBlocks[this._mtNextOutputBlockId]; + this._mtProcessedBlocks.Remove(this._mtNextOutputBlockId); + } + + // check if we got anything to write + if (currentOutput is null) + return false; + + this._mtNextOutputBlockId++; + } + + lock (this._syncRootOutputStream) + { + // update file CRC + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ currentOutput.BlockCrc; + currentOutput.WriteToRealOutputStream(this._bitStream); + } + return true; + } + catch (Exception ex) + { + // set this without any locks... + this._lastWorkerException = ex; + this._unsafeFatalException = true; + + // rethrow exception, hopefully something catches it?... + throw new IOException( + "BZip2 error writing output data! See inner exception for details!", + ex + ); + } + } + + /// + /// Compresses a block of data + /// + /// + /// if compressing the block somehow fails... + private void MultiThreadWorkerAction(object? blockData) + { + try + { + if (blockData is BZip2ParallelOutputDataBlock buffer) + { + // do the work + buffer.CompressBytes(); + + // queue up the finished compressed block data + lock (this._syncRootProcesing) + { + this._mtProcessedBlocks.Add(buffer.BlockId, buffer); + } + } + } + catch (Exception ex) + { + // set this without any locks... + this._lastWorkerException = ex; + this._unsafeFatalException = true; + + throw new IOException( + "BZip2 Processing thread somehow crashed... See inner exception for details!", + ex + ); + } + finally + { + lock (this._syncRootProcesing) + { + this._mtPendingBlocks--; + } + } + } + + /// + /// Writes the BZ2 header to the bit stream + /// + private void WriteBz2Header() + { + lock (this._syncRootOutputStream) + { + // write BZIP file header + this._bitStream.WriteBits(8, 0x42); // B + this._bitStream.WriteBits(8, 0x5A); // Z + this._bitStream.WriteBits(8, 0x68); // h + this._bitStream.WriteBits(8, (uint)(0x30 + this._blockLevel)); // block level digit + } + } + + /// + /// Writes the BZ2 footer tot he bit stream and flushes the stream + /// + private void WriteBz2FooterAndFlush() + { + lock (this._syncRootOutputStream) + { + // end magic + this._bitStream.WriteBits(8, 0x17); + this._bitStream.WriteBits(8, 0x72); + this._bitStream.WriteBits(8, 0x45); + this._bitStream.WriteBits(8, 0x38); + this._bitStream.WriteBits(8, 0x50); + this._bitStream.WriteBits(8, 0x90); + + // write combined CRC + this._bitStream.WriteBits(16, (this._streamCrc >> 16) & 0xFFFF); + this._bitStream.WriteBits(16, this._streamCrc & 0xFFFF); + + // flush all remaining bits + this._bitStream.Flush(); + this._outputStream.Flush(); + } + } + + /// + /// Queues up the current data block if it is not empty + /// + /// True if the data block was queued up, false if the current block is empty + /// if one of the worker threads or writing to the final bit stream experienced an exception + private bool TryEnqueueCurrentBlockBuffer() + { + if (this._unsafeFatalException) + { + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); + } + + // make sure current block has data + if (this._currentBlockBuffer.LoadedBytes <= 0) + return false; + + lock (this._syncRootProcesing) + { + this._mtPendingBlocks++; + ThreadPool.QueueUserWorkItem( + this.MultiThreadWorkerAction, + this._currentBlockBuffer + ); + this._currentBlockBuffer = new BZip2ParallelOutputDataBlock( + this._compressBlockSize, + this._mtNextInputBlockId++ + ); + } + + return true; + } + + /// + /// Waits for any pending data blocks to be written to the bit stream and writes the BZ2 footer + /// + /// If there was an exception in a worker thread or when writing to the final bitstream + private void FinishBitstream() + { + if (this._unsafeFatalException) + { + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); + } + + lock (this._syncRootProcesing) + { + if (this._mtStreamIsFinished) + return; + this._mtStreamIsFinished = true; + } + + // check if there is still data left to write + this.TryEnqueueCurrentBlockBuffer(); + + // decrement next input block since no longer getting another block + this._mtNextInputBlockId--; + + while (true) + { + if (this._unsafeFatalException) + { + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); + } + + lock (this._syncRootProcesing) + { + if (this._mtNextInputBlockId == this._mtNextOutputBlockId) + { + // all done, can safely exit + break; + } + } + + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) { } + Thread.Sleep(1); + } + + // sanity check, this should be impossible and should be caught by some test right?... + lock (this._syncRootProcesing) + { + if (this._mtPendingBlocks != 0 || this._mtProcessedBlocks.Count > 0) + { + throw new IOException("BZip2 dispose operation sanity check failed!..."); + } + } + + // finally, write the footer! + this.WriteBz2FooterAndFlush(); + + //Console.WriteLine($"Max number of data pair entries was {this._debugMaxMetaBufferDataPairCount}"); + } + + #region Implementation of abstract members of Stream + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this.FinishBitstream(); + base.Dispose(disposing); + if (this._isOwner) + { + this._outputStream.Close(); + } + } + + public override void Flush() + { + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) { } + } + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); + + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support 'SetLength(long value)' method." + ); + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method." + ); + + public override void WriteByte(byte value) + { + if (!this._currentBlockBuffer.LoadByte(value)) + { + // byte could not be loaded, this happens when current block buffer is full + + this.TryEnqueueCurrentBlockBuffer(); + this._currentBlockBuffer.LoadByte(value); + + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) { } + } + } + + public override void Write(byte[] data, int offset, int length) + { + while (length > 0) + { + if (!this._currentBlockBuffer.IsFull) + { + int count = this._currentBlockBuffer.LoadBytes(data, offset, length); + offset += count; + length -= count; + } + + if (this._currentBlockBuffer.IsFull) + { + this.TryEnqueueCurrentBlockBuffer(); + } + + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) { } + } + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => this._outputStream.CanWrite; + public override long Length => this._outputStream.Length; + + public override long Position + { + get => this._outputStream.Position; + set => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support Set operation for property 'Position'." + ); + } + +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + } +} diff --git a/src/SharpCompress/Factories/TarFactory.cs b/src/SharpCompress/Factories/TarFactory.cs index d32020fd7..64dd2d611 100644 --- a/src/SharpCompress/Factories/TarFactory.cs +++ b/src/SharpCompress/Factories/TarFactory.cs @@ -6,7 +6,8 @@ using SharpCompress.Archives.Tar; using SharpCompress.Common; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT; +using SharpCompress.Compressors.BZip2MT.InputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.Lzw; @@ -120,8 +121,8 @@ public TestOption( new(CompressionType.None, (stream) => true, (stream) => stream, ["tar"], false), // We always do a test for IsTarFile later new( CompressionType.BZip2, - BZip2Stream.IsBZip2, - (stream) => new BZip2Stream(stream, CompressionMode.Decompress, false), + Compressors.BZip2.BZip2Stream.IsBZip2, + (stream) => new BZip2ParallelInputStream(stream), ["tar.bz2", "tb2", "tbz", "tbz2", "tz2"] ), new( diff --git a/src/SharpCompress/Readers/Tar/TarReader.cs b/src/SharpCompress/Readers/Tar/TarReader.cs index c92188856..3efe0c277 100644 --- a/src/SharpCompress/Readers/Tar/TarReader.cs +++ b/src/SharpCompress/Readers/Tar/TarReader.cs @@ -6,7 +6,7 @@ using SharpCompress.Common; using SharpCompress.Common.Tar; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT.InputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.Lzw; @@ -34,7 +34,7 @@ protected override Stream RequestInitialStream() var stream = base.RequestInitialStream(); return compressionType switch { - CompressionType.BZip2 => new BZip2Stream(stream, CompressionMode.Decompress, false), + CompressionType.BZip2 => new BZip2ParallelInputStream(stream), CompressionType.GZip => new GZipStream(stream, CompressionMode.Decompress), CompressionType.ZStandard => new ZStandardStream(stream), CompressionType.LZip => new LZipStream(stream, CompressionMode.Decompress), @@ -74,10 +74,10 @@ public static TarReader Open(Stream stream, ReaderOptions? options = null) } ((IStreamStack)rewindableStream).StackSeek(pos); - if (BZip2Stream.IsBZip2(rewindableStream)) + if (Compressors.BZip2.BZip2Stream.IsBZip2(rewindableStream)) { ((IStreamStack)rewindableStream).StackSeek(pos); - var testStream = new BZip2Stream(rewindableStream, CompressionMode.Decompress, false); + var testStream = new BZip2ParallelInputStream(rewindableStream); if (TarArchive.IsTarFile(testStream)) { ((IStreamStack)rewindableStream).StackSeek(pos); diff --git a/src/SharpCompress/Writers/Tar/TarWriter.cs b/src/SharpCompress/Writers/Tar/TarWriter.cs index 96346e2bc..e542654cf 100644 --- a/src/SharpCompress/Writers/Tar/TarWriter.cs +++ b/src/SharpCompress/Writers/Tar/TarWriter.cs @@ -6,6 +6,8 @@ using SharpCompress.Common.Tar.Headers; using SharpCompress.Compressors; using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT; +using SharpCompress.Compressors.BZip2MT.OutputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.IO; @@ -35,7 +37,7 @@ public TarWriter(Stream destination, TarWriterOptions options) break; case CompressionType.BZip2: { - destination = new BZip2Stream(destination, CompressionMode.Compress, false); + destination = new BZip2ParallelOutputStream(destination); } break; case CompressionType.GZip: diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index f867c8a91..8580424c1 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -9,7 +9,7 @@ using SharpCompress.Common.Zip; using SharpCompress.Common.Zip.Headers; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT.OutputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.PPMd; @@ -441,7 +441,7 @@ private Stream GetWriteStream(Stream writeStream) } case ZipCompressionMethod.BZip2: { - return new BZip2Stream(counting, CompressionMode.Compress, false); + return new BZip2ParallelOutputStream(counting); } case ZipCompressionMethod.LZMA: { diff --git a/tests/SharpCompress.Test/BZip2MT/BZip2MTReaderTests.cs b/tests/SharpCompress.Test/BZip2MT/BZip2MTReaderTests.cs new file mode 100644 index 000000000..3321c3260 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/BZip2MTReaderTests.cs @@ -0,0 +1,28 @@ +using System.IO; +using SharpCompress.Readers; +using SharpCompress.Readers.Tar; +using Xunit; +namespace SharpCompress.Test.BZip2MT; + +public class BZip2MTReaderTests : ReaderTests +{ + [Fact] + public void BZip2MT_Reader_Factory_ValidTarBz2File() + { + Stream stream = new MemoryStream( + new byte[] + { + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xd2, 0xeb, 0x13, 0xbb, 0x00, 0x00, + 0x2f, 0xdf, 0x81, 0xce, 0x00, 0x40, 0x01, 0x6f, 0x80, 0x20, 0x00, 0x00, 0x20, 0x60, 0x00, 0x1e, + 0x40, 0x02, 0x00, 0x04, 0x00, 0xa0, 0x00, 0x54, 0x42, 0x8f, 0xd5, 0x18, 0x00, 0x03, 0x20, 0x94, + 0x53, 0xc1, 0x40, 0x03, 0x40, 0x5a, 0xbe, 0xa5, 0x09, 0x81, 0x03, 0x52, 0x89, 0x46, 0x40, 0x53, + 0x65, 0xab, 0x3b, 0xb4, 0xe7, 0x20, 0x9c, 0x48, 0x6f, 0x0f, 0xa3, 0x4c, 0x3d, 0xc1, 0x58, 0x06, + 0xdc, 0x7a, 0xc8, 0x99, 0x22, 0x82, 0xc6, 0xd1, 0x55, 0x15, 0x8d, 0x33, 0xe8, 0xbb, 0x92, 0x29, + 0xc2, 0x84, 0x86, 0x97, 0x58, 0x9d, 0xd8, + } + ); + var reader = ReaderFactory.Open(stream); + + Assert.IsType(typeof(TarReader), reader); + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/BZip2MTStreamAsyncTests.cs b/tests/SharpCompress.Test/BZip2MT/BZip2MTStreamAsyncTests.cs new file mode 100644 index 000000000..b6e8c6922 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/BZip2MTStreamAsyncTests.cs @@ -0,0 +1,201 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using SharpCompress.Compressors.BZip2MT.InputStream; +using SharpCompress.Compressors.BZip2MT.OutputStream; +using Xunit; +namespace SharpCompress.Test.BZip2MT; + +public class BZip2MTStreamAsyncTests +{ + private byte[] CreateTestData(int size) + { + var data = new byte[size]; + // Create compressible data with repetitive pattern + for (int i = 0; i < size; i++) + { + data[i] = (byte)('A' + (i % 26)); + } + return data; + } + + [Fact] + public async Task BZip2MTCompressDecompressAsyncTest() + { + var testData = this.CreateTestData(10000); + byte[] compressed; + + // Compress + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream) + ) + { + await bzip2Stream.WriteAsync(testData, 0, testData.Length); + bzip2Stream.Close(); + } + compressed = memoryStream.ToArray(); + } + + // Verify compression occurred + Assert.True(compressed.Length > 0); + Assert.True(compressed.Length < testData.Length); + + // Decompress + byte[] decompressed; + using (var memoryStream = new MemoryStream(compressed)) + { + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + decompressed = new byte[testData.Length]; + var totalRead = 0; + int bytesRead; + while ( + ( + bytesRead = await bzip2Stream.ReadAsync( + decompressed, + totalRead, + testData.Length - totalRead + ) + ) > 0 + ) + { + totalRead += bytesRead; + } + } + } + + // Verify decompression + Assert.Equal(testData, decompressed); + } + + [Fact] + public async Task BZip2MTReadAsyncWithCancellationTest() + { + var testData = Encoding.ASCII.GetBytes(new string('A', 5000)); // Repetitive data compresses well + byte[] compressed; + + // Compress + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream) + ) + { + await bzip2Stream.WriteAsync(testData, 0, testData.Length); + bzip2Stream.Close(); + } + compressed = memoryStream.ToArray(); + } + + // Decompress with cancellation support + using (var memoryStream = new MemoryStream(compressed)) + { + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + var buffer = new byte[1024]; + using var cts = new System.Threading.CancellationTokenSource(); + + // Read should complete without cancellation + var bytesRead = await bzip2Stream.ReadAsync(buffer, 0, buffer.Length, cts.Token); + Assert.True(bytesRead > 0); + } + } + } + + [Fact] + public async Task BZip2MTMultipleAsyncWritesTest() + { + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream, false) + ) + { + var data1 = Encoding.ASCII.GetBytes("Hello "); + var data2 = Encoding.ASCII.GetBytes("World"); + var data3 = Encoding.ASCII.GetBytes("!"); + + await bzip2Stream.WriteAsync(data1, 0, data1.Length); + await bzip2Stream.WriteAsync(data2, 0, data2.Length); + await bzip2Stream.WriteAsync(data3, 0, data3.Length); + + bzip2Stream.Close(); + } + + var compressed = memoryStream.ToArray(); + Assert.True(compressed.Length > 0); + + // Decompress and verify + using (var readStream = new MemoryStream(compressed)) + { + // reset memory stream position + memoryStream.Position = 0; + + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + var result = new StringBuilder(); + var buffer = new byte[256]; + int bytesRead; + while ((bytesRead = await bzip2Stream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + result.Append(Encoding.ASCII.GetString(buffer, 0, bytesRead)); + } + + Assert.Equal("Hello World!", result.ToString()); + } + } + } + } + + [Fact] + public async Task BZip2MTLargeDataAsyncTest() + { + var largeData = this.CreateTestData(100000); + + // Compress + byte[] compressed; + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream) + ) + { + await bzip2Stream.WriteAsync(largeData, 0, largeData.Length); + bzip2Stream.Close(); + } + compressed = memoryStream.ToArray(); + } + + // Decompress + byte[] decompressed; + using (var memoryStream = new MemoryStream(compressed)) + { + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + decompressed = new byte[largeData.Length]; + var totalRead = 0; + int bytesRead; + var buffer = new byte[4096]; + while ((bytesRead = await bzip2Stream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + Array.Copy(buffer, 0, decompressed, totalRead, bytesRead); + totalRead += bytesRead; + } + } + } + + // Verify + Assert.Equal(largeData, decompressed); + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/TestCommon.cs b/tests/SharpCompress.Test/BZip2MT/TestCommon.cs new file mode 100644 index 000000000..e50142952 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/TestCommon.cs @@ -0,0 +1,206 @@ +using System; +using System.Diagnostics; +using System.IO; +using SharpCompress.Compressors.BZip2MT.InputStream; +using SharpCompress.Compressors.BZip2MT.OutputStream; +using Xunit; +using Xunit.Abstractions; +namespace SharpCompress.Test.BZip2MT +{ + public static class TestCommon + { + + /// + /// Common test routine for Multi Threaded compression + Single Threaded decompression + /// + /// + /// Input stream, must be seekable + /// If true, will use Parallel Compressor + /// If true, will use Parallel Decompressor + /// Maximum number of threads to use for compression, if 0 will use Environment.ProcessorCount + /// Size for temporary output memory buffer + /// Size for temporary copy buffer + /// If true, will save inputStream data to a file + /// (Time Compression, Time Decompression) + public static (double, double) GenericTest(ITestOutputHelper console, Stream inputStream, bool compressMultiThread, bool decompressMultiThread, int outputBufferSize = 8388608, int copyBufferSize = 8388608, bool saveFileOnFail = false) + { + void DebugSaveInputStream() + { + string randomFile = Path.GetRandomFileName(); + console.WriteLine($" Saving input data to {randomFile}"); + using FileStream fs = new FileStream(randomFile, FileMode.Create, FileAccess.Write); + inputStream.Position = 0; + inputStream.CopyTo(fs); + fs.Flush(); + fs.Close(); + } + + double timeCompress = 0; + double timeDecompress = 0; + + using MemoryStream output = new MemoryStream(outputBufferSize); + using MemoryStream outputDecompressed = new MemoryStream(outputBufferSize); + + try + { + // compress input + Stopwatch swCompress = new Stopwatch(); + swCompress.Start(); + using Stream compressor = compressMultiThread + ? new BZip2ParallelOutputStream(output, false, 9) + : new BZip2OutputStream(output, false, 9); + inputStream.CopyTo(compressor, copyBufferSize); + compressor.Close(); + swCompress.Stop(); + timeCompress = swCompress.ElapsedMilliseconds; + console.WriteLine($" {timeCompress} ms {(compressMultiThread ? "MT": "ST")} compression time... "); + + // reset output position + output.Position = 0; + + // decompress output + Stopwatch SwDecompress = new Stopwatch(); + SwDecompress.Start(); + using Stream decompressor = decompressMultiThread + ? new BZip2ParallelInputStream(output, false) + : new BZip2InputStream(output, false); + decompressor.CopyTo(outputDecompressed, copyBufferSize); + SwDecompress.Stop(); + timeDecompress = SwDecompress.ElapsedMilliseconds; + console.WriteLine($" {timeDecompress} ms {(decompressMultiThread ? "MT": "ST")} decompression time"); + } catch (Exception ex) + { + if (saveFileOnFail) + { + DebugSaveInputStream(); + } + + Assert.Fail($"Exception was thrown... {ex}"); + } + + if (inputStream.Length != outputDecompressed.Length) + { + Assert.Fail($"Decompressed stream length mismatch, expecting {inputStream.Length}, got {outputDecompressed.Length}"); + } + + inputStream.Position = 0; + outputDecompressed.Position = 0; + + for (int i = 0; i < inputStream.Length; i++) + { + int expect = inputStream.ReadByte(); + int value = outputDecompressed.ReadByte(); + if ( expect != value) + { + if (saveFileOnFail) + { + DebugSaveInputStream(); + Assert.Fail($"bytes differ at position {i}, expected {expect}, got {value}"); + } + } + } + + return (timeCompress, timeDecompress); + } + + public enum RandomDataMode + { + SingleByteValue, + RandomBytes, + RandomBytesRepeat, + } + + public enum TestMode + { + CMT_DST, // multi thread compress, single thread decompress + CST_DST, // single thread compress, single thread decompress + CMT_DMT, // multi thread compress, multi thread decompress + CST_DMT, // single thread compress, multi thread decompress + } + + public static void RandomLongTest_X(ITestOutputHelper console, int repeat, RandomDataMode dataMode, TestMode testMode, int len = 9000000) + { + Random random = new Random(); + + int copyBufferSize = 8388608; + int outBufferSize = 8388608; + int repeatStreaks = 64; + + double totalCompressionTimeMs = 0; + double totalDecompressionTimeMs = 0; + + for (int r = 0; r < repeat; r++) + { + byte[] bigBuffer = new byte[len]; + switch (dataMode) + { + case RandomDataMode.RandomBytes: + { + random.NextBytes(bigBuffer); + break; + } + case RandomDataMode.RandomBytesRepeat: + { + random.NextBytes(bigBuffer); + + int offset = 0; + for (int rs = 0; rs < repeatStreaks; rs++) + { + int newoffset = random.Next(0, (len - 10000) / repeatStreaks); + offset += newoffset; + int count = random.Next(0, 512); + byte val = bigBuffer[offset++]; + for (int i = 0; i < count; i++) + { + bigBuffer[offset++] = val; + } + } + break; + } + case RandomDataMode.SingleByteValue: + { + byte value = (byte)(random.Next() & 0xFF); + for (int i = 0; i < len; i++) + { + bigBuffer[i] = value; + } + break; + } + } + + MemoryStream ms = new MemoryStream(bigBuffer); + double timeC = 0, timeD = 0; + switch (testMode) + { + default: throw new ArgumentException("Unknown test mode"); + case TestMode.CMT_DMT: + { + (timeC, timeD) = GenericTest(console, ms, true, true, outBufferSize, copyBufferSize, true); + break; + } + case TestMode.CST_DMT: + { + (timeC, timeD) = GenericTest(console, ms, false, true, outBufferSize, copyBufferSize, true); + break; + } + case TestMode.CST_DST: + { + (timeC, timeD) = GenericTest(console, ms, false, false, outBufferSize, copyBufferSize, true); + break; + } + case TestMode.CMT_DST: + { + (timeC, timeD) = GenericTest(console, ms, true, false, outBufferSize, copyBufferSize, true); + break; + } + } + + totalCompressionTimeMs += timeC; + totalDecompressionTimeMs += timeD; + } + console.WriteLine($"DONE {testMode}"); + console.WriteLine($"AVERAGE {totalCompressionTimeMs / repeat} ms compression time... "); + console.WriteLine($"AVERAGE {totalDecompressionTimeMs / repeat} ms decompression time... "); + } + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/Tests.cs b/tests/SharpCompress.Test/BZip2MT/Tests.cs new file mode 100644 index 000000000..02db3b341 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/Tests.cs @@ -0,0 +1,139 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.InputStream; +using SharpCompress.Compressors.BZip2MT.OutputStream; +using Xunit; +using Xunit.Abstractions; +namespace SharpCompress.Test.BZip2MT +{ + + /// + /// Unit test for the BZip2 compression library + /// + public class Tests + { + + private const int BufferSizeLarge = 10000000; // Almost 10 Mb + private const int BufferSizeSmall = 100000; // Around 100 Kb + private static byte[] Buffer = new byte[Tests.BufferSizeLarge]; + + private readonly ITestOutputHelper _console; + private readonly Random _random; + + /// + /// Fills the test buffer with random values + /// + public Tests(ITestOutputHelper console) + { + this._random = new Random(); + this._console = console; + + this._random.NextBytes(Tests.Buffer); + } + + /// + /// Performs a CRC check and compare against well-known results + /// The buffer has different values + /// + [Fact] + public void CrcAlgorithmDifferentValues() + { + byte[] buffer = + { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA + }; + + var crc = new CRC32(); + for (int i = 0; i < buffer.Length; i++) + crc.UpdateCrc(buffer[i]); + + Assert.Equal(0x8AEE127A, crc.CRC); + } + + /// + /// Performs a CRC check and compare against well-known results + /// The buffer has different values + /// + [Fact] + public void CrcAlgorithmSameValues() + { + var crc = new CRC32(); + crc.UpdateCrc(0x55, 10); + Assert.Equal(0xA1E07747, crc.CRC); + } + + /// + /// Compresses the full buffer and checks for a reasonable compressed size + /// + [Fact] + public void CompressSmokeLarge() + { + var input = new MemoryStream(Tests.Buffer); + var output = new MemoryStream(); + + var compressor = new BZip2OutputStream(output, false); + input.CopyTo(compressor); + compressor.Close(); + + // Estimated size between inputSize*0.5 and inputSize*1.1 + Assert.True(output.Length > Tests.BufferSizeLarge * 0.5); + Assert.True(output.Length < Tests.BufferSizeLarge * 1.1); + } + + /// + /// Compresses a portion of the buffer and checks for a reasonable compressed size + /// + [Fact] + public void CompressSmokeSmall() + { + var input = new MemoryStream(Tests.Buffer, 0, Tests.BufferSizeSmall); + var output = new MemoryStream(); + + var compressor = new BZip2OutputStream(output, false); + input.CopyTo(compressor); + compressor.Close(); + + // Estimated size between inputSize*0.5 and inputSize*1.1 + Assert.True(output.Length > Tests.BufferSizeSmall * 0.5); + Assert.True(output.Length < Tests.BufferSizeSmall * 1.1); + } + + /// + /// Compresses and decompresses a long random buffer + /// + [Fact] + public void CompressAndDecompress() + { + var input = new MemoryStream(Tests.Buffer); + var output = new MemoryStream(); + + var compressor = new BZip2OutputStream(output, false); + input.CopyTo(compressor); + compressor.Close(); + + Assert.True(output.Length > 4); + + output.Position = 0; + var output2 = new MemoryStream(); + var decompressor = new BZip2InputStream(output, false); + decompressor.CopyTo(output2); + + Assert.Equal(Tests.Buffer.Length, output2.Length); + output2.Position = 0; + for (int i = 0; i < Tests.Buffer.Length; i++) + { + if (Tests.Buffer[i] != (byte)output2.ReadByte()) + { + Assert.Fail($"bytes differ at position {i}"); + } + } + } + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/TestsRandomData.cs b/tests/SharpCompress.Test/BZip2MT/TestsRandomData.cs new file mode 100644 index 000000000..5deec2dce --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/TestsRandomData.cs @@ -0,0 +1,53 @@ +using Xunit; +using Xunit.Abstractions; +namespace SharpCompress.Test.BZip2MT +{ + public class TestsRandomData + { + private readonly ITestOutputHelper _console; + protected int _repeatCount = 1; + + public TestsRandomData(ITestOutputHelper console) + { + this._console = console; + } + + [Fact] + public void RandomSingleByteLongTest_CST_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CST_DST, 100000000); + [Fact] + public void RandomLongTest_CST_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CST_DST); + [Fact] + public void RandomLongTestWithRepeatedValues_CST_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CST_DST); + [Fact] + public void RandomSingleByteLongTest_CMT_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CMT_DST, 100000000); + [Fact] + public void RandomLongTest_CMT_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CMT_DST); + [Fact] + public void RandomLongTestWithRepeatedValues_CMT_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CMT_DST); + + [Fact] + public void RandomSingleByteLongTest_CST_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CST_DMT, 100000000); + [Fact] + public void RandomLongTest_CST_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CST_DMT); + [Fact] + public void RandomLongTestWithRepeatedValues_CST_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CST_DMT); + [Fact] + public void RandomSingleByteLongTest_CMT_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CMT_DMT, 100000000); + [Fact] + public void RandomLongTest_CMT_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CMT_DMT); + [Fact] + public void RandomLongTestWithRepeatedValues_CMT_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CMT_DMT); + } +}