From 0cb46169c79ddedd5434d14344e05f692ca13611 Mon Sep 17 00:00:00 2001 From: drone1400 <> Date: Sat, 6 Dec 2025 16:46:57 +0200 Subject: [PATCH 1/3] Initial Implementation of Multi Threaded BZip2 - currently causes BZip2_Reader_Factory test to fail due to some implementation details, to figure out later... - have not replaced original BZip2 implementation in LZMA Registry.cs due to BZip2MT implementation not having a built in concatenated stream support --- src/SharpCompress/Common/Zip/ZipFilePart.cs | 4 +- .../BZip2MT/Algorithm/BZip2BlockCompressor.cs | 256 ++ .../Algorithm/BZip2BlockDecompressor.cs | 483 ++++ .../BZip2MT/Algorithm/BZip2DivSufSort.cs | 2406 +++++++++++++++++ .../Algorithm/BZip2HuffmanStageDecoder.cs | 155 ++ .../Algorithm/BZip2HuffmanStageEncoder.cs | 344 +++ .../Algorithm/Bzip2MTFAndRLE2StageEncoder.cs | 168 ++ .../Compressors/BZip2MT/Algorithm/CRC32.cs | 74 + .../BZip2MT/Algorithm/HuffmanAllocator.cs | 201 ++ .../BZip2MT/Algorithm/MoveToFront.cs | 60 + .../InputStream/BZip2BitInputStream.cs | 111 + .../BZip2BitInputStreamSplitter.cs | 184 ++ .../BZip2MT/InputStream/BZip2InputStream.cs | 248 ++ .../BZip2ParallelInputDataBlock.cs | 108 + .../InputStream/BZip2ParallelInputStream.cs | 344 +++ .../BZip2MT/Interface/BZip2Constants.cs | 43 + .../BZip2MT/Interface/IBZip2BitInputStream.cs | 28 + .../Interface/IBZip2BitOutputStream.cs | 47 + .../OutputStream/BZip2BitOutputStream.cs | 92 + .../BZip2MT/OutputStream/BZip2OutputStream.cs | 179 ++ .../BZip2ParallelOutputDataBlock.cs | 171 ++ .../OutputStream/BZip2ParallelOutputStream.cs | 382 +++ src/SharpCompress/Factories/TarFactory.cs | 7 +- src/SharpCompress/Readers/Tar/TarReader.cs | 8 +- src/SharpCompress/Writers/Tar/TarWriter.cs | 4 +- src/SharpCompress/Writers/Zip/ZipWriter.cs | 4 +- .../BZip2MT/BZip2MTReaderTests.cs | 28 + .../BZip2MT/BZip2MTStreamAsyncTests.cs | 201 ++ .../SharpCompress.Test/BZip2MT/TestCommon.cs | 206 ++ tests/SharpCompress.Test/BZip2MT/Tests.cs | 139 + .../BZip2MT/TestsRandomData.cs | 53 + 31 files changed, 6726 insertions(+), 12 deletions(-) create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs create mode 100644 src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs create mode 100644 tests/SharpCompress.Test/BZip2MT/BZip2MTReaderTests.cs create mode 100644 tests/SharpCompress.Test/BZip2MT/BZip2MTStreamAsyncTests.cs create mode 100644 tests/SharpCompress.Test/BZip2MT/TestCommon.cs create mode 100644 tests/SharpCompress.Test/BZip2MT/Tests.cs create mode 100644 tests/SharpCompress.Test/BZip2MT/TestsRandomData.cs diff --git a/src/SharpCompress/Common/Zip/ZipFilePart.cs b/src/SharpCompress/Common/Zip/ZipFilePart.cs index 77dc4abba..63d2a39fa 100644 --- a/src/SharpCompress/Common/Zip/ZipFilePart.cs +++ b/src/SharpCompress/Common/Zip/ZipFilePart.cs @@ -4,7 +4,7 @@ using System.Linq; using SharpCompress.Common.Zip.Headers; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT.InputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.Deflate64; using SharpCompress.Compressors.Explode; @@ -122,7 +122,7 @@ protected Stream CreateDecompressionStream(Stream stream, ZipCompressionMethod m } case ZipCompressionMethod.BZip2: { - return new BZip2Stream(stream, CompressionMode.Decompress, false); + return new BZip2ParallelInputStream(stream); } case ZipCompressionMethod.LZMA: { diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs new file mode 100644 index 000000000..260397eef --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs @@ -0,0 +1,256 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// Compresses and writes a single BZip2 block + /// + /// Block encoding consists of the following stages: + /// 1. Run-Length Encoding[1] - write() + /// 2. Burrows Wheeler Transform - close() (through BZip2DivSufSort) + /// 3. Write block header - close() + /// 4. Move To Front Transform - close() (through BZip2HuffmanStageEncoder) + /// 5. Run-Length Encoding[2] - close() (through BZip2HuffmanStageEncoder) + /// 6. Create and write Huffman tables - close() (through BZip2HuffmanStageEncoder) + /// 7. Huffman encode and write data - close() (through BZip2HuffmanStageEncoder) + /// + internal class BZip2BlockCompressor + { + // The stream to which compressed BZip2 data is written + private readonly IBZip2BitOutputStream bitOutputStream; + + // CRC builder for the block + private readonly CRC32 crc = new CRC32(); + + // The RLE'd block data + private readonly byte[] block; + + // Current length of the data within the block array + private int blockLength; + + // A limit beyond which new data will not be accepted into the block + private readonly int blockLengthLimit; + + // The values that are present within the RLE'd block data. For each index, true if that + // value is present within the data, otherwise false + private readonly bool[] blockValuesPresent = new bool[256]; + + // The Burrows Wheeler Transformed block data + private readonly int[] bwtBlock; + + // The current RLE value being accumulated (undefined when rleLength is 0) + private int rleCurrentValue = -1; + + // The repeat count of the current RLE value + private int rleLength; + + /// + /// Determines if any bytes have been written to the block. + /// True if one or more bytes has been written to the block, otherwise false. + /// + public bool IsEmpty + { + get { return ((this.blockLength == 0) && (this.rleLength == 0)); } + } + + /// + /// Gets the CRC of the completed block. Only valid after calling Close(). + /// + public uint CRC + { + get { return this.crc.CRC; } + } + + /// + /// Public constructor + /// + /// The BZip2BitOutputStream to which compressed BZip2 data is written + /// The declared block size in bytes. Up to this many bytes will be accepted + /// into the block after Run-Length Encoding is applied + public BZip2BlockCompressor(IBZip2BitOutputStream bitOutputStream, int blockSize) + { + this.bitOutputStream = bitOutputStream; + + // One extra byte is added to allow for the block wrap applied in close() + this.block = new byte[blockSize + 1]; + this.bwtBlock = new int[blockSize + 1]; + this.blockLengthLimit = blockSize - 6; // 5 bytes for one RLE run plus one byte - see Write(int) + } + + /// + /// Writes a byte to the block, accumulating to an RLE run where possible + /// + /// The byte to write + /// True if the byte was written, or false if the block is already full + public bool Write(int value) + { + if (this.blockLength > this.blockLengthLimit) + return false; + + if (this.rleLength == 0) + { + this.rleCurrentValue = value; + this.rleLength = 1; + } else if (this.rleCurrentValue != value) + { + // This path commits us to write 6 bytes - one RLE run (5 bytes) plus one extra + this.WriteRun(this.rleCurrentValue & 0xff, this.rleLength); + this.rleCurrentValue = value; + this.rleLength = 1; + } else + { + if (this.rleLength == 254) + { + this.WriteRun(this.rleCurrentValue & 0xff, 255); + this.rleLength = 0; + } else + { + this.rleLength++; + } + } + + return true; + } + + /// + /// Writes an array to the block + /// + /// The array to write + /// The offset within the input data to write from + /// The number of bytes of input data to write + /// The actual number of input bytes written. May be less than the number requested, or + /// zero if the block is already full + public int Write(byte[] data, int offset, int length) + { + var written = 0; + + while (length-- > 0) + { + if (!this.Write(data[offset++])) + break; + written++; + } + + return written; + } + + /// Compresses and writes out the block. + /// Exception on any I/O error writing the data + public void CloseBlock() + { + // If an RLE run is in progress, write it out + if (this.rleLength > 0) + this.WriteRun(this.rleCurrentValue & 0xff, this.rleLength); + + // Apply a one byte block wrap required by the BWT implementation + this.block[this.blockLength] = this.block[0]; + + // Perform the Burrows Wheeler Transform + var divSufSort = new BZip2DivSufSort(this.block, this.bwtBlock, this.blockLength); + var bwtStartPointer = divSufSort.BWT(); + + // Write out the block header + this.bitOutputStream.WriteBits(24, BZip2Constants.BLOCK_HEADER_MARKER_1); + this.bitOutputStream.WriteBits(24, BZip2Constants.BLOCK_HEADER_MARKER_2); + this.bitOutputStream.WriteInteger(this.crc.CRC); + this.bitOutputStream.WriteBoolean(false); // Randomised block flag. We never create randomised blocks + this.bitOutputStream.WriteBits(24, (uint)bwtStartPointer); + + // Write out the symbol map + this.WriteSymbolMap(); + + // Perform the Move To Front Transform and Run-Length Encoding[2] stages + var mtfEncoder = new BZip2MTFAndRLE2StageEncoder(this.bwtBlock, this.blockLength, this.blockValuesPresent); + mtfEncoder.Encode(); + + // Perform the Huffman Encoding stage and write out the encoded data + var huffmanEncoder = new BZip2HuffmanStageEncoder(this.bitOutputStream, mtfEncoder.MtfBlock, mtfEncoder.MtfLength, mtfEncoder.MtfAlphabetSize, mtfEncoder.MtfSymbolFrequencies); + huffmanEncoder.Encode(); + } + + /// + /// Write the Huffman symbol to output byte map + /// + /// on any I/O error writing the data + private void WriteSymbolMap() + { + var condensedInUse = new bool[16]; + + for (var i = 0; i < 16; i++) + { + for (int j = 0, k = i << 4; j < 16; j++, k++) + { + if (this.blockValuesPresent[k]) + { + condensedInUse[i] = true; + } + } + } + + for (var i = 0; i < 16; i++) + { + this.bitOutputStream.WriteBoolean(condensedInUse[i]); + } + + for (var i = 0; i < 16; i++) + { + if (condensedInUse[i]) + { + for (int j = 0, k = i * 16; j < 16; j++, k++) + { + this.bitOutputStream.WriteBoolean(this.blockValuesPresent[k]); + } + } + } + } + + /// + /// Writes an RLE run to the block array, updating the block CRC and present values array as required + /// + /// The value to write + /// The run length of the value to write + private void WriteRun ( int value, int runLength) + { + this.blockValuesPresent[value] = true; + this.crc.UpdateCrc (value, runLength); + + var byteValue = (byte)value; + switch (runLength) + { + case 1: + this.block[this.blockLength] = byteValue; + this.blockLength = this.blockLength + 1; + break; + + case 2: + this.block[this.blockLength] = byteValue; + this.block[this.blockLength + 1] = byteValue; + this.blockLength = this.blockLength + 2; + break; + + case 3: + this.block[this.blockLength] = byteValue; + this.block[this.blockLength + 1] = byteValue; + this.block[this.blockLength + 2] = byteValue; + this.blockLength = this.blockLength + 3; + break; + + default: + runLength -= 4; + this.blockValuesPresent[runLength] = true; + this.block[this.blockLength] = byteValue; + this.block[this.blockLength + 1] = byteValue; + this.block[this.blockLength + 2] = byteValue; + this.block[this.blockLength + 3] = byteValue; + this.block[this.blockLength + 4] = (byte)runLength; + this.blockLength = this.blockLength + 5; + break; + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs new file mode 100644 index 000000000..2a61f2ca7 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs @@ -0,0 +1,483 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// Reads and decompresses a single BZip2 block + /// + /// Block decoding consists of the following stages: + /// 1. Read block header - BZip2BlockDecompressor() + /// 2. Read Huffman tables - readHuffmanTables() + /// 3. Read and decode Huffman encoded data - decodeHuffmanData() + /// 4. Run-Length Decoding[2] - decodeHuffmanData() + /// 5. Inverse Move To Front Transform - decodeHuffmanData() + /// 6. Inverse Burrows Wheeler Transform - initialiseInverseBWT() + /// 7. Run-Length Decoding[1] - read() + /// 8. Optional Block De-Randomisation - read() (through decodeNextBWTByte()) + /// + internal class BZip2BlockDecompressor + { + /// + /// The BZip2 specification originally included the optional addition of a slight pseudo-random + /// perturbation to the input data, in order to work around the block sorting algorithm's non- + /// optimal performance on some types of input. The current mainline bzip2 does not require this + /// and will not create randomised blocks, but compatibility is still required for old data (and + /// third party compressors that haven't caught up). When decompressing a randomised block, for + /// each value N in this array, a 1 will be XOR'd onto the output of the Burrows-Wheeler + /// transform stage after N bytes, then the next N taken from the following entry. + /// + private static readonly int[] RNUMS = + { + 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, 985, 724, 205, 454, 863, 491, + 741, 242, 949, 214, 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, 419, 436, + 278, 496, 867, 210, 399, 680, 480, 51, 878, 465, 811, 169, 869, 675, 611, 697, + 867, 561, 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, 150, 238, 59, 379, + 684, 877, 625, 169, 643, 105, 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, + 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, 909, 545, 703, 919, 874, 474, + 882, 500, 594, 612, 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, 161, 604, + 958, 533, 221, 400, 386, 867, 600, 782, 382, 596, 414, 171, 516, 375, 682, 485, + 911, 276, 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, 227, 730, 475, 186, + 263, 647, 537, 686, 600, 224, 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, + 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, 715, 67, 618, 276, 204, 918, + 873, 777, 604, 560, 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, 652, 934, + 970, 447, 318, 353, 859, 672, 112, 785, 645, 863, 803, 350, 139, 93, 354, 99, + 820, 908, 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, 653, 282, 762, 623, + 680, 81, 927, 626, 789, 125, 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, + 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, 857, 956, 358, 619, 580, 124, + 737, 594, 701, 612, 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, 944, 375, + 748, 52, 600, 747, 642, 182, 862, 81, 344, 805, 988, 739, 511, 655, 814, 334, + 249, 515, 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, 433, 837, 553, 268, + 926, 240, 102, 654, 459, 51, 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, + 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, 978, 321, 576, 617, 626, 502, + 894, 679, 243, 440, 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, 707, 151, + 457, 449, 797, 195, 791, 558, 945, 679, 297, 59, 87, 824, 713, 663, 412, 693, + 342, 606, 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, 343, 97, 430, 751, + 497, 314, 983, 374, 822, 928, 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, + 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, 369, 970, 294, 750, 807, 827, + 150, 790, 288, 923, 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, 896, 831, + 547, 261, 524, 462, 293, 465, 502, 56, 661, 821, 976, 991, 658, 869, 905, 758, + 745, 193, 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, 61, 688, 793, 644, + 986, 403, 106, 366, 905, 644, 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, + 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, 920, 176, 193, 713, 857, 265, + 203, 50, 668, 108, 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, 936, 638 + }; + + // Maximum possible number of Huffman table selectors + private const int HUFFMAN_MAXIMUM_SELECTORS = (900000 / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH) + 1; + + // Provides bits of input to decode + private readonly IBZip2BitInputStream bitInputStream; + + // Calculates the block CRC from the fully decoded bytes of the block + private readonly CRC32 crc = new CRC32(); + + // The CRC of the current block as read from the block header + private readonly uint blockCRC; + + // true if the current block is randomised, otherwise false + private readonly bool blockRandomised; + + // + // Huffman Decoding stage + // + + // The end-of-block Huffman symbol. Decoding of the block ends when this is encountered + private int huffmanEndOfBlockSymbol; + + /// + /// A map from Huffman symbol index to output character. Some types of data (e.g. ASCII text) + /// may contain only a limited number of byte values; Huffman symbols are only allocated to + /// those values that actually occur in the uncompressed data. + /// + private readonly byte[] huffmanSymbolMap = new byte[256]; + + // + // Move To Front stage + // + + /// + /// Counts of each byte value within the bwtTransformedArray data. Collected at the Move + /// To Front stage, consumed by the Inverse Burrows Wheeler Transform stage + /// + private readonly int[] bwtByteCounts = new int[256]; + + /// + /// The Burrows-Wheeler Transform processed data. Read at the Move To Front stage, consumed by the + /// Inverse Burrows Wheeler Transform stage + /// + private byte[] bwtBlock; + + // + // Inverse Burrows-Wheeler Transform stage + // + + /// + /// At each position contains the union of :- + /// An output character (8 bits) + /// A pointer from each position to its successor (24 bits, left shifted 8 bits) + /// As the pointer cannot exceed the maximum block size of 900k, 24 bits is more than enough to + /// hold it; Folding the character data into the spare bits while performing the inverse BWT, + /// when both pieces of information are available, saves a large number of memory accesses in + /// the final decoding stages. + /// + private int[] bwtMergedPointers = Array.Empty(); + + // The current merged pointer into the Burrow-Wheeler Transform array + private int bwtCurrentMergedPointer; + + /// + /// The actual length in bytes of the current block at the Inverse Burrows Wheeler Transform + /// stage (before final Run-Length Decoding) + /// + private int bwtBlockLength; + + // The number of output bytes that have been decoded up to the Inverse Burrows Wheeler Transform stage + private int bwtBytesDecoded; + + // + // Run-Length Encoding and Random Perturbation stage + // + + // The most recently RLE decoded byte + private int rleLastDecodedByte = -1; + + /// + /// The number of previous identical output bytes decoded. After 4 identical bytes, the next byte + /// decoded is an RLE repeat count + /// + private int rleAccumulator; + + // The RLE repeat count of the current decoded byte. When this reaches zero, a new byte is decoded + private int rleRepeat; + + // If the current block is randomised, the position within the RNUMS randomisation array + private int randomIndex; + + // If the current block is randomised, the remaining count at the current RNUMS position + private int randomCount = BZip2BlockDecompressor.RNUMS[0] - 1; + + // Minimum number of alternative Huffman tables + public const int HUFFMAN_MINIMUM_TABLES = 2; + + // Maximum number of alternative Huffman tables + public const int HUFFMAN_MAXIMUM_TABLES = 6; + + /// + /// Read and decode the block's Huffman tables + /// @return A decoder for the Huffman stage that uses the decoded tables + /// + /// if the input stream reaches EOF before all table data has been read + private BZip2HuffmanStageDecoder ReadHuffmanTables() + { + var tableCodeLengths = new byte[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE]; + + // Read Huffman symbol to output byte map + uint huffmanUsedRanges = this.bitInputStream.ReadBits(16); + int huffmanSymbolCount = 0; + + for (int i = 0; i < 16; i++) + { + if ((huffmanUsedRanges & ((1 << 15) >> i)) == 0) + continue; + for (int j = 0, k = i << 4; j < 16; j++, k++) + { + if (this.bitInputStream.ReadBoolean()) + { + this.huffmanSymbolMap[huffmanSymbolCount++] = (byte)k; + } + } + } + var endOfBlockSymbol = huffmanSymbolCount + 1; + this.huffmanEndOfBlockSymbol = endOfBlockSymbol; + + // Read total number of tables and selectors + uint totalTables = this.bitInputStream.ReadBits(3); + uint totalSelectors = this.bitInputStream.ReadBits(15); + if ((totalTables < BZip2BlockDecompressor.HUFFMAN_MINIMUM_TABLES) + || (totalTables > BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES) + || (totalSelectors < 1) + || (totalSelectors > BZip2BlockDecompressor.HUFFMAN_MAXIMUM_SELECTORS)) + { + throw new IOException("BZip2 block Huffman tables invalid"); + } + + // Read and decode MTFed Huffman selector list + var tableMTF = new MoveToFront(); + var selectors = new byte[totalSelectors]; + for (var selector = 0; selector < totalSelectors; selector++) + { + selectors[selector] = tableMTF.IndexToFront((int)this.bitInputStream.ReadUnary()); + } + + // Read the Canonical Huffman code lengths for each table + for (var table = 0; table < totalTables; table++) + { + int currentLength = (int)this.bitInputStream.ReadBits(5); + for (var i = 0; i <= endOfBlockSymbol; i++) + { + while (this.bitInputStream.ReadBoolean()) + { + currentLength += this.bitInputStream.ReadBoolean() ? -1 : 1; + } + tableCodeLengths[table, i] = (byte)currentLength; + } + } + + return new BZip2HuffmanStageDecoder(this.bitInputStream, endOfBlockSymbol + 1, tableCodeLengths, selectors); + } + + /// + /// Reads the Huffman encoded data from the input stream, performs Run-Length Decoding and + /// applies the Move To Front transform to reconstruct the Burrows-Wheeler Transform array + /// + /// The Huffman decoder through which symbols are read + /// if an end-of-block symbol was not decoded within the declared block size + private void DecodeHuffmanData(BZip2HuffmanStageDecoder huffmanDecoder) + { + var symbolMTF = new MoveToFront(); + int _bwtBlockLength = 0; + int repeatCount = 0; + int repeatIncrement = 1; + int mtfValue = 0; + + while (true) + { + var nextSymbol = huffmanDecoder.NextSymbol(); + + if (nextSymbol == BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA) + { + repeatCount += repeatIncrement; + repeatIncrement <<= 1; + } else if (nextSymbol == BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB) + { + repeatCount += repeatIncrement << 1; + repeatIncrement <<= 1; + } else + { + byte nextByte; + if (repeatCount > 0) + { + if (_bwtBlockLength + repeatCount > this.bwtBlock.Length) + throw new IOException("BZip2 block exceeds declared block size"); + + nextByte = this.huffmanSymbolMap[mtfValue]; + this.bwtByteCounts[nextByte & 0xff] += repeatCount; + while (--repeatCount >= 0) + { + this.bwtBlock[_bwtBlockLength++] = nextByte; + } + + repeatCount = 0; + repeatIncrement = 1; + } + + if (nextSymbol == this.huffmanEndOfBlockSymbol) + break; + + if (_bwtBlockLength >= this.bwtBlock.Length) + throw new IOException("BZip2 block exceeds declared block size"); + + mtfValue = symbolMTF.IndexToFront(nextSymbol - 1) & 0xff; + + nextByte = this.huffmanSymbolMap[mtfValue]; + this.bwtByteCounts[nextByte & 0xff]++; + this.bwtBlock[_bwtBlockLength++] = nextByte; + } + } + + this.bwtBlockLength = _bwtBlockLength; + } + + /// + /// Set up the Inverse Burrows-Wheeler Transform merged pointer array + /// + /// The start pointer into the BWT array + /// if the given start pointer is invalid + private void InitialiseInverseBWT(uint bwtStartPointer) + { + var _bwtMergedPointers = new int[this.bwtBlockLength]; + var characterBase = new int[256]; + + if ((bwtStartPointer < 0) || (bwtStartPointer >= this.bwtBlockLength)) + throw new IOException("BZip2 start pointer invalid"); + + // Cumulatise character counts + Array.ConstrainedCopy(this.bwtByteCounts, 0, characterBase, 1, 255); + for (int i = 2; i <= 255; i++) + { + characterBase[i] += characterBase[i - 1]; + } + + // Merged-Array Inverse Burrows-Wheeler Transform + // Combining the output characters and forward pointers into a single array here, where we + // have already read both of the corresponding values, cuts down on memory accesses in the + // final walk through the array + for (int i = 0; i < this.bwtBlockLength; i++) + { + int value = this.bwtBlock[i] & 0xff; + _bwtMergedPointers[characterBase[value]++] = (i << 8) + value; + } + + //this.bwtBlock = null; + this.bwtMergedPointers = _bwtMergedPointers; + this.bwtCurrentMergedPointer = _bwtMergedPointers[bwtStartPointer]; + } + + /// + /// Decodes a byte from the Burrows-Wheeler Transform stage. If the block has randomisation + /// applied, reverses the randomisation + /// + /// The decoded byte + private int decodeNextBWTByte() + { + int nextDecodedByte = this.bwtCurrentMergedPointer & 0xff; + this.bwtCurrentMergedPointer = this.bwtMergedPointers[this.bwtCurrentMergedPointer >> 8]; + + if (this.blockRandomised) + { + if (--this.randomCount == 0) + { + nextDecodedByte ^= 1; + this.randomIndex = (this.randomIndex + 1) % 512; + this.randomCount = BZip2BlockDecompressor.RNUMS[this.randomIndex]; + } + } + + this.bwtBytesDecoded++; + + return nextDecodedByte; + } + + /// + /// Public constructor + /// + /// The BZip2BitInputStream to read from + /// The maximum decoded size of the block + /// If the block could not be decoded + public BZip2BlockDecompressor(IBZip2BitInputStream bitInputStream, uint blockSize) + { + this.bitInputStream = bitInputStream; + this.bwtBlock = new byte[blockSize]; + + // Read block header + this.blockCRC = this.bitInputStream.ReadInteger(); //.ReadBits(32); + this.blockRandomised = this.bitInputStream.ReadBoolean(); + uint bwtStartPointer = this.bitInputStream.ReadBits(24); + + // Read block data and decode through to the Inverse Burrows Wheeler Transform stage + var huffmanDecoder = this.ReadHuffmanTables(); + this.DecodeHuffmanData(huffmanDecoder); + this.InitialiseInverseBWT(bwtStartPointer); + } + + + /// + /// Decodes a byte from the final Run-Length Encoding stage, pulling a new byte from the + /// Burrows-Wheeler Transform stage when required + /// + /// The decoded byte, or -1 if there are no more bytes + public int Read() + { + while (this.rleRepeat < 1) + { + if (this.bwtBytesDecoded == this.bwtBlockLength) + return -1; + + int nextByte = this.decodeNextBWTByte(); + if (nextByte != this.rleLastDecodedByte) + { + // New byte, restart accumulation + this.rleLastDecodedByte = nextByte; + this.rleRepeat = 1; + this.rleAccumulator = 1; + this.crc.UpdateCrc(nextByte); + } else + { + if (++this.rleAccumulator == 4) + { + // Accumulation complete, start repetition + int _rleRepeat = this.decodeNextBWTByte() + 1; + this.rleRepeat = _rleRepeat; + this.rleAccumulator = 0; + this.crc.UpdateCrc(nextByte, _rleRepeat); + } else + { + this.rleRepeat = 1; + this.crc.UpdateCrc(nextByte); + } + } + } + + this.rleRepeat--; + + return this.rleLastDecodedByte; + } + + /// + /// Decodes multiple bytes from the final Run-Length Encoding stage, pulling new bytes from the + /// Burrows-Wheeler Transform stage when required + /// + /// The array to write to + /// The starting position within the array + /// The number of bytes to read + /// The number of bytes actually read, or -1 if there are no bytes left in the block + public int Read(byte[] destination, int offset, int length) + { + int i; + for (i = 0; i < length; i++, offset++) + { + var decoded = this.Read(); + if (decoded == -1) + return (i == 0) ? -1 : i; + + destination[offset] = (byte)decoded; + } + return i; + } + + public int Read(Stream x, int maxLength) + { + int i; + for (i = 0; i < maxLength; i++) + { + var decoded = this.Read(); + if (decoded == -1) + return (i == 0) ? -1 : i; + + x.WriteByte((byte)decoded); + } + return i; + } + + public int ReadAll(Stream x) + { + int count = 0; + while (true) + { + var decoded = this.Read(); + if (decoded == -1) + return (count == 0) ? -1 : count; + count++; + x.WriteByte((byte)decoded); + } + } + + /// + /// Verify and return the block CRC. This method may only be called after all of the block's bytes have been read + /// + /// The block CRC + /// if the CRC verification failed + public uint CheckCrc() + { + if (this.blockCRC != this.crc.CRC) + throw new IOException("BZip2 block CRC error"); + + return this.crc.CRC; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs new file mode 100644 index 000000000..b121dfa97 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs @@ -0,0 +1,2406 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// DivSufSort suffix array generator + /// Based on libdivsufsort 1.2.3 patched to support BZip2 + /// + /// + /// This is a simple conversion of the original C with two minor bugfixes applied(see "BUGFIX" + /// comments within the class). Documentation within the class is largely absent. + /// + internal class BZip2DivSufSort + { + private class StackEntry + { + readonly public int a; + readonly public int b; + readonly public int c; + readonly public int d; + + public StackEntry(int a, int b, int c, int d) + { + this.a = a; + this.b = b; + this.c = c; + this.d = d; + } + } + + private class PartitionResult + { + readonly public int first; + readonly public int last; + + public PartitionResult(int first, int last) + { + this.first = first; + this.last = last; + } + } + + private class TRBudget + { + private int budget; + public int chance; + + public bool update(int size, int n) + { + + this.budget -= n; + if (this.budget <= 0) + { + if (--this.chance == 0) + { + return false; + } + this.budget += size; + } + + return true; + } + + public TRBudget(int budget, int chance) + { + this.budget = budget; + this.chance = chance; + } + } + + private const int STACK_SIZE = 64; + private const int BUCKET_A_SIZE = 256; + private const int BUCKET_B_SIZE = 65536; + private const int SS_BLOCKSIZE = 1024; + private const int INSERTIONSORT_THRESHOLD = 8; + + private static readonly int[] log2table = + { + -1, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 + }; + + private readonly int[] SA; + private readonly byte[] T; + private readonly int n; + + /// + /// Public constructor + /// + /// T The input array + /// SA The output array + /// The length of the input data + public BZip2DivSufSort(byte[] T, int[] SA, int n) + { + this.T = T; + this.SA = SA; + this.n = n; + } + + /// + /// Performs a Burrows Wheeler Transform on the input array + /// + /// the index of the first character of the input array within the output array + public int BWT() + { + if (this.n == 0) + return 0; + + if (this.n == 1) + { + this.SA[0] = this.T[0]; + return 0; + } + + var bucketA = new int[BZip2DivSufSort.BUCKET_A_SIZE]; + var bucketB = new int[BZip2DivSufSort.BUCKET_B_SIZE]; + + int m = this.sortTypeBstar(bucketA, bucketB); + return 0 < m ? this.constructBWT(bucketA, bucketB) : 0; + } + + // ReSharper disable LoopVariableIsNeverChangedInsideLoop + private static void swapElements (int[] array1, int index1, int[] array2, int index2) + { + var temp = array1[index1]; + array1[index1] = array2[index2]; + array2[index2] = temp; + } + + private int ssCompare (int p1, int p2, int depth) + { + int U1n, U2n; // pointers within T + int U1, U2; + + for ( + U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = this.SA[p1 + 1] + 2, U2n = this.SA[p2 + 1] + 2; + (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); + ++U1, ++U2 + ) { } + + return U1 < U1n ? (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) : (U2 < U2n ? -1 : 0); + } + + private int ssCompareLast (int PA, int p1, int p2, int depth, int size) + { + int U1, U2, U1n, U2n; + + for ( + U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = size, U2n = this.SA[(p2 + 1)] + 2; + (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); + ++U1, ++U2 + ) { } + + if (U1 < U1n) + return (U2 < U2n) ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1; + + if (U2 == U2n) + return 1; + + for ( + U1 = U1 % size, U1n = this.SA[PA] + 2; + (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); + ++U1, ++U2 + ) { } + + return U1 < U1n ? + (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) + : (U2 < U2n ? -1 : 0); + } + + private void ssInsertionSort (int PA, int first, int last, int depth) + { + int i; // pointer within SA + + for (i = last - 2; first <= i; --i) + { + int j; // pointer within SA + int t; + int r; + for (t = this.SA[i], j = i + 1; 0 < (r = this.ssCompare (PA + t, PA + this.SA[j], depth));) + { + do + { + this.SA[j - 1] = this.SA[j]; + } while ((++j < last) && (this.SA[j] < 0)); + if (last <= j) + { + break; + } + } + if (r == 0) + { + this.SA[j] = ~this.SA[j]; + } + this.SA[j - 1] = t; + } + } + + private void ssFixdown (int Td, int PA, int sa, int i, int size) + { + int j, k; + int v; + int c; + + for (v = this.SA[sa + i], c = (this.T[Td + this.SA[PA + v]]) & 0xff; (j = 2 * i + 1) < size; this.SA[sa + i] = this.SA[sa + k], i = k) + { + int d = this.T[Td + this.SA[PA + this.SA[sa + (k = j++)]]] & 0xff; + int e; + if (d < (e = this.T[Td + this.SA[PA + this.SA[sa + j]]] & 0xff)) + { + k = j; + d = e; + } + if (d <= c) break; + } + this.SA[sa + i] = v; + } + + private void ssHeapSort (int Td, int PA, int sa, int size) + { + int i; + + int m = size; + if ((size % 2) == 0) + { + m--; + if ((this.T[Td + this.SA[PA + this.SA[sa + (m / 2)]]] & 0xff) < (this.T[Td + this.SA[PA + this.SA[sa + m]]] & 0xff)) + { + swapElements (this.SA, sa + m, this.SA, sa + (m / 2)); + } + } + + for (i = m / 2 - 1; 0 <= i; --i) + { + this.ssFixdown (Td, PA, sa, i, m); + } + + if ((size % 2) == 0) + { + swapElements (this.SA, sa, this.SA, sa + m); + this.ssFixdown (Td, PA, sa, 0, m); + } + + for (i = m - 1; 0 < i; --i) + { + int t = this.SA[sa]; + this.SA[sa] = this.SA[sa + i]; + this.ssFixdown (Td, PA, sa, 0, i); + this.SA[sa + i] = t; + } + } + + private int ssMedian3 (int Td, int PA, int v1, int v2, int v3) + { + var T_v1 = this.T[Td + this.SA[PA + this.SA[v1]]] & 0xff; + var T_v2 = this.T[Td + this.SA[PA + this.SA[v2]]] & 0xff; + var T_v3 = this.T[Td + this.SA[PA + this.SA[v3]]] & 0xff; + + if (T_v1 > T_v2) + { + var temp = v1; + v1 = v2; + v2 = temp; + var T_vtemp = T_v1; + T_v1 = T_v2; + T_v2 = T_vtemp; + } + if (T_v2 > T_v3) + { + return T_v1 > T_v3 ? v1 : v3; + } + return v2; + } + + private int ssMedian5 (int Td, int PA, int v1, int v2, int v3, int v4, int v5) + { + var T_v1 = this.T[Td + this.SA[PA + this.SA[v1]]] & 0xff; + var T_v2 = this.T[Td + this.SA[PA + this.SA[v2]]] & 0xff; + var T_v3 = this.T[Td + this.SA[PA + this.SA[v3]]] & 0xff; + var T_v4 = this.T[Td + this.SA[PA + this.SA[v4]]] & 0xff; + var T_v5 = this.T[Td + this.SA[PA + this.SA[v5]]] & 0xff; + int temp; + int T_vtemp; + + // ReSharper disable RedundantAssignment + if (T_v2 > T_v3) + { + temp = v2; + v2 = v3; + v3 = temp; + T_vtemp = T_v2; + T_v2 = T_v3; + T_v3 = T_vtemp; + } + if (T_v4 > T_v5) + { + temp = v4; + v4 = v5; + v5 = temp; + T_vtemp = T_v4; + T_v4 = T_v5; + T_v5 = T_vtemp; + } + if (T_v2 > T_v4) + { + temp = v2; + v2 = v4; + v4 = temp; + T_vtemp = T_v2; + T_v2 = T_v4; + T_v4 = T_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + T_vtemp = T_v3; + T_v3 = T_v5; + T_v5 = T_vtemp; + } + if (T_v1 > T_v3) + { + temp = v1; + v1 = v3; + v3 = temp; + T_vtemp = T_v1; + T_v1 = T_v3; + T_v3 = T_vtemp; + } + if (T_v1 > T_v4) + { + temp = v1; + v1 = v4; + v4 = temp; + T_vtemp = T_v1; + T_v1 = T_v4; + T_v4 = T_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + T_vtemp = T_v3; + T_v3 = T_v5; + T_v5 = T_vtemp; + } + // ReSharper restore RedundantAssignment + + return T_v3 > T_v4 ? v4 : v3; + } + + private int ssPivot (int Td, int PA, int first, int last) + { + int t = last - first; + int middle = first + t / 2; + + if (t <= 512) + { + if (t <= 32) + { + return this.ssMedian3 (Td, PA, first, middle, last - 1); + } + t >>= 2; + return this.ssMedian5 (Td, PA, first, first + t, middle, last - 1 - t, last - 1); + } + t >>= 3; + return this.ssMedian3 ( + Td, PA, + this.ssMedian3 (Td, PA, first, first + t, first + (t << 1)), + this.ssMedian3 (Td, PA, middle - t, middle, middle + t), + this.ssMedian3 (Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1) + ); + } + + private static int ssLog (int x) + { + return ((x & 0xff00) != 0) ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] : BZip2DivSufSort.log2table[x & 0xff]; + } + + private int ssSubstringPartition (int PA, int first, int last, int depth) + { + int a, b; + + for (a = first - 1, b = last;;) + { + for (; (++a < b) && ((this.SA[PA + this.SA[a]] + depth) >= (this.SA[PA + this.SA[a] + 1] + 1));) + { + this.SA[a] = ~this.SA[a]; + } + for (; (a < --b) && ((this.SA[PA + this.SA[b]] + depth) < (this.SA[PA + this.SA[b] + 1] + 1));) { } + if (b <= a) + { + break; + } + int t = ~this.SA[b]; + this.SA[b] = this.SA[a]; + this.SA[a] = t; + } + if (first < a) + this.SA[first] = ~this.SA[first]; + + return a; + } + + private void ssMultiKeyIntroSort (int PA, int first, int last, int depth) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int ssize; + int limit; + int x = 0; + + for (ssize = 0, limit = ssLog (last - first);;) + { + if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) + { + if (1 < (last - first)) + { + this.ssInsertionSort (PA, first, last, depth); + } + if (ssize == 0) return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + depth = entry.c; + limit = entry.d; + continue; + } + + int Td = depth; + if (limit-- == 0) + { + this.ssHeapSort (Td, PA, first, last - first); + } + int a; + int v; + if (limit < 0) + { + for (a = first + 1, v = this.T[Td + this.SA[PA + this.SA[first]]] & 0xff; a < last; ++a) + { + if ((x = (this.T[Td + this.SA[PA + this.SA[a]]] & 0xff)) != v) + { + if (1 < (a - first)) { break; } + v = x; + first = a; + } + } + if ((this.T[Td + this.SA[PA + this.SA[first]] - 1] & 0xff) < v) + { + first = this.ssSubstringPartition (PA, first, a, depth); + } + if ((a - first) <= (last - a)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry (a, last, depth, -1); + last = a; + depth += 1; + limit = ssLog (a - first); + } else + { + first = a; + limit = -1; + } + } else + { + if (1 < (last - a)) + { + stack[ssize++] = new StackEntry (first, a, depth + 1, ssLog (a - first)); + first = a; + limit = -1; + } else + { + last = a; + depth += 1; + limit = ssLog (a - first); + } + } + continue; + } + + a = this.ssPivot (Td, PA, first, last); + v = this.T[Td + this.SA[PA + this.SA[a]]] & 0xff; + swapElements (this.SA, first, this.SA, a); + + int b; + for (b = first; (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) == v); ) { } + if (((a = b) < last) && (x < v)) + { + for (; (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + } + int c; + for (c = last; (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) == v);) { } + int d; + if ((b < (d = c)) && (x > v)) + { + for (; (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c;) + { + swapElements (this.SA, b, this.SA, c); + for (; (++b < c) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + for (; (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + + int s; + int t; + if ((s = a - first) > (t = b - a)) + { + s = t; + } + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + + a = first + (b - a); + c = last - (d - c); + b = (v <= (this.T[Td + this.SA[PA + this.SA[a]] - 1] & 0xff)) ? a : this.ssSubstringPartition (PA, a, c, depth); + + if ((a - first) <= (last - c)) + { + if ((last - c) <= (c - b)) + { + stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); + stack[ssize++] = new StackEntry (c, last, depth, limit); + last = a; + } else if ((a - first) <= (c - b)) + { + stack[ssize++] = new StackEntry (c, last, depth, limit); + stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); + last = a; + } else + { + stack[ssize++] = new StackEntry (c, last, depth, limit); + stack[ssize++] = new StackEntry (first, a, depth, limit); + first = b; + last = c; + depth += 1; + limit = ssLog (c - b); + } + } else + { + if ((a - first) <= (c - b)) + { + stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); + stack[ssize++] = new StackEntry (first, a, depth, limit); + first = c; + } else if ((last - c) <= (c - b)) + { + stack[ssize++] = new StackEntry (first, a, depth, limit); + stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); + first = c; + } else + { + stack[ssize++] = new StackEntry (first, a, depth, limit); + stack[ssize++] = new StackEntry (c, last, depth, limit); + first = b; + last = c; + depth += 1; + limit = ssLog (c - b); + } + } + } else + { + limit += 1; + if ((this.T[Td + this.SA[PA + this.SA[first]] - 1] & 0xff) < v) + { + first = this.ssSubstringPartition (PA, first, last, depth); + limit = ssLog (last - first); + } + depth += 1; + } + } + } + + private static void ssBlockSwap (int[] array1, int first1, int[] array2, int first2, int size) + { + int a, b; + int i; + for (i = size, a = first1, b = first2; 0 < i; --i, ++a, ++b) + { + swapElements (array1, a, array2, b); + } + } + + private void ssMergeForward (int PA, int[] buf, int bufoffset, int first, int middle, int last, int depth) + { + int i, j, k; + int t; + + int bufend = bufoffset + (middle - first) - 1; + ssBlockSwap (buf, bufoffset, this.SA, first, middle - first); + + for (t = this.SA[first], i = first, j = bufoffset, k = middle;;) + { + int r = this.ssCompare (PA + buf[j], PA + this.SA[k], depth); + if (r < 0) + { + do + { + this.SA[i++] = buf[j]; + if (bufend <= j) + { + buf[j] = t; + return; + } + buf[j++] = this.SA[i]; + } while (buf[j] < 0); + } else if (r > 0) + { + do + { + this.SA[i++] = this.SA[k]; + this.SA[k++] = this.SA[i]; + if (last <= k) + { + while (j < bufend) + { + this.SA[i++] = buf[j]; + buf[j++] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + } while (this.SA[k] < 0); + } else + { + this.SA[k] = ~this.SA[k]; + do + { + this.SA[i++] = buf[j]; + if (bufend <= j) + { + buf[j] = t; + return; + } + buf[j++] = this.SA[i]; + } while (buf[j] < 0); + + do + { + this.SA[i++] = this.SA[k]; + this.SA[k++] = this.SA[i]; + if (last <= k) + { + while (j < bufend) + { + this.SA[i++] = buf[j]; + buf[j++] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + } while (this.SA[k] < 0); + } + } + } + + private void ssMergeBackward (int PA, int[] buf, int bufoffset, int first, int middle, int last, int depth) + { + int p1, p2; + int i, j, k; + int t; + + int bufend = bufoffset + (last - middle); + ssBlockSwap (buf, bufoffset, this.SA, middle, last - middle); + + int x = 0; + if (buf[bufend - 1] < 0) + { + x |= 1; + p1 = PA + ~buf[bufend - 1]; + } else + { + p1 = PA + buf[bufend - 1]; + } + if (this.SA[middle - 1] < 0) + { + x |= 2; + p2 = PA + ~this.SA[middle - 1]; + } else + { + p2 = PA + this.SA[middle - 1]; + } + for (t = this.SA[last - 1], i = last - 1, j = bufend - 1, k = middle - 1;;) + { + int r = this.ssCompare (p1, p2, depth); + if (r > 0) + { + if ((x & 1) != 0) + { + do + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } while (buf[j] < 0); + x ^= 1; + } + this.SA[i--] = buf[j]; + if (j <= bufoffset) + { + buf[j] = t; + return; + } + buf[j--] = this.SA[i]; + + if (buf[j] < 0) + { + x |= 1; + p1 = PA + ~buf[j]; + } else + { + p1 = PA + buf[j]; + } + } else if (r < 0) + { + if ((x & 2) != 0) + { + do + { + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + } while (this.SA[k] < 0); + x ^= 2; + } + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + if (k < first) + { + while (bufoffset < j) + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + + if (this.SA[k] < 0) + { + x |= 2; + p2 = PA + ~this.SA[k]; + } else + { + p2 = PA + this.SA[k]; + } + } else + { + if ((x & 1) != 0) + { + do + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } while (buf[j] < 0); + x ^= 1; + } + this.SA[i--] = ~buf[j]; + if (j <= bufoffset) + { + buf[j] = t; + return; + } + buf[j--] = this.SA[i]; + + if ((x & 2) != 0) + { + do + { + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + } while (this.SA[k] < 0); + x ^= 2; + } + this.SA[i--] = this.SA[k]; + this.SA[k--] = this.SA[i]; + if (k < first) + { + while (bufoffset < j) + { + this.SA[i--] = buf[j]; + buf[j--] = this.SA[i]; + } + this.SA[i] = buf[j]; + buf[j] = t; + return; + } + + if (buf[j] < 0) + { + x |= 1; + p1 = PA + ~buf[j]; + } else + { + p1 = PA + buf[j]; + } + if (this.SA[k] < 0) + { + x |= 2; + p2 = PA + ~this.SA[k]; + } else + { + p2 = PA + this.SA[k]; + } + } + } + } + + private static int getIDX (int a) + { + return (0 <= a) ? a : ~a; + } + + private void ssMergeCheckEqual (int PA, int depth, int a) + { + if ((0 <= this.SA[a]) && (this.ssCompare (PA + getIDX (this.SA[a - 1]), PA + this.SA[a], depth) == 0)) + { + this.SA[a] = ~this.SA[a]; + } + } + + private void ssMerge (int PA, int first, int middle, int last, int[] buf, int bufoffset, int bufsize, int depth) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int ssize; + int check; + + for (check = 0, ssize = 0;;) + { + if ((last - middle) <= bufsize) + { + if ((first < middle) && (middle < last)) + { + this.ssMergeBackward (PA, buf, bufoffset, first, middle, last, depth); + } + + if ((check & 1) != 0) + { + this.ssMergeCheckEqual (PA, depth, first); + } + if ((check & 2) != 0) + { + this.ssMergeCheckEqual (PA, depth, last); + } + if (ssize == 0) return; + var entry = stack[--ssize]; + first = entry.a; + middle = entry.b; + last = entry.c; + check = entry.d; + continue; + } + + if ((middle - first) <= bufsize) + { + if (first < middle) + { + this.ssMergeForward ( PA, buf, bufoffset, first, middle, last, depth); + } + if ((check & 1) != 0) + { + this.ssMergeCheckEqual (PA, depth, first); + } + if ((check & 2) != 0) + { + this.ssMergeCheckEqual (PA, depth, last); + } + if (ssize == 0) return; + var entry = stack[--ssize]; + first = entry.a; + middle = entry.b; + last = entry.c; + check = entry.d; + continue; + } + + int m; + int len; + int half; + for ( + m = 0, len = Math.Min (middle - first, last - middle), half = len >> 1; + 0 < len; + len = half, half >>= 1 + ) + { + if (this.ssCompare (PA + getIDX (this.SA[middle + m + half]), + PA + getIDX (this.SA[middle - m - half - 1]), depth) < 0) + { + m += half + 1; + half -= (len & 1) ^ 1; + } + } + + if (0 < m) + { + ssBlockSwap (this.SA, middle - m, this.SA, middle, m); + int j; + int i = j = middle; + int next = 0; + if ((middle + m) < last) + { + if (this.SA[middle + m] < 0) + { + for (; this.SA[i - 1] < 0; --i) + { } + this.SA[middle + m] = ~this.SA[middle + m]; + } + for (j = middle; this.SA[j] < 0; ++j) + { } + next = 1; + } + if ((i - first) <= (last - j)) + { + stack[ssize++] = new StackEntry (j, middle + m, last, (check & 2) | (next & 1)); + middle -= m; + last = i; + check = (check & 1); + } else + { + if ((i == middle) && (middle == j)) + { + next <<= 1; + } + stack[ssize++] = new StackEntry (first, middle - m, i, (check & 1) | (next & 2)); + first = j; + middle += m; + check = (check & 2) | (next & 1); + } + } else + { + if ((check & 1) != 0) + { + this.ssMergeCheckEqual (PA, depth, first); + } + this.ssMergeCheckEqual (PA, depth, middle); + if ((check & 2) != 0) + { + this.ssMergeCheckEqual (PA, depth, last); + } + if (ssize == 0) return; + var entry = stack[--ssize]; + first = entry.a; + middle = entry.b; + last = entry.c; + check = entry.d; + } + } + } + + private void subStringSort (int PA, int first, int last, int[] buf, int bufoffset, int bufsize, int depth, bool lastsuffix, int size) + { + int a; + int i; + int k; + + if (lastsuffix) + ++first; + + for (a = first, i = 0; (a + BZip2DivSufSort.SS_BLOCKSIZE) < last; a += BZip2DivSufSort.SS_BLOCKSIZE, ++i) + { + this.ssMultiKeyIntroSort (PA, a, a + BZip2DivSufSort.SS_BLOCKSIZE, depth); + int[] curbuf = this.SA; + int curbufoffset = a + BZip2DivSufSort.SS_BLOCKSIZE; + int curbufsize = last - (a + BZip2DivSufSort.SS_BLOCKSIZE); + if (curbufsize <= bufsize) + { + curbufsize = bufsize; + curbuf = buf; + curbufoffset = bufoffset; + } + int b; + int j; + for (b = a, k = BZip2DivSufSort.SS_BLOCKSIZE, j = i; (j & 1) != 0; b -= k, k <<= 1, j >>= 1) + this.ssMerge (PA, b - k, b, b + k, curbuf, curbufoffset, curbufsize, depth); + } + + this.ssMultiKeyIntroSort (PA, a, last, depth); + + for (k = BZip2DivSufSort.SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) + { + if ((i & 1) != 0) + { + this.ssMerge (PA, a - k, a, last, buf, bufoffset, bufsize, depth); + a -= k; + } + } + + if (lastsuffix) + { + int r; + for ( + a = first, i = this.SA[first - 1], r = 1; + (a < last) && ((this.SA[a] < 0) || (0 < (r = this.ssCompareLast (PA, PA + i, PA + this.SA[a], depth, size)))); + ++a + ) + { + this.SA[a - 1] = this.SA[a]; + } + if (r == 0) + { + this.SA[a] = ~this.SA[a]; + } + this.SA[a - 1] = i; + } + } + + private int trGetC (int ISA, int ISAd, int ISAn, int p) + { + return (((ISAd + p) < ISAn) ? this.SA[ISAd + p] : this.SA[ISA + ((ISAd - ISA + p) % (ISAn - ISA))]); + } + + private void trFixdown (int ISA, int ISAd, int ISAn, int sa, int i, int size) + { + int j, k; + int v; + int c; + + for (v = this.SA[sa + i], c = this.trGetC (ISA, ISAd, ISAn, v); (j = 2 * i + 1) < size; this.SA[sa + i] = this.SA[sa + k], i = k) + { + k = j++; + int d = this.trGetC (ISA, ISAd, ISAn, this.SA[sa + k]); + int e; + if (d < (e = this.trGetC (ISA, ISAd, ISAn, this.SA[sa + j]))) + { + k = j; + d = e; + } + if (d <= c) + { + break; + } + } + this.SA[sa + i] = v; + } + + private void trHeapSort (int ISA, int ISAd, int ISAn, int sa, int size) + { + int i; + + int m = size; + if ((size % 2) == 0) + { + m--; + if (this.trGetC (ISA, ISAd, ISAn, this.SA[sa + (m / 2)]) < this.trGetC (ISA, ISAd, ISAn, this.SA[sa + m])) + { + swapElements (this.SA, sa + m, this.SA, sa + (m / 2)); + } + } + + for (i = m / 2 - 1; 0 <= i; --i) + { + this.trFixdown (ISA, ISAd, ISAn, sa, i, m); + } + + if ((size % 2) == 0) + { + swapElements (this.SA, sa + 0, this.SA, sa + m); + this.trFixdown (ISA, ISAd, ISAn, sa, 0, m); + } + + for (i = m - 1; 0 < i; --i) + { + int t = this.SA[sa + 0]; + this.SA[sa + 0] = this.SA[sa + i]; + this.trFixdown (ISA, ISAd, ISAn, sa, 0, i); + this.SA[sa + i] = t; + } + } + + private void trInsertionSort (int ISA, int ISAd, int ISAn, int first, int last) + { + int a; + + for (a = first + 1; a < last; ++a) + { + int b; + int t; + int r; + for (t = this.SA[a], b = a - 1; 0 > (r = this.trGetC (ISA, ISAd, ISAn, t) - this.trGetC (ISA, ISAd, ISAn, this.SA[b]));) + { + do + { + this.SA[b + 1] = this.SA[b]; + } while ((first <= --b) && (this.SA[b] < 0)); + if (b < first) + { + break; + } + } + if (r == 0) + { + this.SA[b] = ~this.SA[b]; + } + this.SA[b + 1] = t; + } + } + + private static int trLog (int x) + { + return ((x & 0xffff0000) != 0) ? + (((x & 0xff000000) != 0) ? 24 + BZip2DivSufSort.log2table[(x >> 24) & 0xff] : 16 + BZip2DivSufSort.log2table[(x >> 16) & 0xff]) + : (((x & 0x0000ff00) != 0) ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] : 0 + BZip2DivSufSort.log2table[(x >> 0) & 0xff]); + } + + private int trMedian3 (int ISA, int ISAd, int ISAn, int v1, int v2, int v3) + { + var SA_v1 = this.trGetC (ISA, ISAd, ISAn, this.SA[v1]); + var SA_v2 = this.trGetC (ISA, ISAd, ISAn, this.SA[v2]); + var SA_v3 = this.trGetC (ISA, ISAd, ISAn, this.SA[v3]); + + if (SA_v1 > SA_v2) + { + var temp = v1; + v1 = v2; + v2 = temp; + var SA_vtemp = SA_v1; + SA_v1 = SA_v2; + SA_v2 = SA_vtemp; + } + if (SA_v2 > SA_v3) + { + return SA_v1 > SA_v3 ? v1 : v3; + } + + return v2; + } + + private int trMedian5 (int ISA, int ISAd, int ISAn, int v1, int v2, int v3, int v4, int v5) + { + var SA_v1 = this.trGetC (ISA, ISAd, ISAn, this.SA[v1]); + var SA_v2 = this.trGetC (ISA, ISAd, ISAn, this.SA[v2]); + var SA_v3 = this.trGetC (ISA, ISAd, ISAn, this.SA[v3]); + var SA_v4 = this.trGetC (ISA, ISAd, ISAn, this.SA[v4]); + var SA_v5 = this.trGetC (ISA, ISAd, ISAn, this.SA[v5]); + int temp; + int SA_vtemp; + + // ReSharper disable RedundantAssignment + if (SA_v2 > SA_v3) + { + temp = v2; + v2 = v3; + v3 = temp; + SA_vtemp = SA_v2; + SA_v2 = SA_v3; + SA_v3 = SA_vtemp; + } + if (SA_v4 > SA_v5) + { + temp = v4; + v4 = v5; + v5 = temp; + SA_vtemp = SA_v4; + SA_v4 = SA_v5; + SA_v5 = SA_vtemp; + } + if (SA_v2 > SA_v4) + { + temp = v2; + v2 = v4; + v4 = temp; + SA_vtemp = SA_v2; + SA_v2 = SA_v4; + SA_v4 = SA_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + SA_vtemp = SA_v3; + SA_v3 = SA_v5; + SA_v5 = SA_vtemp; + } + if (SA_v1 > SA_v3) + { + temp = v1; + v1 = v3; + v3 = temp; + SA_vtemp = SA_v1; + SA_v1 = SA_v3; + SA_v3 = SA_vtemp; + } + if (SA_v1 > SA_v4) + { + temp = v1; + v1 = v4; + v4 = temp; + SA_vtemp = SA_v1; + SA_v1 = SA_v4; + SA_v4 = SA_vtemp; + temp = v3; + v3 = v5; + v5 = temp; + SA_vtemp = SA_v3; + SA_v3 = SA_v5; + SA_v5 = SA_vtemp; + } + // ReSharper restore RedundantAssignment + + return SA_v3 > SA_v4 ? v4 : v3; + } + + private int trPivot (int ISA, int ISAd, int ISAn, int first, int last) + { + int t = last - first; + int middle = first + t / 2; + + if (t <= 512) + { + if (t <= 32) + { + return this.trMedian3 (ISA, ISAd, ISAn, first, middle, last - 1); + } + t >>= 2; + return this.trMedian5 ( + ISA, ISAd, ISAn, + first, first + t, + middle, + last - 1 - t, last - 1 + ); + } + t >>= 3; + return this.trMedian3 ( + ISA, ISAd, ISAn, + this.trMedian3 (ISA, ISAd, ISAn, first, first + t, first + (t << 1)), + this.trMedian3 (ISA, ISAd, ISAn, middle - t, middle, middle + t), + this.trMedian3 (ISA, ISAd, ISAn, last - 1 - (t << 1), last - 1 - t, last - 1) + ); + } + + private void lsUpdateGroup (int ISA, int first, int last) + { + int a; + + for (a = first; a < last; ++a) + { + int b; + if (0 <= this.SA[a]) + { + b = a; + do + { + this.SA[ISA + this.SA[a]] = a; + } while ((++a < last) && (0 <= this.SA[a])); + this.SA[b] = b - a; + if (last <= a) + { + break; + } + } + b = a; + do + { + this.SA[a] = ~this.SA[a]; + } while (this.SA[++a] < 0); + int t = a; + do + { + this.SA[ISA + this.SA[b]] = t; + } while (++b <= a); + } + } + + private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int limit; + int x = 0; + int ssize; + + for (ssize = 0, limit = trLog (last - first);;) + { + + if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) + { + if (1 < (last - first)) + { + this.trInsertionSort (ISA, ISAd, ISAn, first, last); + this.lsUpdateGroup (ISA, first, last); + } else if ((last - first) == 1) + { + this.SA[first] = -1; + } + if (ssize == 0) return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + limit = entry.c; + continue; + } + + int a; + int b; + if (limit-- == 0) + { + this.trHeapSort (ISA, ISAd, ISAn, first, last - first); + for (a = last - 1; first < a; a = b) + { + for ( + x = this.trGetC (ISA, ISAd, ISAn, this.SA[a]), b = a - 1; + (first <= b) && (this.trGetC (ISA, ISAd, ISAn, this.SA[b]) == x); + --b + ) + { + this.SA[b] = ~this.SA[b]; + } + } + this.lsUpdateGroup (ISA, first, last); + if (ssize == 0) return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + limit = entry.c; + continue; + } + + a = this.trPivot (ISA, ISAd, ISAn, first, last); + swapElements (this.SA, first, this.SA, a); + int v = this.trGetC (ISA, ISAd, ISAn, this.SA[first]); + + for (b = first; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) == v);) + { } + if (((a = b) < last) && (x < v)) + { + for (; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + } + int c; + for (c = last; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) == v);) + { } + int d; + if ((b < (d = c)) && (x > v)) + { + for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c;) + { + swapElements (this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + + int s; + int t; + if ((s = a - first) > (t = b - a)) + { + s = t; + } + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + + a = first + (b - a); + b = last - (d - c); + + for (c = first, v = a - 1; c < a; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + if (b < last) + { + for (c = a, v = b - 1; c < b; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + if ((b - a) == 1) + { + this.SA[a] = - 1; + } + + if ((a - first) <= (last - b)) + { + if (first < a) + { + stack[ssize++] = new StackEntry (b, last, limit, 0); + last = a; + } else + { + first = b; + } + } else + { + if (b < last) + { + stack[ssize++] = new StackEntry (first, a, limit, 0); + first = b; + } else + { + last = a; + } + } + } else + { + if (ssize == 0) return; + var entry = stack[--ssize]; + first = entry.a; + last = entry.b; + limit = entry.c; + } + } + } + + private void lsSort (int ISA, int x, int depth) + { + int ISAd; + + for (ISAd = ISA + depth; -x < this.SA[0]; ISAd += (ISAd - ISA)) + { + int first = 0; + int skip = 0; + int last; + int t; + do + { + if ((t = this.SA[first]) < 0) + { + first -= t; + skip += t; + } else + { + if (skip != 0) + { + this.SA[first + skip] = skip; + skip = 0; + } + last = this.SA[ISA + t] + 1; + this.lsIntroSort (ISA, ISAd, ISA + x, first, last); + first = last; + } + } while (first < x); + + if (skip != 0) + this.SA[first + skip] = skip; + + if (x < (ISAd - ISA)) + { + first = 0; + do + { + if ((t = this.SA[first]) < 0) + first -= t; + else + { + last = this.SA[ISA + t] + 1; + int i; + for (i = first; i < last; ++i) + this.SA[ISA + this.SA[i]] = i; + first = last; + } + } while (first < x); + break; + } + } + } + + private PartitionResult trPartition (int ISA, int ISAd, int ISAn, int first, int last, int v) + { + int a, b, c, d; + var x = 0; + + for (b = first - 1; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) == v);) + { } + if (((a = b) < last) && (x < v)) + { + for (; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + } + for (c = last; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) == v);) + { } + if ((b < (d = c)) && (x > v)) + { + for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c;) + { + swapElements (this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + int t; + int s; + if ((s = a - first) > (t = b - a)) + s = t; + + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + first += (b - a); + last -= (d - c); + } + + return new PartitionResult (first, last); + } + + private void trCopy (int ISA, int ISAn, int first, int a, int b, int last, int depth) + { + int c, d, e; + int s; + int v = b - 1; + + for (c = first, d = a - 1; c <= d; ++c) + { + if ((s = this.SA[c] - depth) < 0) + { + s += ISAn - ISA; + } + if (this.SA[ISA + s] == v) + { + this.SA[++d] = s; + this.SA[ISA + s] = d; + } + } + for (c = last - 1, e = d + 1, d = b; e < d; --c) + { + if ((s = this.SA[c] - depth) < 0) + { + s += ISAn - ISA; + } + if (this.SA[ISA + s] == v) + { + this.SA[--d] = s; + this.SA[ISA + s] = d; + } + } + } + + private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRBudget budget, int size) + { + var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; + + int s; + int x = 0; + int limit; + int ssize; + + for (ssize = 0, limit = trLog (last - first);;) + { + int a; + int b; + int c; + int v; + int next; + if (limit < 0) + { + if (limit == -1) + { + if (!budget.update (size, last - first)) break; + var result = this.trPartition (ISA, ISAd - 1, ISAn, first, last, last - 1); + a = result.first; + b = result.last; + if ((first < a) || (b < last)) + { + if (a < last) + { + for (c = first, v = a - 1; c < a; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + if (b < last) + { + for (c = a, v = b - 1; c < b; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + + stack[ssize++] = new StackEntry (0, a, b, 0); + stack[ssize++] = new StackEntry (ISAd - 1, first, last, -2); + if ((a - first) <= (last - b)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry (ISAd, b, last, trLog (last - b)); + last = a; + limit = trLog (a - first); + } else if (1 < (last - b)) + { + first = b; + limit = trLog (last - b); + } else + { + if (ssize == 0) return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } else + { + if (1 < (last - b)) + { + stack[ssize++] = new StackEntry (ISAd, first, a, trLog (a - first)); + first = b; + limit = trLog (last - b); + } else if (1 < (a - first)) + { + last = a; + limit = trLog (a - first); + } else + { + if (ssize == 0) return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } + } else + { + for (c = first; c < last; ++c) + { + this.SA[ISA + this.SA[c]] = c; + } + if (ssize == 0) return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } else if (limit == -2) + { + a = stack[--ssize].b; + b = stack[ssize].c; + this.trCopy (ISA, ISAn, first, a, b, last, ISAd - ISA); + if (ssize == 0) return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } else + { + if (0 <= this.SA[first]) + { + a = first; + do + { + this.SA[ISA + this.SA[a]] = a; + } while ((++a < last) && (0 <= this.SA[a])); + first = a; + } + if (first < last) + { + a = first; + do + { + this.SA[a] = ~this.SA[a]; + } while (this.SA[++a] < 0); + next = (this.SA[ISA + this.SA[a]] != this.SA[ISAd + this.SA[a]]) ? trLog (a - first + 1) : -1; + if (++a < last) + { + for (b = first, v = a - 1; b < a; ++b) + { + this.SA[ISA + this.SA[b]] = v; + } + } + + if ((a - first) <= (last - a)) + { + stack[ssize++] = new StackEntry (ISAd, a, last, -3); + ISAd += 1; + last = a; + limit = next; + } else + { + if (1 < (last - a)) + { + stack[ssize++] = new StackEntry (ISAd + 1, first, a, next); + first = a; + limit = -3; + } else + { + ISAd += 1; + last = a; + limit = next; + } + } + } else + { + if (ssize == 0) return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } + continue; + } + + if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) + { + if (!budget.update (size, last - first)) break; + this.trInsertionSort (ISA, ISAd, ISAn, first, last); + limit = -3; + continue; + } + + if (limit-- == 0) + { + if (!budget.update (size, last - first)) break; + this.trHeapSort (ISA, ISAd, ISAn, first, last - first); + for (a = last - 1; first < a; a = b) + { + for ( + x = this.trGetC (ISA, ISAd, ISAn, this.SA[a]), b = a - 1; + (first <= b) && (this.trGetC (ISA, ISAd, ISAn, this.SA[b]) == x); + --b + ) + { + this.SA[b] = ~this.SA[b]; + } + } + limit = -3; + continue; + } + + a = this.trPivot (ISA, ISAd, ISAn, first, last); + + swapElements (this.SA, first, this.SA, a); + v = this.trGetC (ISA, ISAd, ISAn, this.SA[first]); + for (b = first; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) == v);) + { } + if (((a = b) < last) && (x < v)) + { + for (; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + } + for (c = last; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) == v);) + { } + int d; + if ((b < (d = c)) && (x > v)) + { + for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + for (; b < c;) + { + swapElements (this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + { + if (x == v) + { + swapElements (this.SA, b, this.SA, a); + ++a; + } + } + for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + { + if (x == v) + { + swapElements (this.SA, c, this.SA, d); + --d; + } + } + } + + if (a <= d) + { + c = b - 1; + + int t; + if ((s = a - first) > (t = b - a)) + { + s = t; + } + int e; + int f; + for (e = first, f = b - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + if ((s = d - c) > (t = last - d - 1)) + { + s = t; + } + for (e = b, f = last - s; 0 < s; --s, ++e, ++f) + { + swapElements (this.SA, e, this.SA, f); + } + + a = first + (b - a); + b = last - (d - c); + next = (this.SA[ISA + this.SA[a]] != v) ? trLog (b - a) : -1; + + for (c = first, v = a - 1; c < a; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + if (b < last) + { + for (c = a, v = b - 1; c < b; ++c) + { + this.SA[ISA + this.SA[c]] = v; + } + } + + if ((a - first) <= (last - b)) + { + if ((last - b) <= (b - a)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry (ISAd, b, last, limit); + last = a; + } else if (1 < (last - b)) + { + stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + first = b; + } else if (1 < (b - a)) + { + ISAd += 1; + first = a; + last = b; + limit = next; + } else + { + if (ssize == 0) return; + var entry = stack[--ssize]; + ISAd = entry.a; + first = entry.b; + last = entry.c; + limit = entry.d; + } + } else if ((a - first) <= (b - a)) + { + if (1 < (a - first)) + { + stack[ssize++] = new StackEntry (ISAd, b, last, limit); + stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + last = a; + } else if (1 < (b - a)) + { + stack[ssize++] = new StackEntry (ISAd, b, last, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } else + { + first = b; + } + } else + { + if (1 < (b - a)) + { + stack[ssize++] = new StackEntry (ISAd, b, last, limit); + stack[ssize++] = new StackEntry (ISAd, first, a, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } else + { + stack[ssize++] = new StackEntry (ISAd, b, last, limit); + last = a; + } + } + } else + { + if ((a - first) <= (b - a)) + { + if (1 < (last - b)) + { + stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry (ISAd, first, a, limit); + first = b; + } else if (1 < (a - first)) + { + stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + last = a; + } else if (1 < (b - a)) + { + ISAd += 1; + first = a; + last = b; + limit = next; + } else + { + stack[ssize++] = new StackEntry (ISAd, first, last, limit); + } + } else if ((last - b) <= (b - a)) + { + if (1 < (last - b)) + { + stack[ssize++] = new StackEntry (ISAd, first, a, limit); + stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + first = b; + } else if (1 < (b - a)) + { + stack[ssize++] = new StackEntry (ISAd, first, a, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } else + { + last = a; + } + } else + { + if (1 < (b - a)) + { + stack[ssize++] = new StackEntry (ISAd, first, a, limit); + stack[ssize++] = new StackEntry (ISAd, b, last, limit); + ISAd += 1; + first = a; + last = b; + limit = next; + } else + { + stack[ssize++] = new StackEntry (ISAd, first, a, limit); + first = b; + } + } + } + } else + { + if (!budget.update (size, last - first)) break; // BUGFIX : Added to prevent an infinite loop in the original code + limit += 1; + ISAd += 1; + } + } + + for (s = 0; s < ssize; ++s) + { + if (stack[s].d == -3) + { + this.lsUpdateGroup (ISA, stack[s].b, stack[s].c); + } + } + } + + private void trSort (int ISA, int x, int depth) + { + int first = 0; + + if (-x < this.SA[0]) + { + var budget = new TRBudget (x, trLog (x) * 2 / 3 + 1); + do + { + int t; + if ((t = this.SA[first]) < 0) + { + first -= t; + } else + { + int last = this.SA[ISA + t] + 1; + if (1 < (last - first)) + { + this.trIntroSort (ISA, ISA + depth, ISA + x, first, last, budget, x); + if (budget.chance == 0) + { + // Switch to Larsson-Sadakane sorting algorithm. + if (0 < first) + { + this.SA[0] = -first; + } + this.lsSort (ISA, x, depth); + break; + } + } + first = last; + + } + } while (first < x); + } + } + + private static int BUCKET_B (int c0, int c1) + { + return (c1 << 8) | c0; + } + + private static int BUCKET_BSTAR (int c0, int c1) + { + return (c0 << 8) | c1; + } + + private int sortTypeBstar (int[] bucketA, int[] bucketB) + { + var tempbuf = new int[256]; + + int i, j, k, t; + int c0, c1; + int flag; + + for (i = 1, flag = 1; i < this.n; ++i) + { + if (this.T[i - 1] != this.T[i]) + { + if ((this.T[i - 1] & 0xff) > (this.T[i] & 0xff)) + { + flag = 0; + } + break; + } + } + i = this.n - 1; + int m = this.n; + + int ti, ti1, t0; + if (((ti = (this.T[i] & 0xff)) < (t0 = (this.T[0] & 0xff))) || ((this.T[i] == this.T[0]) && (flag != 0))) + { + if (flag == 0) + { + ++bucketB[BUCKET_BSTAR (ti, t0)]; + this.SA[--m] = i; + } else + { + ++bucketB[BUCKET_B (ti, t0)]; + } + for (--i; (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); --i) + { + ++bucketB[BUCKET_B (ti, ti1)]; + } + } + + for (; 0 <= i;) + { + do + { + ++bucketA[this.T[i] & 0xff]; + } while ((0 <= --i) && ((this.T[i] & 0xff) >= (this.T[i + 1] & 0xff))); + + if (0 <= i) + { + ++bucketB[BUCKET_BSTAR (this.T[i] & 0xff, this.T[i + 1] & 0xff)]; + this.SA[--m] = i; + for (--i; (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); --i) + { + ++bucketB[BUCKET_B (ti, ti1)]; + } + } + } + m = this.n - m; + if (m == 0) + { + for (i = 0; i < this.n; ++i) + { + this.SA[i] = i; + } + return 0; + } + + for (c0 = 0, i = -1, j = 0; c0 < 256; ++c0) + { + t = i + bucketA[c0]; + bucketA[c0] = i + j; + i = t + bucketB[BUCKET_B (c0, c0)]; + for (c1 = c0 + 1; c1 < 256; ++c1) + { + j += bucketB[BUCKET_BSTAR (c0, c1)]; + bucketB[(c0 << 8) | c1] = j; + i += bucketB[BUCKET_B (c0, c1)]; + } + } + + int PAb = this.n - m; + int ISAb = m; + for (i = m - 2; 0 <= i; --i) + { + t = this.SA[PAb + i]; + c0 = this.T[t] & 0xff; + c1 = this.T[t + 1] & 0xff; + this.SA[--bucketB[BUCKET_BSTAR (c0, c1)]] = i; + } + t = this.SA[PAb + m - 1]; + c0 = this.T[t] & 0xff; + c1 = this.T[t + 1] & 0xff; + this.SA[--bucketB[BUCKET_BSTAR (c0, c1)]] = m - 1; + + int[] buf = this.SA; + int bufoffset = m; + int bufsize = this.n - (2 * m); + if (bufsize <= 256) + { + buf = tempbuf; + bufoffset = 0; + bufsize = 256; + } + + for (c0 = 255, j = m; 0 < j; --c0) + { + for (c1 = 255; c0 < c1; j = i, --c1) + { + i = bucketB[BUCKET_BSTAR (c0, c1)]; + if (1 < (j - i)) + { + this.subStringSort (PAb, i, j, buf, bufoffset, bufsize, 2, this.SA[i] == (m - 1), this.n); + } + } + } + + for (i = m - 1; 0 <= i; --i) + { + if (0 <= this.SA[i]) + { + j = i; + do + { + this.SA[ISAb + this.SA[i]] = i; + } while ((0 <= --i) && (0 <= this.SA[i])); + this.SA[i + 1] = i - j; + if (i <= 0) + { + break; + } + } + j = i; + do + { + this.SA[ISAb + (this.SA[i] = ~this.SA[i])] = j; + } while (this.SA[--i] < 0); + this.SA[ISAb + this.SA[i]] = j; + } + + this.trSort (ISAb, m, 1); + + i = this.n - 1; + j = m; + if (((this.T[i] & 0xff) < (this.T[0] & 0xff)) || ((this.T[i] == this.T[0]) && (flag != 0))) + { + if (flag == 0) + { + this.SA[this.SA[ISAb + --j]] = i; + } + for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) + { } + } + for (; 0 <= i;) + { + for (--i; (0 <= i) && ((this.T[i] & 0xff) >= (this.T[i + 1] & 0xff)); --i) + { } + if (0 <= i) + { + this.SA[this.SA[ISAb + --j]] = i; + for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) + { } + } + } + + for (c0 = 255, i = this.n - 1, k = m - 1; 0 <= c0; --c0) + { + for (c1 = 255; c0 < c1; --c1) + { + t = i - bucketB[BUCKET_B (c0, c1)]; + bucketB[BUCKET_B (c0, c1)] = i + 1; + + for (i = t, j = bucketB[BUCKET_BSTAR (c0, c1)]; j <= k; --i, --k) + { + this.SA[i] = this.SA[k]; + } + } + t = i - bucketB[BUCKET_B (c0, c0)]; + bucketB[BUCKET_B (c0, c0)] = i + 1; + if (c0 < 255) + { + bucketB[BUCKET_BSTAR (c0, c0 + 1)] = t + 1; + } + i = bucketA[c0]; + } + + return m; + } + + private int constructBWT (int[] bucketA, int[] bucketB) + { + int i; + int t = 0; + int s, s1; + int c0, c1, c2 = 0; + var orig = -1; + + for (c1 = 254; 0 <= c1; --c1) + { + int j; + for (i = bucketB[BUCKET_BSTAR (c1, c1 + 1)], j = bucketA[c1 + 1], t = 0, c2 = -1; i <= j; --j) + { + if (0 <= (s1 = s = this.SA[j])) + { + if (--s < 0) + s = this.n - 1; + + if ((c0 = (this.T[s] & 0xff)) <= c1) + { + this.SA[j] = ~s1; + if ((0 < s) && ((this.T[s - 1] & 0xff) > c0)) + { + s = ~s; + } + if (c2 == c0) + { + this.SA[--t] = s; + } else + { + if (0 <= c2) + bucketB[BUCKET_B (c2, c1)] = t; + + this.SA[t = bucketB[BUCKET_B (c2 = c0, c1)] - 1] = s; + } + } + } else + { + this.SA[j] = ~s; + } + } + } + + for (i = 0; i < this.n; ++i) + { + if (0 <= (s1 = s = this.SA[i])) + { + if (--s < 0) + s = this.n - 1; + + if ((c0 = (this.T[s] & 0xff)) >= (this.T[s + 1] & 0xff)) + { + if ((0 < s) && ((this.T[s - 1] & 0xff) < c0)) + s = ~s; + + if (c0 == c2) + { + this.SA[++t] = s; + } else + { + if (c2 != -1) // BUGFIX: Original code can write to bucketA[-1] + bucketA[c2] = t; + this.SA[t = bucketA[c2 = c0] + 1] = s; + } + } + + } else + { + s1 = ~s1; + } + + if (s1 == 0) + { + this.SA[i] = this.T[this.n - 1]; + orig = i; + } else + { + this.SA[i] = this.T[s1 - 1]; + } + } + + return orig; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs new file mode 100644 index 000000000..790cc7cfb --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs @@ -0,0 +1,155 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// A decoder for the BZip2 Huffman coding stage + /// + internal class BZip2HuffmanStageDecoder + { + // The BZip2BitInputStream from which Huffman codes are read + private readonly IBZip2BitInputStream bitInputStream; + + // The longest Huffman code length accepted by the decoder + private const int HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH = 23; + + // The Huffman table number to use for each group of 50 symbols + private readonly byte[] selectors; + + // The minimum code length for each Huffman table + private readonly int[] minimumLengths = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES]; + + /// + /// An array of values for each Huffman table that must be subtracted from the numerical value of + /// a Huffman code of a given bit length to give its canonical code index + /// + private readonly int[,] codeBases = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2]; + + /// + /// An array of values for each Huffman table that gives the highest numerical value of a Huffman + /// code of a given bit length + /// + private readonly int[,] codeLimits = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 1]; + + // A mapping for each Huffman table from canonical code index to output symbol + private readonly int[,] codeSymbols = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE]; + + // The Huffman table for the current group + private int currentTable; + + // The index of the current group within the selectors array + private int groupIndex = -1; + + // The byte position within the current group. A new group is selected every 50 decoded bytes + private int groupPosition = -1; + + /// + /// Public constructor + /// + /// The BZip2BitInputStream from which Huffman codes are read + /// The total number of codes (uniform for each table) + /// The Canonical Huffman code lengths for each table + /// The Huffman table number to use for each group of 50 symbols + public BZip2HuffmanStageDecoder(IBZip2BitInputStream bitInputStream, int alphabetSize, byte[,] tableCodeLengths, byte[] selectors) + { + this.bitInputStream = bitInputStream; + this.selectors = selectors; + this.currentTable = this.selectors[0]; + this.CreateHuffmanDecodingTables(alphabetSize, tableCodeLengths); + } + + /// + /// Decodes and returns the next symbol + /// + /// The decoded symbol + /// if the end of the input stream is reached while decoding + public int NextSymbol() + { + // Move to next group selector if required + if (((++this.groupPosition % BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH) == 0)) + { + this.groupIndex++; + if (this.groupIndex == this.selectors.Length) + throw new IOException("Error decoding BZip2 block"); + + this.currentTable = this.selectors[this.groupIndex] & 0xff; + } + + var codeLength = this.minimumLengths[this.currentTable]; + + // Starting with the minimum bit length for the table, read additional bits one at a time + // until a complete code is recognised + for (uint codeBits = this.bitInputStream.ReadBits(codeLength); codeLength <= BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH; codeLength++) + { + if (codeBits <= this.codeLimits[this.currentTable, codeLength]) + { + // Convert the code to a symbol index and return + return this.codeSymbols[this.currentTable, codeBits - this.codeBases[this.currentTable, codeLength]]; + } + codeBits = (codeBits << 1) | this.bitInputStream.ReadBits(1); + } + + // A valid code was not recognised + throw new IOException("Error decoding BZip2 block"); + } + + /// + /// Constructs Huffman decoding tables from lists of Canonical Huffman code lengths + /// + /// The total number of codes (uniform for each table) + /// The Canonical Huffman code lengths for each table + private void CreateHuffmanDecodingTables (int alphabetSize, byte[,] tableCodeLengths) + { + + for (int table = 0; table < tableCodeLengths.GetLength(0); table++) + { + int minimumLength = BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH; + int maximumLength = 0; + + // Find the minimum and maximum code length for the table + for (int i = 0; i < alphabetSize; i++) + { + maximumLength = Math.Max (tableCodeLengths[table, i], maximumLength); + minimumLength = Math.Min (tableCodeLengths[table, i], minimumLength); + } + this.minimumLengths[table] = minimumLength; + + // Calculate the first output symbol for each code length + for (int i = 0; i < alphabetSize; i++) + { + this.codeBases[table, tableCodeLengths[table, i] + 1]++; + } + for (int i = 1; i < BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2; i++) + { + this.codeBases[table, i] += this.codeBases[table, i - 1]; + } + + // Calculate the first and last Huffman code for each code length (codes at a given length are sequential in value) + for (int i = minimumLength, code = 0; i <= maximumLength; i++) + { + int base1 = code; + code += this.codeBases[table, i + 1] - this.codeBases[table, i]; + this.codeBases[table, i] = base1 - this.codeBases[table, i]; + this.codeLimits[table, i] = code - 1; + code <<= 1; + } + + // Populate the mapping from canonical code index to output symbol + for (int bitLength = minimumLength, codeIndex = 0; bitLength <= maximumLength; bitLength++) + { + for (int symbol = 0; symbol < alphabetSize; symbol++) + { + if (tableCodeLengths[table, symbol] == bitLength) + this.codeSymbols[table, codeIndex++] = symbol; + } + } + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs new file mode 100644 index 000000000..c03334e30 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs @@ -0,0 +1,344 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// An encoder for the BZip2 Huffman encoding stage + /// + internal class BZip2HuffmanStageEncoder + { + // Used in initial Huffman table generation + private const int HUFFMAN_HIGH_SYMBOL_COST = 15; + + // The longest Huffman code length created by the encoder + private const int HUFFMAN_ENCODE_MAXIMUM_CODE_LENGTH = 20; + + // Number of symbols decoded after which a new Huffman table is selected + public const int HUFFMAN_GROUP_RUN_LENGTH = 50; + + // The BZip2BitOutputStream to which the Huffman tables and data is written + private readonly IBZip2BitOutputStream bitOutputStream; + + // The output of the Move To Front Transform and Run Length Encoding[2] stages + private readonly ushort[] mtfBlock; + + // The actual number of values contained in the mtfBlock array + private readonly int mtfLength; + + // The number of unique values in the mtfBlock array + private readonly int mtfAlphabetSize; + + // The global frequencies of values within the mtfBlock array + private readonly int[] mtfSymbolFrequencies; + + // The Canonical Huffman code lengths for each table + private readonly int[,] huffmanCodeLengths; + + // Merged code symbols for each table. The value at each position is ((code length << 24) | code) + private readonly int[,] huffmanMergedCodeSymbols; + + // The selectors for each segment + private readonly byte[] selectors; + + /// + /// Public constructor + /// + /// The BZip2BitOutputStream to write to + /// The MTF block data + /// The actual length of the MTF block + /// The size of the MTF block's alphabet + /// The frequencies the MTF block's symbols + public BZip2HuffmanStageEncoder(IBZip2BitOutputStream bitOutputStream, ushort[] mtfBlock, int mtfLength, int mtfAlphabetSize, int[] mtfSymbolFrequencies) + { + this.bitOutputStream = bitOutputStream; + this.mtfBlock = mtfBlock; + this.mtfSymbolFrequencies = mtfSymbolFrequencies; + this.mtfAlphabetSize = mtfAlphabetSize; + this.mtfLength = mtfLength; + + var totalTables = selectTableCount(mtfLength); + + this.huffmanCodeLengths = new int[totalTables, mtfAlphabetSize]; + this.huffmanMergedCodeSymbols = new int[totalTables, mtfAlphabetSize]; + this.selectors = new byte[(mtfLength + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH - 1) / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH]; + } + + /// + /// Encodes and writes the block data + /// + /// on any I/O error writing the data + public void Encode() + { + // Create optimised selector list and Huffman tables + this.generateHuffmanOptimisationSeeds(); + for (var i = 3; i >= 0; i--) + { + this.optimiseSelectorsAndHuffmanTables(i == 0); + } + this.assignHuffmanCodeSymbols(); + + // Write out the tables and the block data encoded with them + this.writeSelectorsAndHuffmanTables(); + this.writeBlockData(); + } + + /// + /// Selects an appropriate table count for a given MTF length + /// + /// The length to select a table count for + /// The selected table count + private static int selectTableCount (int mtfLength) + { + if (mtfLength >= 2400) + return 6; + if (mtfLength >= 1200) + return 5; + if (mtfLength >= 600) + return 4; + return mtfLength >= 200 ? 3 : 2; + } + + /// + /// Generate a Huffman code length table for a given list of symbol frequencies + /// + /// The total number of symbols + /// The frequencies of the symbols + /// The array to which the generated code lengths should be written + /// + private static void generateHuffmanCodeLengths (int alphabetSize, int[,] symbolFrequencies, int[,] codeLengths, int index) + { + var mergedFrequenciesAndIndices = new int[alphabetSize]; + var sortedFrequencies = new int[alphabetSize]; + + // The Huffman allocator needs its input symbol frequencies to be sorted, but we need to return code lengths in the same order as the + // corresponding frequencies are passed in + + // The symbol frequency and index are merged into a single array of integers - frequency in the high 23 bits, index in the low 9 bits. + // 2^23 = 8,388,608 which is higher than the maximum possible frequency for one symbol in a block + // 2^9 = 512 which is higher than the maximum possible alphabet size (== 258) + // Sorting this array simultaneously sorts the frequencies and leaves a lookup that can be used to cheaply invert the sort + for (int i = 0; i < alphabetSize; i++) + mergedFrequenciesAndIndices[i] = (symbolFrequencies[index, i] << 9) | i; + + Array.Sort(mergedFrequenciesAndIndices); + for (int i = 0; i < alphabetSize; i++) + sortedFrequencies[i] = mergedFrequenciesAndIndices[i] >> 9; + + // Allocate code lengths - the allocation is in place, so the code lengths will be in the sortedFrequencies array afterwards + HuffmanAllocator.AllocateHuffmanCodeLengths (sortedFrequencies, BZip2HuffmanStageEncoder.HUFFMAN_ENCODE_MAXIMUM_CODE_LENGTH); + + // Reverse the sort to place the code lengths in the same order as the symbols whose frequencies were passed in + for (int i = 0; i < alphabetSize; i++) + codeLengths[index, mergedFrequenciesAndIndices[i] & 0x1ff] = sortedFrequencies[i]; + } + + /// + /// Generate initial Huffman code length tables, giving each table a different low cost section + /// of the alphabet that is roughly equal in overall cumulative frequency. Note that the initial + /// tables are invalid for actual Huffman code generation, and only serve as the seed for later + /// iterative optimisation in optimiseSelectorsAndHuffmanTables(int) + /// + private void generateHuffmanOptimisationSeeds () + { + int totalTables = this.huffmanCodeLengths.GetLength(0); + int remainingLength = this.mtfLength; + int lowCostEnd = -1; + + for (int i = 0; i < totalTables; i++) + { + int targetCumulativeFrequency = remainingLength / (totalTables - i); + int lowCostStart = lowCostEnd + 1; + int actualCumulativeFrequency = 0; + + while ((actualCumulativeFrequency < targetCumulativeFrequency) && (lowCostEnd < (this.mtfAlphabetSize - 1))) + { + actualCumulativeFrequency += this.mtfSymbolFrequencies[++lowCostEnd]; + } + + if ((lowCostEnd > lowCostStart) && (i != 0) && (i != (totalTables - 1)) && (((totalTables - i) & 1) == 0)) + actualCumulativeFrequency -= this.mtfSymbolFrequencies[lowCostEnd--]; + + for (var j = 0; j < this.mtfAlphabetSize; j++) + { + if ((j < lowCostStart) || (j > lowCostEnd)) + this.huffmanCodeLengths[i, j] = BZip2HuffmanStageEncoder.HUFFMAN_HIGH_SYMBOL_COST; + } + + remainingLength -= actualCumulativeFrequency; + } + } + + /// + /// Co-optimise the selector list and the alternative Huffman table code lengths. This method is + /// called repeatedly in the hope that the total encoded size of the selectors, the Huffman code + /// lengths and the block data encoded with them will converge towards a minimum.
+ /// If the data is highly incompressible, it is possible that the total encoded size will + /// instead diverge (increase) slightly.
+ /// + /// If true, write out the (final) chosen selectors + private void optimiseSelectorsAndHuffmanTables (bool storeSelectors) + { + int totalTables = this.huffmanCodeLengths.GetLength(0); + var tableFrequencies = new int[totalTables, this.mtfAlphabetSize]; + + int selectorIndex = 0; + + // Find the best table for each group of 50 block bytes based on the current Huffman code lengths + for (int groupStart = 0; groupStart < this.mtfLength;) + { + int groupEnd = Math.Min(groupStart + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, this.mtfLength) - 1; + + // Calculate the cost of this group when encoded by each table + var cost = new int[totalTables]; + for (int i = groupStart; i <= groupEnd; i++) + { + int value = this.mtfBlock[i]; + for (var j = 0; j < totalTables; j++) + { + cost[j] += this.huffmanCodeLengths[j, value]; + } + } + + // Find the table with the least cost for this group + byte bestTable = 0; + var bestCost = cost[0]; + for (byte i = 1 ; i < totalTables; i++) + { + var tableCost = cost[i]; + if (tableCost < bestCost) + { + bestCost = tableCost; + bestTable = i; + } + } + + // Accumulate symbol frequencies for the table chosen for this block + for (int i = groupStart; i <= groupEnd; i++) + { + tableFrequencies[bestTable, this.mtfBlock[i]]++; + } + + // Store a selector indicating the table chosen for this block + if (storeSelectors) + { + this.selectors[selectorIndex++] = bestTable; + } + + groupStart = groupEnd + 1; + } + + // Generate new Huffman code lengths based on the frequencies for each table accumulated in this iteration + for (int i = 0; i < totalTables; i++) + { + generateHuffmanCodeLengths (this.mtfAlphabetSize, tableFrequencies, this.huffmanCodeLengths, i); + } + } + + // Assigns Canonical Huffman codes based on the calculated lengths + private void assignHuffmanCodeSymbols() + { + int totalTables = this.huffmanCodeLengths.GetLength(0); + + for (int i = 0; i < totalTables; i++) + { + int minimumLength = 32; + int maximumLength = 0; + for (var j = 0; j < this.mtfAlphabetSize; j++) + { + var length = this.huffmanCodeLengths[i, j]; + if (length > maximumLength) + { + maximumLength = length; + } + if (length < minimumLength) + { + minimumLength = length; + } + } + + int code = 0; + for (int j = minimumLength; j <= maximumLength; j++) + { + for (var k = 0; k < this.mtfAlphabetSize; k++) + { + if ((this.huffmanCodeLengths[i, k] & 0xff) == j) + { + this.huffmanMergedCodeSymbols[i, k] = (j << 24) | code; + code++; + } + } + code <<= 1; + } + } + } + + /// + /// Write out the selector list and Huffman tables + /// + /// on any I/O error writing the data + private void writeSelectorsAndHuffmanTables() + { + int totalSelectors = this.selectors.Length; + int totalTables = this.huffmanCodeLengths.GetLength(0); + + this.bitOutputStream.WriteBits (3, (uint)totalTables); + this.bitOutputStream.WriteBits (15, (uint)totalSelectors); + + // Write the selectors + var selectorMTF = new MoveToFront(); + for (int i = 0; i < totalSelectors; i++) + { + this.bitOutputStream.WriteUnary (selectorMTF.ValueToFront (this.selectors[i])); + } + + // Write the Huffman tables + for (int i = 0; i < totalTables; i++) + { + var currentLength = this.huffmanCodeLengths[i, 0]; + + this.bitOutputStream.WriteBits (5, (uint)currentLength); + + for (var j = 0; j < this.mtfAlphabetSize; j++) + { + var codeLength = this.huffmanCodeLengths[i, j]; + var value = (currentLength < codeLength) ? 2u : 3u; + var delta = Math.Abs (codeLength - currentLength); + while (delta-- > 0) + { + this.bitOutputStream.WriteBits (2, value); + } + this.bitOutputStream.WriteBoolean (false); + currentLength = codeLength; + } + } + } + + /// + /// Writes out the encoded block data + /// + /// on any I/O error writing the data + private void writeBlockData() + { + int selectorIndex = 0; + + for (int mtfIndex = 0; mtfIndex < this.mtfLength;) + { + int groupEnd = Math.Min (mtfIndex + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, this.mtfLength) - 1; + + int index = this.selectors [selectorIndex++]; + + while (mtfIndex <= groupEnd) + { + var mergedCodeSymbol = this.huffmanMergedCodeSymbols[index, this.mtfBlock[mtfIndex++]]; + this.bitOutputStream.WriteBits(mergedCodeSymbol >> 24, (uint)mergedCodeSymbol); + } + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs new file mode 100644 index 000000000..be28b7fb1 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs @@ -0,0 +1,168 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// An encoder for the BZip2 Move To Front Transform and Run-Length Encoding[2] stages + /// + /// An encoder for the BZip2 Move To Front Transform and Run-Length Encoding[2] stages. + /// Although conceptually these two stages are separate, it is computationally efficient to perform them in one pass. + /// + internal class BZip2MTFAndRLE2StageEncoder + { + // The Burrows-Wheeler transformed block + private readonly int[] bwtBlock; + + // Actual length of the data in the bwtBlock array + private readonly int bwtLength; + + // At each position, true if the byte value with that index is present within the block, otherwise false + private readonly bool[] bwtValuesInUse; + + // The output of the Move To Front Transform and Run-Length Encoding[2] stages + private readonly ushort[] mtfBlock; + + // The global frequencies of values within the mtfBlock array + private readonly int[] mtfSymbolFrequencies = new int[BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE]; + + // Maximum possible Huffman alphabet size + public const int HUFFMAN_MAXIMUM_ALPHABET_SIZE = 258; + + // Huffman symbol used for run-length encoding + public const ushort RLE_SYMBOL_RUNA = 0; + + // Huffman symbol used for run-length encoding + public const ushort RLE_SYMBOL_RUNB = 1; + + // Gets the encoded MTF block + public ushort[] MtfBlock + { + get + { + return this.mtfBlock; + } + } + + // Gets The actual length of the MTF block + public int MtfLength { get; private set; } + + // Gets the size of the MTF block's alphabet + public int MtfAlphabetSize { get; private set; } + + // Gets the frequencies of the MTF block's symbols + public int[] MtfSymbolFrequencies + { + get + { + return this.mtfSymbolFrequencies; + } + } + + /// + /// Public constructor + /// + /// The Burrows Wheeler Transformed block data + /// The actual length of the BWT data + /// The values that are present within the BWT data. For each index, + /// true if that value is present within the data, otherwise false + public BZip2MTFAndRLE2StageEncoder(int[] bwtBlock, int bwtLength, bool[] bwtValuesPresent) + { + this.bwtBlock = bwtBlock; + this.bwtLength = bwtLength; + this.bwtValuesInUse = bwtValuesPresent; + this.mtfBlock = new ushort[bwtLength + 1]; + } + + // Performs the Move To Front transform and Run Length Encoding[1] stages + public void Encode() + { + var huffmanSymbolMap = new byte[256]; + var symbolMTF = new MoveToFront(); + + int totalUniqueValues = 0; + for (var i = 0; i < 256; i++) + { + if (this.bwtValuesInUse[i]) + huffmanSymbolMap[i] = (byte) totalUniqueValues++; + } + + int endOfBlockSymbol = totalUniqueValues + 1; + int mtfIndex = 0; + int repeatCount = 0; + int totalRunAs = 0; + int totalRunBs = 0; + + for (var i = 0; i < this.bwtLength; i++) + { + // Move To Front + int mtfPosition = symbolMTF.ValueToFront(huffmanSymbolMap[this.bwtBlock[i] & 0xff]); + + // Run Length Encode + if (mtfPosition == 0) + { + repeatCount++; + } else + { + if (repeatCount > 0) + { + repeatCount--; + while (true) + { + if ((repeatCount & 1) == 0) + { + this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA; + totalRunAs++; + } else + { + this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB; + totalRunBs++; + } + + if (repeatCount <= 1) + break; + + repeatCount = (repeatCount - 2) >> 1; + } + repeatCount = 0; + } + + this.mtfBlock[mtfIndex++] = (char) (mtfPosition + 1); + this.mtfSymbolFrequencies[mtfPosition + 1]++; + } + } + + if (repeatCount > 0) + { + repeatCount--; + while (true) + { + if ((repeatCount & 1) == 0) + { + this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA; + totalRunAs++; + } else + { + this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB; + totalRunBs++; + } + + if (repeatCount <= 1) + break; + + repeatCount = (repeatCount - 2) >> 1; + } + } + + this.mtfBlock[mtfIndex] = (ushort)endOfBlockSymbol; + this.mtfSymbolFrequencies[endOfBlockSymbol]++; + this.mtfSymbolFrequencies[BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA] += totalRunAs; + this.mtfSymbolFrequencies[BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB] += totalRunBs; + + this.MtfLength = mtfIndex + 1; + this.MtfAlphabetSize = endOfBlockSymbol + 1; + + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs new file mode 100644 index 000000000..2bbffb9bb --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs @@ -0,0 +1,74 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// A CRC32 calculator + /// + internal class CRC32 + { + /// The CRC lookup table + private static readonly uint[] Crc32Lookup = + { + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, + 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, + 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, + 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, + 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, + 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, + 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, + 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, + 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, + 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, + 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, + 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, + 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, + 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, + 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, + 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, + 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, + 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, + 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, + 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, + 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, + 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + }; + + /// The current CRC + private uint crc = 0xffffffff; + + /// Gets the current CRC + public uint CRC => ~this.crc; + + /// Updates the CRC with a single byte + /// The value to update the CRC with + public void UpdateCrc (int value) + { + this.crc = (this.crc << 8) ^ CRC32.Crc32Lookup[((this.crc >> 24) ^ value) & 0xff]; + } + + /// Update the CRC with a sequence of identical bytes + /// The value to update the CRC with + /// The number of bytes + public void UpdateCrc (int value, int count) + { + while (count-- > 0) + { + this.crc = (this.crc << 8) ^ CRC32.Crc32Lookup[((this.crc >> 24) ^ value) & 0xff]; + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs new file mode 100644 index 000000000..2c5ca5502 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs @@ -0,0 +1,201 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// An in-place, length restricted Canonical Huffman code length allocator + /// + /// Based on the algorithm proposed by R.L.Milidiú, A.A.Pessoa and E.S.Laber + /// in "In-place Length-Restricted Prefix Coding" (see: http://www-di.inf.puc-rio.br/~laber/public/spire98.ps) + /// and incorporating additional ideas from the implementation of "shcodec" by Simakov Alexander + /// (see: http://webcenter.ru/~xander/) + /// + internal static class HuffmanAllocator + { + /// Allocates Canonical Huffman code lengths in place based on a sorted frequency array + /// On input, a sorted array of symbol frequencies; On output, an array of Canonical Huffman code lenghts + /// The maximum code length. Must be at least ceil(log2(array.length)) + public static void AllocateHuffmanCodeLengths(int[] array, int maximumLength) + { + switch (array.Length) + { + case 2: + array[1] = 1; + break; + case 1: + array[0] = 1; + return; + } + + // Pass 1 : Set extended parent pointers + SetExtendedParentPointers(array); + + // Pass 2 : Find number of nodes to relocate in order to achieve maximum code length + int nodesToRelocate = FindNodesToRelocate(array, maximumLength); + + // Pass 3 : Generate code lengths + if ((array[0] % array.Length) >= nodesToRelocate) + { + AllocateNodeLengths(array); + } else + { + var insertDepth = maximumLength - SignificantBits(nodesToRelocate - 1); + AllocateNodeLengthsWithRelocation(array, nodesToRelocate, insertDepth); + } + } + + /// + /// + /// + /// The code length array + /// The input position + /// The number of internal nodes to be relocated + /// The smallest k such that nodesToMove <= k <= i and i <= (array[k] % array.length) + private static int First (int[] array, int i, int nodesToMove) + { + int length = array.Length; + int limit = i; + int k = array.Length - 2; + + while ((i >= nodesToMove) && ((array[i] % length) > limit)) + { + k = i; + i -= (limit - i + 1); + } + i = Math.Max (nodesToMove - 1, i); + + while (k > (i + 1)) + { + var temp = (i + k) >> 1; + if ((array[temp] % length) > limit) + k = temp; + else + i = temp; + } + + return k; + } + + /// + /// Fills the code array with extended parent pointers + /// + /// The code length array + private static void SetExtendedParentPointers(int[] array) + { + int length = array.Length; + + array[0] += array[1]; + + for (int headNode = 0, tailNode = 1, topNode = 2; tailNode < (length - 1); tailNode++) + { + int temp; + if ((topNode >= length) || (array[headNode] < array[topNode])) + { + temp = array[headNode]; + array[headNode++] = tailNode; + } else + { + temp = array[topNode++]; + } + + if ((topNode >= length) || ((headNode < tailNode) && (array[headNode] < array[topNode]))) + { + temp += array[headNode]; + array[headNode++] = tailNode + length; + } else + { + temp += array[topNode++]; + } + + array[tailNode] = temp; + } + } + + /// + /// Finds the number of nodes to relocate in order to achieve a given code length limit + /// + /// The code length array + /// The maximum bit length for the generated codes + /// The number of nodes to relocate + private static int FindNodesToRelocate (int[] array, int maximumLength) + { + int currentNode = array.Length - 2; + + for (var currentDepth = 1; (currentDepth < (maximumLength - 1)) && (currentNode > 1); currentDepth++) + currentNode = First (array, currentNode - 1, 0); + + return currentNode; + } + + /// + /// A final allocation pass with no code length limit + /// + /// The code length array + private static void AllocateNodeLengths (int[] array) + { + int firstNode = array.Length - 2; + int nextNode = array.Length - 1; + + for (int currentDepth = 1, availableNodes = 2; availableNodes > 0; currentDepth++) + { + int lastNode = firstNode; + firstNode = First (array, lastNode - 1, 0); + + for (var i = availableNodes - (lastNode - firstNode); i > 0; i--) + array[nextNode--] = currentDepth; + + availableNodes = (lastNode - firstNode) << 1; + } + } + + /// + /// A final allocation pass that relocates nodes in order to achieve a maximum code length limit + /// + /// The code length array + /// The number of internal nodes to be relocated + /// The depth at which to insert relocated nodes + private static void AllocateNodeLengthsWithRelocation (int[] array, int nodesToMove, int insertDepth) + { + int firstNode = array.Length - 2; + int nextNode = array.Length - 1; + int currentDepth = (insertDepth == 1) ? 2 : 1; + int nodesLeftToMove = (insertDepth == 1) ? nodesToMove - 2 : nodesToMove; + + for (int availableNodes = currentDepth << 1; availableNodes > 0; currentDepth++) + { + int lastNode = firstNode; + firstNode = (firstNode <= nodesToMove) ? firstNode : First (array, lastNode - 1, nodesToMove); + + int offset = 0; + if (currentDepth >= insertDepth) + { + offset = Math.Min (nodesLeftToMove, 1 << (currentDepth - insertDepth)); + } else if (currentDepth == (insertDepth - 1)) + { + offset = 1; + if ((array[firstNode]) == lastNode) + firstNode++; + } + + for (var i = availableNodes - (lastNode - firstNode + offset); i > 0; i--) + array[nextNode--] = currentDepth; + + nodesLeftToMove -= offset; + availableNodes = (lastNode - firstNode + offset) << 1; + } + } + + private static int SignificantBits(int x) + { + int n; + for (n = 0; x > 0; n++) + { + x >>= 1; + } + return n; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs new file mode 100644 index 000000000..047985067 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs @@ -0,0 +1,60 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System; +namespace SharpCompress.Compressors.BZip2MT.Algorithm +{ + /// + /// A 256 entry Move To Front transform + /// + internal class MoveToFront + { + /// The Move To Front list + private readonly byte[] mtf; + + /// Public constructor + public MoveToFront() + { + this.mtf = new byte[256]; + for (int i = 0; i < 256; i++) + this.mtf[i] = (byte)i; + } + + /// Moves a value to the head of the MTF list (forward Move To Front transform) + /// The value to move + /// The position the value moved from + public int ValueToFront(byte value) + { + int index = 0; + byte temp = this.mtf[0]; + + if (value == temp) + return index; + + this.mtf[0] = value; + while (temp != value) + { + index++; + byte temp2 = temp; + temp = this.mtf[index]; + this.mtf[index] = temp2; + } + + return index; + } + + /// Gets the value from a given index and moves it to the front of the MTF list (inverse Move To Front transform) + /// The index to move + /// The value at the given index + public byte IndexToFront(int index) + { + byte value = this.mtf[index]; + Array.ConstrainedCopy(this.mtf, 0, this.mtf, 1, index); + this.mtf[0] = value; + + return value; + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs new file mode 100644 index 000000000..a59594021 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs @@ -0,0 +1,111 @@ +// Bzip2 library for .net +// By Jaime Olivares +// Location: http://github.com/jaime-olivares/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 + +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + /// + /// Implements a bit-wise input stream + /// + internal class BZip2BitInputStream : IBZip2BitInputStream + { + // The stream from which bits are read + private Stream _inputStream; + + // A buffer of bits read from the input stream that have not yet been returned + private uint _bitBuffer; + + // The number of bits currently buffered in bitBuffer + private int _bitCount; + + /// Public constructor + /// The input stream to wrap + public BZip2BitInputStream(Stream inputStream) => this._inputStream = inputStream; + + public void Dispose() + { + // do nothing, the BZip2BitInputStream is not the owner of the _inputStream so don't dispose it + } + + /// Reads a single bit from the wrapped input stream + /// true if the bit read was 1, otherwise false + /// if no more bits are available in the input stream + public bool ReadBoolean() + { + if (this._bitCount > 0) + { + this._bitCount--; + } else + { + int byteRead = this._inputStream.ReadByte(); + + if (byteRead < 0) + throw new IOException("Insufficient data"); + + this._bitBuffer = (this._bitBuffer << 8) | (uint)byteRead; + this._bitCount += 7; + } + + return ((this._bitBuffer & (1 << this._bitCount))) != 0; + } + + /// Reads a zero-terminated unary number from the wrapped input stream + /// The unary number + /// if no more bits are available in the input stream + public uint ReadUnary() + { + for (uint unaryCount = 0; ; unaryCount++) + { + if (this._bitCount > 0) + { + this._bitCount--; + } else + { + var byteRead = this._inputStream.ReadByte(); + + if (byteRead < 0) + throw new IOException("Insufficient data"); + + this._bitBuffer = (this._bitBuffer << 8) | (uint)byteRead; + this._bitCount += 7; + } + + if (((this._bitBuffer & (1 << this._bitCount))) == 0) + return unaryCount; + } + } + + /// Reads up to 32 bits from the wrapped input stream + /// The number of bits to read (maximum 32) + /// The bits requested, right-aligned within the integer + /// if no more bits are available in the input stream + public uint ReadBits(int count) + { + if (this._bitCount < count) + { + while (this._bitCount < count) + { + int byteRead = this._inputStream.ReadByte(); + + if (byteRead < 0) + throw new IOException("Insufficient data"); + + this._bitBuffer = (this._bitBuffer << 8) | (uint)byteRead; + this._bitCount += 8; + } + } + + this._bitCount -= count; + + return (uint)((this._bitBuffer >> this._bitCount) & ((1 << count) - 1)); + } + + /// Reads 32 bits of input as an integer + /// The integer read + /// if 32 bits are not available in the input stream + public uint ReadInteger() => (this.ReadBits(16) << 16) | (this.ReadBits(16)); + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs new file mode 100644 index 000000000..6d4f6866a --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs @@ -0,0 +1,184 @@ +// Added by drone1400, December 2025 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + + /// + /// Helper class for splitting a BZip2 stream into multiple Memory Streams made up of individual BZip2 blocks. + /// The spitting is done by looking for the magic block header 6 byte sequence. + /// The output MemoryStreams bits are left aligned. + /// + internal class BZip2BitInputStreamSplitter : IDisposable + { + + // The stream from which bits are read + private BZip2BitInputStream? _bitInputStream; + + // internal temporary buffer + private MemoryStream? _buffer = null; + + private int _blockLevel = 0; + private int _blockSizeBytes = 0; + private uint _finalCrc = 0; + private ulong _crtVal = 0x00; + private int _bitCount = 0; + + public bool IsStreamComplete => this._bitInputStream is null; + public int BlockLevel => this._blockLevel; + public int BlockSizeBytes => this._blockSizeBytes; + public uint FinalCrc => this._finalCrc; + + public BZip2BitInputStreamSplitter(Stream inputStream, InputStreamHeaderCheckType inputStreamHeaderCheck = InputStreamHeaderCheckType.FULL_HEADER, int manualBlockLevel = 9) + { + this._bitInputStream = new BZip2BitInputStream(inputStream); + this.Initialize(inputStreamHeaderCheck, manualBlockLevel); + } + + private void Initialize(InputStreamHeaderCheckType inputStreamHeaderCheck, int blockLevel) + { + if (this._bitInputStream is null) + return; + + /* Read the stream header */ + try + { + switch (inputStreamHeaderCheck) + { + case InputStreamHeaderCheckType.FULL_HEADER: + { + uint marker1 = this._bitInputStream.ReadBits(16); + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (marker1 != BZip2Constants.STREAM_START_MARKER_1 || + marker2 != BZip2Constants.STREAM_START_MARKER_2 || + blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZ: + { + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (marker2 != BZip2Constants.STREAM_START_MARKER_2 || + blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZH: + { + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + } + + this._blockLevel = blockLevel; + this._blockSizeBytes = blockLevel * 100000; + + this._crtVal = this._bitInputStream.ReadBits(24); + this._crtVal <<= 24; + this._crtVal |= this._bitInputStream.ReadBits(24); + + // expecting the first block header immediately after + if (this._crtVal != BZip2Constants.BLOCK_HEADER_MARKER) + { + throw new IOException("BZip2 stream format error"); + } + + this._buffer = new MemoryStream(this._blockSizeBytes + 100000); + } catch (IOException) + { + // If the stream header was not valid, stop trying to read more data + this._bitInputStream = null; + this._buffer = null; + throw; + } + } + + /// + /// Finds the next BZip2 block and returns an in memory copy of it + /// + /// MemoryStream + public MemoryStream? CopyNextBlock() + { + void FlushByte() + { + this._buffer.WriteByte((byte)(this._crtVal >> 48)); + this._crtVal &= 0x0000FFFF_FFFFFFFF; + this._bitCount = 0; + } + + void FlushPartialByte() + { + byte partialByte = (byte)(this._crtVal >> 48); + partialByte <<= (8 - this._bitCount); + this._buffer.WriteByte(partialByte); + + this._crtVal &= 0x0000FFFF_FFFFFFFF; + this._bitCount = 0; + } + + if (this._bitInputStream is null || this._buffer is null) + return null; + + while (true) + { + this._crtVal <<= 1; + this._crtVal |= this._bitInputStream.ReadBits(1); + ulong testVal = this._crtVal & 0x0000FFFF_FFFFFFFF; + this._bitCount++; + if (this._bitCount == 8) + { + FlushByte(); + } + + if (testVal == BZip2Constants.STREAM_END_MARKER) + { + // we found the magic bit sequence for end of stream, that means we're done! + if (this._bitCount > 0) + { + FlushPartialByte(); + } + + // read the 32 bit CRC at the end of the file + this._finalCrc = this._bitInputStream.ReadInteger(); + + MemoryStream retBuffer = this._buffer; + + // clear buffer and input stream to mark that we are done processing blocks for good + this._buffer = null; + this._bitInputStream = null; + return retBuffer; + } + + if (testVal == BZip2Constants.BLOCK_HEADER_MARKER) + { + // we found the magic bit sequence! + if (this._bitCount > 0) + { + FlushPartialByte(); + } + MemoryStream retBuffer = this._buffer; + this._buffer = new MemoryStream(this._blockSizeBytes); + return retBuffer; + } + } + } + public void Dispose() + { + this._bitInputStream?.Dispose(); + this._buffer?.Dispose(); + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs new file mode 100644 index 000000000..3ac770f22 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs @@ -0,0 +1,248 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + /// An InputStream wrapper that decompresses BZip2 data + /// Instances of this class are not threadsafe + public class BZip2InputStream : Stream + { + // The stream from which compressed BZip2 data is read and decoded + private Stream _inputStream; + + // True if the underlying stream will be closed with the current Stream + private readonly bool _isOwner; + + // An InputStream wrapper that provides bit-level reads + private BZip2BitInputStream _bitInputStream; + + // (@code true} if the end of the compressed stream has been reached, otherwise false + private bool _streamComplete; + + /// + /// The declared block size of the stream (before final run-length decoding). The final block + /// will usually be smaller, but no block in the stream has to be exactly this large, and an + /// encoder could in theory choose to mix blocks of any size up to this value. Its function is + /// therefore as a hint to the decompressor as to how much working space is sufficient to + /// decompress blocks in a given stream + /// + private uint _streamBlockSize; + + // The merged CRC of all blocks decompressed so far + private uint _streamCrc; + + // The decompressor for the current block + private BZip2BlockDecompressor? _blockDecompressor; + + + /// Public constructor + /// The InputStream to wrap + /// if true, will close the stream when done + /// + /// Used when is NoHeader + public BZip2InputStream(Stream inputStream, bool isOwner = true, InputStreamHeaderCheckType inputStreamHeaderCheck = InputStreamHeaderCheckType.FULL_HEADER, int manualBlockLevel = 9) + { + this._inputStream = inputStream; + this._bitInputStream = new BZip2BitInputStream(inputStream); + this._isOwner = isOwner; + + // initialize stream immediately + this.InitializeStream(inputStreamHeaderCheck, manualBlockLevel); + // prepare first block + this.InitializeNextBlock(); + } + + #region Implementation of abstract members of Stream + + #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'Flush()' method."); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); + public override void SetLength(long value) => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'SetLength(long value)' method."); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method."); + public override bool CanRead => this._inputStream.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => this._inputStream.Length; + public override long Position + { + get => this._inputStream.Position; + set =>throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support Set operation for property 'Position'."); + } + + public override int ReadByte() + { + var nextByte = this._blockDecompressor?.Read() ?? -1; + + // if current block has reached its end, prepare next block and try reading again + if (nextByte == -1) + { + if (this.InitializeNextBlock()) + { + nextByte = this._blockDecompressor?.Read() ?? -1; + } + } + + return nextByte; + } + + public override int Read(byte[] destination, int offset, int length) + { + int bytesRead = this._blockDecompressor?.Read(destination, offset, length) ?? 0; + + // if current block has reached its end, prepare next block and try reading again + if (bytesRead == -1) + { + if (this.InitializeNextBlock()) + { + bytesRead = this._blockDecompressor?.Read(destination, offset, length) ?? 0; + } + } + + if (bytesRead == -1) bytesRead = 0; + + return bytesRead; + } + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this._streamComplete = true; + this._blockDecompressor = null; + + this._bitInputStream.Dispose(); + + if (this._isOwner) + { + //this._inputStream.Close(); + this._inputStream.Dispose(); + } + } + + #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + + /// Reads the stream header and checks that the data appears to be a valid BZip2 stream + /// if the stream header is not valid + private void InitializeStream(InputStreamHeaderCheckType inputStreamHeaderCheck, int blockLevel) + { + /* If the stream has been explicitly closed, throw an exception */ + if (this._bitInputStream is null) + throw new IOException("Stream closed"); + + // If we're already at the end of the stream, do nothing + if (this._streamComplete) + return; + + // Read the stream header + try + { + switch (inputStreamHeaderCheck) + { + case InputStreamHeaderCheckType.FULL_HEADER: + { + uint marker1 = this._bitInputStream.ReadBits(16); + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (marker1 != BZip2Constants.STREAM_START_MARKER_1 || + marker2 != BZip2Constants.STREAM_START_MARKER_2 || + blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZ: + { + uint marker2 = this._bitInputStream.ReadBits(8); + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (marker2 != BZip2Constants.STREAM_START_MARKER_2 || + blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + case InputStreamHeaderCheckType.NO_BZH: + { + blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); + if (blockLevel < 1 || blockLevel > 9) + { + throw new IOException("Invalid BZip2 header"); + } + break; + } + } + + this._streamBlockSize = (uint)(blockLevel * 100000); + } catch (IOException) + { + // If the stream header was not valid, stop trying to read more data + this._streamComplete = true; + throw; + } + } + + /// Prepares a new block for decompression if any remain in the stream + /// If a previous block has completed, its CRC is checked and merged into the stream CRC. + /// If the previous block was the final block in the stream, the stream CRC is validated + /// true if a block was successfully initialised, or false if the end of file marker was encountered + /// If either the block or stream CRC check failed, if the following data is + /// not a valid block-header or end-of-file marker, or if the following block could not be decoded + private bool InitializeNextBlock() + { + + // If we're already at the end of the stream, do nothing + if (this._streamComplete) + return false; + + // If a block is complete, check the block CRC and integrate it into the stream CRC + if (this._blockDecompressor != null) + { + uint blockCrc = this._blockDecompressor.CheckCrc(); + this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + } + + // Read block-header or end-of-stream marker + uint marker1 = this._bitInputStream.ReadBits(24); + uint marker2 = this._bitInputStream.ReadBits(24); + + if (marker1 == BZip2Constants.BLOCK_HEADER_MARKER_1 && marker2 == BZip2Constants.BLOCK_HEADER_MARKER_2) + { + // Initialise a new block + try + { + this._blockDecompressor = new BZip2BlockDecompressor(this._bitInputStream, this._streamBlockSize); + } catch (IOException) + { + // If the block could not be decoded, stop trying to read more data + this._streamComplete = true; + throw; + } + return true; + } + if (marker1 == BZip2Constants.STREAM_END_MARKER_1 && marker2 == BZip2Constants.STREAM_END_MARKER_2) + { + // Read and verify the end-of-stream CRC + this._streamComplete = true; + uint storedCombinedCrc = this._bitInputStream.ReadInteger(); // .ReadBits(32); + + if (storedCombinedCrc != this._streamCrc) + throw new IOException("BZip2 stream CRC error"); + + return false; + } + + // If what was read is not a valid block-header or end-of-stream marker, the stream is broken + this._streamComplete = true; + throw new IOException("BZip2 stream format error"); + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs new file mode 100644 index 000000000..d265dfeaa --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs @@ -0,0 +1,108 @@ +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + internal class BZip2ParallelInputDataBlock : IDisposable + { + + private int _blockId = 0; + private MemoryStream? _inputBlockBuffer; + private MemoryStream? _outputBuffer; + private int _outputBufferSize; + private int _blockSizeBytes; + private bool _isCrcOk = false; + private uint _crcValue; + + public long Length => this._outputBuffer?.Length ?? 0; + public long Position => this._outputBuffer?.Position ?? 0; + public bool IsDone => this.Position >= this.Length; + public int BlockId => this._blockId; + public bool IsCrcOk => this._isCrcOk; + public uint CrcValue => this._crcValue; + + public BZip2ParallelInputDataBlock(int blockId, MemoryStream inputBlockBuffer, int blockSizeBytes, int outputBufferSize) + { + this._blockId = blockId; + this._inputBlockBuffer = inputBlockBuffer; + this._blockSizeBytes = blockSizeBytes; + this._outputBufferSize = outputBufferSize; + } + + public void Dispose() + { + this._inputBlockBuffer?.Dispose(); + this._outputBuffer?.Dispose(); + } + + + /// + /// Decompresses the full block + /// + public void Decompress() + { + if (this._inputBlockBuffer is null) + throw new IOException("Attempted to Decompress the same block twice"); + + this._outputBuffer = new MemoryStream(this._outputBufferSize); + + // note: we need to skip the first 6 magic bytes... + this._inputBlockBuffer.Position = 6; + using BZip2BitInputStream inputStream = new BZip2BitInputStream(this._inputBlockBuffer); + BZip2BlockDecompressor blockDecompressor = new BZip2BlockDecompressor(inputStream, (uint)this._blockSizeBytes); + + int readCount = blockDecompressor.ReadAll(this._outputBuffer); + + // reset buffer position + this._outputBuffer.Position = 0; + + try + { + this._crcValue = blockDecompressor.CheckCrc(); + this._isCrcOk = true; + } catch (Exception) + { + this._crcValue = 0; + this._isCrcOk = false; + } + + this._inputBlockBuffer.Dispose(); + this._inputBlockBuffer = null; + } + + /// + /// reads a single byte from the output buffer + /// + /// + public int Read() + { + if (this._outputBuffer is null) + throw new IOException("Attempted to read decompressed data before decompressing block"); + + if (this._isCrcOk == false) + return -1; + if (this._outputBuffer.Position < this._outputBuffer.Length) + return this._outputBuffer.ReadByte(); + return -1; + } + + /// + /// Reads a number of bytes from the output buffer into the destination buffer + /// + /// destination buffer + /// starting position within destination buffer + /// maximum number of bytes to read + /// + public int Read(byte[] destination, int offset, int length) + { + if (this._outputBuffer is null) + throw new IOException("Attempted to read decompressed data before decompressing block"); + + if (this._isCrcOk == false) + return 0; + + return this._outputBuffer.Read(destination, offset, length); + } + + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs new file mode 100644 index 000000000..d7cbf54bf --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs @@ -0,0 +1,344 @@ +// Added by drone1400, July 2025 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.InputStream +{ + /// An InputStream wrapper that decompresses BZip2 data using multiple threads + /// Instances of this class are not threadsafe + public class BZip2ParallelInputStream : Stream + { + #region Private fields + + // block size fields + private readonly int _blockLevel; + private readonly int _blockSizeBytes; + + private int _mtNextInputBlockId = 0; + private int _mtNextOutputBlockId = 0; + + // yikes! sounds bad right? + // if one of the worker threads, this is set to true and any further attempt to + // write/flush/close the stream will throw an exception + private bool _unsafeFatalException = false; + private Exception? _lastWorkerException = null; + + // dictionary of processed blocks + private readonly Dictionary _mtDecodedBlocks = new Dictionary(); + + private int _mtPendingBlocks = 0; + private readonly int _mtMaxPendingBlocks; + + private readonly int _mtWorkerBufferSize; + + private readonly object _syncRootProcesing = new object(); + + + // The stream from which compressed BZip2 data is read and decoded + private Stream _inputStream; + + // True if the underlying stream will be closed with the current Stream + private readonly bool _isOwner; + + // An InputStream wrapper that provides bit-level reads + private BZip2BitInputStreamSplitter _inputStreamSplitter; + + + // The merged CRC of all blocks decompressed so far + private uint _streamCrc; + + #endregion + + #region Public methods + + /// Public constructor + /// The InputStream to wrap + /// True if the underlying stream will be closed with the current Stream + /// + /// Used when is NoHeader + /// Maximum number of blocks to read/decompress at the same time + /// Minimum buffer size for each worker thread during decompression, in bytes. + public BZip2ParallelInputStream(Stream inputStream, bool isOwner = true, InputStreamHeaderCheckType inputStreamHeaderCheck = InputStreamHeaderCheckType.FULL_HEADER, int manualBlockLevel = 9, int maxPendingBlocks = 0, int workerBufferSize = 12582912) + { + if (maxPendingBlocks == 0) + { + maxPendingBlocks = Environment.ProcessorCount; + } + this._mtMaxPendingBlocks = maxPendingBlocks; + this._mtWorkerBufferSize = workerBufferSize; + + // initialize stream + this._inputStream = inputStream; + this._isOwner = isOwner; + this._inputStreamSplitter = new BZip2BitInputStreamSplitter(inputStream, inputStreamHeaderCheck, manualBlockLevel); + + this._blockLevel = this._inputStreamSplitter.BlockLevel; + this._blockSizeBytes = this._inputStreamSplitter.BlockSizeBytes; + } + + #endregion + + /// + /// Decompress a block of data + /// + /// + /// if compressing the block somehow fails... + private void MultiThreadWorkerAction(object? blockData) + { + try + { + if (blockData is BZip2ParallelInputDataBlock dataBlock) + { + // do the work + dataBlock.Decompress(); + + // queue up the finished decompressed block data + lock (this._syncRootProcesing) + { + this._mtDecodedBlocks.Add(dataBlock.BlockId, dataBlock); + } + } + } catch (Exception ex) + { + // set this without any locks... + this._lastWorkerException = ex; + this._unsafeFatalException = true; + + throw new IOException("BZip2 Processing thread somehow crashed... See inner exception for details!", ex); + } finally + { + lock (this._syncRootProcesing) + { + this._mtPendingBlocks --; + } + } + } + + /// + /// If there are not too many pending blocks being processed, reads another block from the input stream and adds it to the work queue. + /// + /// True if the data block was queued up, false otherwise + /// if one of the worker threads or writing to the final bit stream experienced an exception + /// Should only be called from within or + private bool TryToQueueUpAnotherBlock() + { + if (this._unsafeFatalException) + { + throw new IOException("One of the decompression threads somehow failed... This should never happen.", this._lastWorkerException); + } + + if (this._inputStreamSplitter.IsStreamComplete) + { + return false; + } + + lock (this._syncRootProcesing) + { + if (this._mtPendingBlocks >= this._mtMaxPendingBlocks) + return false; + } + + // extract the next block... + MemoryStream? ms = this._inputStreamSplitter.CopyNextBlock(); + + // this should be impossible if this._inputStreamSplitter.IsStreamComplete is not true + if (ms is null) + return false; + + lock (this._syncRootProcesing) + { + this._mtPendingBlocks ++; + BZip2ParallelInputDataBlock data = new BZip2ParallelInputDataBlock(this._mtNextInputBlockId++, ms, this._blockSizeBytes, this._mtWorkerBufferSize); + ThreadPool.QueueUserWorkItem(this.MultiThreadWorkerAction, data); + } + + return true; + } + + /// + /// Checks if we are done processing the stream. This is done by checing if there are any pending blocks to be decoded or read. + /// Also by checking if the input stream splitter helper has reached the end of stream markers. + /// + /// True if we are done, false otherwise + /// If there's a CRC error at end of the stream. + /// /// Should only be called from within or + private bool CheckIfCompletelyDone() + { + if (this._unsafeFatalException) + { + throw new IOException("One of the decompression threads somehow failed... This should never happen.", this._lastWorkerException); + } + + lock (this._syncRootProcesing) + { + if (this._mtPendingBlocks == 0 && + this._mtDecodedBlocks.Count == 0 && + this._inputStreamSplitter.IsStreamComplete) + { + if (this._inputStreamSplitter.FinalCrc != this._streamCrc) + { + throw new IOException("BZip2 stream CRC error"); + } + // we are compeltely done, can no longer do anything! + return true; + } + } + + return false; + } + + #region Implementation of abstract members of Stream + + #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'Flush()' method."); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); + public override void SetLength(long value) => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'SetLength(long value)' method."); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method."); + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => this._inputStream.Length; + public override long Position + { + get => this._inputStream.Position; + set => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support Set operation for property 'Position'."); + } + + public override int ReadByte() + { + while (true) + { + // check if the next output block is done + BZip2ParallelInputDataBlock? data = null; + lock (this._syncRootProcesing) + { + if (this._mtDecodedBlocks.ContainsKey(this._mtNextOutputBlockId)) + { + data = this._mtDecodedBlocks[this._mtNextOutputBlockId]; + } + } + + if (data != null) + { + if (data.IsCrcOk == false) + { + this._unsafeFatalException = true; + throw new IOException("BZip2 block decompression CRC error..."); + } + + if (data.IsDone == false) + { + // can read next byte + return data.Read(); + } + + // decompressor is done! + lock (this._syncRootProcesing) + { + uint blockCrc = data.CrcValue; + this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + + this._mtDecodedBlocks.Remove(this._mtNextOutputBlockId); + this._mtNextOutputBlockId ++; + + // continue to processing next block that is already decoded, skip queueing up more blocks... + continue; + } + } + + // queue up the next block + while (this.TryToQueueUpAnotherBlock()) + { } + + // check if we are completely done + if (this.CheckIfCompletelyDone()) + return -1; + + Thread.Sleep(1); + } + } + + public override int Read(byte[] destination, int offset, int length) + { + int totalRead = 0; + int remaining = length; + + while (true) + { + // check if the next output block is done + BZip2ParallelInputDataBlock? data = null; + lock (this._syncRootProcesing) + { + if (this._mtDecodedBlocks.ContainsKey(this._mtNextOutputBlockId)) + { + data = this._mtDecodedBlocks[this._mtNextOutputBlockId]; + } + } + + if (data != null) + { + if (data.IsCrcOk == false) + { + this._unsafeFatalException = true; + throw new IOException("BZip2 block decompression CRC error..."); + } + + int readCount = data.Read(destination, offset, remaining); + remaining -= readCount; + offset += readCount; + totalRead += readCount; + + if (totalRead >= length) + return totalRead; + + if (data.IsDone) + { + // decompressor is done! + lock (this._syncRootProcesing) + { + uint blockCrc = data.CrcValue; + this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + + this._mtDecodedBlocks.Remove(this._mtNextOutputBlockId); + this._mtNextOutputBlockId ++; + + // continue to processing next block that is already decoded, skip queueing up more blocks... + continue; + } + } + } + + // queue up the next block + while (this.TryToQueueUpAnotherBlock()) + { } + + // check if we are completely done + if (this.CheckIfCompletelyDone()) + return totalRead; + + Thread.Sleep(1); + } + } + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this._mtDecodedBlocks.Clear(); + + if (this._isOwner) + { + //this._inputStream.Close(); + this._inputStream.Dispose(); + } + } + + #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs new file mode 100644 index 000000000..8518911ff --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs @@ -0,0 +1,43 @@ +// Added by drone1400, December 2025 +// Location: https://github.com/drone1400/bzip2 + +namespace SharpCompress.Compressors.BZip2MT.Interface { + internal static class BZip2Constants { + public const uint STREAM_START_MARKER = 0x425A68; + public const ulong STREAM_END_MARKER = 0x177245385090; + public const ulong BLOCK_HEADER_MARKER = 0x314159265359; + + /// The first 2 bytes of a Bzip2 marker, BZ + public const uint STREAM_START_MARKER_1 = 0x425a; + + /// The 'h' that distinguishes BZip from BZip2 + public const uint STREAM_START_MARKER_2 = 0x68; + + /// First three bytes of the end of stream marker + public const uint STREAM_END_MARKER_1 = 0x177245; + + /// Last three bytes of the end of stream marker + public const uint STREAM_END_MARKER_2 = 0x385090; + + // First three bytes of the block header marker + public const uint BLOCK_HEADER_MARKER_1 = 0x314159; + + // Last three bytes of the block header marker + public const uint BLOCK_HEADER_MARKER_2 = 0x265359; + } + + public enum InputStreamHeaderCheckType + { + // check for full header (ex: BZh9 ) + FULL_HEADER, + + // skips BZ part of header + NO_BZ, + + // skips BZh part of header + NO_BZH, + + // skips BZh and block level, aka the whole header + NO_HEADER, + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs new file mode 100644 index 000000000..1501992e7 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs @@ -0,0 +1,28 @@ +using System; +using System.IO; +namespace SharpCompress.Compressors.BZip2MT.Interface +{ + internal interface IBZip2BitInputStream : IDisposable + { + /// Reads a single bit from the wrapped input stream + /// true if the bit read was 1, otherwise false + /// if no more bits are available in the input stream + public bool ReadBoolean(); + + /// Reads a zero-terminated unary number from the wrapped input stream + /// The unary number + /// if no more bits are available in the input stream + public uint ReadUnary(); + + /// Reads up to 32 bits from the wrapped input stream + /// The number of bits to read (maximum 32) + /// The bits requested, right-aligned within the integer + /// if no more bits are available in the input stream + public uint ReadBits(int count); + + /// Reads 32 bits of input as an integer + /// The integer read + /// if 32 bits are not available in the input stream + public uint ReadInteger(); + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs new file mode 100644 index 000000000..4855e7b4a --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs @@ -0,0 +1,47 @@ +// Added by drone1400, July 2022 +// Location: https://github.com/drone1400/bzip2 + +using System; +namespace SharpCompress.Compressors.BZip2MT.Interface +{ + /// + /// Interface for a stream wrapper that implements bit-wise write operations + /// + internal interface IBZip2BitOutputStream : IDisposable + { + /// + /// Writes a single bit to the wrapped output stream + /// + /// The bit to write + /// if an error occurs writing to the stream + public void WriteBoolean (bool value); + + /// + /// Writes a zero-terminated unary number to the wrapped output stream + /// + /// The number to write (must be non-negative) + /// if an error occurs writing to the stream + public void WriteUnary (int value); + + /// + /// Writes up to 24 bits to the wrapped output stream + /// + /// The number of bits to write (maximum 24) + /// The bits to write + /// if an error occurs writing to the stream + public void WriteBits (int count, uint value) ; + + /// + /// Writes an integer as 32 bits of output + /// + /// The integer to write + /// if an error occurs writing to the stream + public void WriteInteger (uint value) ; + + /// + /// Writes any remaining bits to the output stream, zero padding to a whole byte as required + /// + /// if an error occurs writing to the stream + public void Flush() ; + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs new file mode 100644 index 000000000..f947f7680 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs @@ -0,0 +1,92 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System.IO; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + /// Implements a bit-wise output stream + /// + /// Allows the writing of single bit booleans, unary numbers, bit + /// strings of arbitrary length(up to 24 bits), and bit aligned 32-bit integers.A single byte at a + /// time is written to the wrapped stream when sufficient bits have been accumulated + /// + internal class BZip2BitOutputStream : IBZip2BitOutputStream + { + // The stream to which bits are written + private readonly Stream _outputStream; + + // A buffer of bits waiting to be written to the output stream + private uint _bitBuffer; + + // The number of bits currently buffered in bitBuffer + private int _bitCount; + + /// + /// Public constructor + /// + /// The OutputStream to wrap + public BZip2BitOutputStream(Stream outputStream) => this._outputStream = outputStream; + + public void Dispose() + { + // do nothing, the BZip2BitOutputStream is not the owner of the _outputStream so don't dispose it + } + + #region IBZip2BitOutputStream implementation + + public void WriteBoolean (bool value) + { + this._bitCount++; + this._bitBuffer |= ((value ? 1u : 0u) << (32 - this._bitCount)); + + if (this._bitCount == 8) + { + this._outputStream.WriteByte((byte)(this._bitBuffer >> 24)); + this._bitBuffer = 0; + this._bitCount = 0; + } + } + + public void WriteUnary (int value) + { + while (value-- > 0) + { + this.WriteBoolean (true); + } + this.WriteBoolean (false); + } + + public void WriteBits (int count, uint value) + { + this._bitBuffer |= ((value << (32 - count)) >> this._bitCount); + this._bitCount += count; + + while (this._bitCount >= 8) + { + this._outputStream.WriteByte((byte)(this._bitBuffer >> 24)); + this._bitBuffer <<= 8; + this._bitCount -= 8; + } + } + + public void WriteInteger (uint value) + { + this.WriteBits (16, (value >> 16) & 0xffff); + this.WriteBits (16, value & 0xffff); + } + + public void Flush() + { + if (this._bitCount > 0) + { + this.WriteBits (8 - this._bitCount, 0); + } + } + + #endregion + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs new file mode 100644 index 000000000..437803384 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs @@ -0,0 +1,179 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + /// An OutputStream wrapper that compresses BZip2 data + /// Instances of this class are not threadsafe + public class BZip2OutputStream : Stream + { + // The stream to which compressed BZip2 data is written + private Stream _outputStream; + + // An OutputStream wrapper that provides bit-level writes + private readonly BZip2BitOutputStream _bitOutputStream; + + // (@code true} if the compressed stream has been finished, otherwise false + private bool _streamFinished; + + // The declared maximum block size of the stream (before final run-length decoding) + private readonly int _streamBlockSize; + + // The merged CRC of all blocks compressed so far + private uint _streamCrc; + + // The compressor for the current block + private BZip2BlockCompressor? _blockCompressor; + + // True if the underlying stream will be closed with the current Stream + private bool _isOwner; + + /// Public constructor + /// The output stream to write to + /// The BZip2 block size as a multiple of 100,000 bytes (minimum 1, maximum 9) + /// True if the underlying stream will be closed with the current Stream + /// On any I/O error writing to the output stream + /// Larger block sizes require more memory for both compression and decompression, + /// but give better compression ratios. 9 will usually be the best value to use + public BZip2OutputStream(Stream outputStream, bool isOwner = true, int blockSizeMultiplier = 9) + { + if (outputStream is null) + throw new ArgumentException("Null output stream"); + + if ((blockSizeMultiplier < 1) || (blockSizeMultiplier > 9)) + throw new ArgumentException("Invalid BZip2 block size" + blockSizeMultiplier); + + this._streamBlockSize = blockSizeMultiplier * 100000; + this._outputStream = outputStream; + this._bitOutputStream = new BZip2BitOutputStream(this._outputStream); + this._isOwner = isOwner; + + this._bitOutputStream.WriteBits(16, BZip2Constants.STREAM_START_MARKER_1); + this._bitOutputStream.WriteBits(8, BZip2Constants.STREAM_START_MARKER_2); + this._bitOutputStream.WriteBits(8, (uint)('0' + blockSizeMultiplier)); + + this.InitialiseNextBlock(); + } + + #region Implementation of abstract members of Stream + + #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush () => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'Flush()' method."); + + public override int Read (byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method."); + + public override long Seek (long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); + + public override void SetLength (long value) => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'SetLength(long value)' method."); + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => this._outputStream.CanWrite; + + public override long Length => this._outputStream.Length; + + public override long Position + { + get => this._outputStream.Position; + set =>throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support Set operation for property 'Position'."); + } + + public override void WriteByte(byte value) + { + if (this._outputStream is null) + throw new IOException("Stream closed"); + + if (this._blockCompressor is null || this._streamFinished) + throw new IOException("Write beyond end of stream"); + + if (!this._blockCompressor.Write(value & 0xff)) + { + this.CloseBlock(); + this.InitialiseNextBlock(); + this._blockCompressor.Write(value & 0xff); + } + } + + public override void Write(byte[] data, int offset, int length) + { + if (this._outputStream is null) + throw new IOException("Stream closed"); + + if (this._blockCompressor is null || this._streamFinished) + throw new IOException("Write beyond end of stream"); + + while (length > 0) + { + int bytesWritten; + if ((bytesWritten = this._blockCompressor.Write(data, offset, length)) < length) + { + this.CloseBlock(); + this.InitialiseNextBlock(); + } + offset += bytesWritten; + length -= bytesWritten; + } + } + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this.Finish(); + if (this._isOwner) + { + this._outputStream.Dispose(); + } + } + + #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + + /// Initialises a new block for compression + private void InitialiseNextBlock() => this._blockCompressor = new BZip2BlockCompressor (this._bitOutputStream, this._streamBlockSize); + + /// Compress and write out the block currently in progress + /// If no bytes have been written to the block, it is discarded + /// On any I/O error writing to the output stream + private void CloseBlock() + { + if (this._blockCompressor is null || this._blockCompressor.IsEmpty) + return; + + this._blockCompressor.CloseBlock(); + this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ this._blockCompressor.CRC; + } + + /// Compresses and writes out any as yet unwritten data, then writes the end of the BZip2 stream + /// The underlying OutputStream is not closed + /// On any I/O error writing to the output stream + private void Finish() + { + if (!this._streamFinished) + { + this._streamFinished = true; + try + { + this.CloseBlock(); + this._bitOutputStream.WriteBits(24, BZip2Constants.STREAM_END_MARKER_1); + this._bitOutputStream.WriteBits(24, BZip2Constants.STREAM_END_MARKER_2); + this._bitOutputStream.WriteInteger(this._streamCrc); + this._bitOutputStream.Flush(); + this._outputStream.Flush(); + } finally + { + this._blockCompressor = null; + } + } + } + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs new file mode 100644 index 000000000..e9a52e593 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs @@ -0,0 +1,171 @@ +// Added by drone1400, July 2022 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.Interface; +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + /// A collection of bit output data + /// + /// Allows the writing of single bit booleans, unary numbers, bit + /// strings of arbitrary length(up to 24 bits), and bit aligned 32-bit integers.A single byte at a + /// time is written to a list of structures that serves as a buffer for use in parallelized + /// execution of block compression + /// + internal class BZip2ParallelOutputDataBlock : IBZip2BitOutputStream + { + private BZip2BitOutputStream _internalBitStream; + private MemoryStream _buffer; + private long _bitCount = 0; + + /// + /// Compressed block CRC to be stored here when block is finished + /// + public uint BlockCrc => this._blockCrc; + private uint _blockCrc = 0; + + /// + /// Indicates that the compression block is full + /// + public bool IsFull => this._isFull; + private bool _isFull = false; + + /// + /// Block numeric id for distinguishing blocks + /// + public int BlockId => this._blockId; + private int _blockId; + + /// + /// Number of bytes loaded into the block compressor + /// + public int LoadedBytes => this._loadedBytes; + private int _loadedBytes = 0; + private readonly BZip2BlockCompressor _compressor; + + /// + /// Public constructor + /// + /// block size in bytes, also initial internal buffer list capacity + /// Block number id, used to distinguish blocks in multithreadding + public BZip2ParallelOutputDataBlock(int blockSizeBytes, int blockId) + { + this._buffer = new MemoryStream(blockSizeBytes + 100000); + this._internalBitStream = new BZip2BitOutputStream(this._buffer); + this._blockId = blockId; + this._compressor = new BZip2BlockCompressor(this, blockSizeBytes); + } + + public void Dispose() + { + this._internalBitStream.Dispose(); + this._buffer.Dispose(); + } + + /// + /// Loads a byte into the 's first RLE stage + /// + /// Byte + /// True if byte was loaded, false if byte could not be loaded because block compressor is full + public bool LoadByte(byte value) + { + if (this._compressor.Write(value)) + { + this._loadedBytes++; + return true; + } + + // could not load the byte, means block is full + this._isFull = true; + + return false; + } + + /// + /// Loads bytes from a buffer into the 's first RLE stage + /// + /// Byte buffer + /// Byte buffer offset + /// Number of bytes to load + /// Number of bytes actually loaded + public int LoadBytes(byte[] buff, int offset, int length) + { + int count = this._compressor.Write(buff, offset, length); + this._loadedBytes += count; + if (count < length) + { + // could not load all the bytes, means block is full + this._isFull = true; + } + return count; + } + + /// + /// Starts the actual compression + /// + public void CompressBytes() + { + this._compressor.CloseBlock(); + this._blockCrc = this._compressor.CRC; + } + + /// + /// Writes all the buffer data to the real + /// + /// The real bit output stream + /// if an error occurs writing to the stream + public void WriteToRealOutputStream(BZip2BitOutputStream stream) + { + this._internalBitStream.Flush(); + + this._buffer.Position = 0; + while (this._bitCount >= 8) + { + int b = this._buffer.ReadByte(); + stream.WriteBits(8, (uint)b); + this._bitCount -= 8; + } + + if (this._bitCount > 0) + { + int b = this._buffer.ReadByte(); + b = (b >> (8 - (int)this._bitCount)); + stream.WriteBits((int)this._bitCount, (uint)b); + this._bitCount = 0; + } + } + + #region IBZip2BitOutputStream implementation + + public void WriteBoolean(bool value) + { + this._bitCount++; + this._internalBitStream.WriteBoolean(value); + } + public void WriteUnary(int value) + { + while (value-- > 0) + { + this._bitCount++; + this._internalBitStream.WriteBoolean (true); + } + this._bitCount++; + this._internalBitStream.WriteBoolean (false); + } + public void WriteBits(int count, uint value) + { + this._bitCount += count; + this._internalBitStream.WriteBits(count, value); + } + public void WriteInteger(uint value) + { + this._bitCount += 32; + this._internalBitStream.WriteInteger(value); + } + public void Flush() => this._internalBitStream.Flush(); + + #endregion + } +} diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs new file mode 100644 index 000000000..20152c048 --- /dev/null +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs @@ -0,0 +1,382 @@ +// Added by drone1400, July 2022 +// Location: https://github.com/drone1400/bzip2 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +namespace SharpCompress.Compressors.BZip2MT.OutputStream +{ + + /// An OutputStream wrapper that compresses BZip2 data using multiple threads + /// Instances of this class are not threadsafe + public class BZip2ParallelOutputStream : Stream + { + // block size fields + private readonly int _compressBlockSize; + private readonly int _blockLevel; + + private int _mtNextInputBlockId = 0; + private int _mtNextOutputBlockId = 0; + + // flag indicating input stream is finished + private bool _mtStreamIsFinished = false; + + // yikes! sounds bad right? + // if one of the worker threads, this is set to true and any furhter attempt to + // write/flush/close the stream will throw an exception + private bool _unsafeFatalException = false; + private Exception? _lastWorkerException = null; + + // dictionary of processed blocks + private readonly Dictionary _mtProcessedBlocks = new Dictionary(); + + private int _mtPendingBlocks = 0; + + private readonly object _syncRootProcesing = new object(); + private readonly object _syncRootOutputStream = new object(); + + // The output stream + private readonly Stream _outputStream; + + // The bit output stream + private readonly BZip2BitOutputStream _bitStream; + + // The merged CRC of all blocks compressed so far + private uint _streamCrc = 0; + + // + private BZip2ParallelOutputDataBlock _currentBlockBuffer; + + // True if the underlying stream will be closed with the current Stream + private readonly bool _isOwner; + + // for debug purposes... + // private int _debugMaxMetaBufferDataPairCount = 0; + + /// + /// Public constructor + /// + /// The output stream in which to write the compressed data + /// True if the underlying stream will be closed with the current Stream + /// The BZip2 block size as a multiple of 100,000 bytes (minimum 1, maximum 9) + /// For best performance, compressor thread number should be equal to CPU core count, but results may vary + public BZip2ParallelOutputStream(Stream output, bool isOwner = true, int blockLevel = 9) + { + this._outputStream = output; + this._bitStream = new BZip2BitOutputStream(output); + + if (blockLevel < 1 ) blockLevel = 1; + if (blockLevel > 9 ) blockLevel = 9; + this._blockLevel = blockLevel; + + // evaluate thread count + // if (compressorThreads < 1) compressorThreads = 1; + // if (compressorThreads > ABSOLUTE_MAX_THREADS) compressorThreads = ABSOLUTE_MAX_THREADS; + // this._mtCompressorThreads = compressorThreads; + + // supposedly a block can only expand 1.25x, so 0.8 of normal block size should always be safe... + this._compressBlockSize = 100000 * this._blockLevel; + + // initialize initial meta buffer + this._currentBlockBuffer = new BZip2ParallelOutputDataBlock(this._compressBlockSize, this._mtNextInputBlockId++); + + this._isOwner = isOwner; + + // write the bz2 header... + this.WriteBz2Header(); + } + + /// + /// Writes the next pending Output data block to the output bit stream + /// + /// True if a block was written, false if didn't find any pending blocks + /// + private bool TryWriteOutputBlockAndIncrementId() + { + if (this._unsafeFatalException) + { + throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + } + + + try + { + BZip2ParallelOutputDataBlock? currentOutput = null; + + lock (this._syncRootProcesing) + { + // check if the next output block can be extracted + if (this._mtProcessedBlocks.ContainsKey(this._mtNextOutputBlockId)) + { + currentOutput = this._mtProcessedBlocks[this._mtNextOutputBlockId]; + this._mtProcessedBlocks.Remove(this._mtNextOutputBlockId); + } + + // check if we got anything to write + if (currentOutput is null) + return false; + + this._mtNextOutputBlockId++; + } + + lock (this._syncRootOutputStream) + { + // update file CRC + this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ currentOutput.BlockCrc; + currentOutput.WriteToRealOutputStream(this._bitStream); + } + return true; + } catch (Exception ex) + { + // set this without any locks... + this._lastWorkerException = ex; + this._unsafeFatalException = true; + + // rethrow exception, hopefully something catches it?... + throw new IOException("BZip2 error writing output data! See inner exception for details!", ex); + } + } + + /// + /// Compresses a block of data + /// + /// + /// if compressing the block somehow fails... + private void MultiThreadWorkerAction(object? blockData) + { + try + { + if (blockData is BZip2ParallelOutputDataBlock buffer) + { + // do the work + buffer.CompressBytes(); + + // queue up the finished compressed block data + lock (this._syncRootProcesing) + { + this._mtProcessedBlocks.Add(buffer.BlockId, buffer); + } + } + } catch (Exception ex) + { + // set this without any locks... + this._lastWorkerException = ex; + this._unsafeFatalException = true; + + throw new IOException("BZip2 Processing thread somehow crashed... See inner exception for details!", ex); + } finally + { + lock (this._syncRootProcesing) + { + this._mtPendingBlocks --; + } + } + } + + /// + /// Writes the BZ2 header to the bit stream + /// + private void WriteBz2Header() + { + lock (this._syncRootOutputStream) + { + // write BZIP file header + this._bitStream.WriteBits(8, 0x42); // B + this._bitStream.WriteBits(8, 0x5A); // Z + this._bitStream.WriteBits(8, 0x68); // h + this._bitStream.WriteBits(8, (uint)(0x30 + this._blockLevel)); // block level digit + } + } + + /// + /// Writes the BZ2 footer tot he bit stream and flushes the stream + /// + private void WriteBz2FooterAndFlush() + { + lock (this._syncRootOutputStream) + { + // end magic + this._bitStream.WriteBits(8, 0x17); + this._bitStream.WriteBits(8, 0x72); + this._bitStream.WriteBits(8, 0x45); + this._bitStream.WriteBits(8, 0x38); + this._bitStream.WriteBits(8, 0x50); + this._bitStream.WriteBits(8, 0x90); + + // write combined CRC + this._bitStream.WriteBits(16, (this._streamCrc >> 16) & 0xFFFF); + this._bitStream.WriteBits(16, this._streamCrc & 0xFFFF); + + // flush all remaining bits + this._bitStream.Flush(); + this._outputStream.Flush(); + } + } + + /// + /// Queues up the current data block if it is not empty + /// + /// True if the data block was queued up, false if the current block is empty + /// if one of the worker threads or writing to the final bit stream experienced an exception + private bool TryEnqueueCurrentBlockBuffer() + { + if (this._unsafeFatalException) + { + throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + } + + // make sure current block has data + if (this._currentBlockBuffer.LoadedBytes <= 0) + return false; + + lock (this._syncRootProcesing) + { + this._mtPendingBlocks ++; + ThreadPool.QueueUserWorkItem(this.MultiThreadWorkerAction, this._currentBlockBuffer); + this._currentBlockBuffer = new BZip2ParallelOutputDataBlock(this._compressBlockSize, this._mtNextInputBlockId++); + } + + return true; + } + + + /// + /// Waits for any pending data blocks to be written to the bit stream and writes the BZ2 footer + /// + /// If there was an exception in a worker thread or when writing to the final bitstream + private void FinishBitstream() + { + if (this._unsafeFatalException) + { + throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + } + + lock (this._syncRootProcesing) + { + if (this._mtStreamIsFinished) + return; + this._mtStreamIsFinished = true; + } + + // check if there is still data left to write + this.TryEnqueueCurrentBlockBuffer(); + + // decrement next input block since no longer getting another block + this._mtNextInputBlockId--; + + while (true) + { + if (this._unsafeFatalException) + { + throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + } + + lock (this._syncRootProcesing) + { + if (this._mtNextInputBlockId == this._mtNextOutputBlockId) + { + // all done, can safely exit + break; + } + } + + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) + { } + Thread.Sleep(1); + } + + // sanity check, this should be impossible and should be caught by some test right?... + lock (this._syncRootProcesing) + { + if (this._mtPendingBlocks != 0 || + this._mtProcessedBlocks.Count > 0) + { + throw new IOException("BZip2 dispose operation sanity check failed!..."); + } + } + + // finally, write the footer! + this.WriteBz2FooterAndFlush(); + + //Console.WriteLine($"Max number of data pair entries was {this._debugMaxMetaBufferDataPairCount}"); + } + + #region Implementation of abstract members of Stream + + #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + + // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 + protected override void Dispose(bool disposing) + { + this.FinishBitstream(); + base.Dispose(disposing); + if (this._isOwner) + { + this._outputStream.Close(); + } + } + + public override void Flush() + { + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) + { } + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); + public override void SetLength(long value) => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support 'SetLength(long value)' method."); + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method."); + + public override void WriteByte(byte value) + { + if (!this._currentBlockBuffer.LoadByte(value)) + { + // byte could not be loaded, this happens when current block buffer is full + + this.TryEnqueueCurrentBlockBuffer(); + this._currentBlockBuffer.LoadByte(value); + + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) { } + } + } + + public override void Write(byte[] data, int offset, int length) + { + while (length > 0) + { + if (!this._currentBlockBuffer.IsFull) + { + int count = this._currentBlockBuffer.LoadBytes(data, offset, length); + offset += count; + length -= count; + } + + if (this._currentBlockBuffer.IsFull) + { + this.TryEnqueueCurrentBlockBuffer(); + } + + // try writing output block and keep doing so while successful + while (this.TryWriteOutputBlockAndIncrementId()) { } + } + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => this._outputStream.CanWrite; + public override long Length => this._outputStream.Length; + + public override long Position + { + get => this._outputStream.Position; + set => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support Set operation for property 'Position'."); + } + + + #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + + #endregion + } +} diff --git a/src/SharpCompress/Factories/TarFactory.cs b/src/SharpCompress/Factories/TarFactory.cs index d32020fd7..64dd2d611 100644 --- a/src/SharpCompress/Factories/TarFactory.cs +++ b/src/SharpCompress/Factories/TarFactory.cs @@ -6,7 +6,8 @@ using SharpCompress.Archives.Tar; using SharpCompress.Common; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT; +using SharpCompress.Compressors.BZip2MT.InputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.Lzw; @@ -120,8 +121,8 @@ public TestOption( new(CompressionType.None, (stream) => true, (stream) => stream, ["tar"], false), // We always do a test for IsTarFile later new( CompressionType.BZip2, - BZip2Stream.IsBZip2, - (stream) => new BZip2Stream(stream, CompressionMode.Decompress, false), + Compressors.BZip2.BZip2Stream.IsBZip2, + (stream) => new BZip2ParallelInputStream(stream), ["tar.bz2", "tb2", "tbz", "tbz2", "tz2"] ), new( diff --git a/src/SharpCompress/Readers/Tar/TarReader.cs b/src/SharpCompress/Readers/Tar/TarReader.cs index c92188856..3efe0c277 100644 --- a/src/SharpCompress/Readers/Tar/TarReader.cs +++ b/src/SharpCompress/Readers/Tar/TarReader.cs @@ -6,7 +6,7 @@ using SharpCompress.Common; using SharpCompress.Common.Tar; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT.InputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.Lzw; @@ -34,7 +34,7 @@ protected override Stream RequestInitialStream() var stream = base.RequestInitialStream(); return compressionType switch { - CompressionType.BZip2 => new BZip2Stream(stream, CompressionMode.Decompress, false), + CompressionType.BZip2 => new BZip2ParallelInputStream(stream), CompressionType.GZip => new GZipStream(stream, CompressionMode.Decompress), CompressionType.ZStandard => new ZStandardStream(stream), CompressionType.LZip => new LZipStream(stream, CompressionMode.Decompress), @@ -74,10 +74,10 @@ public static TarReader Open(Stream stream, ReaderOptions? options = null) } ((IStreamStack)rewindableStream).StackSeek(pos); - if (BZip2Stream.IsBZip2(rewindableStream)) + if (Compressors.BZip2.BZip2Stream.IsBZip2(rewindableStream)) { ((IStreamStack)rewindableStream).StackSeek(pos); - var testStream = new BZip2Stream(rewindableStream, CompressionMode.Decompress, false); + var testStream = new BZip2ParallelInputStream(rewindableStream); if (TarArchive.IsTarFile(testStream)) { ((IStreamStack)rewindableStream).StackSeek(pos); diff --git a/src/SharpCompress/Writers/Tar/TarWriter.cs b/src/SharpCompress/Writers/Tar/TarWriter.cs index 96346e2bc..e542654cf 100644 --- a/src/SharpCompress/Writers/Tar/TarWriter.cs +++ b/src/SharpCompress/Writers/Tar/TarWriter.cs @@ -6,6 +6,8 @@ using SharpCompress.Common.Tar.Headers; using SharpCompress.Compressors; using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT; +using SharpCompress.Compressors.BZip2MT.OutputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.IO; @@ -35,7 +37,7 @@ public TarWriter(Stream destination, TarWriterOptions options) break; case CompressionType.BZip2: { - destination = new BZip2Stream(destination, CompressionMode.Compress, false); + destination = new BZip2ParallelOutputStream(destination); } break; case CompressionType.GZip: diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index f867c8a91..8580424c1 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -9,7 +9,7 @@ using SharpCompress.Common.Zip; using SharpCompress.Common.Zip.Headers; using SharpCompress.Compressors; -using SharpCompress.Compressors.BZip2; +using SharpCompress.Compressors.BZip2MT.OutputStream; using SharpCompress.Compressors.Deflate; using SharpCompress.Compressors.LZMA; using SharpCompress.Compressors.PPMd; @@ -441,7 +441,7 @@ private Stream GetWriteStream(Stream writeStream) } case ZipCompressionMethod.BZip2: { - return new BZip2Stream(counting, CompressionMode.Compress, false); + return new BZip2ParallelOutputStream(counting); } case ZipCompressionMethod.LZMA: { diff --git a/tests/SharpCompress.Test/BZip2MT/BZip2MTReaderTests.cs b/tests/SharpCompress.Test/BZip2MT/BZip2MTReaderTests.cs new file mode 100644 index 000000000..3321c3260 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/BZip2MTReaderTests.cs @@ -0,0 +1,28 @@ +using System.IO; +using SharpCompress.Readers; +using SharpCompress.Readers.Tar; +using Xunit; +namespace SharpCompress.Test.BZip2MT; + +public class BZip2MTReaderTests : ReaderTests +{ + [Fact] + public void BZip2MT_Reader_Factory_ValidTarBz2File() + { + Stream stream = new MemoryStream( + new byte[] + { + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xd2, 0xeb, 0x13, 0xbb, 0x00, 0x00, + 0x2f, 0xdf, 0x81, 0xce, 0x00, 0x40, 0x01, 0x6f, 0x80, 0x20, 0x00, 0x00, 0x20, 0x60, 0x00, 0x1e, + 0x40, 0x02, 0x00, 0x04, 0x00, 0xa0, 0x00, 0x54, 0x42, 0x8f, 0xd5, 0x18, 0x00, 0x03, 0x20, 0x94, + 0x53, 0xc1, 0x40, 0x03, 0x40, 0x5a, 0xbe, 0xa5, 0x09, 0x81, 0x03, 0x52, 0x89, 0x46, 0x40, 0x53, + 0x65, 0xab, 0x3b, 0xb4, 0xe7, 0x20, 0x9c, 0x48, 0x6f, 0x0f, 0xa3, 0x4c, 0x3d, 0xc1, 0x58, 0x06, + 0xdc, 0x7a, 0xc8, 0x99, 0x22, 0x82, 0xc6, 0xd1, 0x55, 0x15, 0x8d, 0x33, 0xe8, 0xbb, 0x92, 0x29, + 0xc2, 0x84, 0x86, 0x97, 0x58, 0x9d, 0xd8, + } + ); + var reader = ReaderFactory.Open(stream); + + Assert.IsType(typeof(TarReader), reader); + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/BZip2MTStreamAsyncTests.cs b/tests/SharpCompress.Test/BZip2MT/BZip2MTStreamAsyncTests.cs new file mode 100644 index 000000000..b6e8c6922 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/BZip2MTStreamAsyncTests.cs @@ -0,0 +1,201 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using SharpCompress.Compressors.BZip2MT.InputStream; +using SharpCompress.Compressors.BZip2MT.OutputStream; +using Xunit; +namespace SharpCompress.Test.BZip2MT; + +public class BZip2MTStreamAsyncTests +{ + private byte[] CreateTestData(int size) + { + var data = new byte[size]; + // Create compressible data with repetitive pattern + for (int i = 0; i < size; i++) + { + data[i] = (byte)('A' + (i % 26)); + } + return data; + } + + [Fact] + public async Task BZip2MTCompressDecompressAsyncTest() + { + var testData = this.CreateTestData(10000); + byte[] compressed; + + // Compress + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream) + ) + { + await bzip2Stream.WriteAsync(testData, 0, testData.Length); + bzip2Stream.Close(); + } + compressed = memoryStream.ToArray(); + } + + // Verify compression occurred + Assert.True(compressed.Length > 0); + Assert.True(compressed.Length < testData.Length); + + // Decompress + byte[] decompressed; + using (var memoryStream = new MemoryStream(compressed)) + { + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + decompressed = new byte[testData.Length]; + var totalRead = 0; + int bytesRead; + while ( + ( + bytesRead = await bzip2Stream.ReadAsync( + decompressed, + totalRead, + testData.Length - totalRead + ) + ) > 0 + ) + { + totalRead += bytesRead; + } + } + } + + // Verify decompression + Assert.Equal(testData, decompressed); + } + + [Fact] + public async Task BZip2MTReadAsyncWithCancellationTest() + { + var testData = Encoding.ASCII.GetBytes(new string('A', 5000)); // Repetitive data compresses well + byte[] compressed; + + // Compress + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream) + ) + { + await bzip2Stream.WriteAsync(testData, 0, testData.Length); + bzip2Stream.Close(); + } + compressed = memoryStream.ToArray(); + } + + // Decompress with cancellation support + using (var memoryStream = new MemoryStream(compressed)) + { + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + var buffer = new byte[1024]; + using var cts = new System.Threading.CancellationTokenSource(); + + // Read should complete without cancellation + var bytesRead = await bzip2Stream.ReadAsync(buffer, 0, buffer.Length, cts.Token); + Assert.True(bytesRead > 0); + } + } + } + + [Fact] + public async Task BZip2MTMultipleAsyncWritesTest() + { + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream, false) + ) + { + var data1 = Encoding.ASCII.GetBytes("Hello "); + var data2 = Encoding.ASCII.GetBytes("World"); + var data3 = Encoding.ASCII.GetBytes("!"); + + await bzip2Stream.WriteAsync(data1, 0, data1.Length); + await bzip2Stream.WriteAsync(data2, 0, data2.Length); + await bzip2Stream.WriteAsync(data3, 0, data3.Length); + + bzip2Stream.Close(); + } + + var compressed = memoryStream.ToArray(); + Assert.True(compressed.Length > 0); + + // Decompress and verify + using (var readStream = new MemoryStream(compressed)) + { + // reset memory stream position + memoryStream.Position = 0; + + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + var result = new StringBuilder(); + var buffer = new byte[256]; + int bytesRead; + while ((bytesRead = await bzip2Stream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + result.Append(Encoding.ASCII.GetString(buffer, 0, bytesRead)); + } + + Assert.Equal("Hello World!", result.ToString()); + } + } + } + } + + [Fact] + public async Task BZip2MTLargeDataAsyncTest() + { + var largeData = this.CreateTestData(100000); + + // Compress + byte[] compressed; + using (var memoryStream = new MemoryStream()) + { + using ( + var bzip2Stream = new BZip2ParallelOutputStream(memoryStream) + ) + { + await bzip2Stream.WriteAsync(largeData, 0, largeData.Length); + bzip2Stream.Close(); + } + compressed = memoryStream.ToArray(); + } + + // Decompress + byte[] decompressed; + using (var memoryStream = new MemoryStream(compressed)) + { + using ( + var bzip2Stream = new BZip2ParallelInputStream(memoryStream) + ) + { + decompressed = new byte[largeData.Length]; + var totalRead = 0; + int bytesRead; + var buffer = new byte[4096]; + while ((bytesRead = await bzip2Stream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + Array.Copy(buffer, 0, decompressed, totalRead, bytesRead); + totalRead += bytesRead; + } + } + } + + // Verify + Assert.Equal(largeData, decompressed); + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/TestCommon.cs b/tests/SharpCompress.Test/BZip2MT/TestCommon.cs new file mode 100644 index 000000000..e50142952 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/TestCommon.cs @@ -0,0 +1,206 @@ +using System; +using System.Diagnostics; +using System.IO; +using SharpCompress.Compressors.BZip2MT.InputStream; +using SharpCompress.Compressors.BZip2MT.OutputStream; +using Xunit; +using Xunit.Abstractions; +namespace SharpCompress.Test.BZip2MT +{ + public static class TestCommon + { + + /// + /// Common test routine for Multi Threaded compression + Single Threaded decompression + /// + /// + /// Input stream, must be seekable + /// If true, will use Parallel Compressor + /// If true, will use Parallel Decompressor + /// Maximum number of threads to use for compression, if 0 will use Environment.ProcessorCount + /// Size for temporary output memory buffer + /// Size for temporary copy buffer + /// If true, will save inputStream data to a file + /// (Time Compression, Time Decompression) + public static (double, double) GenericTest(ITestOutputHelper console, Stream inputStream, bool compressMultiThread, bool decompressMultiThread, int outputBufferSize = 8388608, int copyBufferSize = 8388608, bool saveFileOnFail = false) + { + void DebugSaveInputStream() + { + string randomFile = Path.GetRandomFileName(); + console.WriteLine($" Saving input data to {randomFile}"); + using FileStream fs = new FileStream(randomFile, FileMode.Create, FileAccess.Write); + inputStream.Position = 0; + inputStream.CopyTo(fs); + fs.Flush(); + fs.Close(); + } + + double timeCompress = 0; + double timeDecompress = 0; + + using MemoryStream output = new MemoryStream(outputBufferSize); + using MemoryStream outputDecompressed = new MemoryStream(outputBufferSize); + + try + { + // compress input + Stopwatch swCompress = new Stopwatch(); + swCompress.Start(); + using Stream compressor = compressMultiThread + ? new BZip2ParallelOutputStream(output, false, 9) + : new BZip2OutputStream(output, false, 9); + inputStream.CopyTo(compressor, copyBufferSize); + compressor.Close(); + swCompress.Stop(); + timeCompress = swCompress.ElapsedMilliseconds; + console.WriteLine($" {timeCompress} ms {(compressMultiThread ? "MT": "ST")} compression time... "); + + // reset output position + output.Position = 0; + + // decompress output + Stopwatch SwDecompress = new Stopwatch(); + SwDecompress.Start(); + using Stream decompressor = decompressMultiThread + ? new BZip2ParallelInputStream(output, false) + : new BZip2InputStream(output, false); + decompressor.CopyTo(outputDecompressed, copyBufferSize); + SwDecompress.Stop(); + timeDecompress = SwDecompress.ElapsedMilliseconds; + console.WriteLine($" {timeDecompress} ms {(decompressMultiThread ? "MT": "ST")} decompression time"); + } catch (Exception ex) + { + if (saveFileOnFail) + { + DebugSaveInputStream(); + } + + Assert.Fail($"Exception was thrown... {ex}"); + } + + if (inputStream.Length != outputDecompressed.Length) + { + Assert.Fail($"Decompressed stream length mismatch, expecting {inputStream.Length}, got {outputDecompressed.Length}"); + } + + inputStream.Position = 0; + outputDecompressed.Position = 0; + + for (int i = 0; i < inputStream.Length; i++) + { + int expect = inputStream.ReadByte(); + int value = outputDecompressed.ReadByte(); + if ( expect != value) + { + if (saveFileOnFail) + { + DebugSaveInputStream(); + Assert.Fail($"bytes differ at position {i}, expected {expect}, got {value}"); + } + } + } + + return (timeCompress, timeDecompress); + } + + public enum RandomDataMode + { + SingleByteValue, + RandomBytes, + RandomBytesRepeat, + } + + public enum TestMode + { + CMT_DST, // multi thread compress, single thread decompress + CST_DST, // single thread compress, single thread decompress + CMT_DMT, // multi thread compress, multi thread decompress + CST_DMT, // single thread compress, multi thread decompress + } + + public static void RandomLongTest_X(ITestOutputHelper console, int repeat, RandomDataMode dataMode, TestMode testMode, int len = 9000000) + { + Random random = new Random(); + + int copyBufferSize = 8388608; + int outBufferSize = 8388608; + int repeatStreaks = 64; + + double totalCompressionTimeMs = 0; + double totalDecompressionTimeMs = 0; + + for (int r = 0; r < repeat; r++) + { + byte[] bigBuffer = new byte[len]; + switch (dataMode) + { + case RandomDataMode.RandomBytes: + { + random.NextBytes(bigBuffer); + break; + } + case RandomDataMode.RandomBytesRepeat: + { + random.NextBytes(bigBuffer); + + int offset = 0; + for (int rs = 0; rs < repeatStreaks; rs++) + { + int newoffset = random.Next(0, (len - 10000) / repeatStreaks); + offset += newoffset; + int count = random.Next(0, 512); + byte val = bigBuffer[offset++]; + for (int i = 0; i < count; i++) + { + bigBuffer[offset++] = val; + } + } + break; + } + case RandomDataMode.SingleByteValue: + { + byte value = (byte)(random.Next() & 0xFF); + for (int i = 0; i < len; i++) + { + bigBuffer[i] = value; + } + break; + } + } + + MemoryStream ms = new MemoryStream(bigBuffer); + double timeC = 0, timeD = 0; + switch (testMode) + { + default: throw new ArgumentException("Unknown test mode"); + case TestMode.CMT_DMT: + { + (timeC, timeD) = GenericTest(console, ms, true, true, outBufferSize, copyBufferSize, true); + break; + } + case TestMode.CST_DMT: + { + (timeC, timeD) = GenericTest(console, ms, false, true, outBufferSize, copyBufferSize, true); + break; + } + case TestMode.CST_DST: + { + (timeC, timeD) = GenericTest(console, ms, false, false, outBufferSize, copyBufferSize, true); + break; + } + case TestMode.CMT_DST: + { + (timeC, timeD) = GenericTest(console, ms, true, false, outBufferSize, copyBufferSize, true); + break; + } + } + + totalCompressionTimeMs += timeC; + totalDecompressionTimeMs += timeD; + } + console.WriteLine($"DONE {testMode}"); + console.WriteLine($"AVERAGE {totalCompressionTimeMs / repeat} ms compression time... "); + console.WriteLine($"AVERAGE {totalDecompressionTimeMs / repeat} ms decompression time... "); + } + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/Tests.cs b/tests/SharpCompress.Test/BZip2MT/Tests.cs new file mode 100644 index 000000000..02db3b341 --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/Tests.cs @@ -0,0 +1,139 @@ +// Bzip2 library for .net +// Modified by drone1400 +// Location: https://github.com/drone1400/bzip2 +// Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 +// Modified from the .net implementation by Jaime Olivares: http://github.com/jaime-olivares/bzip2 + +using System; +using System.IO; +using SharpCompress.Compressors.BZip2MT.Algorithm; +using SharpCompress.Compressors.BZip2MT.InputStream; +using SharpCompress.Compressors.BZip2MT.OutputStream; +using Xunit; +using Xunit.Abstractions; +namespace SharpCompress.Test.BZip2MT +{ + + /// + /// Unit test for the BZip2 compression library + /// + public class Tests + { + + private const int BufferSizeLarge = 10000000; // Almost 10 Mb + private const int BufferSizeSmall = 100000; // Around 100 Kb + private static byte[] Buffer = new byte[Tests.BufferSizeLarge]; + + private readonly ITestOutputHelper _console; + private readonly Random _random; + + /// + /// Fills the test buffer with random values + /// + public Tests(ITestOutputHelper console) + { + this._random = new Random(); + this._console = console; + + this._random.NextBytes(Tests.Buffer); + } + + /// + /// Performs a CRC check and compare against well-known results + /// The buffer has different values + /// + [Fact] + public void CrcAlgorithmDifferentValues() + { + byte[] buffer = + { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA + }; + + var crc = new CRC32(); + for (int i = 0; i < buffer.Length; i++) + crc.UpdateCrc(buffer[i]); + + Assert.Equal(0x8AEE127A, crc.CRC); + } + + /// + /// Performs a CRC check and compare against well-known results + /// The buffer has different values + /// + [Fact] + public void CrcAlgorithmSameValues() + { + var crc = new CRC32(); + crc.UpdateCrc(0x55, 10); + Assert.Equal(0xA1E07747, crc.CRC); + } + + /// + /// Compresses the full buffer and checks for a reasonable compressed size + /// + [Fact] + public void CompressSmokeLarge() + { + var input = new MemoryStream(Tests.Buffer); + var output = new MemoryStream(); + + var compressor = new BZip2OutputStream(output, false); + input.CopyTo(compressor); + compressor.Close(); + + // Estimated size between inputSize*0.5 and inputSize*1.1 + Assert.True(output.Length > Tests.BufferSizeLarge * 0.5); + Assert.True(output.Length < Tests.BufferSizeLarge * 1.1); + } + + /// + /// Compresses a portion of the buffer and checks for a reasonable compressed size + /// + [Fact] + public void CompressSmokeSmall() + { + var input = new MemoryStream(Tests.Buffer, 0, Tests.BufferSizeSmall); + var output = new MemoryStream(); + + var compressor = new BZip2OutputStream(output, false); + input.CopyTo(compressor); + compressor.Close(); + + // Estimated size between inputSize*0.5 and inputSize*1.1 + Assert.True(output.Length > Tests.BufferSizeSmall * 0.5); + Assert.True(output.Length < Tests.BufferSizeSmall * 1.1); + } + + /// + /// Compresses and decompresses a long random buffer + /// + [Fact] + public void CompressAndDecompress() + { + var input = new MemoryStream(Tests.Buffer); + var output = new MemoryStream(); + + var compressor = new BZip2OutputStream(output, false); + input.CopyTo(compressor); + compressor.Close(); + + Assert.True(output.Length > 4); + + output.Position = 0; + var output2 = new MemoryStream(); + var decompressor = new BZip2InputStream(output, false); + decompressor.CopyTo(output2); + + Assert.Equal(Tests.Buffer.Length, output2.Length); + output2.Position = 0; + for (int i = 0; i < Tests.Buffer.Length; i++) + { + if (Tests.Buffer[i] != (byte)output2.ReadByte()) + { + Assert.Fail($"bytes differ at position {i}"); + } + } + } + } +} diff --git a/tests/SharpCompress.Test/BZip2MT/TestsRandomData.cs b/tests/SharpCompress.Test/BZip2MT/TestsRandomData.cs new file mode 100644 index 000000000..5deec2dce --- /dev/null +++ b/tests/SharpCompress.Test/BZip2MT/TestsRandomData.cs @@ -0,0 +1,53 @@ +using Xunit; +using Xunit.Abstractions; +namespace SharpCompress.Test.BZip2MT +{ + public class TestsRandomData + { + private readonly ITestOutputHelper _console; + protected int _repeatCount = 1; + + public TestsRandomData(ITestOutputHelper console) + { + this._console = console; + } + + [Fact] + public void RandomSingleByteLongTest_CST_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CST_DST, 100000000); + [Fact] + public void RandomLongTest_CST_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CST_DST); + [Fact] + public void RandomLongTestWithRepeatedValues_CST_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CST_DST); + [Fact] + public void RandomSingleByteLongTest_CMT_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CMT_DST, 100000000); + [Fact] + public void RandomLongTest_CMT_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CMT_DST); + [Fact] + public void RandomLongTestWithRepeatedValues_CMT_DST() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CMT_DST); + + [Fact] + public void RandomSingleByteLongTest_CST_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CST_DMT, 100000000); + [Fact] + public void RandomLongTest_CST_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CST_DMT); + [Fact] + public void RandomLongTestWithRepeatedValues_CST_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CST_DMT); + [Fact] + public void RandomSingleByteLongTest_CMT_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.SingleByteValue, TestCommon.TestMode.CMT_DMT, 100000000); + [Fact] + public void RandomLongTest_CMT_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytes, TestCommon.TestMode.CMT_DMT); + [Fact] + public void RandomLongTestWithRepeatedValues_CMT_DMT() => + TestCommon.RandomLongTest_X(this._console, this._repeatCount, TestCommon.RandomDataMode.RandomBytesRepeat, TestCommon.TestMode.CMT_DMT); + } +} From f8daa0882db5468465badfda8e62bae8459a2390 Mon Sep 17 00:00:00 2001 From: drone1400 <> Date: Sat, 6 Dec 2025 19:39:34 +0200 Subject: [PATCH 2/3] Fixed oversight in BZip2ParallelInputStream.cs causing very slow reads when reading very few bytes --- .../InputStream/BZip2ParallelInputStream.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs index d7cbf54bf..9196e4e02 100644 --- a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs @@ -251,14 +251,16 @@ public override int ReadByte() } // queue up the next block - while (this.TryToQueueUpAnotherBlock()) - { } + bool queueSuccessful = this.TryToQueueUpAnotherBlock(); // check if we are completely done if (this.CheckIfCompletelyDone()) return -1; - Thread.Sleep(1); + if (!queueSuccessful) + { + Thread.Sleep(1); + } } } @@ -313,14 +315,16 @@ public override int Read(byte[] destination, int offset, int length) } // queue up the next block - while (this.TryToQueueUpAnotherBlock()) - { } + bool queueSuccessful = this.TryToQueueUpAnotherBlock(); // check if we are completely done if (this.CheckIfCompletelyDone()) return totalRead; - Thread.Sleep(1); + if (!queueSuccessful) + { + Thread.Sleep(1); + } } } From 1f2195e13b02258f6bba382f54378c3e87c5c9c3 Mon Sep 17 00:00:00 2001 From: drone1400 <> Date: Mon, 8 Dec 2025 17:51:29 +0200 Subject: [PATCH 3/3] Reformatted modified files with csharpier --- .../BZip2MT/Algorithm/BZip2BlockCompressor.cs | 28 +- .../Algorithm/BZip2BlockDecompressor.cs | 591 ++++++- .../BZip2MT/Algorithm/BZip2DivSufSort.cs | 1461 ++++++++++++----- .../Algorithm/BZip2HuffmanStageDecoder.cs | 59 +- .../Algorithm/BZip2HuffmanStageEncoder.cs | 97 +- .../Algorithm/Bzip2MTFAndRLE2StageEncoder.cs | 40 +- .../Compressors/BZip2MT/Algorithm/CRC32.cs | 300 +++- .../BZip2MT/Algorithm/HuffmanAllocator.cs | 53 +- .../BZip2MT/Algorithm/MoveToFront.cs | 3 +- .../InputStream/BZip2BitInputStream.cs | 7 +- .../BZip2BitInputStreamSplitter.cs | 34 +- .../BZip2MT/InputStream/BZip2InputStream.cs | 93 +- .../BZip2ParallelInputDataBlock.cs | 27 +- .../InputStream/BZip2ParallelInputStream.cs | 108 +- .../BZip2MT/Interface/BZip2Constants.cs | 6 +- .../BZip2MT/Interface/IBZip2BitInputStream.cs | 1 + .../Interface/IBZip2BitOutputStream.cs | 11 +- .../OutputStream/BZip2BitOutputStream.cs | 19 +- .../BZip2MT/OutputStream/BZip2OutputStream.cs | 48 +- .../BZip2ParallelOutputDataBlock.cs | 9 +- .../OutputStream/BZip2ParallelOutputStream.cs | 117 +- 21 files changed, 2358 insertions(+), 754 deletions(-) diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs index 260397eef..d8e703f44 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockCompressor.cs @@ -6,6 +6,7 @@ using System; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.Algorithm { /// Compresses and writes a single BZip2 block @@ -96,19 +97,22 @@ public bool Write(int value) { this.rleCurrentValue = value; this.rleLength = 1; - } else if (this.rleCurrentValue != value) + } + else if (this.rleCurrentValue != value) { // This path commits us to write 6 bytes - one RLE run (5 bytes) plus one extra this.WriteRun(this.rleCurrentValue & 0xff, this.rleLength); this.rleCurrentValue = value; this.rleLength = 1; - } else + } + else { if (this.rleLength == 254) { this.WriteRun(this.rleCurrentValue & 0xff, 255); this.rleLength = 0; - } else + } + else { this.rleLength++; } @@ -165,11 +169,21 @@ public void CloseBlock() this.WriteSymbolMap(); // Perform the Move To Front Transform and Run-Length Encoding[2] stages - var mtfEncoder = new BZip2MTFAndRLE2StageEncoder(this.bwtBlock, this.blockLength, this.blockValuesPresent); + var mtfEncoder = new BZip2MTFAndRLE2StageEncoder( + this.bwtBlock, + this.blockLength, + this.blockValuesPresent + ); mtfEncoder.Encode(); // Perform the Huffman Encoding stage and write out the encoded data - var huffmanEncoder = new BZip2HuffmanStageEncoder(this.bitOutputStream, mtfEncoder.MtfBlock, mtfEncoder.MtfLength, mtfEncoder.MtfAlphabetSize, mtfEncoder.MtfSymbolFrequencies); + var huffmanEncoder = new BZip2HuffmanStageEncoder( + this.bitOutputStream, + mtfEncoder.MtfBlock, + mtfEncoder.MtfLength, + mtfEncoder.MtfAlphabetSize, + mtfEncoder.MtfSymbolFrequencies + ); huffmanEncoder.Encode(); } @@ -214,10 +228,10 @@ private void WriteSymbolMap() /// /// The value to write /// The run length of the value to write - private void WriteRun ( int value, int runLength) + private void WriteRun(int value, int runLength) { this.blockValuesPresent[value] = true; - this.crc.UpdateCrc (value, runLength); + this.crc.UpdateCrc(value, runLength); var byteValue = (byte)value; switch (runLength) diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs index 2a61f2ca7..b1c4bb2c4 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2BlockDecompressor.cs @@ -6,6 +6,7 @@ using System; using System.IO; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.Algorithm { /// Reads and decompresses a single BZip2 block @@ -33,42 +34,523 @@ internal class BZip2BlockDecompressor /// private static readonly int[] RNUMS = { - 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, 985, 724, 205, 454, 863, 491, - 741, 242, 949, 214, 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, 419, 436, - 278, 496, 867, 210, 399, 680, 480, 51, 878, 465, 811, 169, 869, 675, 611, 697, - 867, 561, 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, 150, 238, 59, 379, - 684, 877, 625, 169, 643, 105, 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, - 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, 909, 545, 703, 919, 874, 474, - 882, 500, 594, 612, 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, 161, 604, - 958, 533, 221, 400, 386, 867, 600, 782, 382, 596, 414, 171, 516, 375, 682, 485, - 911, 276, 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, 227, 730, 475, 186, - 263, 647, 537, 686, 600, 224, 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, - 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, 715, 67, 618, 276, 204, 918, - 873, 777, 604, 560, 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, 652, 934, - 970, 447, 318, 353, 859, 672, 112, 785, 645, 863, 803, 350, 139, 93, 354, 99, - 820, 908, 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, 653, 282, 762, 623, - 680, 81, 927, 626, 789, 125, 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, - 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, 857, 956, 358, 619, 580, 124, - 737, 594, 701, 612, 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, 944, 375, - 748, 52, 600, 747, 642, 182, 862, 81, 344, 805, 988, 739, 511, 655, 814, 334, - 249, 515, 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, 433, 837, 553, 268, - 926, 240, 102, 654, 459, 51, 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, - 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, 978, 321, 576, 617, 626, 502, - 894, 679, 243, 440, 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, 707, 151, - 457, 449, 797, 195, 791, 558, 945, 679, 297, 59, 87, 824, 713, 663, 412, 693, - 342, 606, 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, 343, 97, 430, 751, - 497, 314, 983, 374, 822, 928, 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, - 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, 369, 970, 294, 750, 807, 827, - 150, 790, 288, 923, 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, 896, 831, - 547, 261, 524, 462, 293, 465, 502, 56, 661, 821, 976, 991, 658, 869, 905, 758, - 745, 193, 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, 61, 688, 793, 644, - 986, 403, 106, 366, 905, 644, 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, - 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, 920, 176, 193, 713, 857, 265, - 203, 50, 668, 108, 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, 936, 638 + 619, + 720, + 127, + 481, + 931, + 816, + 813, + 233, + 566, + 247, + 985, + 724, + 205, + 454, + 863, + 491, + 741, + 242, + 949, + 214, + 733, + 859, + 335, + 708, + 621, + 574, + 73, + 654, + 730, + 472, + 419, + 436, + 278, + 496, + 867, + 210, + 399, + 680, + 480, + 51, + 878, + 465, + 811, + 169, + 869, + 675, + 611, + 697, + 867, + 561, + 862, + 687, + 507, + 283, + 482, + 129, + 807, + 591, + 733, + 623, + 150, + 238, + 59, + 379, + 684, + 877, + 625, + 169, + 643, + 105, + 170, + 607, + 520, + 932, + 727, + 476, + 693, + 425, + 174, + 647, + 73, + 122, + 335, + 530, + 442, + 853, + 695, + 249, + 445, + 515, + 909, + 545, + 703, + 919, + 874, + 474, + 882, + 500, + 594, + 612, + 641, + 801, + 220, + 162, + 819, + 984, + 589, + 513, + 495, + 799, + 161, + 604, + 958, + 533, + 221, + 400, + 386, + 867, + 600, + 782, + 382, + 596, + 414, + 171, + 516, + 375, + 682, + 485, + 911, + 276, + 98, + 553, + 163, + 354, + 666, + 933, + 424, + 341, + 533, + 870, + 227, + 730, + 475, + 186, + 263, + 647, + 537, + 686, + 600, + 224, + 469, + 68, + 770, + 919, + 190, + 373, + 294, + 822, + 808, + 206, + 184, + 943, + 795, + 384, + 383, + 461, + 404, + 758, + 839, + 887, + 715, + 67, + 618, + 276, + 204, + 918, + 873, + 777, + 604, + 560, + 951, + 160, + 578, + 722, + 79, + 804, + 96, + 409, + 713, + 940, + 652, + 934, + 970, + 447, + 318, + 353, + 859, + 672, + 112, + 785, + 645, + 863, + 803, + 350, + 139, + 93, + 354, + 99, + 820, + 908, + 609, + 772, + 154, + 274, + 580, + 184, + 79, + 626, + 630, + 742, + 653, + 282, + 762, + 623, + 680, + 81, + 927, + 626, + 789, + 125, + 411, + 521, + 938, + 300, + 821, + 78, + 343, + 175, + 128, + 250, + 170, + 774, + 972, + 275, + 999, + 639, + 495, + 78, + 352, + 126, + 857, + 956, + 358, + 619, + 580, + 124, + 737, + 594, + 701, + 612, + 669, + 112, + 134, + 694, + 363, + 992, + 809, + 743, + 168, + 974, + 944, + 375, + 748, + 52, + 600, + 747, + 642, + 182, + 862, + 81, + 344, + 805, + 988, + 739, + 511, + 655, + 814, + 334, + 249, + 515, + 897, + 955, + 664, + 981, + 649, + 113, + 974, + 459, + 893, + 228, + 433, + 837, + 553, + 268, + 926, + 240, + 102, + 654, + 459, + 51, + 686, + 754, + 806, + 760, + 493, + 403, + 415, + 394, + 687, + 700, + 946, + 670, + 656, + 610, + 738, + 392, + 760, + 799, + 887, + 653, + 978, + 321, + 576, + 617, + 626, + 502, + 894, + 679, + 243, + 440, + 680, + 879, + 194, + 572, + 640, + 724, + 926, + 56, + 204, + 700, + 707, + 151, + 457, + 449, + 797, + 195, + 791, + 558, + 945, + 679, + 297, + 59, + 87, + 824, + 713, + 663, + 412, + 693, + 342, + 606, + 134, + 108, + 571, + 364, + 631, + 212, + 174, + 643, + 304, + 329, + 343, + 97, + 430, + 751, + 497, + 314, + 983, + 374, + 822, + 928, + 140, + 206, + 73, + 263, + 980, + 736, + 876, + 478, + 430, + 305, + 170, + 514, + 364, + 692, + 829, + 82, + 855, + 953, + 676, + 246, + 369, + 970, + 294, + 750, + 807, + 827, + 150, + 790, + 288, + 923, + 804, + 378, + 215, + 828, + 592, + 281, + 565, + 555, + 710, + 82, + 896, + 831, + 547, + 261, + 524, + 462, + 293, + 465, + 502, + 56, + 661, + 821, + 976, + 991, + 658, + 869, + 905, + 758, + 745, + 193, + 768, + 550, + 608, + 933, + 378, + 286, + 215, + 979, + 792, + 961, + 61, + 688, + 793, + 644, + 986, + 403, + 106, + 366, + 905, + 644, + 372, + 567, + 466, + 434, + 645, + 210, + 389, + 550, + 919, + 135, + 780, + 773, + 635, + 389, + 707, + 100, + 626, + 958, + 165, + 504, + 920, + 176, + 193, + 713, + 857, + 265, + 203, + 50, + 668, + 108, + 645, + 990, + 626, + 197, + 510, + 357, + 358, + 850, + 858, + 364, + 936, + 638, }; // Maximum possible number of Huffman table selectors - private const int HUFFMAN_MAXIMUM_SELECTORS = (900000 / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH) + 1; + private const int HUFFMAN_MAXIMUM_SELECTORS = + (900000 / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH) + 1; // Provides bits of input to decode private readonly IBZip2BitInputStream bitInputStream; @@ -101,9 +583,9 @@ internal class BZip2BlockDecompressor // /// - /// Counts of each byte value within the bwtTransformedArray data. Collected at the Move - /// To Front stage, consumed by the Inverse Burrows Wheeler Transform stage - /// + /// Counts of each byte value within the bwtTransformedArray data. Collected at the Move + /// To Front stage, consumed by the Inverse Burrows Wheeler Transform stage + /// private readonly int[] bwtByteCounts = new int[256]; /// @@ -174,7 +656,10 @@ internal class BZip2BlockDecompressor /// if the input stream reaches EOF before all table data has been read private BZip2HuffmanStageDecoder ReadHuffmanTables() { - var tableCodeLengths = new byte[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE]; + var tableCodeLengths = new byte[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE + ]; // Read Huffman symbol to output byte map uint huffmanUsedRanges = this.bitInputStream.ReadBits(16); @@ -198,10 +683,12 @@ private BZip2HuffmanStageDecoder ReadHuffmanTables() // Read total number of tables and selectors uint totalTables = this.bitInputStream.ReadBits(3); uint totalSelectors = this.bitInputStream.ReadBits(15); - if ((totalTables < BZip2BlockDecompressor.HUFFMAN_MINIMUM_TABLES) + if ( + (totalTables < BZip2BlockDecompressor.HUFFMAN_MINIMUM_TABLES) || (totalTables > BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES) || (totalSelectors < 1) - || (totalSelectors > BZip2BlockDecompressor.HUFFMAN_MAXIMUM_SELECTORS)) + || (totalSelectors > BZip2BlockDecompressor.HUFFMAN_MAXIMUM_SELECTORS) + ) { throw new IOException("BZip2 block Huffman tables invalid"); } @@ -228,7 +715,12 @@ private BZip2HuffmanStageDecoder ReadHuffmanTables() } } - return new BZip2HuffmanStageDecoder(this.bitInputStream, endOfBlockSymbol + 1, tableCodeLengths, selectors); + return new BZip2HuffmanStageDecoder( + this.bitInputStream, + endOfBlockSymbol + 1, + tableCodeLengths, + selectors + ); } /// @@ -253,11 +745,13 @@ private void DecodeHuffmanData(BZip2HuffmanStageDecoder huffmanDecoder) { repeatCount += repeatIncrement; repeatIncrement <<= 1; - } else if (nextSymbol == BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB) + } + else if (nextSymbol == BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB) { repeatCount += repeatIncrement << 1; repeatIncrement <<= 1; - } else + } + else { byte nextByte; if (repeatCount > 0) @@ -336,7 +830,9 @@ private void InitialiseInverseBWT(uint bwtStartPointer) private int decodeNextBWTByte() { int nextDecodedByte = this.bwtCurrentMergedPointer & 0xff; - this.bwtCurrentMergedPointer = this.bwtMergedPointers[this.bwtCurrentMergedPointer >> 8]; + this.bwtCurrentMergedPointer = this.bwtMergedPointers[ + this.bwtCurrentMergedPointer >> 8 + ]; if (this.blockRandomised) { @@ -365,7 +861,7 @@ public BZip2BlockDecompressor(IBZip2BitInputStream bitInputStream, uint blockSiz this.bwtBlock = new byte[blockSize]; // Read block header - this.blockCRC = this.bitInputStream.ReadInteger(); //.ReadBits(32); + this.blockCRC = this.bitInputStream.ReadInteger(); //.ReadBits(32); this.blockRandomised = this.bitInputStream.ReadBoolean(); uint bwtStartPointer = this.bitInputStream.ReadBits(24); @@ -375,7 +871,6 @@ public BZip2BlockDecompressor(IBZip2BitInputStream bitInputStream, uint blockSiz this.InitialiseInverseBWT(bwtStartPointer); } - /// /// Decodes a byte from the final Run-Length Encoding stage, pulling a new byte from the /// Burrows-Wheeler Transform stage when required @@ -396,7 +891,8 @@ public int Read() this.rleRepeat = 1; this.rleAccumulator = 1; this.crc.UpdateCrc(nextByte); - } else + } + else { if (++this.rleAccumulator == 4) { @@ -405,7 +901,8 @@ public int Read() this.rleRepeat = _rleRepeat; this.rleAccumulator = 0; this.crc.UpdateCrc(nextByte, _rleRepeat); - } else + } + else { this.rleRepeat = 1; this.crc.UpdateCrc(nextByte); diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs index b121dfa97..6d52a1ab6 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2DivSufSort.cs @@ -4,6 +4,7 @@ // Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 using System; + namespace SharpCompress.Compressors.BZip2MT.Algorithm { /// @@ -18,10 +19,10 @@ internal class BZip2DivSufSort { private class StackEntry { - readonly public int a; - readonly public int b; - readonly public int c; - readonly public int d; + public readonly int a; + public readonly int b; + public readonly int c; + public readonly int d; public StackEntry(int a, int b, int c, int d) { @@ -34,8 +35,8 @@ public StackEntry(int a, int b, int c, int d) private class PartitionResult { - readonly public int first; - readonly public int last; + public readonly int first; + public readonly int last; public PartitionResult(int first, int last) { @@ -51,7 +52,6 @@ private class TRBudget public bool update(int size, int n) { - this.budget -= n; if (this.budget <= 0) { @@ -80,14 +80,262 @@ public TRBudget(int budget, int chance) private static readonly int[] log2table = { - -1, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 + -1, + 0, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, }; private readonly int[] SA; @@ -130,33 +378,42 @@ public int BWT() } // ReSharper disable LoopVariableIsNeverChangedInsideLoop - private static void swapElements (int[] array1, int index1, int[] array2, int index2) + private static void swapElements(int[] array1, int index1, int[] array2, int index2) { var temp = array1[index1]; array1[index1] = array2[index2]; array2[index2] = temp; } - private int ssCompare (int p1, int p2, int depth) + private int ssCompare(int p1, int p2, int depth) { - int U1n, U2n; // pointers within T - int U1, U2; + int U1n, + U2n; // pointers within T + int U1, + U2; for ( - U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = this.SA[p1 + 1] + 2, U2n = this.SA[p2 + 1] + 2; + U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = this.SA[p1 + 1] + 2, U2n = + this.SA[p2 + 1] + 2; (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); ++U1, ++U2 ) { } - return U1 < U1n ? (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) : (U2 < U2n ? -1 : 0); + return U1 < U1n + ? (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) + : (U2 < U2n ? -1 : 0); } - private int ssCompareLast (int PA, int p1, int p2, int depth, int size) + private int ssCompareLast(int PA, int p1, int p2, int depth, int size) { - int U1, U2, U1n, U2n; + int U1, + U2, + U1n, + U2n; for ( - U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = size, U2n = this.SA[(p2 + 1)] + 2; + U1 = depth + this.SA[p1], U2 = depth + this.SA[p2], U1n = size, U2n = + this.SA[(p2 + 1)] + 2; (U1 < U1n) && (U2 < U2n) && (this.T[U1] == this.T[U2]); ++U1, ++U2 ) { } @@ -173,12 +430,12 @@ private int ssCompareLast (int PA, int p1, int p2, int depth, int size) ++U1, ++U2 ) { } - return U1 < U1n ? - (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) + return U1 < U1n + ? (U2 < U2n ? (this.T[U1] & 0xff) - (this.T[U2] & 0xff) : 1) : (U2 < U2n ? -1 : 0); } - private void ssInsertionSort (int PA, int first, int last, int depth) + private void ssInsertionSort(int PA, int first, int last, int depth) { int i; // pointer within SA @@ -187,7 +444,11 @@ private void ssInsertionSort (int PA, int first, int last, int depth) int j; // pointer within SA int t; int r; - for (t = this.SA[i], j = i + 1; 0 < (r = this.ssCompare (PA + t, PA + this.SA[j], depth));) + for ( + t = this.SA[i], j = i + 1; + 0 < (r = this.ssCompare(PA + t, PA + this.SA[j], depth)); + + ) { do { @@ -206,13 +467,18 @@ private void ssInsertionSort (int PA, int first, int last, int depth) } } - private void ssFixdown (int Td, int PA, int sa, int i, int size) + private void ssFixdown(int Td, int PA, int sa, int i, int size) { - int j, k; + int j, + k; int v; int c; - for (v = this.SA[sa + i], c = (this.T[Td + this.SA[PA + v]]) & 0xff; (j = 2 * i + 1) < size; this.SA[sa + i] = this.SA[sa + k], i = k) + for ( + v = this.SA[sa + i], c = (this.T[Td + this.SA[PA + v]]) & 0xff; + (j = 2 * i + 1) < size; + this.SA[sa + i] = this.SA[sa + k], i = k + ) { int d = this.T[Td + this.SA[PA + this.SA[sa + (k = j++)]]] & 0xff; int e; @@ -221,12 +487,13 @@ private void ssFixdown (int Td, int PA, int sa, int i, int size) k = j; d = e; } - if (d <= c) break; + if (d <= c) + break; } this.SA[sa + i] = v; } - private void ssHeapSort (int Td, int PA, int sa, int size) + private void ssHeapSort(int Td, int PA, int sa, int size) { int i; @@ -234,33 +501,36 @@ private void ssHeapSort (int Td, int PA, int sa, int size) if ((size % 2) == 0) { m--; - if ((this.T[Td + this.SA[PA + this.SA[sa + (m / 2)]]] & 0xff) < (this.T[Td + this.SA[PA + this.SA[sa + m]]] & 0xff)) + if ( + (this.T[Td + this.SA[PA + this.SA[sa + (m / 2)]]] & 0xff) + < (this.T[Td + this.SA[PA + this.SA[sa + m]]] & 0xff) + ) { - swapElements (this.SA, sa + m, this.SA, sa + (m / 2)); + swapElements(this.SA, sa + m, this.SA, sa + (m / 2)); } } for (i = m / 2 - 1; 0 <= i; --i) { - this.ssFixdown (Td, PA, sa, i, m); + this.ssFixdown(Td, PA, sa, i, m); } if ((size % 2) == 0) { - swapElements (this.SA, sa, this.SA, sa + m); - this.ssFixdown (Td, PA, sa, 0, m); + swapElements(this.SA, sa, this.SA, sa + m); + this.ssFixdown(Td, PA, sa, 0, m); } for (i = m - 1; 0 < i; --i) { int t = this.SA[sa]; this.SA[sa] = this.SA[sa + i]; - this.ssFixdown (Td, PA, sa, 0, i); + this.ssFixdown(Td, PA, sa, 0, i); this.SA[sa + i] = t; } } - private int ssMedian3 (int Td, int PA, int v1, int v2, int v3) + private int ssMedian3(int Td, int PA, int v1, int v2, int v3) { var T_v1 = this.T[Td + this.SA[PA + this.SA[v1]]] & 0xff; var T_v2 = this.T[Td + this.SA[PA + this.SA[v2]]] & 0xff; @@ -282,7 +552,7 @@ private int ssMedian3 (int Td, int PA, int v1, int v2, int v3) return v2; } - private int ssMedian5 (int Td, int PA, int v1, int v2, int v3, int v4, int v5) + private int ssMedian5(int Td, int PA, int v1, int v2, int v3, int v4, int v5) { var T_v1 = this.T[Td + this.SA[PA + this.SA[v1]]] & 0xff; var T_v2 = this.T[Td + this.SA[PA + this.SA[v2]]] & 0xff; @@ -355,7 +625,7 @@ private int ssMedian5 (int Td, int PA, int v1, int v2, int v3, int v4, int v5) return T_v3 > T_v4 ? v4 : v3; } - private int ssPivot (int Td, int PA, int first, int last) + private int ssPivot(int Td, int PA, int first, int last) { int t = last - first; int middle = first + t / 2; @@ -364,36 +634,54 @@ private int ssPivot (int Td, int PA, int first, int last) { if (t <= 32) { - return this.ssMedian3 (Td, PA, first, middle, last - 1); + return this.ssMedian3(Td, PA, first, middle, last - 1); } t >>= 2; - return this.ssMedian5 (Td, PA, first, first + t, middle, last - 1 - t, last - 1); + return this.ssMedian5(Td, PA, first, first + t, middle, last - 1 - t, last - 1); } t >>= 3; - return this.ssMedian3 ( - Td, PA, - this.ssMedian3 (Td, PA, first, first + t, first + (t << 1)), - this.ssMedian3 (Td, PA, middle - t, middle, middle + t), - this.ssMedian3 (Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1) - ); + return this.ssMedian3( + Td, + PA, + this.ssMedian3(Td, PA, first, first + t, first + (t << 1)), + this.ssMedian3(Td, PA, middle - t, middle, middle + t), + this.ssMedian3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1) + ); } - private static int ssLog (int x) + private static int ssLog(int x) { - return ((x & 0xff00) != 0) ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] : BZip2DivSufSort.log2table[x & 0xff]; + return ((x & 0xff00) != 0) + ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] + : BZip2DivSufSort.log2table[x & 0xff]; } - private int ssSubstringPartition (int PA, int first, int last, int depth) + private int ssSubstringPartition(int PA, int first, int last, int depth) { - int a, b; + int a, + b; - for (a = first - 1, b = last;;) + for (a = first - 1, b = last; ; ) { - for (; (++a < b) && ((this.SA[PA + this.SA[a]] + depth) >= (this.SA[PA + this.SA[a] + 1] + 1));) + for ( + ; + (++a < b) + && ( + (this.SA[PA + this.SA[a]] + depth) >= (this.SA[PA + this.SA[a] + 1] + 1) + ); + + ) { this.SA[a] = ~this.SA[a]; } - for (; (a < --b) && ((this.SA[PA + this.SA[b]] + depth) < (this.SA[PA + this.SA[b] + 1] + 1));) { } + for ( + ; + (a < --b) + && ( + (this.SA[PA + this.SA[b]] + depth) < (this.SA[PA + this.SA[b] + 1] + 1) + ); + + ) { } if (b <= a) { break; @@ -408,7 +696,7 @@ private int ssSubstringPartition (int PA, int first, int last, int depth) return a; } - private void ssMultiKeyIntroSort (int PA, int first, int last, int depth) + private void ssMultiKeyIntroSort(int PA, int first, int last, int depth) { var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; @@ -416,15 +704,16 @@ private void ssMultiKeyIntroSort (int PA, int first, int last, int depth) int limit; int x = 0; - for (ssize = 0, limit = ssLog (last - first);;) + for (ssize = 0, limit = ssLog(last - first); ; ) { if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) { if (1 < (last - first)) { - this.ssInsertionSort (PA, first, last, depth); + this.ssInsertionSort(PA, first, last, depth); } - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; first = entry.a; last = entry.b; @@ -436,102 +725,136 @@ private void ssMultiKeyIntroSort (int PA, int first, int last, int depth) int Td = depth; if (limit-- == 0) { - this.ssHeapSort (Td, PA, first, last - first); + this.ssHeapSort(Td, PA, first, last - first); } int a; int v; if (limit < 0) { - for (a = first + 1, v = this.T[Td + this.SA[PA + this.SA[first]]] & 0xff; a < last; ++a) + for ( + a = first + 1, v = this.T[Td + this.SA[PA + this.SA[first]]] & 0xff; + a < last; + ++a + ) { if ((x = (this.T[Td + this.SA[PA + this.SA[a]]] & 0xff)) != v) { - if (1 < (a - first)) { break; } + if (1 < (a - first)) + { + break; + } v = x; first = a; } } if ((this.T[Td + this.SA[PA + this.SA[first]] - 1] & 0xff) < v) { - first = this.ssSubstringPartition (PA, first, a, depth); + first = this.ssSubstringPartition(PA, first, a, depth); } if ((a - first) <= (last - a)) { if (1 < (a - first)) { - stack[ssize++] = new StackEntry (a, last, depth, -1); + stack[ssize++] = new StackEntry(a, last, depth, -1); last = a; depth += 1; - limit = ssLog (a - first); - } else + limit = ssLog(a - first); + } + else { first = a; limit = -1; } - } else + } + else { if (1 < (last - a)) { - stack[ssize++] = new StackEntry (first, a, depth + 1, ssLog (a - first)); + stack[ssize++] = new StackEntry(first, a, depth + 1, ssLog(a - first)); first = a; limit = -1; - } else + } + else { last = a; depth += 1; - limit = ssLog (a - first); + limit = ssLog(a - first); } } continue; } - a = this.ssPivot (Td, PA, first, last); + a = this.ssPivot(Td, PA, first, last); v = this.T[Td + this.SA[PA + this.SA[a]]] & 0xff; - swapElements (this.SA, first, this.SA, a); + swapElements(this.SA, first, this.SA, a); int b; - for (b = first; (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) == v); ) { } + for ( + b = first; + (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) == v); + + ) { } if (((a = b) < last) && (x < v)) { - for (; (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v);) + for ( + ; + (++b < last) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v); + + ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } } int c; - for (c = last; (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) == v);) { } + for ( + c = last; + (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) == v); + + ) { } int d; if ((b < (d = c)) && (x > v)) { - for (; (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v);) + for ( + ; + (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v); + + ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } } - for (; b < c;) + for (; b < c; ) { - swapElements (this.SA, b, this.SA, c); - for (; (++b < c) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v);) + swapElements(this.SA, b, this.SA, c); + for ( + ; + (++b < c) && ((x = (this.T[Td + this.SA[PA + this.SA[b]]] & 0xff)) <= v); + + ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } - for (; (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v);) + for ( + ; + (b < --c) && ((x = (this.T[Td + this.SA[PA + this.SA[c]]] & 0xff)) >= v); + + ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } @@ -551,7 +874,7 @@ private void ssMultiKeyIntroSort (int PA, int first, int last, int depth) int f; for (e = first, f = b - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } if ((s = d - c) > (t = last - d - 1)) { @@ -559,90 +882,116 @@ private void ssMultiKeyIntroSort (int PA, int first, int last, int depth) } for (e = b, f = last - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } a = first + (b - a); c = last - (d - c); - b = (v <= (this.T[Td + this.SA[PA + this.SA[a]] - 1] & 0xff)) ? a : this.ssSubstringPartition (PA, a, c, depth); + b = + (v <= (this.T[Td + this.SA[PA + this.SA[a]] - 1] & 0xff)) + ? a + : this.ssSubstringPartition(PA, a, c, depth); if ((a - first) <= (last - c)) { if ((last - c) <= (c - b)) { - stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); - stack[ssize++] = new StackEntry (c, last, depth, limit); + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); + stack[ssize++] = new StackEntry(c, last, depth, limit); last = a; - } else if ((a - first) <= (c - b)) + } + else if ((a - first) <= (c - b)) { - stack[ssize++] = new StackEntry (c, last, depth, limit); - stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); + stack[ssize++] = new StackEntry(c, last, depth, limit); + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); last = a; - } else + } + else { - stack[ssize++] = new StackEntry (c, last, depth, limit); - stack[ssize++] = new StackEntry (first, a, depth, limit); + stack[ssize++] = new StackEntry(c, last, depth, limit); + stack[ssize++] = new StackEntry(first, a, depth, limit); first = b; last = c; depth += 1; - limit = ssLog (c - b); + limit = ssLog(c - b); } - } else + } + else { if ((a - first) <= (c - b)) { - stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); - stack[ssize++] = new StackEntry (first, a, depth, limit); + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); + stack[ssize++] = new StackEntry(first, a, depth, limit); first = c; - } else if ((last - c) <= (c - b)) + } + else if ((last - c) <= (c - b)) { - stack[ssize++] = new StackEntry (first, a, depth, limit); - stack[ssize++] = new StackEntry (b, c, depth + 1, ssLog (c - b)); + stack[ssize++] = new StackEntry(first, a, depth, limit); + stack[ssize++] = new StackEntry(b, c, depth + 1, ssLog(c - b)); first = c; - } else + } + else { - stack[ssize++] = new StackEntry (first, a, depth, limit); - stack[ssize++] = new StackEntry (c, last, depth, limit); + stack[ssize++] = new StackEntry(first, a, depth, limit); + stack[ssize++] = new StackEntry(c, last, depth, limit); first = b; last = c; depth += 1; - limit = ssLog (c - b); + limit = ssLog(c - b); } } - } else + } + else { limit += 1; if ((this.T[Td + this.SA[PA + this.SA[first]] - 1] & 0xff) < v) { - first = this.ssSubstringPartition (PA, first, last, depth); - limit = ssLog (last - first); + first = this.ssSubstringPartition(PA, first, last, depth); + limit = ssLog(last - first); } depth += 1; } } } - private static void ssBlockSwap (int[] array1, int first1, int[] array2, int first2, int size) + private static void ssBlockSwap( + int[] array1, + int first1, + int[] array2, + int first2, + int size + ) { - int a, b; + int a, + b; int i; for (i = size, a = first1, b = first2; 0 < i; --i, ++a, ++b) { - swapElements (array1, a, array2, b); + swapElements(array1, a, array2, b); } } - private void ssMergeForward (int PA, int[] buf, int bufoffset, int first, int middle, int last, int depth) + private void ssMergeForward( + int PA, + int[] buf, + int bufoffset, + int first, + int middle, + int last, + int depth + ) { - int i, j, k; + int i, + j, + k; int t; int bufend = bufoffset + (middle - first) - 1; - ssBlockSwap (buf, bufoffset, this.SA, first, middle - first); + ssBlockSwap(buf, bufoffset, this.SA, first, middle - first); - for (t = this.SA[first], i = first, j = bufoffset, k = middle;;) + for (t = this.SA[first], i = first, j = bufoffset, k = middle; ; ) { - int r = this.ssCompare (PA + buf[j], PA + this.SA[k], depth); + int r = this.ssCompare(PA + buf[j], PA + this.SA[k], depth); if (r < 0) { do @@ -655,7 +1004,8 @@ private void ssMergeForward (int PA, int[] buf, int bufoffset, int first, int } buf[j++] = this.SA[i]; } while (buf[j] < 0); - } else if (r > 0) + } + else if (r > 0) { do { @@ -673,7 +1023,8 @@ private void ssMergeForward (int PA, int[] buf, int bufoffset, int first, int return; } } while (this.SA[k] < 0); - } else + } + else { this.SA[k] = ~this.SA[k]; do @@ -707,35 +1058,48 @@ private void ssMergeForward (int PA, int[] buf, int bufoffset, int first, int } } - private void ssMergeBackward (int PA, int[] buf, int bufoffset, int first, int middle, int last, int depth) + private void ssMergeBackward( + int PA, + int[] buf, + int bufoffset, + int first, + int middle, + int last, + int depth + ) { - int p1, p2; - int i, j, k; + int p1, + p2; + int i, + j, + k; int t; int bufend = bufoffset + (last - middle); - ssBlockSwap (buf, bufoffset, this.SA, middle, last - middle); + ssBlockSwap(buf, bufoffset, this.SA, middle, last - middle); int x = 0; if (buf[bufend - 1] < 0) { - x |= 1; + x |= 1; p1 = PA + ~buf[bufend - 1]; - } else + } + else { - p1 = PA + buf[bufend - 1]; + p1 = PA + buf[bufend - 1]; } if (this.SA[middle - 1] < 0) { - x |= 2; + x |= 2; p2 = PA + ~this.SA[middle - 1]; - } else + } + else { - p2 = PA + this.SA[middle - 1]; + p2 = PA + this.SA[middle - 1]; } - for (t = this.SA[last - 1], i = last - 1, j = bufend - 1, k = middle - 1;;) + for (t = this.SA[last - 1], i = last - 1, j = bufend - 1, k = middle - 1; ; ) { - int r = this.ssCompare (p1, p2, depth); + int r = this.ssCompare(p1, p2, depth); if (r > 0) { if ((x & 1) != 0) @@ -757,13 +1121,15 @@ private void ssMergeBackward (int PA, int[] buf, int bufoffset, int first, in if (buf[j] < 0) { - x |= 1; + x |= 1; p1 = PA + ~buf[j]; - } else + } + else { - p1 = PA + buf[j]; + p1 = PA + buf[j]; } - } else if (r < 0) + } + else if (r < 0) { if ((x & 2) != 0) { @@ -790,13 +1156,15 @@ private void ssMergeBackward (int PA, int[] buf, int bufoffset, int first, in if (this.SA[k] < 0) { - x |= 2; + x |= 2; p2 = PA + ~this.SA[k]; - } else + } + else { - p2 = PA + this.SA[k]; + p2 = PA + this.SA[k]; } - } else + } + else { if ((x & 1) != 0) { @@ -840,62 +1208,77 @@ private void ssMergeBackward (int PA, int[] buf, int bufoffset, int first, in if (buf[j] < 0) { - x |= 1; + x |= 1; p1 = PA + ~buf[j]; - } else + } + else { - p1 = PA + buf[j]; + p1 = PA + buf[j]; } if (this.SA[k] < 0) { - x |= 2; + x |= 2; p2 = PA + ~this.SA[k]; - } else + } + else { - p2 = PA + this.SA[k]; + p2 = PA + this.SA[k]; } } } } - private static int getIDX (int a) + private static int getIDX(int a) { return (0 <= a) ? a : ~a; } - private void ssMergeCheckEqual (int PA, int depth, int a) + private void ssMergeCheckEqual(int PA, int depth, int a) { - if ((0 <= this.SA[a]) && (this.ssCompare (PA + getIDX (this.SA[a - 1]), PA + this.SA[a], depth) == 0)) + if ( + (0 <= this.SA[a]) + && (this.ssCompare(PA + getIDX(this.SA[a - 1]), PA + this.SA[a], depth) == 0) + ) { this.SA[a] = ~this.SA[a]; } } - private void ssMerge (int PA, int first, int middle, int last, int[] buf, int bufoffset, int bufsize, int depth) + private void ssMerge( + int PA, + int first, + int middle, + int last, + int[] buf, + int bufoffset, + int bufsize, + int depth + ) { var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; int ssize; int check; - for (check = 0, ssize = 0;;) + for (check = 0, ssize = 0; ; ) { if ((last - middle) <= bufsize) { if ((first < middle) && (middle < last)) { - this.ssMergeBackward (PA, buf, bufoffset, first, middle, last, depth); + this.ssMergeBackward(PA, buf, bufoffset, first, middle, last, depth); } if ((check & 1) != 0) { - this.ssMergeCheckEqual (PA, depth, first); + this.ssMergeCheckEqual(PA, depth, first); } if ((check & 2) != 0) { - this.ssMergeCheckEqual (PA, depth, last); + this.ssMergeCheckEqual(PA, depth, last); } - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; first = entry.a; middle = entry.b; @@ -908,17 +1291,18 @@ private void ssMerge (int PA, int first, int middle, int last, int[] buf, int b { if (first < middle) { - this.ssMergeForward ( PA, buf, bufoffset, first, middle, last, depth); + this.ssMergeForward(PA, buf, bufoffset, first, middle, last, depth); } if ((check & 1) != 0) { - this.ssMergeCheckEqual (PA, depth, first); + this.ssMergeCheckEqual(PA, depth, first); } if ((check & 2) != 0) { - this.ssMergeCheckEqual (PA, depth, last); + this.ssMergeCheckEqual(PA, depth, last); } - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; first = entry.a; middle = entry.b; @@ -931,13 +1315,18 @@ private void ssMerge (int PA, int first, int middle, int last, int[] buf, int b int len; int half; for ( - m = 0, len = Math.Min (middle - first, last - middle), half = len >> 1; + m = 0, len = Math.Min(middle - first, last - middle), half = len >> 1; 0 < len; len = half, half >>= 1 ) { - if (this.ssCompare (PA + getIDX (this.SA[middle + m + half]), - PA + getIDX (this.SA[middle - m - half - 1]), depth) < 0) + if ( + this.ssCompare( + PA + getIDX(this.SA[middle + m + half]), + PA + getIDX(this.SA[middle - m - half - 1]), + depth + ) < 0 + ) { m += half + 1; half -= (len & 1) ^ 1; @@ -946,7 +1335,7 @@ private void ssMerge (int PA, int first, int middle, int last, int[] buf, int b if (0 < m) { - ssBlockSwap (this.SA, middle - m, this.SA, middle, m); + ssBlockSwap(this.SA, middle - m, this.SA, middle, m); int j; int i = j = middle; int next = 0; @@ -954,43 +1343,54 @@ private void ssMerge (int PA, int first, int middle, int last, int[] buf, int b { if (this.SA[middle + m] < 0) { - for (; this.SA[i - 1] < 0; --i) - { } + for (; this.SA[i - 1] < 0; --i) { } this.SA[middle + m] = ~this.SA[middle + m]; } - for (j = middle; this.SA[j] < 0; ++j) - { } + for (j = middle; this.SA[j] < 0; ++j) { } next = 1; } if ((i - first) <= (last - j)) { - stack[ssize++] = new StackEntry (j, middle + m, last, (check & 2) | (next & 1)); + stack[ssize++] = new StackEntry( + j, + middle + m, + last, + (check & 2) | (next & 1) + ); middle -= m; last = i; check = (check & 1); - } else + } + else { if ((i == middle) && (middle == j)) { next <<= 1; } - stack[ssize++] = new StackEntry (first, middle - m, i, (check & 1) | (next & 2)); + stack[ssize++] = new StackEntry( + first, + middle - m, + i, + (check & 1) | (next & 2) + ); first = j; middle += m; check = (check & 2) | (next & 1); } - } else + } + else { if ((check & 1) != 0) { - this.ssMergeCheckEqual (PA, depth, first); + this.ssMergeCheckEqual(PA, depth, first); } - this.ssMergeCheckEqual (PA, depth, middle); + this.ssMergeCheckEqual(PA, depth, middle); if ((check & 2) != 0) { - this.ssMergeCheckEqual (PA, depth, last); + this.ssMergeCheckEqual(PA, depth, last); } - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; first = entry.a; middle = entry.b; @@ -1000,7 +1400,17 @@ private void ssMerge (int PA, int first, int middle, int last, int[] buf, int b } } - private void subStringSort (int PA, int first, int last, int[] buf, int bufoffset, int bufsize, int depth, bool lastsuffix, int size) + private void subStringSort( + int PA, + int first, + int last, + int[] buf, + int bufoffset, + int bufsize, + int depth, + bool lastsuffix, + int size + ) { int a; int i; @@ -1009,9 +1419,13 @@ private void subStringSort (int PA, int first, int last, int[] buf, int bufof if (lastsuffix) ++first; - for (a = first, i = 0; (a + BZip2DivSufSort.SS_BLOCKSIZE) < last; a += BZip2DivSufSort.SS_BLOCKSIZE, ++i) + for ( + a = first, i = 0; + (a + BZip2DivSufSort.SS_BLOCKSIZE) < last; + a += BZip2DivSufSort.SS_BLOCKSIZE, ++i + ) { - this.ssMultiKeyIntroSort (PA, a, a + BZip2DivSufSort.SS_BLOCKSIZE, depth); + this.ssMultiKeyIntroSort(PA, a, a + BZip2DivSufSort.SS_BLOCKSIZE, depth); int[] curbuf = this.SA; int curbufoffset = a + BZip2DivSufSort.SS_BLOCKSIZE; int curbufsize = last - (a + BZip2DivSufSort.SS_BLOCKSIZE); @@ -1023,17 +1437,21 @@ private void subStringSort (int PA, int first, int last, int[] buf, int bufof } int b; int j; - for (b = a, k = BZip2DivSufSort.SS_BLOCKSIZE, j = i; (j & 1) != 0; b -= k, k <<= 1, j >>= 1) - this.ssMerge (PA, b - k, b, b + k, curbuf, curbufoffset, curbufsize, depth); + for ( + b = a, k = BZip2DivSufSort.SS_BLOCKSIZE, j = i; + (j & 1) != 0; + b -= k, k <<= 1, j >>= 1 + ) + this.ssMerge(PA, b - k, b, b + k, curbuf, curbufoffset, curbufsize, depth); } - this.ssMultiKeyIntroSort (PA, a, last, depth); + this.ssMultiKeyIntroSort(PA, a, last, depth); for (k = BZip2DivSufSort.SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) { if ((i & 1) != 0) { - this.ssMerge (PA, a - k, a, last, buf, bufoffset, bufsize, depth); + this.ssMerge(PA, a - k, a, last, buf, bufoffset, bufsize, depth); a -= k; } } @@ -1043,7 +1461,14 @@ private void subStringSort (int PA, int first, int last, int[] buf, int bufof int r; for ( a = first, i = this.SA[first - 1], r = 1; - (a < last) && ((this.SA[a] < 0) || (0 < (r = this.ssCompareLast (PA, PA + i, PA + this.SA[a], depth, size)))); + (a < last) + && ( + (this.SA[a] < 0) + || ( + 0 + < (r = this.ssCompareLast(PA, PA + i, PA + this.SA[a], depth, size)) + ) + ); ++a ) { @@ -1057,23 +1482,32 @@ private void subStringSort (int PA, int first, int last, int[] buf, int bufof } } - private int trGetC (int ISA, int ISAd, int ISAn, int p) + private int trGetC(int ISA, int ISAd, int ISAn, int p) { - return (((ISAd + p) < ISAn) ? this.SA[ISAd + p] : this.SA[ISA + ((ISAd - ISA + p) % (ISAn - ISA))]); + return ( + ((ISAd + p) < ISAn) + ? this.SA[ISAd + p] + : this.SA[ISA + ((ISAd - ISA + p) % (ISAn - ISA))] + ); } - private void trFixdown (int ISA, int ISAd, int ISAn, int sa, int i, int size) + private void trFixdown(int ISA, int ISAd, int ISAn, int sa, int i, int size) { - int j, k; + int j, + k; int v; int c; - for (v = this.SA[sa + i], c = this.trGetC (ISA, ISAd, ISAn, v); (j = 2 * i + 1) < size; this.SA[sa + i] = this.SA[sa + k], i = k) + for ( + v = this.SA[sa + i], c = this.trGetC(ISA, ISAd, ISAn, v); + (j = 2 * i + 1) < size; + this.SA[sa + i] = this.SA[sa + k], i = k + ) { k = j++; - int d = this.trGetC (ISA, ISAd, ISAn, this.SA[sa + k]); + int d = this.trGetC(ISA, ISAd, ISAn, this.SA[sa + k]); int e; - if (d < (e = this.trGetC (ISA, ISAd, ISAn, this.SA[sa + j]))) + if (d < (e = this.trGetC(ISA, ISAd, ISAn, this.SA[sa + j]))) { k = j; d = e; @@ -1086,7 +1520,7 @@ private void trFixdown (int ISA, int ISAd, int ISAn, int sa, int i, int size this.SA[sa + i] = v; } - private void trHeapSort (int ISA, int ISAd, int ISAn, int sa, int size) + private void trHeapSort(int ISA, int ISAd, int ISAn, int sa, int size) { int i; @@ -1094,33 +1528,36 @@ private void trHeapSort (int ISA, int ISAd, int ISAn, int sa, int size) if ((size % 2) == 0) { m--; - if (this.trGetC (ISA, ISAd, ISAn, this.SA[sa + (m / 2)]) < this.trGetC (ISA, ISAd, ISAn, this.SA[sa + m])) + if ( + this.trGetC(ISA, ISAd, ISAn, this.SA[sa + (m / 2)]) + < this.trGetC(ISA, ISAd, ISAn, this.SA[sa + m]) + ) { - swapElements (this.SA, sa + m, this.SA, sa + (m / 2)); + swapElements(this.SA, sa + m, this.SA, sa + (m / 2)); } } for (i = m / 2 - 1; 0 <= i; --i) { - this.trFixdown (ISA, ISAd, ISAn, sa, i, m); + this.trFixdown(ISA, ISAd, ISAn, sa, i, m); } if ((size % 2) == 0) { - swapElements (this.SA, sa + 0, this.SA, sa + m); - this.trFixdown (ISA, ISAd, ISAn, sa, 0, m); + swapElements(this.SA, sa + 0, this.SA, sa + m); + this.trFixdown(ISA, ISAd, ISAn, sa, 0, m); } for (i = m - 1; 0 < i; --i) { int t = this.SA[sa + 0]; this.SA[sa + 0] = this.SA[sa + i]; - this.trFixdown (ISA, ISAd, ISAn, sa, 0, i); + this.trFixdown(ISA, ISAd, ISAn, sa, 0, i); this.SA[sa + i] = t; } } - private void trInsertionSort (int ISA, int ISAd, int ISAn, int first, int last) + private void trInsertionSort(int ISA, int ISAd, int ISAn, int first, int last) { int a; @@ -1129,7 +1566,16 @@ private void trInsertionSort (int ISA, int ISAd, int ISAn, int first, int last int b; int t; int r; - for (t = this.SA[a], b = a - 1; 0 > (r = this.trGetC (ISA, ISAd, ISAn, t) - this.trGetC (ISA, ISAd, ISAn, this.SA[b]));) + for ( + t = this.SA[a], b = a - 1; + 0 + > ( + r = + this.trGetC(ISA, ISAd, ISAn, t) + - this.trGetC(ISA, ISAd, ISAn, this.SA[b]) + ); + + ) { do { @@ -1148,18 +1594,26 @@ private void trInsertionSort (int ISA, int ISAd, int ISAn, int first, int last } } - private static int trLog (int x) + private static int trLog(int x) { - return ((x & 0xffff0000) != 0) ? - (((x & 0xff000000) != 0) ? 24 + BZip2DivSufSort.log2table[(x >> 24) & 0xff] : 16 + BZip2DivSufSort.log2table[(x >> 16) & 0xff]) - : (((x & 0x0000ff00) != 0) ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] : 0 + BZip2DivSufSort.log2table[(x >> 0) & 0xff]); + return ((x & 0xffff0000) != 0) + ? ( + ((x & 0xff000000) != 0) + ? 24 + BZip2DivSufSort.log2table[(x >> 24) & 0xff] + : 16 + BZip2DivSufSort.log2table[(x >> 16) & 0xff] + ) + : ( + ((x & 0x0000ff00) != 0) + ? 8 + BZip2DivSufSort.log2table[(x >> 8) & 0xff] + : 0 + BZip2DivSufSort.log2table[(x >> 0) & 0xff] + ); } - private int trMedian3 (int ISA, int ISAd, int ISAn, int v1, int v2, int v3) + private int trMedian3(int ISA, int ISAd, int ISAn, int v1, int v2, int v3) { - var SA_v1 = this.trGetC (ISA, ISAd, ISAn, this.SA[v1]); - var SA_v2 = this.trGetC (ISA, ISAd, ISAn, this.SA[v2]); - var SA_v3 = this.trGetC (ISA, ISAd, ISAn, this.SA[v3]); + var SA_v1 = this.trGetC(ISA, ISAd, ISAn, this.SA[v1]); + var SA_v2 = this.trGetC(ISA, ISAd, ISAn, this.SA[v2]); + var SA_v3 = this.trGetC(ISA, ISAd, ISAn, this.SA[v3]); if (SA_v1 > SA_v2) { @@ -1178,13 +1632,13 @@ private int trMedian3 (int ISA, int ISAd, int ISAn, int v1, int v2, int v3) return v2; } - private int trMedian5 (int ISA, int ISAd, int ISAn, int v1, int v2, int v3, int v4, int v5) + private int trMedian5(int ISA, int ISAd, int ISAn, int v1, int v2, int v3, int v4, int v5) { - var SA_v1 = this.trGetC (ISA, ISAd, ISAn, this.SA[v1]); - var SA_v2 = this.trGetC (ISA, ISAd, ISAn, this.SA[v2]); - var SA_v3 = this.trGetC (ISA, ISAd, ISAn, this.SA[v3]); - var SA_v4 = this.trGetC (ISA, ISAd, ISAn, this.SA[v4]); - var SA_v5 = this.trGetC (ISA, ISAd, ISAn, this.SA[v5]); + var SA_v1 = this.trGetC(ISA, ISAd, ISAn, this.SA[v1]); + var SA_v2 = this.trGetC(ISA, ISAd, ISAn, this.SA[v2]); + var SA_v3 = this.trGetC(ISA, ISAd, ISAn, this.SA[v3]); + var SA_v4 = this.trGetC(ISA, ISAd, ISAn, this.SA[v4]); + var SA_v5 = this.trGetC(ISA, ISAd, ISAn, this.SA[v5]); int temp; int SA_vtemp; @@ -1251,7 +1705,7 @@ private int trMedian5 (int ISA, int ISAd, int ISAn, int v1, int v2, int v3, in return SA_v3 > SA_v4 ? v4 : v3; } - private int trPivot (int ISA, int ISAd, int ISAn, int first, int last) + private int trPivot(int ISA, int ISAd, int ISAn, int first, int last) { int t = last - first; int middle = first + t / 2; @@ -1260,26 +1714,32 @@ private int trPivot (int ISA, int ISAd, int ISAn, int first, int last) { if (t <= 32) { - return this.trMedian3 (ISA, ISAd, ISAn, first, middle, last - 1); + return this.trMedian3(ISA, ISAd, ISAn, first, middle, last - 1); } t >>= 2; - return this.trMedian5 ( - ISA, ISAd, ISAn, - first, first + t, + return this.trMedian5( + ISA, + ISAd, + ISAn, + first, + first + t, middle, - last - 1 - t, last - 1 - ); + last - 1 - t, + last - 1 + ); } t >>= 3; - return this.trMedian3 ( - ISA, ISAd, ISAn, - this.trMedian3 (ISA, ISAd, ISAn, first, first + t, first + (t << 1)), - this.trMedian3 (ISA, ISAd, ISAn, middle - t, middle, middle + t), - this.trMedian3 (ISA, ISAd, ISAn, last - 1 - (t << 1), last - 1 - t, last - 1) - ); + return this.trMedian3( + ISA, + ISAd, + ISAn, + this.trMedian3(ISA, ISAd, ISAn, first, first + t, first + (t << 1)), + this.trMedian3(ISA, ISAd, ISAn, middle - t, middle, middle + t), + this.trMedian3(ISA, ISAd, ISAn, last - 1 - (t << 1), last - 1 - t, last - 1) + ); } - private void lsUpdateGroup (int ISA, int first, int last) + private void lsUpdateGroup(int ISA, int first, int last) { int a; @@ -1312,7 +1772,7 @@ private void lsUpdateGroup (int ISA, int first, int last) } } - private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) + private void lsIntroSort(int ISA, int ISAd, int ISAn, int first, int last) { var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; @@ -1320,20 +1780,21 @@ private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) int x = 0; int ssize; - for (ssize = 0, limit = trLog (last - first);;) + for (ssize = 0, limit = trLog(last - first); ; ) { - if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) { if (1 < (last - first)) { - this.trInsertionSort (ISA, ISAd, ISAn, first, last); - this.lsUpdateGroup (ISA, first, last); - } else if ((last - first) == 1) + this.trInsertionSort(ISA, ISAd, ISAn, first, last); + this.lsUpdateGroup(ISA, first, last); + } + else if ((last - first) == 1) { this.SA[first] = -1; } - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; first = entry.a; last = entry.b; @@ -1345,20 +1806,21 @@ private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) int b; if (limit-- == 0) { - this.trHeapSort (ISA, ISAd, ISAn, first, last - first); + this.trHeapSort(ISA, ISAd, ISAn, first, last - first); for (a = last - 1; first < a; a = b) { for ( - x = this.trGetC (ISA, ISAd, ISAn, this.SA[a]), b = a - 1; - (first <= b) && (this.trGetC (ISA, ISAd, ISAn, this.SA[b]) == x); + x = this.trGetC(ISA, ISAd, ISAn, this.SA[a]), b = a - 1; + (first <= b) && (this.trGetC(ISA, ISAd, ISAn, this.SA[b]) == x); --b ) { this.SA[b] = ~this.SA[b]; } } - this.lsUpdateGroup (ISA, first, last); - if (ssize == 0) return; + this.lsUpdateGroup(ISA, first, last); + if (ssize == 0) + return; var entry = stack[--ssize]; first = entry.a; last = entry.b; @@ -1366,54 +1828,57 @@ private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) continue; } - a = this.trPivot (ISA, ISAd, ISAn, first, last); - swapElements (this.SA, first, this.SA, a); - int v = this.trGetC (ISA, ISAd, ISAn, this.SA[first]); + a = this.trPivot(ISA, ISAd, ISAn, first, last); + swapElements(this.SA, first, this.SA, a); + int v = this.trGetC(ISA, ISAd, ISAn, this.SA[first]); - for (b = first; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) == v);) - { } + for ( + b = first; + (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) == v); + + ) { } if (((a = b) < last) && (x < v)) { - for (; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + for (; (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } } int c; - for (c = last; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) == v);) + for (c = last; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) == v); ) { } int d; if ((b < (d = c)) && (x > v)) { - for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } } - for (; b < c;) + for (; b < c; ) { - swapElements (this.SA, b, this.SA, c); - for (; (++b < c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + swapElements(this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } - for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } @@ -1433,7 +1898,7 @@ private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) int f; for (e = first, f = b - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } if ((s = d - c) > (t = last - d - 1)) { @@ -1441,7 +1906,7 @@ private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) } for (e = b, f = last - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } a = first + (b - a); @@ -1460,33 +1925,38 @@ private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) } if ((b - a) == 1) { - this.SA[a] = - 1; + this.SA[a] = -1; } if ((a - first) <= (last - b)) { if (first < a) { - stack[ssize++] = new StackEntry (b, last, limit, 0); + stack[ssize++] = new StackEntry(b, last, limit, 0); last = a; - } else + } + else { first = b; } - } else + } + else { if (b < last) { - stack[ssize++] = new StackEntry (first, a, limit, 0); + stack[ssize++] = new StackEntry(first, a, limit, 0); first = b; - } else + } + else { last = a; } } - } else + } + else { - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; first = entry.a; last = entry.b; @@ -1495,7 +1965,7 @@ private void lsIntroSort (int ISA, int ISAd, int ISAn, int first, int last) } } - private void lsSort (int ISA, int x, int depth) + private void lsSort(int ISA, int x, int depth) { int ISAd; @@ -1511,7 +1981,8 @@ private void lsSort (int ISA, int x, int depth) { first -= t; skip += t; - } else + } + else { if (skip != 0) { @@ -1519,7 +1990,7 @@ private void lsSort (int ISA, int x, int depth) skip = 0; } last = this.SA[ISA + t] + 1; - this.lsIntroSort (ISA, ISAd, ISA + x, first, last); + this.lsIntroSort(ISA, ISAd, ISA + x, first, last); first = last; } } while (first < x); @@ -1548,53 +2019,58 @@ private void lsSort (int ISA, int x, int depth) } } - private PartitionResult trPartition (int ISA, int ISAd, int ISAn, int first, int last, int v) + private PartitionResult trPartition(int ISA, int ISAd, int ISAn, int first, int last, int v) { - int a, b, c, d; + int a, + b, + c, + d; var x = 0; - for (b = first - 1; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) == v);) - { } + for ( + b = first - 1; + (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) == v); + + ) { } if (((a = b) < last) && (x < v)) { - for (; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + for (; (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } } - for (c = last; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) == v);) - { } + for (c = last; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) == v); ) { } if ((b < (d = c)) && (x > v)) { - for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } } - for (; b < c;) + for (; b < c; ) { - swapElements (this.SA, b, this.SA, c); - for (; (++b < c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + swapElements(this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } - for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } @@ -1612,7 +2088,7 @@ private PartitionResult trPartition (int ISA, int ISAd, int ISAn, int first, i int f; for (e = first, f = b - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } if ((s = d - c) > (t = last - d - 1)) { @@ -1620,18 +2096,20 @@ private PartitionResult trPartition (int ISA, int ISAd, int ISAn, int first, i } for (e = b, f = last - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } first += (b - a); last -= (d - c); } - return new PartitionResult (first, last); + return new PartitionResult(first, last); } - private void trCopy (int ISA, int ISAn, int first, int a, int b, int last, int depth) + private void trCopy(int ISA, int ISAn, int first, int a, int b, int last, int depth) { - int c, d, e; + int c, + d, + e; int s; int v = b - 1; @@ -1661,7 +2139,15 @@ private void trCopy (int ISA, int ISAn, int first, int a, int b, int last, } } - private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRBudget budget, int size) + private void trIntroSort( + int ISA, + int ISAd, + int ISAn, + int first, + int last, + TRBudget budget, + int size + ) { var stack = new StackEntry[BZip2DivSufSort.STACK_SIZE]; @@ -1670,7 +2156,7 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB int limit; int ssize; - for (ssize = 0, limit = trLog (last - first);;) + for (ssize = 0, limit = trLog(last - first); ; ) { int a; int b; @@ -1681,8 +2167,9 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB { if (limit == -1) { - if (!budget.update (size, last - first)) break; - var result = this.trPartition (ISA, ISAd - 1, ISAn, first, last, last - 1); + if (!budget.update(size, last - first)) + break; + var result = this.trPartition(ISA, ISAd - 1, ISAn, first, last, last - 1); a = result.first; b = result.last; if ((first < a) || (b < last)) @@ -1702,42 +2189,54 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB } } - stack[ssize++] = new StackEntry (0, a, b, 0); - stack[ssize++] = new StackEntry (ISAd - 1, first, last, -2); + stack[ssize++] = new StackEntry(0, a, b, 0); + stack[ssize++] = new StackEntry(ISAd - 1, first, last, -2); if ((a - first) <= (last - b)) { if (1 < (a - first)) { - stack[ssize++] = new StackEntry (ISAd, b, last, trLog (last - b)); + stack[ssize++] = new StackEntry(ISAd, b, last, trLog(last - b)); last = a; - limit = trLog (a - first); - } else if (1 < (last - b)) + limit = trLog(a - first); + } + else if (1 < (last - b)) { first = b; - limit = trLog (last - b); - } else + limit = trLog(last - b); + } + else { - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; ISAd = entry.a; first = entry.b; last = entry.c; limit = entry.d; } - } else + } + else { if (1 < (last - b)) { - stack[ssize++] = new StackEntry (ISAd, first, a, trLog (a - first)); + stack[ssize++] = new StackEntry( + ISAd, + first, + a, + trLog(a - first) + ); first = b; - limit = trLog (last - b); - } else if (1 < (a - first)) + limit = trLog(last - b); + } + else if (1 < (a - first)) { last = a; - limit = trLog (a - first); - } else + limit = trLog(a - first); + } + else { - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; ISAd = entry.a; first = entry.b; @@ -1745,31 +2244,36 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB limit = entry.d; } } - } else + } + else { for (c = first; c < last; ++c) { this.SA[ISA + this.SA[c]] = c; } - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; ISAd = entry.a; first = entry.b; last = entry.c; limit = entry.d; } - } else if (limit == -2) + } + else if (limit == -2) { a = stack[--ssize].b; b = stack[ssize].c; - this.trCopy (ISA, ISAn, first, a, b, last, ISAd - ISA); - if (ssize == 0) return; + this.trCopy(ISA, ISAn, first, a, b, last, ISAd - ISA); + if (ssize == 0) + return; var entry = stack[--ssize]; ISAd = entry.a; first = entry.b; last = entry.c; limit = entry.d; - } else + } + else { if (0 <= this.SA[first]) { @@ -1787,7 +2291,10 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB { this.SA[a] = ~this.SA[a]; } while (this.SA[++a] < 0); - next = (this.SA[ISA + this.SA[a]] != this.SA[ISAd + this.SA[a]]) ? trLog (a - first + 1) : -1; + next = + (this.SA[ISA + this.SA[a]] != this.SA[ISAd + this.SA[a]]) + ? trLog(a - first + 1) + : -1; if (++a < last) { for (b = first, v = a - 1; b < a; ++b) @@ -1798,27 +2305,31 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB if ((a - first) <= (last - a)) { - stack[ssize++] = new StackEntry (ISAd, a, last, -3); + stack[ssize++] = new StackEntry(ISAd, a, last, -3); ISAd += 1; last = a; limit = next; - } else + } + else { if (1 < (last - a)) { - stack[ssize++] = new StackEntry (ISAd + 1, first, a, next); + stack[ssize++] = new StackEntry(ISAd + 1, first, a, next); first = a; limit = -3; - } else + } + else { ISAd += 1; last = a; limit = next; } } - } else + } + else { - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; ISAd = entry.a; first = entry.b; @@ -1831,21 +2342,23 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB if ((last - first) <= BZip2DivSufSort.INSERTIONSORT_THRESHOLD) { - if (!budget.update (size, last - first)) break; - this.trInsertionSort (ISA, ISAd, ISAn, first, last); + if (!budget.update(size, last - first)) + break; + this.trInsertionSort(ISA, ISAd, ISAn, first, last); limit = -3; continue; } if (limit-- == 0) { - if (!budget.update (size, last - first)) break; - this.trHeapSort (ISA, ISAd, ISAn, first, last - first); + if (!budget.update(size, last - first)) + break; + this.trHeapSort(ISA, ISAd, ISAn, first, last - first); for (a = last - 1; first < a; a = b) { for ( - x = this.trGetC (ISA, ISAd, ISAn, this.SA[a]), b = a - 1; - (first <= b) && (this.trGetC (ISA, ISAd, ISAn, this.SA[b]) == x); + x = this.trGetC(ISA, ISAd, ISAn, this.SA[a]), b = a - 1; + (first <= b) && (this.trGetC(ISA, ISAd, ISAn, this.SA[b]) == x); --b ) { @@ -1856,53 +2369,56 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB continue; } - a = this.trPivot (ISA, ISAd, ISAn, first, last); + a = this.trPivot(ISA, ISAd, ISAn, first, last); - swapElements (this.SA, first, this.SA, a); - v = this.trGetC (ISA, ISAd, ISAn, this.SA[first]); - for (b = first; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) == v);) - { } + swapElements(this.SA, first, this.SA, a); + v = this.trGetC(ISA, ISAd, ISAn, this.SA[first]); + for ( + b = first; + (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) == v); + + ) { } if (((a = b) < last) && (x < v)) { - for (; (++b < last) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + for (; (++b < last) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } } - for (c = last; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) == v);) + for (c = last; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) == v); ) { } int d; if ((b < (d = c)) && (x > v)) { - for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } } - for (; b < c;) + for (; b < c; ) { - swapElements (this.SA, b, this.SA, c); - for (; (++b < c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[b])) <= v);) + swapElements(this.SA, b, this.SA, c); + for (; (++b < c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[b])) <= v); ) { if (x == v) { - swapElements (this.SA, b, this.SA, a); + swapElements(this.SA, b, this.SA, a); ++a; } } - for (; (b < --c) && ((x = this.trGetC (ISA, ISAd, ISAn, this.SA[c])) >= v);) + for (; (b < --c) && ((x = this.trGetC(ISA, ISAd, ISAn, this.SA[c])) >= v); ) { if (x == v) { - swapElements (this.SA, c, this.SA, d); + swapElements(this.SA, c, this.SA, d); --d; } } @@ -1921,7 +2437,7 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB int f; for (e = first, f = b - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } if ((s = d - c) > (t = last - d - 1)) { @@ -1929,12 +2445,12 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB } for (e = b, f = last - s; 0 < s; --s, ++e, ++f) { - swapElements (this.SA, e, this.SA, f); + swapElements(this.SA, e, this.SA, f); } a = first + (b - a); b = last - (d - c); - next = (this.SA[ISA + this.SA[a]] != v) ? trLog (b - a) : -1; + next = (this.SA[ISA + this.SA[a]] != v) ? trLog(b - a) : -1; for (c = first, v = a - 1; c < a; ++c) { @@ -1954,123 +2470,143 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB { if (1 < (a - first)) { - stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); - stack[ssize++] = new StackEntry (ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); last = a; - } else if (1 < (last - b)) + } + else if (1 < (last - b)) { - stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); first = b; - } else if (1 < (b - a)) + } + else if (1 < (b - a)) { ISAd += 1; first = a; last = b; limit = next; - } else + } + else { - if (ssize == 0) return; + if (ssize == 0) + return; var entry = stack[--ssize]; ISAd = entry.a; first = entry.b; last = entry.c; limit = entry.d; } - } else if ((a - first) <= (b - a)) + } + else if ((a - first) <= (b - a)) { if (1 < (a - first)) { - stack[ssize++] = new StackEntry (ISAd, b, last, limit); - stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); last = a; - } else if (1 < (b - a)) + } + else if (1 < (b - a)) { - stack[ssize++] = new StackEntry (ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); ISAd += 1; first = a; last = b; limit = next; - } else + } + else { first = b; } - } else + } + else { if (1 < (b - a)) { - stack[ssize++] = new StackEntry (ISAd, b, last, limit); - stack[ssize++] = new StackEntry (ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); ISAd += 1; first = a; last = b; limit = next; - } else + } + else { - stack[ssize++] = new StackEntry (ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); last = a; } } - } else + } + else { if ((a - first) <= (b - a)) { if (1 < (last - b)) { - stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); - stack[ssize++] = new StackEntry (ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); first = b; - } else if (1 < (a - first)) + } + else if (1 < (a - first)) { - stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); last = a; - } else if (1 < (b - a)) + } + else if (1 < (b - a)) { ISAd += 1; first = a; last = b; limit = next; - } else + } + else { - stack[ssize++] = new StackEntry (ISAd, first, last, limit); + stack[ssize++] = new StackEntry(ISAd, first, last, limit); } - } else if ((last - b) <= (b - a)) + } + else if ((last - b) <= (b - a)) { if (1 < (last - b)) { - stack[ssize++] = new StackEntry (ISAd, first, a, limit); - stack[ssize++] = new StackEntry (ISAd + 1, a, b, next); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd + 1, a, b, next); first = b; - } else if (1 < (b - a)) + } + else if (1 < (b - a)) { - stack[ssize++] = new StackEntry (ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); ISAd += 1; first = a; last = b; limit = next; - } else + } + else { last = a; } - } else + } + else { if (1 < (b - a)) { - stack[ssize++] = new StackEntry (ISAd, first, a, limit); - stack[ssize++] = new StackEntry (ISAd, b, last, limit); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd, b, last, limit); ISAd += 1; first = a; last = b; limit = next; - } else + } + else { - stack[ssize++] = new StackEntry (ISAd, first, a, limit); + stack[ssize++] = new StackEntry(ISAd, first, a, limit); first = b; } } } - } else + } + else { - if (!budget.update (size, last - first)) break; // BUGFIX : Added to prevent an infinite loop in the original code + if (!budget.update(size, last - first)) + break; // BUGFIX : Added to prevent an infinite loop in the original code limit += 1; ISAd += 1; } @@ -2080,30 +2616,31 @@ private void trIntroSort (int ISA, int ISAd, int ISAn, int first, int last, TRB { if (stack[s].d == -3) { - this.lsUpdateGroup (ISA, stack[s].b, stack[s].c); + this.lsUpdateGroup(ISA, stack[s].b, stack[s].c); } } } - private void trSort (int ISA, int x, int depth) + private void trSort(int ISA, int x, int depth) { int first = 0; if (-x < this.SA[0]) { - var budget = new TRBudget (x, trLog (x) * 2 / 3 + 1); + var budget = new TRBudget(x, trLog(x) * 2 / 3 + 1); do { int t; if ((t = this.SA[first]) < 0) { first -= t; - } else + } + else { int last = this.SA[ISA + t] + 1; if (1 < (last - first)) { - this.trIntroSort (ISA, ISA + depth, ISA + x, first, last, budget, x); + this.trIntroSort(ISA, ISA + depth, ISA + x, first, last, budget, x); if (budget.chance == 0) { // Switch to Larsson-Sadakane sorting algorithm. @@ -2111,33 +2648,36 @@ private void trSort (int ISA, int x, int depth) { this.SA[0] = -first; } - this.lsSort (ISA, x, depth); + this.lsSort(ISA, x, depth); break; } } first = last; - } } while (first < x); } } - private static int BUCKET_B (int c0, int c1) + private static int BUCKET_B(int c0, int c1) { return (c1 << 8) | c0; } - private static int BUCKET_BSTAR (int c0, int c1) + private static int BUCKET_BSTAR(int c0, int c1) { return (c0 << 8) | c1; } - private int sortTypeBstar (int[] bucketA, int[] bucketB) + private int sortTypeBstar(int[] bucketA, int[] bucketB) { var tempbuf = new int[256]; - int i, j, k, t; - int c0, c1; + int i, + j, + k, + t; + int c0, + c1; int flag; for (i = 1, flag = 1; i < this.n; ++i) @@ -2154,24 +2694,34 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) i = this.n - 1; int m = this.n; - int ti, ti1, t0; - if (((ti = (this.T[i] & 0xff)) < (t0 = (this.T[0] & 0xff))) || ((this.T[i] == this.T[0]) && (flag != 0))) + int ti, + ti1, + t0; + if ( + ((ti = (this.T[i] & 0xff)) < (t0 = (this.T[0] & 0xff))) + || ((this.T[i] == this.T[0]) && (flag != 0)) + ) { if (flag == 0) { - ++bucketB[BUCKET_BSTAR (ti, t0)]; + ++bucketB[BUCKET_BSTAR(ti, t0)]; this.SA[--m] = i; - } else + } + else { - ++bucketB[BUCKET_B (ti, t0)]; + ++bucketB[BUCKET_B(ti, t0)]; } - for (--i; (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); --i) + for ( + --i; + (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); + --i + ) { - ++bucketB[BUCKET_B (ti, ti1)]; + ++bucketB[BUCKET_B(ti, ti1)]; } } - for (; 0 <= i;) + for (; 0 <= i; ) { do { @@ -2180,11 +2730,15 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) if (0 <= i) { - ++bucketB[BUCKET_BSTAR (this.T[i] & 0xff, this.T[i + 1] & 0xff)]; + ++bucketB[BUCKET_BSTAR(this.T[i] & 0xff, this.T[i + 1] & 0xff)]; this.SA[--m] = i; - for (--i; (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); --i) + for ( + --i; + (0 <= i) && ((ti = (this.T[i] & 0xff)) <= (ti1 = (this.T[i + 1] & 0xff))); + --i + ) { - ++bucketB[BUCKET_B (ti, ti1)]; + ++bucketB[BUCKET_B(ti, ti1)]; } } } @@ -2202,12 +2756,12 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) { t = i + bucketA[c0]; bucketA[c0] = i + j; - i = t + bucketB[BUCKET_B (c0, c0)]; + i = t + bucketB[BUCKET_B(c0, c0)]; for (c1 = c0 + 1; c1 < 256; ++c1) { - j += bucketB[BUCKET_BSTAR (c0, c1)]; + j += bucketB[BUCKET_BSTAR(c0, c1)]; bucketB[(c0 << 8) | c1] = j; - i += bucketB[BUCKET_B (c0, c1)]; + i += bucketB[BUCKET_B(c0, c1)]; } } @@ -2218,12 +2772,12 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) t = this.SA[PAb + i]; c0 = this.T[t] & 0xff; c1 = this.T[t + 1] & 0xff; - this.SA[--bucketB[BUCKET_BSTAR (c0, c1)]] = i; + this.SA[--bucketB[BUCKET_BSTAR(c0, c1)]] = i; } t = this.SA[PAb + m - 1]; c0 = this.T[t] & 0xff; c1 = this.T[t + 1] & 0xff; - this.SA[--bucketB[BUCKET_BSTAR (c0, c1)]] = m - 1; + this.SA[--bucketB[BUCKET_BSTAR(c0, c1)]] = m - 1; int[] buf = this.SA; int bufoffset = m; @@ -2239,10 +2793,20 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) { for (c1 = 255; c0 < c1; j = i, --c1) { - i = bucketB[BUCKET_BSTAR (c0, c1)]; + i = bucketB[BUCKET_BSTAR(c0, c1)]; if (1 < (j - i)) { - this.subStringSort (PAb, i, j, buf, bufoffset, bufsize, 2, this.SA[i] == (m - 1), this.n); + this.subStringSort( + PAb, + i, + j, + buf, + bufoffset, + bufsize, + 2, + this.SA[i] == (m - 1), + this.n + ); } } } @@ -2270,28 +2834,28 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) this.SA[ISAb + this.SA[i]] = j; } - this.trSort (ISAb, m, 1); + this.trSort(ISAb, m, 1); i = this.n - 1; j = m; - if (((this.T[i] & 0xff) < (this.T[0] & 0xff)) || ((this.T[i] == this.T[0]) && (flag != 0))) + if ( + ((this.T[i] & 0xff) < (this.T[0] & 0xff)) + || ((this.T[i] == this.T[0]) && (flag != 0)) + ) { if (flag == 0) { this.SA[this.SA[ISAb + --j]] = i; } - for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) - { } + for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) { } } - for (; 0 <= i;) + for (; 0 <= i; ) { - for (--i; (0 <= i) && ((this.T[i] & 0xff) >= (this.T[i + 1] & 0xff)); --i) - { } + for (--i; (0 <= i) && ((this.T[i] & 0xff) >= (this.T[i + 1] & 0xff)); --i) { } if (0 <= i) { this.SA[this.SA[ISAb + --j]] = i; - for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) - { } + for (--i; (0 <= i) && ((this.T[i] & 0xff) <= (this.T[i + 1] & 0xff)); --i) { } } } @@ -2299,19 +2863,19 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) { for (c1 = 255; c0 < c1; --c1) { - t = i - bucketB[BUCKET_B (c0, c1)]; - bucketB[BUCKET_B (c0, c1)] = i + 1; + t = i - bucketB[BUCKET_B(c0, c1)]; + bucketB[BUCKET_B(c0, c1)] = i + 1; - for (i = t, j = bucketB[BUCKET_BSTAR (c0, c1)]; j <= k; --i, --k) + for (i = t, j = bucketB[BUCKET_BSTAR(c0, c1)]; j <= k; --i, --k) { this.SA[i] = this.SA[k]; } } - t = i - bucketB[BUCKET_B (c0, c0)]; - bucketB[BUCKET_B (c0, c0)] = i + 1; + t = i - bucketB[BUCKET_B(c0, c0)]; + bucketB[BUCKET_B(c0, c0)] = i + 1; if (c0 < 255) { - bucketB[BUCKET_BSTAR (c0, c0 + 1)] = t + 1; + bucketB[BUCKET_BSTAR(c0, c0 + 1)] = t + 1; } i = bucketA[c0]; } @@ -2319,18 +2883,25 @@ private int sortTypeBstar (int[] bucketA, int[] bucketB) return m; } - private int constructBWT (int[] bucketA, int[] bucketB) + private int constructBWT(int[] bucketA, int[] bucketB) { int i; int t = 0; - int s, s1; - int c0, c1, c2 = 0; + int s, + s1; + int c0, + c1, + c2 = 0; var orig = -1; for (c1 = 254; 0 <= c1; --c1) { int j; - for (i = bucketB[BUCKET_BSTAR (c1, c1 + 1)], j = bucketA[c1 + 1], t = 0, c2 = -1; i <= j; --j) + for ( + i = bucketB[BUCKET_BSTAR(c1, c1 + 1)], j = bucketA[c1 + 1], t = 0, c2 = -1; + i <= j; + --j + ) { if (0 <= (s1 = s = this.SA[j])) { @@ -2347,15 +2918,17 @@ private int constructBWT (int[] bucketA, int[] bucketB) if (c2 == c0) { this.SA[--t] = s; - } else + } + else { if (0 <= c2) - bucketB[BUCKET_B (c2, c1)] = t; + bucketB[BUCKET_B(c2, c1)] = t; - this.SA[t = bucketB[BUCKET_B (c2 = c0, c1)] - 1] = s; + this.SA[t = bucketB[BUCKET_B(c2 = c0, c1)] - 1] = s; } } - } else + } + else { this.SA[j] = ~s; } @@ -2377,15 +2950,16 @@ private int constructBWT (int[] bucketA, int[] bucketB) if (c0 == c2) { this.SA[++t] = s; - } else + } + else { if (c2 != -1) // BUGFIX: Original code can write to bucketA[-1] bucketA[c2] = t; this.SA[t = bucketA[c2 = c0] + 1] = s; } } - - } else + } + else { s1 = ~s1; } @@ -2394,7 +2968,8 @@ private int constructBWT (int[] bucketA, int[] bucketB) { this.SA[i] = this.T[this.n - 1]; orig = i; - } else + } + else { this.SA[i] = this.T[s1 - 1]; } diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs index 790cc7cfb..20a302799 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageDecoder.cs @@ -6,6 +6,7 @@ using System; using System.IO; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.Algorithm { /// @@ -23,22 +24,33 @@ internal class BZip2HuffmanStageDecoder private readonly byte[] selectors; // The minimum code length for each Huffman table - private readonly int[] minimumLengths = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES]; + private readonly int[] minimumLengths = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES + ]; /// /// An array of values for each Huffman table that must be subtracted from the numerical value of /// a Huffman code of a given bit length to give its canonical code index /// - private readonly int[,] codeBases = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2]; + private readonly int[,] codeBases = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2 + ]; /// /// An array of values for each Huffman table that gives the highest numerical value of a Huffman /// code of a given bit length /// - private readonly int[,] codeLimits = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 1]; + private readonly int[,] codeLimits = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 1 + ]; // A mapping for each Huffman table from canonical code index to output symbol - private readonly int[,] codeSymbols = new int[BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE]; + private readonly int[,] codeSymbols = new int[ + BZip2BlockDecompressor.HUFFMAN_MAXIMUM_TABLES, + BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE + ]; // The Huffman table for the current group private int currentTable; @@ -56,7 +68,12 @@ internal class BZip2HuffmanStageDecoder /// The total number of codes (uniform for each table) /// The Canonical Huffman code lengths for each table /// The Huffman table number to use for each group of 50 symbols - public BZip2HuffmanStageDecoder(IBZip2BitInputStream bitInputStream, int alphabetSize, byte[,] tableCodeLengths, byte[] selectors) + public BZip2HuffmanStageDecoder( + IBZip2BitInputStream bitInputStream, + int alphabetSize, + byte[,] tableCodeLengths, + byte[] selectors + ) { this.bitInputStream = bitInputStream; this.selectors = selectors; @@ -85,12 +102,19 @@ public int NextSymbol() // Starting with the minimum bit length for the table, read additional bits one at a time // until a complete code is recognised - for (uint codeBits = this.bitInputStream.ReadBits(codeLength); codeLength <= BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH; codeLength++) + for ( + uint codeBits = this.bitInputStream.ReadBits(codeLength); + codeLength <= BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH; + codeLength++ + ) { if (codeBits <= this.codeLimits[this.currentTable, codeLength]) { // Convert the code to a symbol index and return - return this.codeSymbols[this.currentTable, codeBits - this.codeBases[this.currentTable, codeLength]]; + return this.codeSymbols[ + this.currentTable, + codeBits - this.codeBases[this.currentTable, codeLength] + ]; } codeBits = (codeBits << 1) | this.bitInputStream.ReadBits(1); } @@ -98,15 +122,14 @@ public int NextSymbol() // A valid code was not recognised throw new IOException("Error decoding BZip2 block"); } - + /// /// Constructs Huffman decoding tables from lists of Canonical Huffman code lengths /// /// The total number of codes (uniform for each table) /// The Canonical Huffman code lengths for each table - private void CreateHuffmanDecodingTables (int alphabetSize, byte[,] tableCodeLengths) + private void CreateHuffmanDecodingTables(int alphabetSize, byte[,] tableCodeLengths) { - for (int table = 0; table < tableCodeLengths.GetLength(0); table++) { int minimumLength = BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH; @@ -115,8 +138,8 @@ private void CreateHuffmanDecodingTables (int alphabetSize, byte[,] tableCodeLe // Find the minimum and maximum code length for the table for (int i = 0; i < alphabetSize; i++) { - maximumLength = Math.Max (tableCodeLengths[table, i], maximumLength); - minimumLength = Math.Min (tableCodeLengths[table, i], minimumLength); + maximumLength = Math.Max(tableCodeLengths[table, i], maximumLength); + minimumLength = Math.Min(tableCodeLengths[table, i], minimumLength); } this.minimumLengths[table] = minimumLength; @@ -125,7 +148,11 @@ private void CreateHuffmanDecodingTables (int alphabetSize, byte[,] tableCodeLe { this.codeBases[table, tableCodeLengths[table, i] + 1]++; } - for (int i = 1; i < BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2; i++) + for ( + int i = 1; + i < BZip2HuffmanStageDecoder.HUFFMAN_DECODE_MAXIMUM_CODE_LENGTH + 2; + i++ + ) { this.codeBases[table, i] += this.codeBases[table, i - 1]; } @@ -141,7 +168,11 @@ private void CreateHuffmanDecodingTables (int alphabetSize, byte[,] tableCodeLe } // Populate the mapping from canonical code index to output symbol - for (int bitLength = minimumLength, codeIndex = 0; bitLength <= maximumLength; bitLength++) + for ( + int bitLength = minimumLength, codeIndex = 0; + bitLength <= maximumLength; + bitLength++ + ) { for (int symbol = 0; symbol < alphabetSize; symbol++) { diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs index c03334e30..9f83d67a5 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/BZip2HuffmanStageEncoder.cs @@ -6,11 +6,12 @@ using System; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.Algorithm { /// /// An encoder for the BZip2 Huffman encoding stage - /// + /// internal class BZip2HuffmanStageEncoder { // Used in initial Huffman table generation @@ -45,7 +46,7 @@ internal class BZip2HuffmanStageEncoder // The selectors for each segment private readonly byte[] selectors; - + /// /// Public constructor /// @@ -54,7 +55,13 @@ internal class BZip2HuffmanStageEncoder /// The actual length of the MTF block /// The size of the MTF block's alphabet /// The frequencies the MTF block's symbols - public BZip2HuffmanStageEncoder(IBZip2BitOutputStream bitOutputStream, ushort[] mtfBlock, int mtfLength, int mtfAlphabetSize, int[] mtfSymbolFrequencies) + public BZip2HuffmanStageEncoder( + IBZip2BitOutputStream bitOutputStream, + ushort[] mtfBlock, + int mtfLength, + int mtfAlphabetSize, + int[] mtfSymbolFrequencies + ) { this.bitOutputStream = bitOutputStream; this.mtfBlock = mtfBlock; @@ -66,7 +73,10 @@ public BZip2HuffmanStageEncoder(IBZip2BitOutputStream bitOutputStream, ushort[] this.huffmanCodeLengths = new int[totalTables, mtfAlphabetSize]; this.huffmanMergedCodeSymbols = new int[totalTables, mtfAlphabetSize]; - this.selectors = new byte[(mtfLength + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH - 1) / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH]; + this.selectors = new byte[ + (mtfLength + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH - 1) + / BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH + ]; } /// @@ -93,7 +103,7 @@ public void Encode() /// /// The length to select a table count for /// The selected table count - private static int selectTableCount (int mtfLength) + private static int selectTableCount(int mtfLength) { if (mtfLength >= 2400) return 6; @@ -111,7 +121,12 @@ private static int selectTableCount (int mtfLength) /// The frequencies of the symbols /// The array to which the generated code lengths should be written /// - private static void generateHuffmanCodeLengths (int alphabetSize, int[,] symbolFrequencies, int[,] codeLengths, int index) + private static void generateHuffmanCodeLengths( + int alphabetSize, + int[,] symbolFrequencies, + int[,] codeLengths, + int index + ) { var mergedFrequenciesAndIndices = new int[alphabetSize]; var sortedFrequencies = new int[alphabetSize]; @@ -131,7 +146,10 @@ private static void generateHuffmanCodeLengths (int alphabetSize, int[,] symbol sortedFrequencies[i] = mergedFrequenciesAndIndices[i] >> 9; // Allocate code lengths - the allocation is in place, so the code lengths will be in the sortedFrequencies array afterwards - HuffmanAllocator.AllocateHuffmanCodeLengths (sortedFrequencies, BZip2HuffmanStageEncoder.HUFFMAN_ENCODE_MAXIMUM_CODE_LENGTH); + HuffmanAllocator.AllocateHuffmanCodeLengths( + sortedFrequencies, + BZip2HuffmanStageEncoder.HUFFMAN_ENCODE_MAXIMUM_CODE_LENGTH + ); // Reverse the sort to place the code lengths in the same order as the symbols whose frequencies were passed in for (int i = 0; i < alphabetSize; i++) @@ -144,7 +162,7 @@ private static void generateHuffmanCodeLengths (int alphabetSize, int[,] symbol /// tables are invalid for actual Huffman code generation, and only serve as the seed for later /// iterative optimisation in optimiseSelectorsAndHuffmanTables(int) /// - private void generateHuffmanOptimisationSeeds () + private void generateHuffmanOptimisationSeeds() { int totalTables = this.huffmanCodeLengths.GetLength(0); int remainingLength = this.mtfLength; @@ -156,18 +174,27 @@ private void generateHuffmanOptimisationSeeds () int lowCostStart = lowCostEnd + 1; int actualCumulativeFrequency = 0; - while ((actualCumulativeFrequency < targetCumulativeFrequency) && (lowCostEnd < (this.mtfAlphabetSize - 1))) + while ( + (actualCumulativeFrequency < targetCumulativeFrequency) + && (lowCostEnd < (this.mtfAlphabetSize - 1)) + ) { actualCumulativeFrequency += this.mtfSymbolFrequencies[++lowCostEnd]; } - if ((lowCostEnd > lowCostStart) && (i != 0) && (i != (totalTables - 1)) && (((totalTables - i) & 1) == 0)) + if ( + (lowCostEnd > lowCostStart) + && (i != 0) + && (i != (totalTables - 1)) + && (((totalTables - i) & 1) == 0) + ) actualCumulativeFrequency -= this.mtfSymbolFrequencies[lowCostEnd--]; for (var j = 0; j < this.mtfAlphabetSize; j++) { if ((j < lowCostStart) || (j > lowCostEnd)) - this.huffmanCodeLengths[i, j] = BZip2HuffmanStageEncoder.HUFFMAN_HIGH_SYMBOL_COST; + this.huffmanCodeLengths[i, j] = + BZip2HuffmanStageEncoder.HUFFMAN_HIGH_SYMBOL_COST; } remainingLength -= actualCumulativeFrequency; @@ -182,7 +209,7 @@ private void generateHuffmanOptimisationSeeds () /// instead diverge (increase) slightly.
/// /// If true, write out the (final) chosen selectors - private void optimiseSelectorsAndHuffmanTables (bool storeSelectors) + private void optimiseSelectorsAndHuffmanTables(bool storeSelectors) { int totalTables = this.huffmanCodeLengths.GetLength(0); var tableFrequencies = new int[totalTables, this.mtfAlphabetSize]; @@ -190,9 +217,13 @@ private void optimiseSelectorsAndHuffmanTables (bool storeSelectors) int selectorIndex = 0; // Find the best table for each group of 50 block bytes based on the current Huffman code lengths - for (int groupStart = 0; groupStart < this.mtfLength;) + for (int groupStart = 0; groupStart < this.mtfLength; ) { - int groupEnd = Math.Min(groupStart + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, this.mtfLength) - 1; + int groupEnd = + Math.Min( + groupStart + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, + this.mtfLength + ) - 1; // Calculate the cost of this group when encoded by each table var cost = new int[totalTables]; @@ -208,7 +239,7 @@ private void optimiseSelectorsAndHuffmanTables (bool storeSelectors) // Find the table with the least cost for this group byte bestTable = 0; var bestCost = cost[0]; - for (byte i = 1 ; i < totalTables; i++) + for (byte i = 1; i < totalTables; i++) { var tableCost = cost[i]; if (tableCost < bestCost) @@ -236,7 +267,12 @@ private void optimiseSelectorsAndHuffmanTables (bool storeSelectors) // Generate new Huffman code lengths based on the frequencies for each table accumulated in this iteration for (int i = 0; i < totalTables; i++) { - generateHuffmanCodeLengths (this.mtfAlphabetSize, tableFrequencies, this.huffmanCodeLengths, i); + generateHuffmanCodeLengths( + this.mtfAlphabetSize, + tableFrequencies, + this.huffmanCodeLengths, + i + ); } } @@ -287,14 +323,14 @@ private void writeSelectorsAndHuffmanTables() int totalSelectors = this.selectors.Length; int totalTables = this.huffmanCodeLengths.GetLength(0); - this.bitOutputStream.WriteBits (3, (uint)totalTables); - this.bitOutputStream.WriteBits (15, (uint)totalSelectors); + this.bitOutputStream.WriteBits(3, (uint)totalTables); + this.bitOutputStream.WriteBits(15, (uint)totalSelectors); // Write the selectors var selectorMTF = new MoveToFront(); for (int i = 0; i < totalSelectors; i++) { - this.bitOutputStream.WriteUnary (selectorMTF.ValueToFront (this.selectors[i])); + this.bitOutputStream.WriteUnary(selectorMTF.ValueToFront(this.selectors[i])); } // Write the Huffman tables @@ -302,18 +338,18 @@ private void writeSelectorsAndHuffmanTables() { var currentLength = this.huffmanCodeLengths[i, 0]; - this.bitOutputStream.WriteBits (5, (uint)currentLength); + this.bitOutputStream.WriteBits(5, (uint)currentLength); for (var j = 0; j < this.mtfAlphabetSize; j++) { var codeLength = this.huffmanCodeLengths[i, j]; var value = (currentLength < codeLength) ? 2u : 3u; - var delta = Math.Abs (codeLength - currentLength); + var delta = Math.Abs(codeLength - currentLength); while (delta-- > 0) { - this.bitOutputStream.WriteBits (2, value); + this.bitOutputStream.WriteBits(2, value); } - this.bitOutputStream.WriteBoolean (false); + this.bitOutputStream.WriteBoolean(false); currentLength = codeLength; } } @@ -327,15 +363,22 @@ private void writeBlockData() { int selectorIndex = 0; - for (int mtfIndex = 0; mtfIndex < this.mtfLength;) + for (int mtfIndex = 0; mtfIndex < this.mtfLength; ) { - int groupEnd = Math.Min (mtfIndex + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, this.mtfLength) - 1; + int groupEnd = + Math.Min( + mtfIndex + BZip2HuffmanStageEncoder.HUFFMAN_GROUP_RUN_LENGTH, + this.mtfLength + ) - 1; - int index = this.selectors [selectorIndex++]; + int index = this.selectors[selectorIndex++]; while (mtfIndex <= groupEnd) { - var mergedCodeSymbol = this.huffmanMergedCodeSymbols[index, this.mtfBlock[mtfIndex++]]; + var mergedCodeSymbol = this.huffmanMergedCodeSymbols[ + index, + this.mtfBlock[mtfIndex++] + ]; this.bitOutputStream.WriteBits(mergedCodeSymbol >> 24, (uint)mergedCodeSymbol); } } diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs index be28b7fb1..5985c6f4a 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/Bzip2MTFAndRLE2StageEncoder.cs @@ -18,15 +18,17 @@ internal class BZip2MTFAndRLE2StageEncoder // Actual length of the data in the bwtBlock array private readonly int bwtLength; - // At each position, true if the byte value with that index is present within the block, otherwise false + // At each position, true if the byte value with that index is present within the block, otherwise false private readonly bool[] bwtValuesInUse; // The output of the Move To Front Transform and Run-Length Encoding[2] stages private readonly ushort[] mtfBlock; // The global frequencies of values within the mtfBlock array - private readonly int[] mtfSymbolFrequencies = new int[BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE]; - + private readonly int[] mtfSymbolFrequencies = new int[ + BZip2MTFAndRLE2StageEncoder.HUFFMAN_MAXIMUM_ALPHABET_SIZE + ]; + // Maximum possible Huffman alphabet size public const int HUFFMAN_MAXIMUM_ALPHABET_SIZE = 258; @@ -35,14 +37,11 @@ internal class BZip2MTFAndRLE2StageEncoder // Huffman symbol used for run-length encoding public const ushort RLE_SYMBOL_RUNB = 1; - + // Gets the encoded MTF block public ushort[] MtfBlock { - get - { - return this.mtfBlock; - } + get { return this.mtfBlock; } } // Gets The actual length of the MTF block @@ -54,10 +53,7 @@ public ushort[] MtfBlock // Gets the frequencies of the MTF block's symbols public int[] MtfSymbolFrequencies { - get - { - return this.mtfSymbolFrequencies; - } + get { return this.mtfSymbolFrequencies; } } /// @@ -85,7 +81,7 @@ public void Encode() for (var i = 0; i < 256; i++) { if (this.bwtValuesInUse[i]) - huffmanSymbolMap[i] = (byte) totalUniqueValues++; + huffmanSymbolMap[i] = (byte)totalUniqueValues++; } int endOfBlockSymbol = totalUniqueValues + 1; @@ -103,7 +99,8 @@ public void Encode() if (mtfPosition == 0) { repeatCount++; - } else + } + else { if (repeatCount > 0) { @@ -112,11 +109,14 @@ public void Encode() { if ((repeatCount & 1) == 0) { - this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA; + this.mtfBlock[mtfIndex++] = + BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA; totalRunAs++; - } else + } + else { - this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB; + this.mtfBlock[mtfIndex++] = + BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB; totalRunBs++; } @@ -128,7 +128,7 @@ public void Encode() repeatCount = 0; } - this.mtfBlock[mtfIndex++] = (char) (mtfPosition + 1); + this.mtfBlock[mtfIndex++] = (char)(mtfPosition + 1); this.mtfSymbolFrequencies[mtfPosition + 1]++; } } @@ -142,7 +142,8 @@ public void Encode() { this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNA; totalRunAs++; - } else + } + else { this.mtfBlock[mtfIndex++] = BZip2MTFAndRLE2StageEncoder.RLE_SYMBOL_RUNB; totalRunBs++; @@ -162,7 +163,6 @@ public void Encode() this.MtfLength = mtfIndex + 1; this.MtfAlphabetSize = endOfBlockSymbol + 1; - } } } diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs index 2bbffb9bb..31594e263 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/CRC32.cs @@ -10,60 +10,284 @@ namespace SharpCompress.Compressors.BZip2MT.Algorithm /// internal class CRC32 { - /// The CRC lookup table + /// The CRC lookup table private static readonly uint[] Crc32Lookup = { - 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, - 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, - 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, - 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, - 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, - 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, - 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, - 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, - 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, - 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, - 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, - 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, - 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, - 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, - 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, - 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, - 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, - 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, - 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, - 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, - 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, - 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, - 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, - 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, - 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, - 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, - 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, - 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, - 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, - 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, - 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, - 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + 0x00000000, + 0x04c11db7, + 0x09823b6e, + 0x0d4326d9, + 0x130476dc, + 0x17c56b6b, + 0x1a864db2, + 0x1e475005, + 0x2608edb8, + 0x22c9f00f, + 0x2f8ad6d6, + 0x2b4bcb61, + 0x350c9b64, + 0x31cd86d3, + 0x3c8ea00a, + 0x384fbdbd, + 0x4c11db70, + 0x48d0c6c7, + 0x4593e01e, + 0x4152fda9, + 0x5f15adac, + 0x5bd4b01b, + 0x569796c2, + 0x52568b75, + 0x6a1936c8, + 0x6ed82b7f, + 0x639b0da6, + 0x675a1011, + 0x791d4014, + 0x7ddc5da3, + 0x709f7b7a, + 0x745e66cd, + 0x9823b6e0, + 0x9ce2ab57, + 0x91a18d8e, + 0x95609039, + 0x8b27c03c, + 0x8fe6dd8b, + 0x82a5fb52, + 0x8664e6e5, + 0xbe2b5b58, + 0xbaea46ef, + 0xb7a96036, + 0xb3687d81, + 0xad2f2d84, + 0xa9ee3033, + 0xa4ad16ea, + 0xa06c0b5d, + 0xd4326d90, + 0xd0f37027, + 0xddb056fe, + 0xd9714b49, + 0xc7361b4c, + 0xc3f706fb, + 0xceb42022, + 0xca753d95, + 0xf23a8028, + 0xf6fb9d9f, + 0xfbb8bb46, + 0xff79a6f1, + 0xe13ef6f4, + 0xe5ffeb43, + 0xe8bccd9a, + 0xec7dd02d, + 0x34867077, + 0x30476dc0, + 0x3d044b19, + 0x39c556ae, + 0x278206ab, + 0x23431b1c, + 0x2e003dc5, + 0x2ac12072, + 0x128e9dcf, + 0x164f8078, + 0x1b0ca6a1, + 0x1fcdbb16, + 0x018aeb13, + 0x054bf6a4, + 0x0808d07d, + 0x0cc9cdca, + 0x7897ab07, + 0x7c56b6b0, + 0x71159069, + 0x75d48dde, + 0x6b93dddb, + 0x6f52c06c, + 0x6211e6b5, + 0x66d0fb02, + 0x5e9f46bf, + 0x5a5e5b08, + 0x571d7dd1, + 0x53dc6066, + 0x4d9b3063, + 0x495a2dd4, + 0x44190b0d, + 0x40d816ba, + 0xaca5c697, + 0xa864db20, + 0xa527fdf9, + 0xa1e6e04e, + 0xbfa1b04b, + 0xbb60adfc, + 0xb6238b25, + 0xb2e29692, + 0x8aad2b2f, + 0x8e6c3698, + 0x832f1041, + 0x87ee0df6, + 0x99a95df3, + 0x9d684044, + 0x902b669d, + 0x94ea7b2a, + 0xe0b41de7, + 0xe4750050, + 0xe9362689, + 0xedf73b3e, + 0xf3b06b3b, + 0xf771768c, + 0xfa325055, + 0xfef34de2, + 0xc6bcf05f, + 0xc27dede8, + 0xcf3ecb31, + 0xcbffd686, + 0xd5b88683, + 0xd1799b34, + 0xdc3abded, + 0xd8fba05a, + 0x690ce0ee, + 0x6dcdfd59, + 0x608edb80, + 0x644fc637, + 0x7a089632, + 0x7ec98b85, + 0x738aad5c, + 0x774bb0eb, + 0x4f040d56, + 0x4bc510e1, + 0x46863638, + 0x42472b8f, + 0x5c007b8a, + 0x58c1663d, + 0x558240e4, + 0x51435d53, + 0x251d3b9e, + 0x21dc2629, + 0x2c9f00f0, + 0x285e1d47, + 0x36194d42, + 0x32d850f5, + 0x3f9b762c, + 0x3b5a6b9b, + 0x0315d626, + 0x07d4cb91, + 0x0a97ed48, + 0x0e56f0ff, + 0x1011a0fa, + 0x14d0bd4d, + 0x19939b94, + 0x1d528623, + 0xf12f560e, + 0xf5ee4bb9, + 0xf8ad6d60, + 0xfc6c70d7, + 0xe22b20d2, + 0xe6ea3d65, + 0xeba91bbc, + 0xef68060b, + 0xd727bbb6, + 0xd3e6a601, + 0xdea580d8, + 0xda649d6f, + 0xc423cd6a, + 0xc0e2d0dd, + 0xcda1f604, + 0xc960ebb3, + 0xbd3e8d7e, + 0xb9ff90c9, + 0xb4bcb610, + 0xb07daba7, + 0xae3afba2, + 0xaafbe615, + 0xa7b8c0cc, + 0xa379dd7b, + 0x9b3660c6, + 0x9ff77d71, + 0x92b45ba8, + 0x9675461f, + 0x8832161a, + 0x8cf30bad, + 0x81b02d74, + 0x857130c3, + 0x5d8a9099, + 0x594b8d2e, + 0x5408abf7, + 0x50c9b640, + 0x4e8ee645, + 0x4a4ffbf2, + 0x470cdd2b, + 0x43cdc09c, + 0x7b827d21, + 0x7f436096, + 0x7200464f, + 0x76c15bf8, + 0x68860bfd, + 0x6c47164a, + 0x61043093, + 0x65c52d24, + 0x119b4be9, + 0x155a565e, + 0x18197087, + 0x1cd86d30, + 0x029f3d35, + 0x065e2082, + 0x0b1d065b, + 0x0fdc1bec, + 0x3793a651, + 0x3352bbe6, + 0x3e119d3f, + 0x3ad08088, + 0x2497d08d, + 0x2056cd3a, + 0x2d15ebe3, + 0x29d4f654, + 0xc5a92679, + 0xc1683bce, + 0xcc2b1d17, + 0xc8ea00a0, + 0xd6ad50a5, + 0xd26c4d12, + 0xdf2f6bcb, + 0xdbee767c, + 0xe3a1cbc1, + 0xe760d676, + 0xea23f0af, + 0xeee2ed18, + 0xf0a5bd1d, + 0xf464a0aa, + 0xf9278673, + 0xfde69bc4, + 0x89b8fd09, + 0x8d79e0be, + 0x803ac667, + 0x84fbdbd0, + 0x9abc8bd5, + 0x9e7d9662, + 0x933eb0bb, + 0x97ffad0c, + 0xafb010b1, + 0xab710d06, + 0xa6322bdf, + 0xa2f33668, + 0xbcb4666d, + 0xb8757bda, + 0xb5365d03, + 0xb1f740b4, }; /// The current CRC private uint crc = 0xffffffff; - - /// Gets the current CRC + + /// Gets the current CRC public uint CRC => ~this.crc; /// Updates the CRC with a single byte /// The value to update the CRC with - public void UpdateCrc (int value) + public void UpdateCrc(int value) { this.crc = (this.crc << 8) ^ CRC32.Crc32Lookup[((this.crc >> 24) ^ value) & 0xff]; } - /// Update the CRC with a sequence of identical bytes + /// Update the CRC with a sequence of identical bytes /// The value to update the CRC with /// The number of bytes - public void UpdateCrc (int value, int count) + public void UpdateCrc(int value, int count) { while (count-- > 0) { diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs index 2c5ca5502..bc1eea761 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/HuffmanAllocator.cs @@ -4,11 +4,12 @@ // Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 using System; + namespace SharpCompress.Compressors.BZip2MT.Algorithm { /// An in-place, length restricted Canonical Huffman code length allocator /// - /// Based on the algorithm proposed by R.L.Milidiú, A.A.Pessoa and E.S.Laber + /// Based on the algorithm proposed by R.L.Milidiú, A.A.Pessoa and E.S.Laber /// in "In-place Length-Restricted Prefix Coding" (see: http://www-di.inf.puc-rio.br/~laber/public/spire98.ps) /// and incorporating additional ideas from the implementation of "shcodec" by Simakov Alexander /// (see: http://webcenter.ru/~xander/) @@ -40,7 +41,8 @@ public static void AllocateHuffmanCodeLengths(int[] array, int maximumLength) if ((array[0] % array.Length) >= nodesToRelocate) { AllocateNodeLengths(array); - } else + } + else { var insertDepth = maximumLength - SignificantBits(nodesToRelocate - 1); AllocateNodeLengthsWithRelocation(array, nodesToRelocate, insertDepth); @@ -48,13 +50,13 @@ public static void AllocateHuffmanCodeLengths(int[] array, int maximumLength) } /// - /// + /// /// /// The code length array /// The input position /// The number of internal nodes to be relocated /// The smallest k such that nodesToMove <= k <= i and i <= (array[k] % array.length) - private static int First (int[] array, int i, int nodesToMove) + private static int First(int[] array, int i, int nodesToMove) { int length = array.Length; int limit = i; @@ -65,7 +67,7 @@ private static int First (int[] array, int i, int nodesToMove) k = i; i -= (limit - i + 1); } - i = Math.Max (nodesToMove - 1, i); + i = Math.Max(nodesToMove - 1, i); while (k > (i + 1)) { @@ -96,16 +98,21 @@ private static void SetExtendedParentPointers(int[] array) { temp = array[headNode]; array[headNode++] = tailNode; - } else + } + else { temp = array[topNode++]; } - if ((topNode >= length) || ((headNode < tailNode) && (array[headNode] < array[topNode]))) + if ( + (topNode >= length) + || ((headNode < tailNode) && (array[headNode] < array[topNode])) + ) { temp += array[headNode]; array[headNode++] = tailNode + length; - } else + } + else { temp += array[topNode++]; } @@ -120,12 +127,16 @@ private static void SetExtendedParentPointers(int[] array) /// The code length array /// The maximum bit length for the generated codes /// The number of nodes to relocate - private static int FindNodesToRelocate (int[] array, int maximumLength) + private static int FindNodesToRelocate(int[] array, int maximumLength) { int currentNode = array.Length - 2; - for (var currentDepth = 1; (currentDepth < (maximumLength - 1)) && (currentNode > 1); currentDepth++) - currentNode = First (array, currentNode - 1, 0); + for ( + var currentDepth = 1; + (currentDepth < (maximumLength - 1)) && (currentNode > 1); + currentDepth++ + ) + currentNode = First(array, currentNode - 1, 0); return currentNode; } @@ -134,7 +145,7 @@ private static int FindNodesToRelocate (int[] array, int maximumLength) /// A final allocation pass with no code length limit /// /// The code length array - private static void AllocateNodeLengths (int[] array) + private static void AllocateNodeLengths(int[] array) { int firstNode = array.Length - 2; int nextNode = array.Length - 1; @@ -142,7 +153,7 @@ private static void AllocateNodeLengths (int[] array) for (int currentDepth = 1, availableNodes = 2; availableNodes > 0; currentDepth++) { int lastNode = firstNode; - firstNode = First (array, lastNode - 1, 0); + firstNode = First(array, lastNode - 1, 0); for (var i = availableNodes - (lastNode - firstNode); i > 0; i--) array[nextNode--] = currentDepth; @@ -157,7 +168,11 @@ private static void AllocateNodeLengths (int[] array) /// The code length array /// The number of internal nodes to be relocated /// The depth at which to insert relocated nodes - private static void AllocateNodeLengthsWithRelocation (int[] array, int nodesToMove, int insertDepth) + private static void AllocateNodeLengthsWithRelocation( + int[] array, + int nodesToMove, + int insertDepth + ) { int firstNode = array.Length - 2; int nextNode = array.Length - 1; @@ -167,13 +182,17 @@ private static void AllocateNodeLengthsWithRelocation (int[] array, int nodesTo for (int availableNodes = currentDepth << 1; availableNodes > 0; currentDepth++) { int lastNode = firstNode; - firstNode = (firstNode <= nodesToMove) ? firstNode : First (array, lastNode - 1, nodesToMove); + firstNode = + (firstNode <= nodesToMove) + ? firstNode + : First(array, lastNode - 1, nodesToMove); int offset = 0; if (currentDepth >= insertDepth) { - offset = Math.Min (nodesLeftToMove, 1 << (currentDepth - insertDepth)); - } else if (currentDepth == (insertDepth - 1)) + offset = Math.Min(nodesLeftToMove, 1 << (currentDepth - insertDepth)); + } + else if (currentDepth == (insertDepth - 1)) { offset = 1; if ((array[firstNode]) == lastNode) diff --git a/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs b/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs index 047985067..a9fa23251 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Algorithm/MoveToFront.cs @@ -4,6 +4,7 @@ // Ported from the Java implementation by Matthew Francis: https://github.com/MateuszBartosiewicz/bzip2 using System; + namespace SharpCompress.Compressors.BZip2MT.Algorithm { /// @@ -11,7 +12,7 @@ namespace SharpCompress.Compressors.BZip2MT.Algorithm /// internal class MoveToFront { - /// The Move To Front list + /// The Move To Front list private readonly byte[] mtf; /// Public constructor diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs index a59594021..d9f9e73de 100644 --- a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStream.cs @@ -5,6 +5,7 @@ using System.IO; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.InputStream { /// @@ -38,7 +39,8 @@ public bool ReadBoolean() if (this._bitCount > 0) { this._bitCount--; - } else + } + else { int byteRead = this._inputStream.ReadByte(); @@ -62,7 +64,8 @@ public uint ReadUnary() if (this._bitCount > 0) { this._bitCount--; - } else + } + else { var byteRead = this._inputStream.ReadByte(); diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs index 6d4f6866a..6e8c70159 100644 --- a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2BitInputStreamSplitter.cs @@ -4,9 +4,9 @@ using System; using System.IO; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.InputStream { - /// /// Helper class for splitting a BZip2 stream into multiple Memory Streams made up of individual BZip2 blocks. /// The spitting is done by looking for the magic block header 6 byte sequence. @@ -14,7 +14,6 @@ namespace SharpCompress.Compressors.BZip2MT.InputStream /// internal class BZip2BitInputStreamSplitter : IDisposable { - // The stream from which bits are read private BZip2BitInputStream? _bitInputStream; @@ -32,9 +31,14 @@ internal class BZip2BitInputStreamSplitter : IDisposable public int BlockSizeBytes => this._blockSizeBytes; public uint FinalCrc => this._finalCrc; - public BZip2BitInputStreamSplitter(Stream inputStream, InputStreamHeaderCheckType inputStreamHeaderCheck = InputStreamHeaderCheckType.FULL_HEADER, int manualBlockLevel = 9) + public BZip2BitInputStreamSplitter( + Stream inputStream, + InputStreamHeaderCheckType inputStreamHeaderCheck = + InputStreamHeaderCheckType.FULL_HEADER, + int manualBlockLevel = 9 + ) { - this._bitInputStream = new BZip2BitInputStream(inputStream); + this._bitInputStream = new BZip2BitInputStream(inputStream); this.Initialize(inputStreamHeaderCheck, manualBlockLevel); } @@ -53,9 +57,12 @@ private void Initialize(InputStreamHeaderCheckType inputStreamHeaderCheck, int b uint marker1 = this._bitInputStream.ReadBits(16); uint marker2 = this._bitInputStream.ReadBits(8); blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); - if (marker1 != BZip2Constants.STREAM_START_MARKER_1 || - marker2 != BZip2Constants.STREAM_START_MARKER_2 || - blockLevel < 1 || blockLevel > 9) + if ( + marker1 != BZip2Constants.STREAM_START_MARKER_1 + || marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) { throw new IOException("Invalid BZip2 header"); } @@ -65,8 +72,11 @@ private void Initialize(InputStreamHeaderCheckType inputStreamHeaderCheck, int b { uint marker2 = this._bitInputStream.ReadBits(8); blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); - if (marker2 != BZip2Constants.STREAM_START_MARKER_2 || - blockLevel < 1 || blockLevel > 9) + if ( + marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) { throw new IOException("Invalid BZip2 header"); } @@ -75,7 +85,7 @@ private void Initialize(InputStreamHeaderCheckType inputStreamHeaderCheck, int b case InputStreamHeaderCheckType.NO_BZH: { blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); - if (blockLevel < 1 || blockLevel > 9) + if (blockLevel < 1 || blockLevel > 9) { throw new IOException("Invalid BZip2 header"); } @@ -97,7 +107,8 @@ private void Initialize(InputStreamHeaderCheckType inputStreamHeaderCheck, int b } this._buffer = new MemoryStream(this._blockSizeBytes + 100000); - } catch (IOException) + } + catch (IOException) { // If the stream header was not valid, stop trying to read more data this._bitInputStream = null; @@ -175,6 +186,7 @@ void FlushPartialByte() } } } + public void Dispose() { this._bitInputStream?.Dispose(); diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs index 3ac770f22..2950e925b 100644 --- a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2InputStream.cs @@ -8,6 +8,7 @@ using System.IO; using SharpCompress.Compressors.BZip2MT.Algorithm; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.InputStream { /// An InputStream wrapper that decompresses BZip2 data @@ -41,13 +42,18 @@ public class BZip2InputStream : Stream // The decompressor for the current block private BZip2BlockDecompressor? _blockDecompressor; - /// Public constructor /// The InputStream to wrap /// if true, will close the stream when done /// /// Used when is NoHeader - public BZip2InputStream(Stream inputStream, bool isOwner = true, InputStreamHeaderCheckType inputStreamHeaderCheck = InputStreamHeaderCheckType.FULL_HEADER, int manualBlockLevel = 9) + public BZip2InputStream( + Stream inputStream, + bool isOwner = true, + InputStreamHeaderCheckType inputStreamHeaderCheck = + InputStreamHeaderCheckType.FULL_HEADER, + int manualBlockLevel = 9 + ) { this._inputStream = inputStream; this._bitInputStream = new BZip2BitInputStream(inputStream); @@ -61,11 +67,27 @@ public BZip2InputStream(Stream inputStream, bool isOwner = true, InputStreamHead #region Implementation of abstract members of Stream - #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member - public override void Flush() => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'Flush()' method."); - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); - public override void SetLength(long value) => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'SetLength(long value)' method."); - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method."); +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'Flush()' method." + ); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); + + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'SetLength(long value)' method." + ); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method." + ); + public override bool CanRead => this._inputStream.CanRead; public override bool CanSeek => false; public override bool CanWrite => false; @@ -73,7 +95,10 @@ public BZip2InputStream(Stream inputStream, bool isOwner = true, InputStreamHead public override long Position { get => this._inputStream.Position; - set =>throw new NotSupportedException($"{nameof(BZip2InputStream)} does not support Set operation for property 'Position'."); + set => + throw new NotSupportedException( + $"{nameof(BZip2InputStream)} does not support Set operation for property 'Position'." + ); } public override int ReadByte() @@ -92,7 +117,7 @@ public override int ReadByte() return nextByte; } - public override int Read(byte[] destination, int offset, int length) + public override int Read(byte[] destination, int offset, int length) { int bytesRead = this._blockDecompressor?.Read(destination, offset, length) ?? 0; @@ -105,7 +130,8 @@ public override int Read(byte[] destination, int offset, int length) } } - if (bytesRead == -1) bytesRead = 0; + if (bytesRead == -1) + bytesRead = 0; return bytesRead; } @@ -125,13 +151,16 @@ protected override void Dispose(bool disposing) } } - #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member #endregion /// Reads the stream header and checks that the data appears to be a valid BZip2 stream /// if the stream header is not valid - private void InitializeStream(InputStreamHeaderCheckType inputStreamHeaderCheck, int blockLevel) + private void InitializeStream( + InputStreamHeaderCheckType inputStreamHeaderCheck, + int blockLevel + ) { /* If the stream has been explicitly closed, throw an exception */ if (this._bitInputStream is null) @@ -151,9 +180,12 @@ private void InitializeStream(InputStreamHeaderCheckType inputStreamHeaderCheck, uint marker1 = this._bitInputStream.ReadBits(16); uint marker2 = this._bitInputStream.ReadBits(8); blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); - if (marker1 != BZip2Constants.STREAM_START_MARKER_1 || - marker2 != BZip2Constants.STREAM_START_MARKER_2 || - blockLevel < 1 || blockLevel > 9) + if ( + marker1 != BZip2Constants.STREAM_START_MARKER_1 + || marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) { throw new IOException("Invalid BZip2 header"); } @@ -163,8 +195,11 @@ private void InitializeStream(InputStreamHeaderCheckType inputStreamHeaderCheck, { uint marker2 = this._bitInputStream.ReadBits(8); blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); - if (marker2 != BZip2Constants.STREAM_START_MARKER_2 || - blockLevel < 1 || blockLevel > 9) + if ( + marker2 != BZip2Constants.STREAM_START_MARKER_2 + || blockLevel < 1 + || blockLevel > 9 + ) { throw new IOException("Invalid BZip2 header"); } @@ -173,7 +208,7 @@ private void InitializeStream(InputStreamHeaderCheckType inputStreamHeaderCheck, case InputStreamHeaderCheckType.NO_BZH: { blockLevel = ((int)this._bitInputStream.ReadBits(8) - '0'); - if (blockLevel < 1 || blockLevel > 9) + if (blockLevel < 1 || blockLevel > 9) { throw new IOException("Invalid BZip2 header"); } @@ -182,7 +217,8 @@ private void InitializeStream(InputStreamHeaderCheckType inputStreamHeaderCheck, } this._streamBlockSize = (uint)(blockLevel * 100000); - } catch (IOException) + } + catch (IOException) { // If the stream header was not valid, stop trying to read more data this._streamComplete = true; @@ -198,7 +234,6 @@ private void InitializeStream(InputStreamHeaderCheckType inputStreamHeaderCheck, /// not a valid block-header or end-of-file marker, or if the following block could not be decoded private bool InitializeNextBlock() { - // If we're already at the end of the stream, do nothing if (this._streamComplete) return false; @@ -214,13 +249,20 @@ private bool InitializeNextBlock() uint marker1 = this._bitInputStream.ReadBits(24); uint marker2 = this._bitInputStream.ReadBits(24); - if (marker1 == BZip2Constants.BLOCK_HEADER_MARKER_1 && marker2 == BZip2Constants.BLOCK_HEADER_MARKER_2) + if ( + marker1 == BZip2Constants.BLOCK_HEADER_MARKER_1 + && marker2 == BZip2Constants.BLOCK_HEADER_MARKER_2 + ) { // Initialise a new block try { - this._blockDecompressor = new BZip2BlockDecompressor(this._bitInputStream, this._streamBlockSize); - } catch (IOException) + this._blockDecompressor = new BZip2BlockDecompressor( + this._bitInputStream, + this._streamBlockSize + ); + } + catch (IOException) { // If the block could not be decoded, stop trying to read more data this._streamComplete = true; @@ -228,7 +270,10 @@ private bool InitializeNextBlock() } return true; } - if (marker1 == BZip2Constants.STREAM_END_MARKER_1 && marker2 == BZip2Constants.STREAM_END_MARKER_2) + if ( + marker1 == BZip2Constants.STREAM_END_MARKER_1 + && marker2 == BZip2Constants.STREAM_END_MARKER_2 + ) { // Read and verify the end-of-stream CRC this._streamComplete = true; diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs index d265dfeaa..27742b331 100644 --- a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputDataBlock.cs @@ -1,11 +1,11 @@ using System; using System.IO; using SharpCompress.Compressors.BZip2MT.Algorithm; + namespace SharpCompress.Compressors.BZip2MT.InputStream { internal class BZip2ParallelInputDataBlock : IDisposable { - private int _blockId = 0; private MemoryStream? _inputBlockBuffer; private MemoryStream? _outputBuffer; @@ -21,7 +21,12 @@ internal class BZip2ParallelInputDataBlock : IDisposable public bool IsCrcOk => this._isCrcOk; public uint CrcValue => this._crcValue; - public BZip2ParallelInputDataBlock(int blockId, MemoryStream inputBlockBuffer, int blockSizeBytes, int outputBufferSize) + public BZip2ParallelInputDataBlock( + int blockId, + MemoryStream inputBlockBuffer, + int blockSizeBytes, + int outputBufferSize + ) { this._blockId = blockId; this._inputBlockBuffer = inputBlockBuffer; @@ -35,7 +40,6 @@ public void Dispose() this._outputBuffer?.Dispose(); } - /// /// Decompresses the full block /// @@ -49,7 +53,10 @@ public void Decompress() // note: we need to skip the first 6 magic bytes... this._inputBlockBuffer.Position = 6; using BZip2BitInputStream inputStream = new BZip2BitInputStream(this._inputBlockBuffer); - BZip2BlockDecompressor blockDecompressor = new BZip2BlockDecompressor(inputStream, (uint)this._blockSizeBytes); + BZip2BlockDecompressor blockDecompressor = new BZip2BlockDecompressor( + inputStream, + (uint)this._blockSizeBytes + ); int readCount = blockDecompressor.ReadAll(this._outputBuffer); @@ -60,7 +67,8 @@ public void Decompress() { this._crcValue = blockDecompressor.CheckCrc(); this._isCrcOk = true; - } catch (Exception) + } + catch (Exception) { this._crcValue = 0; this._isCrcOk = false; @@ -77,7 +85,9 @@ public void Decompress() public int Read() { if (this._outputBuffer is null) - throw new IOException("Attempted to read decompressed data before decompressing block"); + throw new IOException( + "Attempted to read decompressed data before decompressing block" + ); if (this._isCrcOk == false) return -1; @@ -96,13 +106,14 @@ public int Read() public int Read(byte[] destination, int offset, int length) { if (this._outputBuffer is null) - throw new IOException("Attempted to read decompressed data before decompressing block"); + throw new IOException( + "Attempted to read decompressed data before decompressing block" + ); if (this._isCrcOk == false) return 0; return this._outputBuffer.Read(destination, offset, length); } - } } diff --git a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs index 9196e4e02..d2fdd10a0 100644 --- a/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/InputStream/BZip2ParallelInputStream.cs @@ -6,6 +6,7 @@ using System.IO; using System.Threading; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.InputStream { /// An InputStream wrapper that decompresses BZip2 data using multiple threads @@ -28,7 +29,8 @@ public class BZip2ParallelInputStream : Stream private Exception? _lastWorkerException = null; // dictionary of processed blocks - private readonly Dictionary _mtDecodedBlocks = new Dictionary(); + private readonly Dictionary _mtDecodedBlocks = + new Dictionary(); private int _mtPendingBlocks = 0; private readonly int _mtMaxPendingBlocks; @@ -37,7 +39,6 @@ public class BZip2ParallelInputStream : Stream private readonly object _syncRootProcesing = new object(); - // The stream from which compressed BZip2 data is read and decoded private Stream _inputStream; @@ -47,7 +48,6 @@ public class BZip2ParallelInputStream : Stream // An InputStream wrapper that provides bit-level reads private BZip2BitInputStreamSplitter _inputStreamSplitter; - // The merged CRC of all blocks decompressed so far private uint _streamCrc; @@ -62,7 +62,15 @@ public class BZip2ParallelInputStream : Stream /// Used when is NoHeader /// Maximum number of blocks to read/decompress at the same time /// Minimum buffer size for each worker thread during decompression, in bytes. - public BZip2ParallelInputStream(Stream inputStream, bool isOwner = true, InputStreamHeaderCheckType inputStreamHeaderCheck = InputStreamHeaderCheckType.FULL_HEADER, int manualBlockLevel = 9, int maxPendingBlocks = 0, int workerBufferSize = 12582912) + public BZip2ParallelInputStream( + Stream inputStream, + bool isOwner = true, + InputStreamHeaderCheckType inputStreamHeaderCheck = + InputStreamHeaderCheckType.FULL_HEADER, + int manualBlockLevel = 9, + int maxPendingBlocks = 0, + int workerBufferSize = 12582912 + ) { if (maxPendingBlocks == 0) { @@ -74,7 +82,11 @@ public BZip2ParallelInputStream(Stream inputStream, bool isOwner = true, InputSt // initialize stream this._inputStream = inputStream; this._isOwner = isOwner; - this._inputStreamSplitter = new BZip2BitInputStreamSplitter(inputStream, inputStreamHeaderCheck, manualBlockLevel); + this._inputStreamSplitter = new BZip2BitInputStreamSplitter( + inputStream, + inputStreamHeaderCheck, + manualBlockLevel + ); this._blockLevel = this._inputStreamSplitter.BlockLevel; this._blockSizeBytes = this._inputStreamSplitter.BlockSizeBytes; @@ -102,18 +114,23 @@ private void MultiThreadWorkerAction(object? blockData) this._mtDecodedBlocks.Add(dataBlock.BlockId, dataBlock); } } - } catch (Exception ex) + } + catch (Exception ex) { // set this without any locks... this._lastWorkerException = ex; this._unsafeFatalException = true; - throw new IOException("BZip2 Processing thread somehow crashed... See inner exception for details!", ex); - } finally + throw new IOException( + "BZip2 Processing thread somehow crashed... See inner exception for details!", + ex + ); + } + finally { lock (this._syncRootProcesing) { - this._mtPendingBlocks --; + this._mtPendingBlocks--; } } } @@ -128,7 +145,10 @@ private bool TryToQueueUpAnotherBlock() { if (this._unsafeFatalException) { - throw new IOException("One of the decompression threads somehow failed... This should never happen.", this._lastWorkerException); + throw new IOException( + "One of the decompression threads somehow failed... This should never happen.", + this._lastWorkerException + ); } if (this._inputStreamSplitter.IsStreamComplete) @@ -151,8 +171,13 @@ private bool TryToQueueUpAnotherBlock() lock (this._syncRootProcesing) { - this._mtPendingBlocks ++; - BZip2ParallelInputDataBlock data = new BZip2ParallelInputDataBlock(this._mtNextInputBlockId++, ms, this._blockSizeBytes, this._mtWorkerBufferSize); + this._mtPendingBlocks++; + BZip2ParallelInputDataBlock data = new BZip2ParallelInputDataBlock( + this._mtNextInputBlockId++, + ms, + this._blockSizeBytes, + this._mtWorkerBufferSize + ); ThreadPool.QueueUserWorkItem(this.MultiThreadWorkerAction, data); } @@ -170,14 +195,19 @@ private bool CheckIfCompletelyDone() { if (this._unsafeFatalException) { - throw new IOException("One of the decompression threads somehow failed... This should never happen.", this._lastWorkerException); + throw new IOException( + "One of the decompression threads somehow failed... This should never happen.", + this._lastWorkerException + ); } lock (this._syncRootProcesing) { - if (this._mtPendingBlocks == 0 && - this._mtDecodedBlocks.Count == 0 && - this._inputStreamSplitter.IsStreamComplete) + if ( + this._mtPendingBlocks == 0 + && this._mtDecodedBlocks.Count == 0 + && this._inputStreamSplitter.IsStreamComplete + ) { if (this._inputStreamSplitter.FinalCrc != this._streamCrc) { @@ -193,11 +223,27 @@ private bool CheckIfCompletelyDone() #region Implementation of abstract members of Stream - #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member - public override void Flush() => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'Flush()' method."); - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); - public override void SetLength(long value) => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'SetLength(long value)' method."); - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method."); +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'Flush()' method." + ); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); + + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'SetLength(long value)' method." + ); + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support 'Write(byte[] buffer, int offset, int count)' method." + ); + public override bool CanRead => true; public override bool CanSeek => false; public override bool CanWrite => false; @@ -205,7 +251,10 @@ private bool CheckIfCompletelyDone() public override long Position { get => this._inputStream.Position; - set => throw new NotSupportedException($"{nameof(BZip2ParallelInputStream)} does not support Set operation for property 'Position'."); + set => + throw new NotSupportedException( + $"{nameof(BZip2ParallelInputStream)} does not support Set operation for property 'Position'." + ); } public override int ReadByte() @@ -240,10 +289,11 @@ public override int ReadByte() lock (this._syncRootProcesing) { uint blockCrc = data.CrcValue; - this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; this._mtDecodedBlocks.Remove(this._mtNextOutputBlockId); - this._mtNextOutputBlockId ++; + this._mtNextOutputBlockId++; // continue to processing next block that is already decoded, skip queueing up more blocks... continue; @@ -264,7 +314,7 @@ public override int ReadByte() } } - public override int Read(byte[] destination, int offset, int length) + public override int Read(byte[] destination, int offset, int length) { int totalRead = 0; int remaining = length; @@ -303,10 +353,11 @@ public override int Read(byte[] destination, int offset, int length) lock (this._syncRootProcesing) { uint blockCrc = data.CrcValue; - this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ blockCrc; this._mtDecodedBlocks.Remove(this._mtNextOutputBlockId); - this._mtNextOutputBlockId ++; + this._mtNextOutputBlockId++; // continue to processing next block that is already decoded, skip queueing up more blocks... continue; @@ -340,9 +391,8 @@ protected override void Dispose(bool disposing) } } - #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member #endregion - } } diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs index 8518911ff..3a6d24067 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/BZip2Constants.cs @@ -1,8 +1,10 @@ // Added by drone1400, December 2025 // Location: https://github.com/drone1400/bzip2 -namespace SharpCompress.Compressors.BZip2MT.Interface { - internal static class BZip2Constants { +namespace SharpCompress.Compressors.BZip2MT.Interface +{ + internal static class BZip2Constants + { public const uint STREAM_START_MARKER = 0x425A68; public const ulong STREAM_END_MARKER = 0x177245385090; public const ulong BLOCK_HEADER_MARKER = 0x314159265359; diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs index 1501992e7..1da856cbe 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitInputStream.cs @@ -1,5 +1,6 @@ using System; using System.IO; + namespace SharpCompress.Compressors.BZip2MT.Interface { internal interface IBZip2BitInputStream : IDisposable diff --git a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs index 4855e7b4a..6aa5651da 100644 --- a/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/Interface/IBZip2BitOutputStream.cs @@ -2,6 +2,7 @@ // Location: https://github.com/drone1400/bzip2 using System; + namespace SharpCompress.Compressors.BZip2MT.Interface { /// @@ -14,14 +15,14 @@ internal interface IBZip2BitOutputStream : IDisposable /// /// The bit to write /// if an error occurs writing to the stream - public void WriteBoolean (bool value); + public void WriteBoolean(bool value); /// /// Writes a zero-terminated unary number to the wrapped output stream /// /// The number to write (must be non-negative) /// if an error occurs writing to the stream - public void WriteUnary (int value); + public void WriteUnary(int value); /// /// Writes up to 24 bits to the wrapped output stream @@ -29,19 +30,19 @@ internal interface IBZip2BitOutputStream : IDisposable /// The number of bits to write (maximum 24) /// The bits to write /// if an error occurs writing to the stream - public void WriteBits (int count, uint value) ; + public void WriteBits(int count, uint value); /// /// Writes an integer as 32 bits of output /// /// The integer to write /// if an error occurs writing to the stream - public void WriteInteger (uint value) ; + public void WriteInteger(uint value); /// /// Writes any remaining bits to the output stream, zero padding to a whole byte as required /// /// if an error occurs writing to the stream - public void Flush() ; + public void Flush(); } } diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs index f947f7680..3e6fccb3f 100644 --- a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2BitOutputStream.cs @@ -6,6 +6,7 @@ using System.IO; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.OutputStream { /// Implements a bit-wise output stream @@ -38,7 +39,7 @@ public void Dispose() #region IBZip2BitOutputStream implementation - public void WriteBoolean (bool value) + public void WriteBoolean(bool value) { this._bitCount++; this._bitBuffer |= ((value ? 1u : 0u) << (32 - this._bitCount)); @@ -51,16 +52,16 @@ public void WriteBoolean (bool value) } } - public void WriteUnary (int value) + public void WriteUnary(int value) { while (value-- > 0) { - this.WriteBoolean (true); + this.WriteBoolean(true); } - this.WriteBoolean (false); + this.WriteBoolean(false); } - public void WriteBits (int count, uint value) + public void WriteBits(int count, uint value) { this._bitBuffer |= ((value << (32 - count)) >> this._bitCount); this._bitCount += count; @@ -73,17 +74,17 @@ public void WriteBits (int count, uint value) } } - public void WriteInteger (uint value) + public void WriteInteger(uint value) { - this.WriteBits (16, (value >> 16) & 0xffff); - this.WriteBits (16, value & 0xffff); + this.WriteBits(16, (value >> 16) & 0xffff); + this.WriteBits(16, value & 0xffff); } public void Flush() { if (this._bitCount > 0) { - this.WriteBits (8 - this._bitCount, 0); + this.WriteBits(8 - this._bitCount, 0); } } diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs index 437803384..4288ce2a6 100644 --- a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2OutputStream.cs @@ -8,6 +8,7 @@ using System.IO; using SharpCompress.Compressors.BZip2MT.Algorithm; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.OutputStream { /// An OutputStream wrapper that compresses BZip2 data @@ -42,7 +43,11 @@ public class BZip2OutputStream : Stream /// On any I/O error writing to the output stream /// Larger block sizes require more memory for both compression and decompression, /// but give better compression ratios. 9 will usually be the best value to use - public BZip2OutputStream(Stream outputStream, bool isOwner = true, int blockSizeMultiplier = 9) + public BZip2OutputStream( + Stream outputStream, + bool isOwner = true, + int blockSizeMultiplier = 9 + ) { if (outputStream is null) throw new ArgumentException("Null output stream"); @@ -64,14 +69,26 @@ public BZip2OutputStream(Stream outputStream, bool isOwner = true, int blockSize #region Implementation of abstract members of Stream - #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member - public override void Flush () => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'Flush()' method."); +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public override void Flush() => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'Flush()' method." + ); - public override int Read (byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method."); + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method." + ); - public override long Seek (long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); - public override void SetLength (long value) => throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support 'SetLength(long value)' method."); + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support 'SetLength(long value)' method." + ); public override bool CanRead => false; @@ -84,7 +101,10 @@ public BZip2OutputStream(Stream outputStream, bool isOwner = true, int blockSize public override long Position { get => this._outputStream.Position; - set =>throw new NotSupportedException($"{nameof(BZip2OutputStream)} does not support Set operation for property 'Position'."); + set => + throw new NotSupportedException( + $"{nameof(BZip2OutputStream)} does not support Set operation for property 'Position'." + ); } public override void WriteByte(byte value) @@ -134,12 +154,16 @@ protected override void Dispose(bool disposing) } } - #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member #endregion /// Initialises a new block for compression - private void InitialiseNextBlock() => this._blockCompressor = new BZip2BlockCompressor (this._bitOutputStream, this._streamBlockSize); + private void InitialiseNextBlock() => + this._blockCompressor = new BZip2BlockCompressor( + this._bitOutputStream, + this._streamBlockSize + ); /// Compress and write out the block currently in progress /// If no bytes have been written to the block, it is discarded @@ -150,7 +174,8 @@ private void CloseBlock() return; this._blockCompressor.CloseBlock(); - this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ this._blockCompressor.CRC; + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ this._blockCompressor.CRC; } /// Compresses and writes out any as yet unwritten data, then writes the end of the BZip2 stream @@ -169,7 +194,8 @@ private void Finish() this._bitOutputStream.WriteInteger(this._streamCrc); this._bitOutputStream.Flush(); this._outputStream.Flush(); - } finally + } + finally { this._blockCompressor = null; } diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs index e9a52e593..58de1843d 100644 --- a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputDataBlock.cs @@ -5,6 +5,7 @@ using System.IO; using SharpCompress.Compressors.BZip2MT.Algorithm; using SharpCompress.Compressors.BZip2MT.Interface; + namespace SharpCompress.Compressors.BZip2MT.OutputStream { /// A collection of bit output data @@ -144,26 +145,30 @@ public void WriteBoolean(bool value) this._bitCount++; this._internalBitStream.WriteBoolean(value); } + public void WriteUnary(int value) { while (value-- > 0) { this._bitCount++; - this._internalBitStream.WriteBoolean (true); + this._internalBitStream.WriteBoolean(true); } this._bitCount++; - this._internalBitStream.WriteBoolean (false); + this._internalBitStream.WriteBoolean(false); } + public void WriteBits(int count, uint value) { this._bitCount += count; this._internalBitStream.WriteBits(count, value); } + public void WriteInteger(uint value) { this._bitCount += 32; this._internalBitStream.WriteInteger(value); } + public void Flush() => this._internalBitStream.Flush(); #endregion diff --git a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs index 20152c048..9e370b800 100644 --- a/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs +++ b/src/SharpCompress/Compressors/BZip2MT/OutputStream/BZip2ParallelOutputStream.cs @@ -5,9 +5,9 @@ using System.Collections.Generic; using System.IO; using System.Threading; + namespace SharpCompress.Compressors.BZip2MT.OutputStream { - /// An OutputStream wrapper that compresses BZip2 data using multiple threads /// Instances of this class are not threadsafe public class BZip2ParallelOutputStream : Stream @@ -29,7 +29,8 @@ public class BZip2ParallelOutputStream : Stream private Exception? _lastWorkerException = null; // dictionary of processed blocks - private readonly Dictionary _mtProcessedBlocks = new Dictionary(); + private readonly Dictionary _mtProcessedBlocks = + new Dictionary(); private int _mtPendingBlocks = 0; @@ -66,8 +67,10 @@ public BZip2ParallelOutputStream(Stream output, bool isOwner = true, int blockLe this._outputStream = output; this._bitStream = new BZip2BitOutputStream(output); - if (blockLevel < 1 ) blockLevel = 1; - if (blockLevel > 9 ) blockLevel = 9; + if (blockLevel < 1) + blockLevel = 1; + if (blockLevel > 9) + blockLevel = 9; this._blockLevel = blockLevel; // evaluate thread count @@ -79,7 +82,10 @@ public BZip2ParallelOutputStream(Stream output, bool isOwner = true, int blockLe this._compressBlockSize = 100000 * this._blockLevel; // initialize initial meta buffer - this._currentBlockBuffer = new BZip2ParallelOutputDataBlock(this._compressBlockSize, this._mtNextInputBlockId++); + this._currentBlockBuffer = new BZip2ParallelOutputDataBlock( + this._compressBlockSize, + this._mtNextInputBlockId++ + ); this._isOwner = isOwner; @@ -96,10 +102,12 @@ private bool TryWriteOutputBlockAndIncrementId() { if (this._unsafeFatalException) { - throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); } - try { BZip2ParallelOutputDataBlock? currentOutput = null; @@ -123,18 +131,23 @@ private bool TryWriteOutputBlockAndIncrementId() lock (this._syncRootOutputStream) { // update file CRC - this._streamCrc = ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ currentOutput.BlockCrc; + this._streamCrc = + ((this._streamCrc << 1) | (this._streamCrc >> 31)) ^ currentOutput.BlockCrc; currentOutput.WriteToRealOutputStream(this._bitStream); } return true; - } catch (Exception ex) + } + catch (Exception ex) { // set this without any locks... this._lastWorkerException = ex; this._unsafeFatalException = true; // rethrow exception, hopefully something catches it?... - throw new IOException("BZip2 error writing output data! See inner exception for details!", ex); + throw new IOException( + "BZip2 error writing output data! See inner exception for details!", + ex + ); } } @@ -158,18 +171,23 @@ private void MultiThreadWorkerAction(object? blockData) this._mtProcessedBlocks.Add(buffer.BlockId, buffer); } } - } catch (Exception ex) + } + catch (Exception ex) { // set this without any locks... this._lastWorkerException = ex; this._unsafeFatalException = true; - throw new IOException("BZip2 Processing thread somehow crashed... See inner exception for details!", ex); - } finally + throw new IOException( + "BZip2 Processing thread somehow crashed... See inner exception for details!", + ex + ); + } + finally { lock (this._syncRootProcesing) { - this._mtPendingBlocks --; + this._mtPendingBlocks--; } } } @@ -182,9 +200,9 @@ private void WriteBz2Header() lock (this._syncRootOutputStream) { // write BZIP file header - this._bitStream.WriteBits(8, 0x42); // B - this._bitStream.WriteBits(8, 0x5A); // Z - this._bitStream.WriteBits(8, 0x68); // h + this._bitStream.WriteBits(8, 0x42); // B + this._bitStream.WriteBits(8, 0x5A); // Z + this._bitStream.WriteBits(8, 0x68); // h this._bitStream.WriteBits(8, (uint)(0x30 + this._blockLevel)); // block level digit } } @@ -223,7 +241,10 @@ private bool TryEnqueueCurrentBlockBuffer() { if (this._unsafeFatalException) { - throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); } // make sure current block has data @@ -232,15 +253,20 @@ private bool TryEnqueueCurrentBlockBuffer() lock (this._syncRootProcesing) { - this._mtPendingBlocks ++; - ThreadPool.QueueUserWorkItem(this.MultiThreadWorkerAction, this._currentBlockBuffer); - this._currentBlockBuffer = new BZip2ParallelOutputDataBlock(this._compressBlockSize, this._mtNextInputBlockId++); + this._mtPendingBlocks++; + ThreadPool.QueueUserWorkItem( + this.MultiThreadWorkerAction, + this._currentBlockBuffer + ); + this._currentBlockBuffer = new BZip2ParallelOutputDataBlock( + this._compressBlockSize, + this._mtNextInputBlockId++ + ); } return true; } - /// /// Waits for any pending data blocks to be written to the bit stream and writes the BZ2 footer /// @@ -249,7 +275,10 @@ private void FinishBitstream() { if (this._unsafeFatalException) { - throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); } lock (this._syncRootProcesing) @@ -269,7 +298,10 @@ private void FinishBitstream() { if (this._unsafeFatalException) { - throw new IOException("One of the compression threads somehow failed... This should never happen.", this._lastWorkerException); + throw new IOException( + "One of the compression threads somehow failed... This should never happen.", + this._lastWorkerException + ); } lock (this._syncRootProcesing) @@ -282,16 +314,14 @@ private void FinishBitstream() } // try writing output block and keep doing so while successful - while (this.TryWriteOutputBlockAndIncrementId()) - { } + while (this.TryWriteOutputBlockAndIncrementId()) { } Thread.Sleep(1); } // sanity check, this should be impossible and should be caught by some test right?... lock (this._syncRootProcesing) { - if (this._mtPendingBlocks != 0 || - this._mtProcessedBlocks.Count > 0) + if (this._mtPendingBlocks != 0 || this._mtProcessedBlocks.Count > 0) { throw new IOException("BZip2 dispose operation sanity check failed!..."); } @@ -305,7 +335,7 @@ private void FinishBitstream() #region Implementation of abstract members of Stream - #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member // overriding Dispose instead of Close as recommended in https://docs.microsoft.com/en-us/dotnet/api/system.io.stream.close?view=net-6.0 protected override void Dispose(bool disposing) @@ -321,12 +351,23 @@ protected override void Dispose(bool disposing) public override void Flush() { // try writing output block and keep doing so while successful - while (this.TryWriteOutputBlockAndIncrementId()) - { } + while (this.TryWriteOutputBlockAndIncrementId()) { } } - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method."); - public override void SetLength(long value) => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support 'SetLength(long value)' method."); - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method."); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support 'Seek(long offset, SeekOrigin origin)' method." + ); + + public override void SetLength(long value) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support 'SetLength(long value)' method." + ); + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support 'Read(byte[] buffer, int offset, int count)' method." + ); public override void WriteByte(byte value) { @@ -371,11 +412,13 @@ public override void Write(byte[] data, int offset, int length) public override long Position { get => this._outputStream.Position; - set => throw new NotSupportedException($"{nameof(BZip2ParallelOutputStream)} does not support Set operation for property 'Position'."); + set => + throw new NotSupportedException( + $"{nameof(BZip2ParallelOutputStream)} does not support Set operation for property 'Position'." + ); } - - #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member #endregion }