Skip to content
This repository has been archived by the owner on Dec 20, 2021. It is now read-only.

HuffmanDecompress: use BitMuncher instead of BitStream #1126

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

HuffmanDecompress: use BitMuncher instead of BitStream #1126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3260022
HuffmanDecompress: use d2datautils.BitMuncher instead of d2datautils.…
gucio321 May 16, 2021
e685b92
bitMuncher: add EnusreBits method
gucio321 May 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
37 changes: 22 additions & 15 deletions d2common/d2data/d2compression/huffman.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,8 @@ func getPrimes() [][]byte {
0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x01, 0x01, 0x02, 0x02, 0x02, 0x06, 0x4B,
}, {
},
{
// Compression type 2 //nolint:dupl // it doesnt matter here
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x27, 0x00, 0x00, 0x23, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
Expand All @@ -138,7 +139,8 @@ func getPrimes() [][]byte {
0x03, 0x01, 0x03, 0x06, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x02, 0x01, 0x01,
0x01, 0x29, 0x07, 0x16, 0x12, 0x40, 0x0A, 0x0A, 0x11, 0x25, 0x01, 0x03, 0x17, 0x10, 0x26, 0x2A,
0x10, 0x01, 0x23, 0x23, 0x2F, 0x10, 0x06, 0x07, 0x02, 0x09, 0x01, 0x01, 0x01, 0x01, 0x01,
}, { //nolint:dupl // it doesnt matter here
},
{ //nolint:dupl // it doesnt matter here
// Compression type 3 //nolint:dupl // it doesnt matter here
0xFF, 0x0B, 0x07, 0x05, 0x0B, 0x02, 0x02, 0x02, 0x06, 0x02, 0x02, 0x01, 0x04, 0x02, 0x01, 0x03,
0x09, 0x01, 0x01, 0x01, 0x03, 0x04, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01,
Expand All @@ -156,14 +158,17 @@ func getPrimes() [][]byte {
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x07, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01,
0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x11,
}, { // Compression type 4 //nolint:dupl // it doesnt matter here
},
{ // Compression type 4 //nolint:dupl // it doesnt matter here
0xFF, 0xFB, 0x98, 0x9A, 0x84, 0x85, 0x63, 0x64, 0x3E, 0x3E, 0x22, 0x22, 0x13, 0x13, 0x18, 0x17,
}, { // Compression type 5 //nolint:dupl // it doesnt matter here
},
{ // Compression type 5 //nolint:dupl // it doesnt matter here
0xFF, 0xF1, 0x9D, 0x9E, 0x9A, 0x9B, 0x9A, 0x97, 0x93, 0x93, 0x8C, 0x8E, 0x86, 0x88, 0x80, 0x82,
0x7C, 0x7C, 0x72, 0x73, 0x69, 0x6B, 0x5F, 0x60, 0x55, 0x56, 0x4A, 0x4B, 0x40, 0x41, 0x37, 0x37,
0x2F, 0x2F, 0x27, 0x27, 0x21, 0x21, 0x1B, 0x1C, 0x17, 0x17, 0x13, 0x13, 0x10, 0x10, 0x0D, 0x0D,
0x0B, 0x0B, 0x09, 0x09, 0x08, 0x08, 0x07, 0x07, 0x06, 0x05, 0x05, 0x04, 0x04, 0x04, 0x19, 0x18,
}, { //nolint:dupl // it doesnt matter here
},
{ //nolint:dupl // it doesnt matter here
// Compression type 6
0xC3, 0xCB, 0xF5, 0x41, 0xFF, 0x7B, 0xF7, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
Expand All @@ -174,7 +179,8 @@ func getPrimes() [][]byte {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x7A, 0x46,
}, { //nolint:dupl // it doesnt matter here
},
{ //nolint:dupl // it doesnt matter here
// Compression type 7
0xC3, 0xD9, 0xEF, 0x3D, 0xF9, 0x7C, 0xE9, 0x1E, 0xFD, 0xAB, 0xF1, 0x2C, 0xFC, 0x5B, 0xFE, 0x17,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
Expand All @@ -185,7 +191,8 @@ func getPrimes() [][]byte {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x70, 0x6C,
}, { // Compression type 8
},
{ // Compression type 8
0xBA, 0xC5, 0xDA, 0x33, 0xE3, 0x6D, 0xD8, 0x18, 0xE5, 0x94, 0xDA, 0x23, 0xDF, 0x4A, 0xD1, 0x10,
0xEE, 0xAF, 0xE4, 0x2C, 0xEA, 0x5A, 0xDE, 0x15, 0xF4, 0x87, 0xE9, 0x21, 0xF6, 0x43, 0xFC, 0x12,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
Expand All @@ -199,15 +206,15 @@ func getPrimes() [][]byte {
}
}

func decode(input *d2datautils.BitStream, head *linkedNode) *linkedNode {
func decode(input *d2datautils.BitMuncher, head *linkedNode) *linkedNode {
node := head

for node.child0 != nil {
bit := input.ReadBits(1)
if bit == -1 {
log.Fatal("unexpected end of file")
if !input.EnsureBits(1) {
log.Fatal("Unexpected end of file")
}

bit := input.GetBit()
if bit == 0 {
node = node.child0
continue
Expand Down Expand Up @@ -387,7 +394,7 @@ func HuffmanDecompress(data []byte) []byte {
head := buildTree(tail)

outputstream := d2datautils.CreateStreamWriter()
bitstream := d2datautils.CreateBitStream(data[1:])
bitstream := d2datautils.CreateBitMuncher(data[1:], 0)

var decoded int

Expand All @@ -399,10 +406,10 @@ Loop:
case 256:
break Loop
case 257:
newvalue := bitstream.ReadBits(8)
newvalue := bitstream.GetByte()

outputstream.PushBytes(byte(newvalue))
tail = insertNode(tail, newvalue)
outputstream.PushBytes(newvalue)
tail = insertNode(tail, int(newvalue))
default:
outputstream.PushBytes(byte(decoded))
}
Expand Down
11 changes: 11 additions & 0 deletions d2common/d2datautils/bitmuncher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,3 +136,14 @@ func (v *BitMuncher) MakeSigned(value uint32, bits int) int32 {
// Force casting to a signed value
return int32(result)
}

// EnsureBits checks, if `count` bits is available
func (v *BitMuncher) EnsureBits(count int) bool {
bytesRead := v.offset / byteLen
bitOffset := v.offset % byteLen
numBytes := len(v.data)
remainingBytes := numBytes - bytesRead
remainingBits := remainingBytes*byteLen - bitOffset

return count <= remainingBits
}
22 changes: 17 additions & 5 deletions d2common/d2datautils/bitmuncher_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestBitmuncherReadBit(t *testing.T) {

var result byte

for i := 0; i < bitsPerByte; i++ {
for i := 0; i < byteLen; i++ {
v := bm.GetBit()
result |= byte(v) << byte(i)
}
Expand All @@ -47,7 +47,7 @@ func TestBitmuncherReadBit(t *testing.T) {
func TestBitmuncherGetBits(t *testing.T) {
bm := CreateBitMuncher(testData, 0)

assert.Equal(t, byte(bm.GetBits(bitsPerByte)), testData[0], "get bits didn't return expected value")
assert.Equal(t, byte(bm.GetBits(byteLen)), testData[0], "get bits didn't return expected value")
}

func TestBitmuncherGetNoBits(t *testing.T) {
Expand Down Expand Up @@ -77,7 +77,7 @@ func TestBitmuncherGetOneSignedBit(t *testing.T) {
func TestBitmuncherSkipBits(t *testing.T) {
bm := CreateBitMuncher(testData, 0)

bm.SkipBits(bitsPerByte)
bm.SkipBits(byteLen)

assert.Equal(t, bm.GetByte(), testData[1], "skipping 8 bits didn't moved bit muncher's position into next byte")
}
Expand All @@ -88,7 +88,7 @@ func TestBitmuncherGetInt32(t *testing.T) {
var testInt int32

for i := 0; i < bytesPerint32; i++ {
testInt |= int32(testData[i]) << int32(bitsPerByte*i)
testInt |= int32(testData[i]) << int32(byteLen*i)
}

assert.Equal(t, bm.GetInt32(), testInt, "int32 value wasn't returned properly")
Expand All @@ -100,8 +100,20 @@ func TestBitmuncherGetUint32(t *testing.T) {
var testUint uint32

for i := 0; i < bytesPerint32; i++ {
testUint |= uint32(testData[i]) << uint32(bitsPerByte*i)
testUint |= uint32(testData[i]) << uint32(byteLen*i)
}

assert.Equal(t, bm.GetUInt32(), testUint, "uint32 value wasn't returned properly")
}

func TestBitMuncherEnsureBits(t *testing.T) {
bm := CreateBitMuncher(testData, 0)

assert.Equal(t, true, bm.EnsureBits(byteLen*len(testData)), "unexpected value returned by EnsureBits")
assert.Equal(t, false, bm.EnsureBits(byteLen*len(testData)+1), "unexpected value returned by EnsureBits")

bm.SkipBits(5)

assert.Equal(t, true, bm.EnsureBits(byteLen*len(testData)-5), "unexpected value returned by EnsureBits")
assert.Equal(t, false, bm.EnsureBits(byteLen*len(testData)-4), "unexpected value returned by EnsureBits")
}
82 changes: 0 additions & 82 deletions d2common/d2datautils/bitstream.go
View file
Open in desktop

This file was deleted.

36 changes: 0 additions & 36 deletions d2common/d2datautils/bitstream_test.go
View file
Open in desktop

This file was deleted.

8 changes: 4 additions & 4 deletions d2common/d2datautils/stream_writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func (v *StreamWriter) PushBit(b bool) {
}
v.bitOffset++

if v.bitOffset != bitsPerByte {
if v.bitOffset != byteLen {
return
}

Expand All @@ -53,7 +53,7 @@ func (v *StreamWriter) PushBit(b bool) {

// PushBits pushes bits (with max range 8)
func (v *StreamWriter) PushBits(b byte, bits int) {
if bits > bitsPerByte {
if bits > byteLen {
log.Print("input bits number must be less (or equal) than 8")
}

Expand All @@ -67,7 +67,7 @@ func (v *StreamWriter) PushBits(b byte, bits int) {

// PushBits16 pushes bits (with max range 16)
func (v *StreamWriter) PushBits16(b uint16, bits int) {
if bits > bitsPerByte*bytesPerint16 {
if bits > byteLen*bytesPerint16 {
log.Print("input bits number must be less (or equal) than 16")
}

Expand All @@ -81,7 +81,7 @@ func (v *StreamWriter) PushBits16(b uint16, bits int) {

// PushBits32 pushes bits (with max range 32)
func (v *StreamWriter) PushBits32(b uint32, bits int) {
if bits > bitsPerByte*bytesPerint32 {
if bits > byteLen*bytesPerint32 {
log.Print("input bits number must be less (or equal) than 32")
}

Expand Down
6 changes: 3 additions & 3 deletions d2common/d2datautils/stream_writer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ func TestStreamWriterBits(t *testing.T) {
data := []byte{221, 19}

for _, i := range data {
sr.PushBits(i, bitsPerByte)
sr.PushBits(i, byteLen)
}

output := sr.GetBytes()
Expand All @@ -25,7 +25,7 @@ func TestStreamWriterBits16(t *testing.T) {
data := []uint16{1024, 19}

for _, i := range data {
sr.PushBits16(i, bitsPerByte*bytesPerint16)
sr.PushBits16(i, byteLen*bytesPerint16)
}

output := sr.GetBytes()
Expand All @@ -45,7 +45,7 @@ func TestStreamWriterBits32(t *testing.T) {
data := []uint32{19324, 87}

for _, i := range data {
sr.PushBits32(i, bitsPerByte*bytesPerint32)
sr.PushBits32(i, byteLen*bytesPerint32)
}

output := sr.GetBytes()
Expand Down