Skip to content
This repository has been archived by the owner on Sep 29, 2024. It is now read-only.

Fix(ish) for unicode encoding in packet #608

Merged
merged 5 commits into from
Aug 21, 2023
Merged

Fix(ish) for unicode encoding in packet #608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ad12016
Fix-ish the unicode issues
grahamjenson Aug 21, 2023
718f303
removing unneded spaces
grahamjenson Aug 21, 2023
469c486
fixing documentation
grahamjenson Aug 21, 2023
f8d2cfd
fixed linting
grahamjenson Aug 21, 2023
454d163
fixing benchmarking of decode
grahamjenson Aug 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 32 additions & 5 deletions engineio/payload/data_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,18 @@ var tests = []struct {
{frame.String, packet.OPEN, []byte{}},
},
},
{true, []byte{0x00, 0x01, 0x03, 0xff, '4', 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd},
{true, []byte{0x00, 0x09, 0xff, '4', 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd},
[]Packet{
{frame.String, packet.MESSAGE, []byte("hello 你好")},
},
},
{true, []byte{0x01, 0x01, 0x03, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd}, []Packet{
{true, []byte{0x01, 0x09, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', ' ', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd}, []Packet{
{frame.Binary, packet.MESSAGE, []byte("hello 你好")},
},
},
{true, []byte{
0x01, 0x07, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', '\n',
0x00, 0x08, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
0x00, 0x04, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
0x00, 0x06, 0xff, '2', 'p', 'r', 'o', 'b', 'e',
}, []Packet{
{frame.Binary, packet.MESSAGE, []byte("hello\n")},
Expand All @@ -43,18 +43,45 @@ var tests = []struct {
{frame.String, packet.OPEN, []byte{}},
},
},
{false, []byte("13:4hello 你好"), []Packet{
{false, []byte("9:4hello 你好"), []Packet{
{frame.String, packet.MESSAGE, []byte("hello 你好")},
},
},
{false, []byte("18:b4aGVsbG8g5L2g5aW9"), []Packet{
{frame.Binary, packet.MESSAGE, []byte("hello 你好")},
},
},
{false, []byte("10:b4aGVsbG8K8:4你好\n6:2probe"), []Packet{
{false, []byte("10:b4aGVsbG8K4:4你好\n6:2probe"), []Packet{
{frame.Binary, packet.MESSAGE, []byte("hello\n")},
{frame.String, packet.MESSAGE, []byte("你好\n")},
{frame.String, packet.PING, []byte("probe")},
},
},
// ↓ is 3 bytes, JavaScript `.length` 1 See https://socket.io/docs/v4/engine-io-protocol/#from-v3-to-v4
{false, []byte("6:412↓453:41↓"), []Packet{
{frame.String, packet.MESSAGE, []byte("12↓45")},
{frame.String, packet.MESSAGE, []byte("1↓")},
},
},
// 🇩🇪 is 8 bytes, 2 unicode chars JavaScript `.length` 4
{false, []byte("6:4hello6:4🇩🇪a5:41234"), []Packet{
{frame.String, packet.MESSAGE, []byte("hello")},
{frame.String, packet.MESSAGE, []byte("🇩🇪a")},
{frame.String, packet.MESSAGE, []byte("1234")},
},
},
// € is 3 bytes, JavaScript `.length` 1
{false, []byte("2:4h3:4€a2:41"), []Packet{
{frame.String, packet.MESSAGE, []byte("h")},
{frame.String, packet.MESSAGE, []byte("€a")},
{frame.String, packet.MESSAGE, []byte("1")},
},
},
//👍 is 4 bytes, JavaScript `.length` 2
{false, []byte("2:4h4:4👍a2:41"), []Packet{
{frame.String, packet.MESSAGE, []byte("h")},
{frame.String, packet.MESSAGE, []byte("👍a")},
{frame.String, packet.MESSAGE, []byte("1")},
},
},
}
19 changes: 18 additions & 1 deletion engineio/payload/decoder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,24 @@ func (d *decoder) Read(p []byte) (int, error) {
if d.b64Reader != nil {
return d.b64Reader.Read(p)
}
return d.limitReader.Read(p)
dd, err := d.limitReader.Read(p)
unicodeCount := 0
for i := range p[:dd] {
b := p[i]
if b>>3 == 30 {
// starts with 11110 4 byte unicode char, probably 2 length in JS
unicodeCount = unicodeCount + 2
} else if b>>4 == 14 {
// starts with 1110 3 byte unicode char, probably 2 length in JS
unicodeCount = unicodeCount + 2
} else if b>>5 == 6 {
// starts with 110 2 byte unicode char, , probably 1 length in JS
unicodeCount = unicodeCount + 1
}
}

d.limitReader.N = d.limitReader.N + int64(unicodeCount)
return dd, err
}

func (d *decoder) Close() error {
Expand Down
4 changes: 2 additions & 2 deletions engineio/payload/decoder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func TestDecoderNextReaderError(t *testing.T) {

func BenchmarkStringDecoder(b *testing.B) {
feeder := fakeReaderFeeder{
data: []byte("8:4你好\n6:2probe"),
data: []byte("4:4你好\n6:2probe"),
supportBinary: false,
}
d := decoder{
Expand Down Expand Up @@ -148,7 +148,7 @@ func BenchmarkBinaryDecoder(b *testing.B) {
feeder := fakeReaderFeeder{
data: []byte{
0x01, 0x07, 0xff, 0x04, 'h', 'e', 'l', 'l', 'o', '\n',
0x00, 0x08, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
0x00, 0x04, 0xff, '4', 0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, '\n',
0x00, 0x06, 0xff, '2', 'p', 'r', 'o', 'b', 'e',
},
supportBinary: true,
Expand Down
36 changes: 32 additions & 4 deletions engineio/payload/encoder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"encoding/base64"
"io"
"unicode/utf8"

"github.com/googollee/go-socket.io/engineio/frame"
"github.com/googollee/go-socket.io/engineio/packet"
Expand Down Expand Up @@ -56,6 +57,7 @@ func (e *encoder) Write(p []byte) (int, error) {
if e.b64Writer != nil {
return e.b64Writer.Write(p)
}
// Need to scan here and add unicode chars to the list
return e.frameCache.Write(p)
}

Expand Down Expand Up @@ -92,16 +94,16 @@ func (e *encoder) Close() error {
}

func (e *encoder) writeTextHeader() error {
l := int64(e.frameCache.Len() + 1) // length for packet type
err := writeTextLen(l, &e.header)

err := writeTextLen(e.calcCodeUnitLength(), &e.header)
if err == nil {
err = e.header.WriteByte(e.pt.StringByte())
}
return err
}

func (e *encoder) writeB64Header() error {
l := int64(e.frameCache.Len() + 2) // length for 'b' and packet type
l := int64(utf8.RuneCount(e.frameCache.Bytes()) + 2) // length for 'b' and packet type
err := writeTextLen(l, &e.header)
if err == nil {
err = e.header.WriteByte('b')
Expand All @@ -112,8 +114,34 @@ func (e *encoder) writeB64Header() error {
return err
}

func (e *encoder) calcCodeUnitLength() int64 {
var l int64 = 1
var codeUnitSize int64
bytes := e.frameCache.Bytes()
for i := range bytes {
b := bytes[i]
if b>>3 == 30 {
// starts with 11110 4 byte unicode char, probably 2 length in JS
codeUnitSize = 2
} else if b>>4 == 14 {
// starts with 1110 3 byte unicode char, probably 1 length in JS
codeUnitSize = 1
} else if b>>5 == 6 {
// starts with 110 2 byte unicode char, , probably 1 length in JS
codeUnitSize = 1
} else if b>>6 == 2 {
// starts with 10 just unicode byte
codeUnitSize = 0
} else {
codeUnitSize = 1
}
l = l + codeUnitSize
}

return int64(l)
}
func (e *encoder) writeBinaryHeader() error {
l := int64(e.frameCache.Len() + 1) // length for packet type
l := int64(e.calcCodeUnitLength()) // length for packet type
b := e.pt.StringByte()
if e.ft == frame.Binary {
b = e.pt.BinaryByte()
Expand Down
Loading