feat(stdlibs): add encoding, encoding/{base32,binary,csv}

gnovm/stdlibs/encoding/binary/varint.gno

@@ -0,0 +1,142 @@
+package binary
+
+import (
+	"errors"
+	"io"
+)
+
+// MaxVariantLenN is the maximum length of a varint-encoded N-bit integer.
+const (
+	MaxVarintLen16 = 3
+	MaxVarintLen32 = 5
+	MaxVarintLen64 = 10
+)
+
+// AppendUvarint appends the variant-encoded form of x.
+// as generated by PutUvarint, to buffer and returns the extended buffer.
+func AppendUvarint(buf []byte, x uint64) []byte {
+	for x >= 0x80 {
+		buf = append(buf, byte(x)|0x80)
+		x >>= 7
+	}
+	return append(buf, byte(x))
+}
+
+// PutUvarint encodes a uint64 into buffer and returns the number of bytes written.
+// If the buffer is too small, PutUvarint will panic.
+func PutUvarint(buf []byte, x uint64) int {
+	i := 0
+	for x >= 0x80 {
+		buf[i] = byte(x) | 0x80
+		x >>= 7
+		i++
+	}
+	buf[i] = byte(x)
+	return i + 1
+}
+
+// Uvarint decodes a uint64 from buf and returns that value and the
+// number of bytes read (> 0). If an error occurred, the value is 0
+// and the number of bytes n is <= 0 meaning:
+//
+//	n == 0: buf too small
+//	n  < 0: value larger than 64 bits (overflow)
+//	        and -n is the number of bytes read
+func Uvarint(buf []byte) (uint64, int) {
+	var x uint64
+	var s uint
+	for i, b := range buf {
+		if i == MaxVarintLen64 {
+			// Catch byte reads past MaxVarintLen64.
+			// See issue https://golang.org/issues/41185
+			return 0, -(i + 1) // overflow
+		}
+		if b < 0x80 {
+			if i == MaxVarintLen64-1 && b > 1 {
+				return 0, -(i + 1) // overflow
+			}
+			return x | uint64(b)<<s, i + 1
+		}
+		x |= uint64(b&0x7f) << s
+		s += 7
+	}
+	return 0, 0
+}
+
+// AppendVarint appends the variant-encoded form of x,
+// as generated by PutVarint, to buffer and returns the extended buffer.
+func AppendVarint(buf []byte, x int64) []byte {
+	ux := uint64(x) << 1
+	if x < 0 {
+		ux = ^ux
+	}
+	return AppendUvarint(buf, ux)
+}
+
+// PutVarint encodes an int64 into buf and returns the number of bytes written.
+// If the buffer is too small, PutVarint will panic.
+func PutVarint(buf []byte, x int64) int {
+	ux := uint64(x) << 1
+	if x < 0 {
+		ux = ^ux
+	}
+	return PutUvarint(buf, ux)
+}
+
+// Varint decodes an int64 from buffer and returns that value and the
+// number of bytes read (> 0). If an error occurred, the value is `0`
+// and the number of bytes n <= 0 with the follwing meaning:
+//
+//	n == 0: buf too small
+//	n  < 0: value larger than 64 bits (overflow)
+//	        and -n is the number of bytes read
+func Varint(buf []byte) (int64, int) {
+	ux, n := Uvarint(buf) // ok to continue in presence of error
+	x := int64(ux >> 1)
+	if ux&1 != 0 {
+		x = ^x
+	}
+	return x, n
+}
+
+var errOverflow = errors.New("binary: varint overflows a 64-bit integer")
+
+// ReadUvarint reads an encoded unsigned integer from r and returns it as a uint64.
+// The error is [io.EOF] only if no bytes were read.
+// If an [io.EOF] happens after reading some but not all the bytes,
+// ReadUvarint returns [io.ErrUnexpectedEOF].
+func ReadUvarint(r io.ByteReader) (uint64, error) {
+	var x uint64
+	var s uint
+	for i := 0; i < MaxVarintLen64; i++ {
+		b, err := r.ReadByte()
+		if err != nil {
+			if i > 0 && err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			return x, err
+		}
+		if b < 0x80 {
+			if i == MaxVarintLen64-1 && b > 1 {
+				return x, errOverflow
+			}
+			return x | uint64(b)<<s, nil
+		}
+		x |= uint64(b&0x7f) << s
+		s += 7
+	}
+	return x, errOverflow
+}
+
+// ReadVarint reads an encoded signed integer from r and returns it as an int64.
+// The error is [io.EOF] only if no bytes were read.
+// If an [io.EOF] happens after reading some but not all the bytes,
+// ReadVarint returns [io.ErrUnexpectedEOF].
+func ReadVarint(r io.ByteReader) (int64, error) {
+	ux, err := ReadUvarint(r) // ok to continue in presence of error
+	x := int64(ux >> 1)
+	if ux&1 != 0 {
+		x = ^x
+	}
+	return x, err
+}

gnovm/stdlibs/encoding/binary/varint_test.gno

@@ -0,0 +1,256 @@
+package binary
+
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+import (
+	"bytes"
+	"io"
+	"math"
+	"testing"
+)
+
+func testConstant(t *testing.T, w uint, max int) {
+	buf := make([]byte, MaxVarintLen64)
+	n := PutUvarint(buf, 1<<w-1)
+	if n != max {
+		t.Errorf("MaxVarintLen%d = %d; want %d", w, max, n)
+	}
+}
+
+func TestConstants(t *testing.T) {
+	testConstant(t, 16, MaxVarintLen16)
+	testConstant(t, 32, MaxVarintLen32)
+	testConstant(t, 64, MaxVarintLen64)
+}
+
+// XXX panic: runtime error: invalid memory address or nil pointer dereference
+//
+// func testVarint(t *testing.T, x int64) {
+// 	buf := make([]byte, MaxVarintLen64)
+// 	n := PutVarint(buf, x)
+// 	y, m := Varint(buf[0:n])
+// 	if x != y {
+// 		t.Errorf("Varint(%d): got %d", x, y)
+// 	}
+// 	if n != m {
+// 		t.Errorf("Varint(%d): got n = %d; want %d", x, m, n)
+// 	}
+
+// 	buf2 := []byte("prefix")
+// 	buf2 = AppendVarint(buf2, x)
+// 	if string(buf2) != "prefix"+string(buf[:n]) {
+// 		t.Errorf("AppendVarint(%d): got %q, want %q", x, buf2, "prefix"+string(buf[:n]))
+// 	}
+
+// 	y, err := ReadVarint(bytes.NewReader(buf))
+// 	if err != nil {
+// 		t.Errorf("ReadVarint(%d): %s", x, err)
+// 	}
+// 	if x != y {
+// 		t.Errorf("ReadVarint(%d): got %d", x, y)
+// 	}
+// }
+
+// func testUvarint(t *testing.T, x uint64) {
+// 	buf := make([]byte, MaxVarintLen64)
+// 	n := PutUvarint(buf, x)
+// 	y, m := Uvarint(buf[0:n])
+// 	if x != y {
+// 		t.Errorf("Uvarint(%d): got %d", x, y)
+// 	}
+// 	if n != m {
+// 		t.Errorf("Uvarint(%d): got n = %d; want %d", x, m, n)
+// 	}
+
+// 	buf2 := []byte("prefix")
+// 	buf2 = AppendUvarint(buf2, x)
+// 	if string(buf2) != "prefix"+string(buf[:n]) {
+// 		t.Errorf("AppendUvarint(%d): got %q, want %q", x, buf2, "prefix"+string(buf[:n]))
+// 	}
+
+// 	y, err := ReadUvarint(bytes.NewReader(buf))
+// 	if err != nil {
+// 		t.Errorf("ReadUvarint(%d): %s", x, err)
+// 	}
+// 	if x != y {
+// 		t.Errorf("ReadUvarint(%d): got %d", x, y)
+// 	}
+// }
+
+// var tests = []int64{
+// 	-1 << 63,
+// 	-1<<63 + 1,
+// 	-1,
+// 	0,
+// 	1,
+// 	2,
+// 	10,
+// 	20,
+// 	63,
+// 	64,
+// 	65,
+// 	127,
+// 	128,
+// 	129,
+// 	255,
+// 	256,
+// 	257,
+// 	1<<63 - 1,
+// }
+
+// func TestVarint(t *testing.T) {
+// 	for _, x := range tests {
+// 		testVarint(t, x)
+// 		testVarint(t, -x)
+// 	}
+// 	for x := int64(0x7); x != 0; x <<= 1 {
+// 		testVarint(t, x)
+// 		testVarint(t, -x)
+// 	}
+// }
+
+// func TestUvarint(t *testing.T) {
+// 	for _, x := range tests {
+// 		testUvarint(t, uint64(x))
+// 	}
+// 	for x := uint64(0x7); x != 0; x <<= 1 {
+// 		testUvarint(t, x)
+// 	}
+// }
+
+// func TestBufferTooSmall(t *testing.T) {
+// 	buf := []byte{0x80, 0x80, 0x80, 0x80}
+// 	for i := 0; i <= len(buf); i++ {
+// 		buf := buf[0:i]
+// 		x, n := Uvarint(buf)
+// 		if x != 0 || n != 0 {
+// 			t.Errorf("Uvarint(%v): got x = %d, n = %d", buf, x, n)
+// 		}
+
+// 		x, err := ReadUvarint(bytes.NewReader(buf))
+// 		wantErr := io.EOF
+// 		if i > 0 {
+// 			wantErr = io.ErrUnexpectedEOF
+// 		}
+// 		if x != 0 || err != wantErr {
+// 			t.Errorf("ReadUvarint(%v): got x = %d, err = %s", buf, x, err)
+// 		}
+// 	}
+// }
+
+// XXX panic: reflect: reflect.Value.SetString using value obtained using unexported field
+//
+// Ensure that we catch overflows of bytes going past MaxVarintLen64.
+// See issue https://golang.org/issues/41185
+// func TestBufferTooBigWithOverflow(t *testing.T) {
+// 	tests := []struct {
+// 		in        []byte
+// 		name      string
+// 		wantN     int
+// 		wantValue uint64
+// 	}{
+// 		{
+// 			name: "invalid: 1000 bytes",
+// 			in: func() []byte {
+// 				b := make([]byte, 1000)
+// 				for i := range b {
+// 					b[i] = 0xff
+// 				}
+// 				b[999] = 0
+// 				return b
+// 			}(),
+// 			wantN:     -11,
+// 			wantValue: 0,
+// 		},
+// 		{
+// 			name:      "valid: math.MaxUint64-40",
+// 			in:        []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01},
+// 			wantValue: math.MaxUint64 - 40,
+// 			wantN:     10,
+// 		},
+// 		{
+// 			name:      "invalid: with more than MaxVarintLen64 bytes",
+// 			in:        []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01},
+// 			wantN:     -11,
+// 			wantValue: 0,
+// 		},
+// 		{
+// 			name:      "invalid: 10th byte",
+// 			in:        []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f},
+// 			wantN:     -10,
+// 			wantValue: 0,
+// 		},
+// 	}
+
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			x, n := Uvarint(tt.in)
+// 			if x != tt.wantValue || n != tt.wantN {
+// 				t.Errorf("Uvarint(%v): got x = %d, n = %d; want %d, %d", tt.in, x, n, tt.wantValue, tt.wantN)
+// 			}
+
+// 			r := bytes.NewReader(tt.in)
+// 			len := r.Len()
+// 			x, err := ReadUvarint(r)
+// 			if x != tt.wantValue || err != errOverflow {
+// 				t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want %d, %s", tt.in, x, err, tt.wantValue, errOverflow)
+// 			}
+// 			if read := len - r.Len(); read > MaxVarintLen64 {
+// 				t.Errorf("ReadUvarint(%v): read more than MaxVarintLen64 bytes, got %d", tt.in, read)
+// 			}
+// 		})
+// 	}
+// }
+
+func testOverflow(t *testing.T, buf []byte, x0 uint64, n0 int, err0 error) {
+	x, n := Uvarint(buf)
+	if x != 0 || n != n0 {
+		t.Errorf("Uvarint(% X): got x = %d, n = %d; want 0, %d", buf, x, n, n0)
+	}
+
+	r := bytes.NewReader(buf)
+	len := r.Len()
+	x, err := ReadUvarint(r)
+	if x != x0 || err != err0 {
+		t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want %d, %s", buf, x, err, x0, err0)
+	}
+	if read := len - r.Len(); read > MaxVarintLen64 {
+		t.Errorf("ReadUvarint(%v): read more than MaxVarintLen64 bytes, got %d", buf, read)
+	}
+}
+
+func TestOverflow(t *testing.T) {
+	testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2}, 0, -10, errOverflow)
+	testOverflow(t, []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x1, 0, 0}, 0, -11, errOverflow)
+	testOverflow(t, []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 1<<64-1, -11, errOverflow) // 11 bytes, should overflow
+}
+
+func TestNonCanonicalZero(t *testing.T) {
+	buf := []byte{0x80, 0x80, 0x80, 0}
+	x, n := Uvarint(buf)
+	if x != 0 || n != 4 {
+		t.Errorf("Uvarint(%v): got x = %d, n = %d; want 0, 4", buf, x, n)
+	}
+}
+
+// func BenchmarkPutUvarint32(b *testing.B) {
+// 	buf := make([]byte, MaxVarintLen32)
+// 	b.SetBytes(4)
+// 	for i := 0; i < b.N; i++ {
+// 		for j := uint(0); j < MaxVarintLen32; j++ {
+// 			PutUvarint(buf, 1<<(j*7))
+// 		}
+// 	}
+// }
+
+// func BenchmarkPutUvarint64(b *testing.B) {
+// 	buf := make([]byte, MaxVarintLen64)
+// 	b.SetBytes(8)
+// 	for i := 0; i < b.N; i++ {
+// 		for j := uint(0); j < MaxVarintLen64; j++ {
+// 			PutUvarint(buf, 1<<(j*7))
+// 		}
+// 	}
+// }