From e195b58fd0a5440a82ade22e8f6d2da4227b5edf Mon Sep 17 00:00:00 2001 From: Thomas Pelletier Date: Wed, 9 Nov 2022 16:12:39 -0500 Subject: [PATCH] Expose parser API as unstable (#827) --- README.md | 11 + decode.go | 88 +++--- errors.go | 32 +- errors_test.go | 7 +- internal/ast/builder.go | 51 --- internal/characters/ascii.go | 42 +++ utf8.go => internal/characters/utf8.go | 53 +--- internal/tracker/key.go | 12 +- internal/tracker/seen.go | 28 +- localtime.go | 6 +- marshaler.go | 4 +- strict.go | 34 +- unmarshaler.go | 116 +++---- {internal/ast => unstable}/ast.go | 68 ++-- .../benchmark_test.go | 4 +- unstable/builder.go | 71 +++++ unstable/doc.go | 3 + {internal/ast => unstable}/kind.go | 12 +- parser.go => unstable/parser.go | 293 +++++++++++------- parser_test.go => unstable/parser_test.go | 166 +++++----- scanner.go => unstable/scanner.go | 50 +-- 21 files changed, 628 insertions(+), 523 deletions(-) delete mode 100644 internal/ast/builder.go create mode 100644 internal/characters/ascii.go rename utf8.go => internal/characters/utf8.go (87%) rename {internal/ast => unstable}/ast.go (60%) rename benchmark_test.go => unstable/benchmark_test.go (98%) create mode 100644 unstable/builder.go create mode 100644 unstable/doc.go rename {internal/ast => unstable}/kind.go (81%) rename parser.go => unstable/parser.go (70%) rename parser_test.go => unstable/parser_test.go (75%) rename scanner.go => unstable/scanner.go (79%) diff --git a/README.md b/README.md index a63c3a79..9f8439cc 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,17 @@ fmt.Println(string(b)) [marshal]: https://pkg.go.dev/github.com/pelletier/go-toml/v2#Marshal +## Unstable API + +This API does not yet follow the backward compatibility guarantees of this +library. They provide early access to features that may have rough edges or an +API subject to change. + +### Parser + +Parser is the unstable API that allows iterative parsing of a TOML document at +the AST level. See https://pkg.go.dev/github.com/pelletier/go-toml/v2/unstable. + ## Benchmarks Execution time speedup compared to other Go TOML libraries: diff --git a/decode.go b/decode.go index 4af96536..3a860d0f 100644 --- a/decode.go +++ b/decode.go @@ -5,6 +5,8 @@ import ( "math" "strconv" "time" + + "github.com/pelletier/go-toml/v2/unstable" ) func parseInteger(b []byte) (int64, error) { @@ -32,7 +34,7 @@ func parseLocalDate(b []byte) (LocalDate, error) { var date LocalDate if len(b) != 10 || b[4] != '-' || b[7] != '-' { - return date, newDecodeError(b, "dates are expected to have the format YYYY-MM-DD") + return date, unstable.NewParserError(b, "dates are expected to have the format YYYY-MM-DD") } var err error @@ -53,7 +55,7 @@ func parseLocalDate(b []byte) (LocalDate, error) { } if !isValidDate(date.Year, date.Month, date.Day) { - return LocalDate{}, newDecodeError(b, "impossible date") + return LocalDate{}, unstable.NewParserError(b, "impossible date") } return date, nil @@ -64,7 +66,7 @@ func parseDecimalDigits(b []byte) (int, error) { for i, c := range b { if c < '0' || c > '9' { - return 0, newDecodeError(b[i:i+1], "expected digit (0-9)") + return 0, unstable.NewParserError(b[i:i+1], "expected digit (0-9)") } v *= 10 v += int(c - '0') @@ -97,7 +99,7 @@ func parseDateTime(b []byte) (time.Time, error) { } else { const dateTimeByteLen = 6 if len(b) != dateTimeByteLen { - return time.Time{}, newDecodeError(b, "invalid date-time timezone") + return time.Time{}, unstable.NewParserError(b, "invalid date-time timezone") } var direction int switch b[0] { @@ -106,11 +108,11 @@ func parseDateTime(b []byte) (time.Time, error) { case '+': direction = +1 default: - return time.Time{}, newDecodeError(b[:1], "invalid timezone offset character") + return time.Time{}, unstable.NewParserError(b[:1], "invalid timezone offset character") } if b[3] != ':' { - return time.Time{}, newDecodeError(b[3:4], "expected a : separator") + return time.Time{}, unstable.NewParserError(b[3:4], "expected a : separator") } hours, err := parseDecimalDigits(b[1:3]) @@ -118,7 +120,7 @@ func parseDateTime(b []byte) (time.Time, error) { return time.Time{}, err } if hours > 23 { - return time.Time{}, newDecodeError(b[:1], "invalid timezone offset hours") + return time.Time{}, unstable.NewParserError(b[:1], "invalid timezone offset hours") } minutes, err := parseDecimalDigits(b[4:6]) @@ -126,7 +128,7 @@ func parseDateTime(b []byte) (time.Time, error) { return time.Time{}, err } if minutes > 59 { - return time.Time{}, newDecodeError(b[:1], "invalid timezone offset minutes") + return time.Time{}, unstable.NewParserError(b[:1], "invalid timezone offset minutes") } seconds := direction * (hours*3600 + minutes*60) @@ -139,7 +141,7 @@ func parseDateTime(b []byte) (time.Time, error) { } if len(b) > 0 { - return time.Time{}, newDecodeError(b, "extra bytes at the end of the timezone") + return time.Time{}, unstable.NewParserError(b, "extra bytes at the end of the timezone") } t := time.Date( @@ -160,7 +162,7 @@ func parseLocalDateTime(b []byte) (LocalDateTime, []byte, error) { const localDateTimeByteMinLen = 11 if len(b) < localDateTimeByteMinLen { - return dt, nil, newDecodeError(b, "local datetimes are expected to have the format YYYY-MM-DDTHH:MM:SS[.NNNNNNNNN]") + return dt, nil, unstable.NewParserError(b, "local datetimes are expected to have the format YYYY-MM-DDTHH:MM:SS[.NNNNNNNNN]") } date, err := parseLocalDate(b[:10]) @@ -171,7 +173,7 @@ func parseLocalDateTime(b []byte) (LocalDateTime, []byte, error) { sep := b[10] if sep != 'T' && sep != ' ' && sep != 't' { - return dt, nil, newDecodeError(b[10:11], "datetime separator is expected to be T or a space") + return dt, nil, unstable.NewParserError(b[10:11], "datetime separator is expected to be T or a space") } t, rest, err := parseLocalTime(b[11:]) @@ -195,7 +197,7 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) { // check if b matches to have expected format HH:MM:SS[.NNNNNN] const localTimeByteLen = 8 if len(b) < localTimeByteLen { - return t, nil, newDecodeError(b, "times are expected to have the format HH:MM:SS[.NNNNNN]") + return t, nil, unstable.NewParserError(b, "times are expected to have the format HH:MM:SS[.NNNNNN]") } var err error @@ -206,10 +208,10 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) { } if t.Hour > 23 { - return t, nil, newDecodeError(b[0:2], "hour cannot be greater 23") + return t, nil, unstable.NewParserError(b[0:2], "hour cannot be greater 23") } if b[2] != ':' { - return t, nil, newDecodeError(b[2:3], "expecting colon between hours and minutes") + return t, nil, unstable.NewParserError(b[2:3], "expecting colon between hours and minutes") } t.Minute, err = parseDecimalDigits(b[3:5]) @@ -217,10 +219,10 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) { return t, nil, err } if t.Minute > 59 { - return t, nil, newDecodeError(b[3:5], "minutes cannot be greater 59") + return t, nil, unstable.NewParserError(b[3:5], "minutes cannot be greater 59") } if b[5] != ':' { - return t, nil, newDecodeError(b[5:6], "expecting colon between minutes and seconds") + return t, nil, unstable.NewParserError(b[5:6], "expecting colon between minutes and seconds") } t.Second, err = parseDecimalDigits(b[6:8]) @@ -229,7 +231,7 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) { } if t.Second > 60 { - return t, nil, newDecodeError(b[6:8], "seconds cannot be greater 60") + return t, nil, unstable.NewParserError(b[6:8], "seconds cannot be greater 60") } b = b[8:] @@ -242,7 +244,7 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) { for i, c := range b[1:] { if !isDigit(c) { if i == 0 { - return t, nil, newDecodeError(b[0:1], "need at least one digit after fraction point") + return t, nil, unstable.NewParserError(b[0:1], "need at least one digit after fraction point") } break } @@ -266,7 +268,7 @@ func parseLocalTime(b []byte) (LocalTime, []byte, error) { } if precision == 0 { - return t, nil, newDecodeError(b[:1], "nanoseconds need at least one digit") + return t, nil, unstable.NewParserError(b[:1], "nanoseconds need at least one digit") } t.Nanosecond = frac * nspow[precision] @@ -289,24 +291,24 @@ func parseFloat(b []byte) (float64, error) { } if cleaned[0] == '.' { - return 0, newDecodeError(b, "float cannot start with a dot") + return 0, unstable.NewParserError(b, "float cannot start with a dot") } if cleaned[len(cleaned)-1] == '.' { - return 0, newDecodeError(b, "float cannot end with a dot") + return 0, unstable.NewParserError(b, "float cannot end with a dot") } dotAlreadySeen := false for i, c := range cleaned { if c == '.' { if dotAlreadySeen { - return 0, newDecodeError(b[i:i+1], "float can have at most one decimal point") + return 0, unstable.NewParserError(b[i:i+1], "float can have at most one decimal point") } if !isDigit(cleaned[i-1]) { - return 0, newDecodeError(b[i-1:i+1], "float decimal point must be preceded by a digit") + return 0, unstable.NewParserError(b[i-1:i+1], "float decimal point must be preceded by a digit") } if !isDigit(cleaned[i+1]) { - return 0, newDecodeError(b[i:i+2], "float decimal point must be followed by a digit") + return 0, unstable.NewParserError(b[i:i+2], "float decimal point must be followed by a digit") } dotAlreadySeen = true } @@ -317,12 +319,12 @@ func parseFloat(b []byte) (float64, error) { start = 1 } if cleaned[start] == '0' && isDigit(cleaned[start+1]) { - return 0, newDecodeError(b, "float integer part cannot have leading zeroes") + return 0, unstable.NewParserError(b, "float integer part cannot have leading zeroes") } f, err := strconv.ParseFloat(string(cleaned), 64) if err != nil { - return 0, newDecodeError(b, "unable to parse float: %w", err) + return 0, unstable.NewParserError(b, "unable to parse float: %w", err) } return f, nil @@ -336,7 +338,7 @@ func parseIntHex(b []byte) (int64, error) { i, err := strconv.ParseInt(string(cleaned), 16, 64) if err != nil { - return 0, newDecodeError(b, "couldn't parse hexadecimal number: %w", err) + return 0, unstable.NewParserError(b, "couldn't parse hexadecimal number: %w", err) } return i, nil @@ -350,7 +352,7 @@ func parseIntOct(b []byte) (int64, error) { i, err := strconv.ParseInt(string(cleaned), 8, 64) if err != nil { - return 0, newDecodeError(b, "couldn't parse octal number: %w", err) + return 0, unstable.NewParserError(b, "couldn't parse octal number: %w", err) } return i, nil @@ -364,7 +366,7 @@ func parseIntBin(b []byte) (int64, error) { i, err := strconv.ParseInt(string(cleaned), 2, 64) if err != nil { - return 0, newDecodeError(b, "couldn't parse binary number: %w", err) + return 0, unstable.NewParserError(b, "couldn't parse binary number: %w", err) } return i, nil @@ -387,12 +389,12 @@ func parseIntDec(b []byte) (int64, error) { } if len(cleaned) > startIdx+1 && cleaned[startIdx] == '0' { - return 0, newDecodeError(b, "leading zero not allowed on decimal number") + return 0, unstable.NewParserError(b, "leading zero not allowed on decimal number") } i, err := strconv.ParseInt(string(cleaned), 10, 64) if err != nil { - return 0, newDecodeError(b, "couldn't parse decimal number: %w", err) + return 0, unstable.NewParserError(b, "couldn't parse decimal number: %w", err) } return i, nil @@ -409,11 +411,11 @@ func checkAndRemoveUnderscoresIntegers(b []byte) ([]byte, error) { } if b[start] == '_' { - return nil, newDecodeError(b[start:start+1], "number cannot start with underscore") + return nil, unstable.NewParserError(b[start:start+1], "number cannot start with underscore") } if b[len(b)-1] == '_' { - return nil, newDecodeError(b[len(b)-1:], "number cannot end with underscore") + return nil, unstable.NewParserError(b[len(b)-1:], "number cannot end with underscore") } // fast path @@ -435,7 +437,7 @@ func checkAndRemoveUnderscoresIntegers(b []byte) ([]byte, error) { c := b[i] if c == '_' { if !before { - return nil, newDecodeError(b[i-1:i+1], "number must have at least one digit between underscores") + return nil, unstable.NewParserError(b[i-1:i+1], "number must have at least one digit between underscores") } before = false } else { @@ -449,11 +451,11 @@ func checkAndRemoveUnderscoresIntegers(b []byte) ([]byte, error) { func checkAndRemoveUnderscoresFloats(b []byte) ([]byte, error) { if b[0] == '_' { - return nil, newDecodeError(b[0:1], "number cannot start with underscore") + return nil, unstable.NewParserError(b[0:1], "number cannot start with underscore") } if b[len(b)-1] == '_' { - return nil, newDecodeError(b[len(b)-1:], "number cannot end with underscore") + return nil, unstable.NewParserError(b[len(b)-1:], "number cannot end with underscore") } // fast path @@ -476,10 +478,10 @@ func checkAndRemoveUnderscoresFloats(b []byte) ([]byte, error) { switch c { case '_': if !before { - return nil, newDecodeError(b[i-1:i+1], "number must have at least one digit between underscores") + return nil, unstable.NewParserError(b[i-1:i+1], "number must have at least one digit between underscores") } if i < len(b)-1 && (b[i+1] == 'e' || b[i+1] == 'E') { - return nil, newDecodeError(b[i+1:i+2], "cannot have underscore before exponent") + return nil, unstable.NewParserError(b[i+1:i+2], "cannot have underscore before exponent") } before = false case '+', '-': @@ -488,15 +490,15 @@ func checkAndRemoveUnderscoresFloats(b []byte) ([]byte, error) { before = false case 'e', 'E': if i < len(b)-1 && b[i+1] == '_' { - return nil, newDecodeError(b[i+1:i+2], "cannot have underscore after exponent") + return nil, unstable.NewParserError(b[i+1:i+2], "cannot have underscore after exponent") } cleaned = append(cleaned, c) case '.': if i < len(b)-1 && b[i+1] == '_' { - return nil, newDecodeError(b[i+1:i+2], "cannot have underscore after decimal point") + return nil, unstable.NewParserError(b[i+1:i+2], "cannot have underscore after decimal point") } if i > 0 && b[i-1] == '_' { - return nil, newDecodeError(b[i-1:i], "cannot have underscore before decimal point") + return nil, unstable.NewParserError(b[i-1:i], "cannot have underscore before decimal point") } cleaned = append(cleaned, c) default: @@ -542,3 +544,7 @@ func daysIn(m int, year int) int { func isLeap(year int) bool { return year%4 == 0 && (year%100 != 0 || year%400 == 0) } + +func isDigit(r byte) bool { + return r >= '0' && r <= '9' +} diff --git a/errors.go b/errors.go index 2e7f0ffd..309733f1 100644 --- a/errors.go +++ b/errors.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/pelletier/go-toml/v2/internal/danger" + "github.com/pelletier/go-toml/v2/unstable" ) // DecodeError represents an error encountered during the parsing or decoding @@ -55,25 +56,6 @@ func (s *StrictMissingError) String() string { type Key []string -// internal version of DecodeError that is used as the base to create a -// DecodeError with full context. -type decodeError struct { - highlight []byte - message string - key Key // optional -} - -func (de *decodeError) Error() string { - return de.message -} - -func newDecodeError(highlight []byte, format string, args ...interface{}) error { - return &decodeError{ - highlight: highlight, - message: fmt.Errorf(format, args...).Error(), - } -} - // Error returns the error message contained in the DecodeError. func (e *DecodeError) Error() string { return "toml: " + e.message @@ -105,12 +87,12 @@ func (e *DecodeError) Key() Key { // highlight can be freely deallocated. // //nolint:funlen -func wrapDecodeError(document []byte, de *decodeError) *DecodeError { - offset := danger.SubsliceOffset(document, de.highlight) +func wrapDecodeError(document []byte, de *unstable.ParserError) *DecodeError { + offset := danger.SubsliceOffset(document, de.Highlight) errMessage := de.Error() errLine, errColumn := positionAtEnd(document[:offset]) - before, after := linesOfContext(document, de.highlight, offset, 3) + before, after := linesOfContext(document, de.Highlight, offset, 3) var buf strings.Builder @@ -140,7 +122,7 @@ func wrapDecodeError(document []byte, de *decodeError) *DecodeError { buf.Write(before[0]) } - buf.Write(de.highlight) + buf.Write(de.Highlight) if len(after) > 0 { buf.Write(after[0]) @@ -158,7 +140,7 @@ func wrapDecodeError(document []byte, de *decodeError) *DecodeError { buf.WriteString(strings.Repeat(" ", len(before[0]))) } - buf.WriteString(strings.Repeat("~", len(de.highlight))) + buf.WriteString(strings.Repeat("~", len(de.Highlight))) if len(errMessage) > 0 { buf.WriteString(" ") @@ -183,7 +165,7 @@ func wrapDecodeError(document []byte, de *decodeError) *DecodeError { message: errMessage, line: errLine, column: errColumn, - key: de.key, + key: de.Key, human: buf.String(), } } diff --git a/errors_test.go b/errors_test.go index c100353a..2d3750b2 100644 --- a/errors_test.go +++ b/errors_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/pelletier/go-toml/v2/unstable" "github.com/stretchr/testify/assert" ) @@ -170,9 +171,9 @@ line 5`, doc := b.Bytes() hl := doc[start:end] - err := wrapDecodeError(doc, &decodeError{ - highlight: hl, - message: e.msg, + err := wrapDecodeError(doc, &unstable.ParserError{ + Highlight: hl, + Message: e.msg, }) var derr *DecodeError diff --git a/internal/ast/builder.go b/internal/ast/builder.go deleted file mode 100644 index 120f16e5..00000000 --- a/internal/ast/builder.go +++ /dev/null @@ -1,51 +0,0 @@ -package ast - -type Reference int - -const InvalidReference Reference = -1 - -func (r Reference) Valid() bool { - return r != InvalidReference -} - -type Builder struct { - tree Root - lastIdx int -} - -func (b *Builder) Tree() *Root { - return &b.tree -} - -func (b *Builder) NodeAt(ref Reference) *Node { - return b.tree.at(ref) -} - -func (b *Builder) Reset() { - b.tree.nodes = b.tree.nodes[:0] - b.lastIdx = 0 -} - -func (b *Builder) Push(n Node) Reference { - b.lastIdx = len(b.tree.nodes) - b.tree.nodes = append(b.tree.nodes, n) - return Reference(b.lastIdx) -} - -func (b *Builder) PushAndChain(n Node) Reference { - newIdx := len(b.tree.nodes) - b.tree.nodes = append(b.tree.nodes, n) - if b.lastIdx >= 0 { - b.tree.nodes[b.lastIdx].next = newIdx - b.lastIdx - } - b.lastIdx = newIdx - return Reference(b.lastIdx) -} - -func (b *Builder) AttachChild(parent Reference, child Reference) { - b.tree.nodes[parent].child = int(child) - int(parent) -} - -func (b *Builder) Chain(from Reference, to Reference) { - b.tree.nodes[from].next = int(to) - int(from) -} diff --git a/internal/characters/ascii.go b/internal/characters/ascii.go new file mode 100644 index 00000000..80f698db --- /dev/null +++ b/internal/characters/ascii.go @@ -0,0 +1,42 @@ +package characters + +var invalidAsciiTable = [256]bool{ + 0x00: true, + 0x01: true, + 0x02: true, + 0x03: true, + 0x04: true, + 0x05: true, + 0x06: true, + 0x07: true, + 0x08: true, + // 0x09 TAB + // 0x0A LF + 0x0B: true, + 0x0C: true, + // 0x0D CR + 0x0E: true, + 0x0F: true, + 0x10: true, + 0x11: true, + 0x12: true, + 0x13: true, + 0x14: true, + 0x15: true, + 0x16: true, + 0x17: true, + 0x18: true, + 0x19: true, + 0x1A: true, + 0x1B: true, + 0x1C: true, + 0x1D: true, + 0x1E: true, + 0x1F: true, + // 0x20 - 0x7E Printable ASCII characters + 0x7F: true, +} + +func InvalidAscii(b byte) bool { + return invalidAsciiTable[b] +} diff --git a/utf8.go b/internal/characters/utf8.go similarity index 87% rename from utf8.go rename to internal/characters/utf8.go index d47a4f20..db4f45ac 100644 --- a/utf8.go +++ b/internal/characters/utf8.go @@ -1,4 +1,4 @@ -package toml +package characters import ( "unicode/utf8" @@ -32,7 +32,7 @@ func (u utf8Err) Zero() bool { // 0x9 => tab, ok // 0xA - 0x1F => invalid // 0x7F => invalid -func utf8TomlValidAlreadyEscaped(p []byte) (err utf8Err) { +func Utf8TomlValidAlreadyEscaped(p []byte) (err utf8Err) { // Fast path. Check for and skip 8 bytes of ASCII characters per iteration. offset := 0 for len(p) >= 8 { @@ -48,7 +48,7 @@ func utf8TomlValidAlreadyEscaped(p []byte) (err utf8Err) { } for i, b := range p[:8] { - if invalidAscii(b) { + if InvalidAscii(b) { err.Index = offset + i err.Size = 1 return @@ -62,7 +62,7 @@ func utf8TomlValidAlreadyEscaped(p []byte) (err utf8Err) { for i := 0; i < n; { pi := p[i] if pi < utf8.RuneSelf { - if invalidAscii(pi) { + if InvalidAscii(pi) { err.Index = offset + i err.Size = 1 return @@ -106,11 +106,11 @@ func utf8TomlValidAlreadyEscaped(p []byte) (err utf8Err) { } // Return the size of the next rune if valid, 0 otherwise. -func utf8ValidNext(p []byte) int { +func Utf8ValidNext(p []byte) int { c := p[0] if c < utf8.RuneSelf { - if invalidAscii(c) { + if InvalidAscii(c) { return 0 } return 1 @@ -140,47 +140,6 @@ func utf8ValidNext(p []byte) int { return size } -var invalidAsciiTable = [256]bool{ - 0x00: true, - 0x01: true, - 0x02: true, - 0x03: true, - 0x04: true, - 0x05: true, - 0x06: true, - 0x07: true, - 0x08: true, - // 0x09 TAB - // 0x0A LF - 0x0B: true, - 0x0C: true, - // 0x0D CR - 0x0E: true, - 0x0F: true, - 0x10: true, - 0x11: true, - 0x12: true, - 0x13: true, - 0x14: true, - 0x15: true, - 0x16: true, - 0x17: true, - 0x18: true, - 0x19: true, - 0x1A: true, - 0x1B: true, - 0x1C: true, - 0x1D: true, - 0x1E: true, - 0x1F: true, - // 0x20 - 0x7E Printable ASCII characters - 0x7F: true, -} - -func invalidAscii(b byte) bool { - return invalidAsciiTable[b] -} - // acceptRange gives the range of valid values for the second byte in a UTF-8 // sequence. type acceptRange struct { diff --git a/internal/tracker/key.go b/internal/tracker/key.go index 7c148f48..149b17f5 100644 --- a/internal/tracker/key.go +++ b/internal/tracker/key.go @@ -1,8 +1,6 @@ package tracker -import ( - "github.com/pelletier/go-toml/v2/internal/ast" -) +import "github.com/pelletier/go-toml/v2/unstable" // KeyTracker is a tracker that keeps track of the current Key as the AST is // walked. @@ -11,19 +9,19 @@ type KeyTracker struct { } // UpdateTable sets the state of the tracker with the AST table node. -func (t *KeyTracker) UpdateTable(node *ast.Node) { +func (t *KeyTracker) UpdateTable(node *unstable.Node) { t.reset() t.Push(node) } // UpdateArrayTable sets the state of the tracker with the AST array table node. -func (t *KeyTracker) UpdateArrayTable(node *ast.Node) { +func (t *KeyTracker) UpdateArrayTable(node *unstable.Node) { t.reset() t.Push(node) } // Push the given key on the stack. -func (t *KeyTracker) Push(node *ast.Node) { +func (t *KeyTracker) Push(node *unstable.Node) { it := node.Key() for it.Next() { t.k = append(t.k, string(it.Node().Data)) @@ -31,7 +29,7 @@ func (t *KeyTracker) Push(node *ast.Node) { } // Pop key from stack. -func (t *KeyTracker) Pop(node *ast.Node) { +func (t *KeyTracker) Pop(node *unstable.Node) { it := node.Key() for it.Next() { t.k = t.k[:len(t.k)-1] diff --git a/internal/tracker/seen.go b/internal/tracker/seen.go index a7ee05ba..40e23f83 100644 --- a/internal/tracker/seen.go +++ b/internal/tracker/seen.go @@ -5,7 +5,7 @@ import ( "fmt" "sync" - "github.com/pelletier/go-toml/v2/internal/ast" + "github.com/pelletier/go-toml/v2/unstable" ) type keyKind uint8 @@ -150,23 +150,23 @@ func (s *SeenTracker) setExplicitFlag(parentIdx int) { // CheckExpression takes a top-level node and checks that it does not contain // keys that have been seen in previous calls, and validates that types are // consistent. -func (s *SeenTracker) CheckExpression(node *ast.Node) error { +func (s *SeenTracker) CheckExpression(node *unstable.Node) error { if s.entries == nil { s.reset() } switch node.Kind { - case ast.KeyValue: + case unstable.KeyValue: return s.checkKeyValue(node) - case ast.Table: + case unstable.Table: return s.checkTable(node) - case ast.ArrayTable: + case unstable.ArrayTable: return s.checkArrayTable(node) default: panic(fmt.Errorf("this should not be a top level node type: %s", node.Kind)) } } -func (s *SeenTracker) checkTable(node *ast.Node) error { +func (s *SeenTracker) checkTable(node *unstable.Node) error { if s.currentIdx >= 0 { s.setExplicitFlag(s.currentIdx) } @@ -219,7 +219,7 @@ func (s *SeenTracker) checkTable(node *ast.Node) error { return nil } -func (s *SeenTracker) checkArrayTable(node *ast.Node) error { +func (s *SeenTracker) checkArrayTable(node *unstable.Node) error { if s.currentIdx >= 0 { s.setExplicitFlag(s.currentIdx) } @@ -267,7 +267,7 @@ func (s *SeenTracker) checkArrayTable(node *ast.Node) error { return nil } -func (s *SeenTracker) checkKeyValue(node *ast.Node) error { +func (s *SeenTracker) checkKeyValue(node *unstable.Node) error { parentIdx := s.currentIdx it := node.Key() @@ -297,26 +297,26 @@ func (s *SeenTracker) checkKeyValue(node *ast.Node) error { value := node.Value() switch value.Kind { - case ast.InlineTable: + case unstable.InlineTable: return s.checkInlineTable(value) - case ast.Array: + case unstable.Array: return s.checkArray(value) } return nil } -func (s *SeenTracker) checkArray(node *ast.Node) error { +func (s *SeenTracker) checkArray(node *unstable.Node) error { it := node.Children() for it.Next() { n := it.Node() switch n.Kind { - case ast.InlineTable: + case unstable.InlineTable: err := s.checkInlineTable(n) if err != nil { return err } - case ast.Array: + case unstable.Array: err := s.checkArray(n) if err != nil { return err @@ -326,7 +326,7 @@ func (s *SeenTracker) checkArray(node *ast.Node) error { return nil } -func (s *SeenTracker) checkInlineTable(node *ast.Node) error { +func (s *SeenTracker) checkInlineTable(node *unstable.Node) error { if pool.New == nil { pool.New = func() interface{} { return &SeenTracker{} diff --git a/localtime.go b/localtime.go index 30a31dcb..a856bfdb 100644 --- a/localtime.go +++ b/localtime.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" "time" + + "github.com/pelletier/go-toml/v2/unstable" ) // LocalDate represents a calendar day in no specific timezone. @@ -75,7 +77,7 @@ func (d LocalTime) MarshalText() ([]byte, error) { func (d *LocalTime) UnmarshalText(b []byte) error { res, left, err := parseLocalTime(b) if err == nil && len(left) != 0 { - err = newDecodeError(left, "extra characters") + err = unstable.NewParserError(left, "extra characters") } if err != nil { return err @@ -109,7 +111,7 @@ func (d LocalDateTime) MarshalText() ([]byte, error) { func (d *LocalDateTime) UnmarshalText(data []byte) error { res, left, err := parseLocalDateTime(data) if err == nil && len(left) != 0 { - err = newDecodeError(left, "extra characters") + err = unstable.NewParserError(left, "extra characters") } if err != nil { return err diff --git a/marshaler.go b/marshaler.go index acb28831..07aceb90 100644 --- a/marshaler.go +++ b/marshaler.go @@ -12,6 +12,8 @@ import ( "strings" "time" "unicode" + + "github.com/pelletier/go-toml/v2/internal/characters" ) // Marshal serializes a Go value as a TOML document. @@ -437,7 +439,7 @@ func (enc *Encoder) encodeString(b []byte, v string, options valueOptions) []byt func needsQuoting(v string) bool { // TODO: vectorize for _, b := range []byte(v) { - if b == '\'' || b == '\r' || b == '\n' || invalidAscii(b) { + if b == '\'' || b == '\r' || b == '\n' || characters.InvalidAscii(b) { return true } } diff --git a/strict.go b/strict.go index b7830d13..802e7e4d 100644 --- a/strict.go +++ b/strict.go @@ -1,9 +1,9 @@ package toml import ( - "github.com/pelletier/go-toml/v2/internal/ast" "github.com/pelletier/go-toml/v2/internal/danger" "github.com/pelletier/go-toml/v2/internal/tracker" + "github.com/pelletier/go-toml/v2/unstable" ) type strict struct { @@ -12,10 +12,10 @@ type strict struct { // Tracks the current key being processed. key tracker.KeyTracker - missing []decodeError + missing []unstable.ParserError } -func (s *strict) EnterTable(node *ast.Node) { +func (s *strict) EnterTable(node *unstable.Node) { if !s.Enabled { return } @@ -23,7 +23,7 @@ func (s *strict) EnterTable(node *ast.Node) { s.key.UpdateTable(node) } -func (s *strict) EnterArrayTable(node *ast.Node) { +func (s *strict) EnterArrayTable(node *unstable.Node) { if !s.Enabled { return } @@ -31,7 +31,7 @@ func (s *strict) EnterArrayTable(node *ast.Node) { s.key.UpdateArrayTable(node) } -func (s *strict) EnterKeyValue(node *ast.Node) { +func (s *strict) EnterKeyValue(node *unstable.Node) { if !s.Enabled { return } @@ -39,7 +39,7 @@ func (s *strict) EnterKeyValue(node *ast.Node) { s.key.Push(node) } -func (s *strict) ExitKeyValue(node *ast.Node) { +func (s *strict) ExitKeyValue(node *unstable.Node) { if !s.Enabled { return } @@ -47,27 +47,27 @@ func (s *strict) ExitKeyValue(node *ast.Node) { s.key.Pop(node) } -func (s *strict) MissingTable(node *ast.Node) { +func (s *strict) MissingTable(node *unstable.Node) { if !s.Enabled { return } - s.missing = append(s.missing, decodeError{ - highlight: keyLocation(node), - message: "missing table", - key: s.key.Key(), + s.missing = append(s.missing, unstable.ParserError{ + Highlight: keyLocation(node), + Message: "missing table", + Key: s.key.Key(), }) } -func (s *strict) MissingField(node *ast.Node) { +func (s *strict) MissingField(node *unstable.Node) { if !s.Enabled { return } - s.missing = append(s.missing, decodeError{ - highlight: keyLocation(node), - message: "missing field", - key: s.key.Key(), + s.missing = append(s.missing, unstable.ParserError{ + Highlight: keyLocation(node), + Message: "missing field", + Key: s.key.Key(), }) } @@ -88,7 +88,7 @@ func (s *strict) Error(doc []byte) error { return err } -func keyLocation(node *ast.Node) []byte { +func keyLocation(node *unstable.Node) []byte { k := node.Key() hasOne := k.Next() diff --git a/unmarshaler.go b/unmarshaler.go index 32206ff2..70f6ec57 100644 --- a/unmarshaler.go +++ b/unmarshaler.go @@ -12,16 +12,16 @@ import ( "sync/atomic" "time" - "github.com/pelletier/go-toml/v2/internal/ast" "github.com/pelletier/go-toml/v2/internal/danger" "github.com/pelletier/go-toml/v2/internal/tracker" + "github.com/pelletier/go-toml/v2/unstable" ) // Unmarshal deserializes a TOML document into a Go value. // // It is a shortcut for Decoder.Decode() with the default options. func Unmarshal(data []byte, v interface{}) error { - p := parser{} + p := unstable.Parser{} p.Reset(data) d := decoder{p: &p} @@ -101,7 +101,7 @@ func (d *Decoder) Decode(v interface{}) error { return fmt.Errorf("toml: %w", err) } - p := parser{} + p := unstable.Parser{} p.Reset(b) dec := decoder{ p: &p, @@ -115,7 +115,7 @@ func (d *Decoder) Decode(v interface{}) error { type decoder struct { // Which parser instance in use for this decoding session. - p *parser + p *unstable.Parser // Flag indicating that the current expression is stashed. // If set to true, calling nextExpr will not actually pull a new expression @@ -157,7 +157,7 @@ func (d *decoder) typeMismatchError(toml string, target reflect.Type) error { return fmt.Errorf("toml: cannot decode TOML %s into a Go value of type %s", toml, target) } -func (d *decoder) expr() *ast.Node { +func (d *decoder) expr() *unstable.Node { return d.p.Expression() } @@ -208,12 +208,12 @@ func (d *decoder) FromParser(v interface{}) error { err := d.fromParser(r) if err == nil { - return d.strict.Error(d.p.data) + return d.strict.Error(d.p.Data()) } - var e *decodeError + var e *unstable.ParserError if errors.As(err, &e) { - return wrapDecodeError(d.p.data, e) + return wrapDecodeError(d.p.Data(), e) } return err @@ -234,16 +234,16 @@ func (d *decoder) fromParser(root reflect.Value) error { Rules for the unmarshal code: - The stack is used to keep track of which values need to be set where. -- handle* functions <=> switch on a given ast.Kind. +- handle* functions <=> switch on a given unstable.Kind. - unmarshalX* functions need to unmarshal a node of kind X. - An "object" is either a struct or a map. */ -func (d *decoder) handleRootExpression(expr *ast.Node, v reflect.Value) error { +func (d *decoder) handleRootExpression(expr *unstable.Node, v reflect.Value) error { var x reflect.Value var err error - if !(d.skipUntilTable && expr.Kind == ast.KeyValue) { + if !(d.skipUntilTable && expr.Kind == unstable.KeyValue) { err = d.seen.CheckExpression(expr) if err != nil { return err @@ -251,16 +251,16 @@ func (d *decoder) handleRootExpression(expr *ast.Node, v reflect.Value) error { } switch expr.Kind { - case ast.KeyValue: + case unstable.KeyValue: if d.skipUntilTable { return nil } x, err = d.handleKeyValue(expr, v) - case ast.Table: + case unstable.Table: d.skipUntilTable = false d.strict.EnterTable(expr) x, err = d.handleTable(expr.Key(), v) - case ast.ArrayTable: + case unstable.ArrayTable: d.skipUntilTable = false d.strict.EnterArrayTable(expr) x, err = d.handleArrayTable(expr.Key(), v) @@ -269,7 +269,7 @@ func (d *decoder) handleRootExpression(expr *ast.Node, v reflect.Value) error { } if d.skipUntilTable { - if expr.Kind == ast.Table || expr.Kind == ast.ArrayTable { + if expr.Kind == unstable.Table || expr.Kind == unstable.ArrayTable { d.strict.MissingTable(expr) } } else if err == nil && x.IsValid() { @@ -279,14 +279,14 @@ func (d *decoder) handleRootExpression(expr *ast.Node, v reflect.Value) error { return err } -func (d *decoder) handleArrayTable(key ast.Iterator, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleArrayTable(key unstable.Iterator, v reflect.Value) (reflect.Value, error) { if key.Next() { return d.handleArrayTablePart(key, v) } return d.handleKeyValues(v) } -func (d *decoder) handleArrayTableCollectionLast(key ast.Iterator, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleArrayTableCollectionLast(key unstable.Iterator, v reflect.Value) (reflect.Value, error) { switch v.Kind() { case reflect.Interface: elem := v.Elem() @@ -353,7 +353,7 @@ func (d *decoder) handleArrayTableCollectionLast(key ast.Iterator, v reflect.Val // evaluated like a normal key, but if it returns a collection, it also needs to // point to the last element of the collection. Unless it is the last part of // the key, then it needs to create a new element at the end. -func (d *decoder) handleArrayTableCollection(key ast.Iterator, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleArrayTableCollection(key unstable.Iterator, v reflect.Value) (reflect.Value, error) { if key.IsLast() { return d.handleArrayTableCollectionLast(key, v) } @@ -400,7 +400,7 @@ func (d *decoder) handleArrayTableCollection(key ast.Iterator, v reflect.Value) return d.handleArrayTable(key, v) } -func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handlerFn, makeFn valueMakerFn) (reflect.Value, error) { +func (d *decoder) handleKeyPart(key unstable.Iterator, v reflect.Value, nextFn handlerFn, makeFn valueMakerFn) (reflect.Value, error) { var rv reflect.Value // First, dispatch over v to make sure it is a valid object. @@ -518,7 +518,7 @@ func (d *decoder) handleKeyPart(key ast.Iterator, v reflect.Value, nextFn handle // HandleArrayTablePart navigates the Go structure v using the key v. It is // only used for the prefix (non-last) parts of an array-table. When // encountering a collection, it should go to the last element. -func (d *decoder) handleArrayTablePart(key ast.Iterator, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleArrayTablePart(key unstable.Iterator, v reflect.Value) (reflect.Value, error) { var makeFn valueMakerFn if key.IsLast() { makeFn = makeSliceInterface @@ -530,10 +530,10 @@ func (d *decoder) handleArrayTablePart(key ast.Iterator, v reflect.Value) (refle // HandleTable returns a reference when it has checked the next expression but // cannot handle it. -func (d *decoder) handleTable(key ast.Iterator, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleTable(key unstable.Iterator, v reflect.Value) (reflect.Value, error) { if v.Kind() == reflect.Slice { if v.Len() == 0 { - return reflect.Value{}, newDecodeError(key.Node().Data, "cannot store a table in a slice") + return reflect.Value{}, unstable.NewParserError(key.Node().Data, "cannot store a table in a slice") } elem := v.Index(v.Len() - 1) x, err := d.handleTable(key, elem) @@ -560,7 +560,7 @@ func (d *decoder) handleKeyValues(v reflect.Value) (reflect.Value, error) { var rv reflect.Value for d.nextExpr() { expr := d.expr() - if expr.Kind != ast.KeyValue { + if expr.Kind != unstable.KeyValue { // Stash the expression so that fromParser can just loop and use // the right handler. // We could just recurse ourselves here, but at least this gives a @@ -587,7 +587,7 @@ func (d *decoder) handleKeyValues(v reflect.Value) (reflect.Value, error) { } type ( - handlerFn func(key ast.Iterator, v reflect.Value) (reflect.Value, error) + handlerFn func(key unstable.Iterator, v reflect.Value) (reflect.Value, error) valueMakerFn func() reflect.Value ) @@ -599,11 +599,11 @@ func makeSliceInterface() reflect.Value { return reflect.MakeSlice(sliceInterfaceType, 0, 16) } -func (d *decoder) handleTablePart(key ast.Iterator, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleTablePart(key unstable.Iterator, v reflect.Value) (reflect.Value, error) { return d.handleKeyPart(key, v, d.handleTable, makeMapStringInterface) } -func (d *decoder) tryTextUnmarshaler(node *ast.Node, v reflect.Value) (bool, error) { +func (d *decoder) tryTextUnmarshaler(node *unstable.Node, v reflect.Value) (bool, error) { // Special case for time, because we allow to unmarshal to it from // different kind of AST nodes. if v.Type() == timeType { @@ -613,7 +613,7 @@ func (d *decoder) tryTextUnmarshaler(node *ast.Node, v reflect.Value) (bool, err if v.CanAddr() && v.Addr().Type().Implements(textUnmarshalerType) { err := v.Addr().Interface().(encoding.TextUnmarshaler).UnmarshalText(node.Data) if err != nil { - return false, newDecodeError(d.p.Raw(node.Raw), "%w", err) + return false, unstable.NewParserError(d.p.Raw(node.Raw), "%w", err) } return true, nil @@ -622,7 +622,7 @@ func (d *decoder) tryTextUnmarshaler(node *ast.Node, v reflect.Value) (bool, err return false, nil } -func (d *decoder) handleValue(value *ast.Node, v reflect.Value) error { +func (d *decoder) handleValue(value *unstable.Node, v reflect.Value) error { for v.Kind() == reflect.Ptr { v = initAndDereferencePointer(v) } @@ -633,32 +633,32 @@ func (d *decoder) handleValue(value *ast.Node, v reflect.Value) error { } switch value.Kind { - case ast.String: + case unstable.String: return d.unmarshalString(value, v) - case ast.Integer: + case unstable.Integer: return d.unmarshalInteger(value, v) - case ast.Float: + case unstable.Float: return d.unmarshalFloat(value, v) - case ast.Bool: + case unstable.Bool: return d.unmarshalBool(value, v) - case ast.DateTime: + case unstable.DateTime: return d.unmarshalDateTime(value, v) - case ast.LocalDate: + case unstable.LocalDate: return d.unmarshalLocalDate(value, v) - case ast.LocalTime: + case unstable.LocalTime: return d.unmarshalLocalTime(value, v) - case ast.LocalDateTime: + case unstable.LocalDateTime: return d.unmarshalLocalDateTime(value, v) - case ast.InlineTable: + case unstable.InlineTable: return d.unmarshalInlineTable(value, v) - case ast.Array: + case unstable.Array: return d.unmarshalArray(value, v) default: panic(fmt.Errorf("handleValue not implemented for %s", value.Kind)) } } -func (d *decoder) unmarshalArray(array *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalArray(array *unstable.Node, v reflect.Value) error { switch v.Kind() { case reflect.Slice: if v.IsNil() { @@ -729,7 +729,7 @@ func (d *decoder) unmarshalArray(array *ast.Node, v reflect.Value) error { return nil } -func (d *decoder) unmarshalInlineTable(itable *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalInlineTable(itable *unstable.Node, v reflect.Value) error { // Make sure v is an initialized object. switch v.Kind() { case reflect.Map: @@ -746,7 +746,7 @@ func (d *decoder) unmarshalInlineTable(itable *ast.Node, v reflect.Value) error } return d.unmarshalInlineTable(itable, elem) default: - return newDecodeError(itable.Data, "cannot store inline table in Go type %s", v.Kind()) + return unstable.NewParserError(itable.Data, "cannot store inline table in Go type %s", v.Kind()) } it := itable.Children() @@ -765,7 +765,7 @@ func (d *decoder) unmarshalInlineTable(itable *ast.Node, v reflect.Value) error return nil } -func (d *decoder) unmarshalDateTime(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalDateTime(value *unstable.Node, v reflect.Value) error { dt, err := parseDateTime(value.Data) if err != nil { return err @@ -775,7 +775,7 @@ func (d *decoder) unmarshalDateTime(value *ast.Node, v reflect.Value) error { return nil } -func (d *decoder) unmarshalLocalDate(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalLocalDate(value *unstable.Node, v reflect.Value) error { ld, err := parseLocalDate(value.Data) if err != nil { return err @@ -792,28 +792,28 @@ func (d *decoder) unmarshalLocalDate(value *ast.Node, v reflect.Value) error { return nil } -func (d *decoder) unmarshalLocalTime(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalLocalTime(value *unstable.Node, v reflect.Value) error { lt, rest, err := parseLocalTime(value.Data) if err != nil { return err } if len(rest) > 0 { - return newDecodeError(rest, "extra characters at the end of a local time") + return unstable.NewParserError(rest, "extra characters at the end of a local time") } v.Set(reflect.ValueOf(lt)) return nil } -func (d *decoder) unmarshalLocalDateTime(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalLocalDateTime(value *unstable.Node, v reflect.Value) error { ldt, rest, err := parseLocalDateTime(value.Data) if err != nil { return err } if len(rest) > 0 { - return newDecodeError(rest, "extra characters at the end of a local date time") + return unstable.NewParserError(rest, "extra characters at the end of a local date time") } if v.Type() == timeType { @@ -828,7 +828,7 @@ func (d *decoder) unmarshalLocalDateTime(value *ast.Node, v reflect.Value) error return nil } -func (d *decoder) unmarshalBool(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalBool(value *unstable.Node, v reflect.Value) error { b := value.Data[0] == 't' switch v.Kind() { @@ -837,13 +837,13 @@ func (d *decoder) unmarshalBool(value *ast.Node, v reflect.Value) error { case reflect.Interface: v.Set(reflect.ValueOf(b)) default: - return newDecodeError(value.Data, "cannot assign boolean to a %t", b) + return unstable.NewParserError(value.Data, "cannot assign boolean to a %t", b) } return nil } -func (d *decoder) unmarshalFloat(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalFloat(value *unstable.Node, v reflect.Value) error { f, err := parseFloat(value.Data) if err != nil { return err @@ -854,13 +854,13 @@ func (d *decoder) unmarshalFloat(value *ast.Node, v reflect.Value) error { v.SetFloat(f) case reflect.Float32: if f > math.MaxFloat32 { - return newDecodeError(value.Data, "number %f does not fit in a float32", f) + return unstable.NewParserError(value.Data, "number %f does not fit in a float32", f) } v.SetFloat(f) case reflect.Interface: v.Set(reflect.ValueOf(f)) default: - return newDecodeError(value.Data, "float cannot be assigned to %s", v.Kind()) + return unstable.NewParserError(value.Data, "float cannot be assigned to %s", v.Kind()) } return nil @@ -886,7 +886,7 @@ func init() { } } -func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalInteger(value *unstable.Node, v reflect.Value) error { i, err := parseInteger(value.Data) if err != nil { return err @@ -967,20 +967,20 @@ func (d *decoder) unmarshalInteger(value *ast.Node, v reflect.Value) error { return nil } -func (d *decoder) unmarshalString(value *ast.Node, v reflect.Value) error { +func (d *decoder) unmarshalString(value *unstable.Node, v reflect.Value) error { switch v.Kind() { case reflect.String: v.SetString(string(value.Data)) case reflect.Interface: v.Set(reflect.ValueOf(string(value.Data))) default: - return newDecodeError(d.p.Raw(value.Raw), "cannot store TOML string into a Go %s", v.Kind()) + return unstable.NewParserError(d.p.Raw(value.Raw), "cannot store TOML string into a Go %s", v.Kind()) } return nil } -func (d *decoder) handleKeyValue(expr *ast.Node, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleKeyValue(expr *unstable.Node, v reflect.Value) (reflect.Value, error) { d.strict.EnterKeyValue(expr) v, err := d.handleKeyValueInner(expr.Key(), expr.Value(), v) @@ -994,7 +994,7 @@ func (d *decoder) handleKeyValue(expr *ast.Node, v reflect.Value) (reflect.Value return v, err } -func (d *decoder) handleKeyValueInner(key ast.Iterator, value *ast.Node, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleKeyValueInner(key unstable.Iterator, value *unstable.Node, v reflect.Value) (reflect.Value, error) { if key.Next() { // Still scoping the key return d.handleKeyValuePart(key, value, v) @@ -1004,7 +1004,7 @@ func (d *decoder) handleKeyValueInner(key ast.Iterator, value *ast.Node, v refle return reflect.Value{}, d.handleValue(value, v) } -func (d *decoder) handleKeyValuePart(key ast.Iterator, value *ast.Node, v reflect.Value) (reflect.Value, error) { +func (d *decoder) handleKeyValuePart(key unstable.Iterator, value *unstable.Node, v reflect.Value) (reflect.Value, error) { // contains the replacement for v var rv reflect.Value diff --git a/internal/ast/ast.go b/unstable/ast.go similarity index 60% rename from internal/ast/ast.go rename to unstable/ast.go index 9dec2e00..b60d9bfd 100644 --- a/internal/ast/ast.go +++ b/unstable/ast.go @@ -1,4 +1,4 @@ -package ast +package unstable import ( "fmt" @@ -7,13 +7,16 @@ import ( "github.com/pelletier/go-toml/v2/internal/danger" ) -// Iterator starts uninitialized, you need to call Next() first. +// Iterator over a sequence of nodes. +// +// Starts uninitialized, you need to call Next() first. // // For example: // // it := n.Children() // for it.Next() { -// it.Node() +// n := it.Node() +// // do something with n // } type Iterator struct { started bool @@ -32,42 +35,31 @@ func (c *Iterator) Next() bool { } // IsLast returns true if the current node of the iterator is the last -// one. Subsequent call to Next() will return false. +// one. Subsequent calls to Next() will return false. func (c *Iterator) IsLast() bool { return c.node.next == 0 } -// Node returns a copy of the node pointed at by the iterator. +// Node returns a pointer to the node pointed at by the iterator. func (c *Iterator) Node() *Node { return c.node } -// Root contains a full AST. +// Node in a TOML expression AST. // -// It is immutable once constructed with Builder. -type Root struct { - nodes []Node -} - -// Iterator over the top level nodes. -func (r *Root) Iterator() Iterator { - it := Iterator{} - if len(r.nodes) > 0 { - it.node = &r.nodes[0] - } - return it -} - -func (r *Root) at(idx Reference) *Node { - return &r.nodes[idx] -} - -// Arrays have one child per element in the array. InlineTables have -// one child per key-value pair in the table. KeyValues have at least -// two children. The first one is the value. The rest make a -// potentially dotted key. Table and Array table have one child per -// element of the key they represent (same as KeyValue, but without -// the last node being the value). +// Depending on Kind, its sequence of children should be interpreted +// differently. +// +// - Array have one child per element in the array. +// - InlineTable have one child per key-value in the table (each of kind +// InlineTable). +// - KeyValue have at least two children. The first one is the value. The rest +// make a potentially dotted key. +// - Table and ArrayTable's children represent a dotted key (same as +// KeyValue, but without the first node being the value). +// +// When relevant, Raw describes the range of bytes this node is refering to in +// the input document. Use Parser.Raw() to retrieve the actual bytes. type Node struct { Kind Kind Raw Range // Raw bytes from the input. @@ -80,13 +72,13 @@ type Node struct { child int // 0 if no child } +// Range of bytes in the document. type Range struct { Offset uint32 Length uint32 } -// Next returns a copy of the next node, or an invalid Node if there -// is no next node. +// Next returns a pointer to the next node, or nil if there is no next node. func (n *Node) Next() *Node { if n.next == 0 { return nil @@ -96,9 +88,9 @@ func (n *Node) Next() *Node { return (*Node)(danger.Stride(ptr, size, n.next)) } -// Child returns a copy of the first child node of this node. Other -// children can be accessed calling Next on the first child. Returns -// an invalid Node if there is none. +// Child returns a pointer to the first child node of this node. Other children +// can be accessed calling Next on the first child. Returns an nil if this Node +// has no child. func (n *Node) Child() *Node { if n.child == 0 { return nil @@ -113,9 +105,9 @@ func (n *Node) Valid() bool { return n != nil } -// Key returns the child nodes making the Key on a supported -// node. Panics otherwise. They are guaranteed to be all be of the -// Kind Key. A simple key would return just one element. +// Key returns the children nodes making the Key on a supported node. Panics +// otherwise. They are guaranteed to be all be of the Kind Key. A simple key +// would return just one element. func (n *Node) Key() Iterator { switch n.Kind { case KeyValue: diff --git a/benchmark_test.go b/unstable/benchmark_test.go similarity index 98% rename from benchmark_test.go rename to unstable/benchmark_test.go index 831acaca..79c9bea1 100644 --- a/benchmark_test.go +++ b/unstable/benchmark_test.go @@ -1,4 +1,4 @@ -package toml +package unstable import ( "bytes" @@ -55,7 +55,7 @@ func BenchmarkParseLiteralStringValid(b *testing.B) { for name, input := range inputs { b.Run(name, func(b *testing.B) { - p := parser{} + p := Parser{} b.SetBytes(int64(len(input))) b.ReportAllocs() b.ResetTimer() diff --git a/unstable/builder.go b/unstable/builder.go new file mode 100644 index 00000000..9538e30d --- /dev/null +++ b/unstable/builder.go @@ -0,0 +1,71 @@ +package unstable + +// root contains a full AST. +// +// It is immutable once constructed with Builder. +type root struct { + nodes []Node +} + +// Iterator over the top level nodes. +func (r *root) Iterator() Iterator { + it := Iterator{} + if len(r.nodes) > 0 { + it.node = &r.nodes[0] + } + return it +} + +func (r *root) at(idx reference) *Node { + return &r.nodes[idx] +} + +type reference int + +const invalidReference reference = -1 + +func (r reference) Valid() bool { + return r != invalidReference +} + +type builder struct { + tree root + lastIdx int +} + +func (b *builder) Tree() *root { + return &b.tree +} + +func (b *builder) NodeAt(ref reference) *Node { + return b.tree.at(ref) +} + +func (b *builder) Reset() { + b.tree.nodes = b.tree.nodes[:0] + b.lastIdx = 0 +} + +func (b *builder) Push(n Node) reference { + b.lastIdx = len(b.tree.nodes) + b.tree.nodes = append(b.tree.nodes, n) + return reference(b.lastIdx) +} + +func (b *builder) PushAndChain(n Node) reference { + newIdx := len(b.tree.nodes) + b.tree.nodes = append(b.tree.nodes, n) + if b.lastIdx >= 0 { + b.tree.nodes[b.lastIdx].next = newIdx - b.lastIdx + } + b.lastIdx = newIdx + return reference(b.lastIdx) +} + +func (b *builder) AttachChild(parent reference, child reference) { + b.tree.nodes[parent].child = int(child) - int(parent) +} + +func (b *builder) Chain(from reference, to reference) { + b.tree.nodes[from].next = int(to) - int(from) +} diff --git a/unstable/doc.go b/unstable/doc.go new file mode 100644 index 00000000..7ff26c53 --- /dev/null +++ b/unstable/doc.go @@ -0,0 +1,3 @@ +// Package unstable provides APIs that do not meet the backward compatibility +// guarantees yet. +package unstable diff --git a/internal/ast/kind.go b/unstable/kind.go similarity index 81% rename from internal/ast/kind.go rename to unstable/kind.go index 2b50c67f..ff9df1be 100644 --- a/internal/ast/kind.go +++ b/unstable/kind.go @@ -1,25 +1,26 @@ -package ast +package unstable import "fmt" +// Kind represents the type of TOML structure contained in a given Node. type Kind int const ( - // meta + // Meta Invalid Kind = iota Comment Key - // top level structures + // Top level structures Table ArrayTable KeyValue - // containers values + // Containers values Array InlineTable - // values + // Values String Bool Float @@ -30,6 +31,7 @@ const ( DateTime ) +// String implementation of fmt.Stringer. func (k Kind) String() string { switch k { case Invalid: diff --git a/parser.go b/unstable/parser.go similarity index 70% rename from parser.go rename to unstable/parser.go index 9859a795..52db88e7 100644 --- a/parser.go +++ b/unstable/parser.go @@ -1,50 +1,108 @@ -package toml +package unstable import ( "bytes" + "fmt" "unicode" - "github.com/pelletier/go-toml/v2/internal/ast" + "github.com/pelletier/go-toml/v2/internal/characters" "github.com/pelletier/go-toml/v2/internal/danger" ) -type parser struct { - builder ast.Builder - ref ast.Reference +// ParserError describes an error relative to the content of the document. +// +// It cannot outlive the instance of Parser it refers to, and may cause panics +// if the parser is reset. +type ParserError struct { + Highlight []byte + Message string + Key []string // optional +} + +// Error is the implementation of the error interface. +func (e *ParserError) Error() string { + return e.Message +} + +// NewParserError is a convenience function to create a ParserError +// +// Warning: Highlight needs to be a subslice of Parser.data, so only slices +// returned by Parser.Raw are valid candidates. +func NewParserError(highlight []byte, format string, args ...interface{}) error { + return &ParserError{ + Highlight: highlight, + Message: fmt.Errorf(format, args...).Error(), + } +} + +// Parser scans over a TOML-encoded document and generates an iterative AST. +// +// To prime the Parser, first reset it with the contents of a TOML document. +// Then, process all top-level expressions sequentially. See Example. +// +// Don't forget to check Error() after you're done parsing. +// +// Each top-level expression needs to be fully processed before calling +// NextExpression() again. Otherwise, calls to various Node methods may panic if +// the parser has moved on the next expression. +// +// For performance reasons, go-toml doesn't make a copy of the input bytes to +// the parser. Make sure to copy all the bytes you need to outlive the slice +// given to the parser. +// +// The parser doesn't provide nodes for comments yet, nor for whitespace. +type Parser struct { data []byte + builder builder + ref reference left []byte err error first bool } -func (p *parser) Range(b []byte) ast.Range { - return ast.Range{ +// Data returns the slice provided to the last call to Reset. +func (p *Parser) Data() []byte { + return p.data +} + +// Range returns a range description that corresponds to a given slice of the +// input. If the argument is not a subslice of the parser input, this function +// panics. +func (p *Parser) Range(b []byte) Range { + return Range{ Offset: uint32(danger.SubsliceOffset(p.data, b)), Length: uint32(len(b)), } } -func (p *parser) Raw(raw ast.Range) []byte { +// Raw returns the slice corresponding to the bytes in the given range. +func (p *Parser) Raw(raw Range) []byte { return p.data[raw.Offset : raw.Offset+raw.Length] } -func (p *parser) Reset(b []byte) { +// Reset brings the parser to its initial state for a given input. It wipes an +// reuses internal storage to reduce allocation. +func (p *Parser) Reset(b []byte) { p.builder.Reset() - p.ref = ast.InvalidReference + p.ref = invalidReference p.data = b p.left = b p.err = nil p.first = true } -//nolint:cyclop -func (p *parser) NextExpression() bool { +// NextExpression parses the next top-level expression. If an expression was +// successfully parsed, it returns true. If the parser is at the end of the +// document or an error occurred, it returns false. +// +// Retrieve the parsed expression with Expression(). +func (p *Parser) NextExpression() bool { if len(p.left) == 0 || p.err != nil { return false } p.builder.Reset() - p.ref = ast.InvalidReference + p.ref = invalidReference for { if len(p.left) == 0 || p.err != nil { @@ -73,15 +131,18 @@ func (p *parser) NextExpression() bool { } } -func (p *parser) Expression() *ast.Node { +// Expression returns a pointer to the node representing the last successfully +// parsed expresion. +func (p *Parser) Expression() *Node { return p.builder.NodeAt(p.ref) } -func (p *parser) Error() error { +// Error returns any error that has occured during parsing. +func (p *Parser) Error() error { return p.err } -func (p *parser) parseNewline(b []byte) ([]byte, error) { +func (p *Parser) parseNewline(b []byte) ([]byte, error) { if b[0] == '\n' { return b[1:], nil } @@ -91,14 +152,14 @@ func (p *parser) parseNewline(b []byte) ([]byte, error) { return rest, err } - return nil, newDecodeError(b[0:1], "expected newline but got %#U", b[0]) + return nil, NewParserError(b[0:1], "expected newline but got %#U", b[0]) } -func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseExpression(b []byte) (reference, []byte, error) { // expression = ws [ comment ] // expression =/ ws keyval ws [ comment ] // expression =/ ws table ws [ comment ] - ref := ast.InvalidReference + ref := invalidReference b = p.parseWhitespace(b) @@ -136,7 +197,7 @@ func (p *parser) parseExpression(b []byte) (ast.Reference, []byte, error) { return ref, b, nil } -func (p *parser) parseTable(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseTable(b []byte) (reference, []byte, error) { // table = std-table / array-table if len(b) > 1 && b[1] == '[' { return p.parseArrayTable(b) @@ -145,12 +206,12 @@ func (p *parser) parseTable(b []byte) (ast.Reference, []byte, error) { return p.parseStdTable(b) } -func (p *parser) parseArrayTable(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseArrayTable(b []byte) (reference, []byte, error) { // array-table = array-table-open key array-table-close // array-table-open = %x5B.5B ws ; [[ Double left square bracket // array-table-close = ws %x5D.5D ; ]] Double right square bracket - ref := p.builder.Push(ast.Node{ - Kind: ast.ArrayTable, + ref := p.builder.Push(Node{ + Kind: ArrayTable, }) b = b[2:] @@ -174,12 +235,12 @@ func (p *parser) parseArrayTable(b []byte) (ast.Reference, []byte, error) { return ref, b, err } -func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseStdTable(b []byte) (reference, []byte, error) { // std-table = std-table-open key std-table-close // std-table-open = %x5B ws ; [ Left square bracket // std-table-close = ws %x5D ; ] Right square bracket - ref := p.builder.Push(ast.Node{ - Kind: ast.Table, + ref := p.builder.Push(Node{ + Kind: Table, }) b = b[1:] @@ -199,15 +260,15 @@ func (p *parser) parseStdTable(b []byte) (ast.Reference, []byte, error) { return ref, b, err } -func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseKeyval(b []byte) (reference, []byte, error) { // keyval = key keyval-sep val - ref := p.builder.Push(ast.Node{ - Kind: ast.KeyValue, + ref := p.builder.Push(Node{ + Kind: KeyValue, }) key, b, err := p.parseKey(b) if err != nil { - return ast.InvalidReference, nil, err + return invalidReference, nil, err } // keyval-sep = ws %x3D ws ; = @@ -215,12 +276,12 @@ func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) { b = p.parseWhitespace(b) if len(b) == 0 { - return ast.InvalidReference, nil, newDecodeError(b, "expected = after a key, but the document ends there") + return invalidReference, nil, NewParserError(b, "expected = after a key, but the document ends there") } b, err = expect('=', b) if err != nil { - return ast.InvalidReference, nil, err + return invalidReference, nil, err } b = p.parseWhitespace(b) @@ -237,12 +298,12 @@ func (p *parser) parseKeyval(b []byte) (ast.Reference, []byte, error) { } //nolint:cyclop,funlen -func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseVal(b []byte) (reference, []byte, error) { // val = string / boolean / array / inline-table / date-time / float / integer - ref := ast.InvalidReference + ref := invalidReference if len(b) == 0 { - return ref, nil, newDecodeError(b, "expected value, not eof") + return ref, nil, NewParserError(b, "expected value, not eof") } var err error @@ -259,8 +320,8 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) { } if err == nil { - ref = p.builder.Push(ast.Node{ - Kind: ast.String, + ref = p.builder.Push(Node{ + Kind: String, Raw: p.Range(raw), Data: v, }) @@ -277,8 +338,8 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) { } if err == nil { - ref = p.builder.Push(ast.Node{ - Kind: ast.String, + ref = p.builder.Push(Node{ + Kind: String, Raw: p.Range(raw), Data: v, }) @@ -287,22 +348,22 @@ func (p *parser) parseVal(b []byte) (ast.Reference, []byte, error) { return ref, b, err case 't': if !scanFollowsTrue(b) { - return ref, nil, newDecodeError(atmost(b, 4), "expected 'true'") + return ref, nil, NewParserError(atmost(b, 4), "expected 'true'") } - ref = p.builder.Push(ast.Node{ - Kind: ast.Bool, + ref = p.builder.Push(Node{ + Kind: Bool, Data: b[:4], }) return ref, b[4:], nil case 'f': if !scanFollowsFalse(b) { - return ref, nil, newDecodeError(atmost(b, 5), "expected 'false'") + return ref, nil, NewParserError(atmost(b, 5), "expected 'false'") } - ref = p.builder.Push(ast.Node{ - Kind: ast.Bool, + ref = p.builder.Push(Node{ + Kind: Bool, Data: b[:5], }) @@ -324,7 +385,7 @@ func atmost(b []byte, n int) []byte { return b[:n] } -func (p *parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) { +func (p *Parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) { v, rest, err := scanLiteralString(b) if err != nil { return nil, nil, nil, err @@ -333,19 +394,19 @@ func (p *parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) { return v, v[1 : len(v)-1], rest, nil } -func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseInlineTable(b []byte) (reference, []byte, error) { // inline-table = inline-table-open [ inline-table-keyvals ] inline-table-close // inline-table-open = %x7B ws ; { // inline-table-close = ws %x7D ; } // inline-table-sep = ws %x2C ws ; , Comma // inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ] - parent := p.builder.Push(ast.Node{ - Kind: ast.InlineTable, + parent := p.builder.Push(Node{ + Kind: InlineTable, }) first := true - var child ast.Reference + var child reference b = b[1:] @@ -356,7 +417,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) { b = p.parseWhitespace(b) if len(b) == 0 { - return parent, nil, newDecodeError(previousB[:1], "inline table is incomplete") + return parent, nil, NewParserError(previousB[:1], "inline table is incomplete") } if b[0] == '}' { @@ -371,7 +432,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) { b = p.parseWhitespace(b) } - var kv ast.Reference + var kv reference kv, b, err = p.parseKeyval(b) if err != nil { @@ -394,7 +455,7 @@ func (p *parser) parseInlineTable(b []byte) (ast.Reference, []byte, error) { } //nolint:funlen,cyclop -func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseValArray(b []byte) (reference, []byte, error) { // array = array-open [ array-values ] ws-comment-newline array-close // array-open = %x5B ; [ // array-close = %x5D ; ] @@ -405,13 +466,13 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { arrayStart := b b = b[1:] - parent := p.builder.Push(ast.Node{ - Kind: ast.Array, + parent := p.builder.Push(Node{ + Kind: Array, }) first := true - var lastChild ast.Reference + var lastChild reference var err error for len(b) > 0 { @@ -421,7 +482,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { } if len(b) == 0 { - return parent, nil, newDecodeError(arrayStart[:1], "array is incomplete") + return parent, nil, NewParserError(arrayStart[:1], "array is incomplete") } if b[0] == ']' { @@ -430,7 +491,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { if b[0] == ',' { if first { - return parent, nil, newDecodeError(b[0:1], "array cannot start with comma") + return parent, nil, NewParserError(b[0:1], "array cannot start with comma") } b = b[1:] @@ -439,7 +500,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { return parent, nil, err } } else if !first { - return parent, nil, newDecodeError(b[0:1], "array elements must be separated by commas") + return parent, nil, NewParserError(b[0:1], "array elements must be separated by commas") } // TOML allows trailing commas in arrays. @@ -447,7 +508,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { break } - var valueRef ast.Reference + var valueRef reference valueRef, b, err = p.parseVal(b) if err != nil { return parent, nil, err @@ -472,7 +533,7 @@ func (p *parser) parseValArray(b []byte) (ast.Reference, []byte, error) { return parent, rest, err } -func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error) { +func (p *Parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error) { for len(b) > 0 { var err error b = p.parseWhitespace(b) @@ -501,7 +562,7 @@ func (p *parser) parseOptionalWhitespaceCommentNewline(b []byte) ([]byte, error) return b, nil } -func (p *parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) { +func (p *Parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) { token, rest, err := scanMultilineLiteralString(b) if err != nil { return nil, nil, nil, err @@ -520,7 +581,7 @@ func (p *parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, } //nolint:funlen,gocognit,cyclop -func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, error) { +func (p *Parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, error) { // ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body // ml-basic-string-delim // ml-basic-string-delim = 3quotation-mark @@ -551,11 +612,11 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er if !escaped { str := token[startIdx:endIdx] - verr := utf8TomlValidAlreadyEscaped(str) + verr := characters.Utf8TomlValidAlreadyEscaped(str) if verr.Zero() { return token, str, rest, nil } - return nil, nil, nil, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8") + return nil, nil, nil, NewParserError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8") } var builder bytes.Buffer @@ -635,13 +696,13 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er builder.WriteRune(x) i += 8 default: - return nil, nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c) + return nil, nil, nil, NewParserError(token[i:i+1], "invalid escaped character %#U", c) } i++ } else { - size := utf8ValidNext(token[i:]) + size := characters.Utf8ValidNext(token[i:]) if size == 0 { - return nil, nil, nil, newDecodeError(token[i:i+1], "invalid character %#U", c) + return nil, nil, nil, NewParserError(token[i:i+1], "invalid character %#U", c) } builder.Write(token[i : i+size]) i += size @@ -651,7 +712,7 @@ func (p *parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, er return token, builder.Bytes(), rest, nil } -func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseKey(b []byte) (reference, []byte, error) { // key = simple-key / dotted-key // simple-key = quoted-key / unquoted-key // @@ -662,11 +723,11 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) { // dot-sep = ws %x2E ws ; . Period raw, key, b, err := p.parseSimpleKey(b) if err != nil { - return ast.InvalidReference, nil, err + return invalidReference, nil, err } - ref := p.builder.Push(ast.Node{ - Kind: ast.Key, + ref := p.builder.Push(Node{ + Kind: Key, Raw: p.Range(raw), Data: key, }) @@ -681,8 +742,8 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) { return ref, nil, err } - p.builder.PushAndChain(ast.Node{ - Kind: ast.Key, + p.builder.PushAndChain(Node{ + Kind: Key, Raw: p.Range(raw), Data: key, }) @@ -694,9 +755,9 @@ func (p *parser) parseKey(b []byte) (ast.Reference, []byte, error) { return ref, b, nil } -func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) { +func (p *Parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) { if len(b) == 0 { - return nil, nil, nil, newDecodeError(b, "expected key but found none") + return nil, nil, nil, NewParserError(b, "expected key but found none") } // simple-key = quoted-key / unquoted-key @@ -711,12 +772,12 @@ func (p *parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) { key, rest = scanUnquotedKey(b) return key, key, rest, nil default: - return nil, nil, nil, newDecodeError(b[0:1], "invalid character at start of key: %c", b[0]) + return nil, nil, nil, NewParserError(b[0:1], "invalid character at start of key: %c", b[0]) } } //nolint:funlen,cyclop -func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) { +func (p *Parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) { // basic-string = quotation-mark *basic-char quotation-mark // quotation-mark = %x22 ; " // basic-char = basic-unescaped / escaped @@ -744,11 +805,11 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) { // validate the string and return a direct reference to the buffer. if !escaped { str := token[startIdx:endIdx] - verr := utf8TomlValidAlreadyEscaped(str) + verr := characters.Utf8TomlValidAlreadyEscaped(str) if verr.Zero() { return token, str, rest, nil } - return nil, nil, nil, newDecodeError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8") + return nil, nil, nil, NewParserError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8") } i := startIdx @@ -795,13 +856,13 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) { builder.WriteRune(x) i += 8 default: - return nil, nil, nil, newDecodeError(token[i:i+1], "invalid escaped character %#U", c) + return nil, nil, nil, NewParserError(token[i:i+1], "invalid escaped character %#U", c) } i++ } else { - size := utf8ValidNext(token[i:]) + size := characters.Utf8ValidNext(token[i:]) if size == 0 { - return nil, nil, nil, newDecodeError(token[i:i+1], "invalid character %#U", c) + return nil, nil, nil, NewParserError(token[i:i+1], "invalid character %#U", c) } builder.Write(token[i : i+size]) i += size @@ -813,7 +874,7 @@ func (p *parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) { func hexToRune(b []byte, length int) (rune, error) { if len(b) < length { - return -1, newDecodeError(b, "unicode point needs %d character, not %d", length, len(b)) + return -1, NewParserError(b, "unicode point needs %d character, not %d", length, len(b)) } b = b[:length] @@ -828,19 +889,19 @@ func hexToRune(b []byte, length int) (rune, error) { case 'A' <= c && c <= 'F': d = uint32(c - 'A' + 10) default: - return -1, newDecodeError(b[i:i+1], "non-hex character") + return -1, NewParserError(b[i:i+1], "non-hex character") } r = r*16 + d } if r > unicode.MaxRune || 0xD800 <= r && r < 0xE000 { - return -1, newDecodeError(b, "escape sequence is invalid Unicode code point") + return -1, NewParserError(b, "escape sequence is invalid Unicode code point") } return rune(r), nil } -func (p *parser) parseWhitespace(b []byte) []byte { +func (p *Parser) parseWhitespace(b []byte) []byte { // ws = *wschar // wschar = %x20 ; Space // wschar =/ %x09 ; Horizontal tab @@ -850,24 +911,24 @@ func (p *parser) parseWhitespace(b []byte) []byte { } //nolint:cyclop -func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) parseIntOrFloatOrDateTime(b []byte) (reference, []byte, error) { switch b[0] { case 'i': if !scanFollowsInf(b) { - return ast.InvalidReference, nil, newDecodeError(atmost(b, 3), "expected 'inf'") + return invalidReference, nil, NewParserError(atmost(b, 3), "expected 'inf'") } - return p.builder.Push(ast.Node{ - Kind: ast.Float, + return p.builder.Push(Node{ + Kind: Float, Data: b[:3], }), b[3:], nil case 'n': if !scanFollowsNan(b) { - return ast.InvalidReference, nil, newDecodeError(atmost(b, 3), "expected 'nan'") + return invalidReference, nil, NewParserError(atmost(b, 3), "expected 'nan'") } - return p.builder.Push(ast.Node{ - Kind: ast.Float, + return p.builder.Push(Node{ + Kind: Float, Data: b[:3], }), b[3:], nil case '+', '-': @@ -898,7 +959,7 @@ func (p *parser) parseIntOrFloatOrDateTime(b []byte) (ast.Reference, []byte, err return p.scanIntOrFloat(b) } -func (p *parser) scanDateTime(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) scanDateTime(b []byte) (reference, []byte, error) { // scans for contiguous characters in [0-9T:Z.+-], and up to one space if // followed by a digit. hasDate := false @@ -941,30 +1002,30 @@ byteLoop: } } - var kind ast.Kind + var kind Kind if hasTime { if hasDate { if hasTz { - kind = ast.DateTime + kind = DateTime } else { - kind = ast.LocalDateTime + kind = LocalDateTime } } else { - kind = ast.LocalTime + kind = LocalTime } } else { - kind = ast.LocalDate + kind = LocalDate } - return p.builder.Push(ast.Node{ + return p.builder.Push(Node{ Kind: kind, Data: b[:i], }), b[i:], nil } //nolint:funlen,gocognit,cyclop -func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) { +func (p *Parser) scanIntOrFloat(b []byte) (reference, []byte, error) { i := 0 if len(b) > 2 && b[0] == '0' && b[1] != '.' && b[1] != 'e' && b[1] != 'E' { @@ -990,8 +1051,8 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) { } } - return p.builder.Push(ast.Node{ - Kind: ast.Integer, + return p.builder.Push(Node{ + Kind: Integer, Data: b[:i], }), b[i:], nil } @@ -1013,40 +1074,40 @@ func (p *parser) scanIntOrFloat(b []byte) (ast.Reference, []byte, error) { if c == 'i' { if scanFollowsInf(b[i:]) { - return p.builder.Push(ast.Node{ - Kind: ast.Float, + return p.builder.Push(Node{ + Kind: Float, Data: b[:i+3], }), b[i+3:], nil } - return ast.InvalidReference, nil, newDecodeError(b[i:i+1], "unexpected character 'i' while scanning for a number") + return invalidReference, nil, NewParserError(b[i:i+1], "unexpected character 'i' while scanning for a number") } if c == 'n' { if scanFollowsNan(b[i:]) { - return p.builder.Push(ast.Node{ - Kind: ast.Float, + return p.builder.Push(Node{ + Kind: Float, Data: b[:i+3], }), b[i+3:], nil } - return ast.InvalidReference, nil, newDecodeError(b[i:i+1], "unexpected character 'n' while scanning for a number") + return invalidReference, nil, NewParserError(b[i:i+1], "unexpected character 'n' while scanning for a number") } break } if i == 0 { - return ast.InvalidReference, b, newDecodeError(b, "incomplete number") + return invalidReference, b, NewParserError(b, "incomplete number") } - kind := ast.Integer + kind := Integer if isFloat { - kind = ast.Float + kind = Float } - return p.builder.Push(ast.Node{ + return p.builder.Push(Node{ Kind: kind, Data: b[:i], }), b[i:], nil @@ -1075,11 +1136,11 @@ func isValidBinaryRune(r byte) bool { func expect(x byte, b []byte) ([]byte, error) { if len(b) == 0 { - return nil, newDecodeError(b, "expected character %c but the document ended here", x) + return nil, NewParserError(b, "expected character %c but the document ended here", x) } if b[0] != x { - return nil, newDecodeError(b[0:1], "expected character %c", x) + return nil, NewParserError(b[0:1], "expected character %c", x) } return b[1:], nil diff --git a/parser_test.go b/unstable/parser_test.go similarity index 75% rename from parser_test.go rename to unstable/parser_test.go index 541c2f49..5e201cda 100644 --- a/parser_test.go +++ b/unstable/parser_test.go @@ -1,143 +1,142 @@ -package toml +package unstable import ( + "fmt" "strconv" "strings" "testing" - "github.com/pelletier/go-toml/v2/internal/ast" "github.com/stretchr/testify/require" ) -//nolint:funlen func TestParser_AST_Numbers(t *testing.T) { examples := []struct { desc string input string - kind ast.Kind + kind Kind err bool }{ { desc: "integer just digits", input: `1234`, - kind: ast.Integer, + kind: Integer, }, { desc: "integer zero", input: `0`, - kind: ast.Integer, + kind: Integer, }, { desc: "integer sign", input: `+99`, - kind: ast.Integer, + kind: Integer, }, { desc: "integer hex uppercase", input: `0xDEADBEEF`, - kind: ast.Integer, + kind: Integer, }, { desc: "integer hex lowercase", input: `0xdead_beef`, - kind: ast.Integer, + kind: Integer, }, { desc: "integer octal", input: `0o01234567`, - kind: ast.Integer, + kind: Integer, }, { desc: "integer binary", input: `0b11010110`, - kind: ast.Integer, + kind: Integer, }, { desc: "float zero", input: `0.0`, - kind: ast.Float, + kind: Float, }, { desc: "float positive zero", input: `+0.0`, - kind: ast.Float, + kind: Float, }, { desc: "float negative zero", input: `-0.0`, - kind: ast.Float, + kind: Float, }, { desc: "float pi", input: `3.1415`, - kind: ast.Float, + kind: Float, }, { desc: "float negative", input: `-0.01`, - kind: ast.Float, + kind: Float, }, { desc: "float signed exponent", input: `5e+22`, - kind: ast.Float, + kind: Float, }, { desc: "float exponent lowercase", input: `1e06`, - kind: ast.Float, + kind: Float, }, { desc: "float exponent uppercase", input: `-2E-2`, - kind: ast.Float, + kind: Float, }, { desc: "float fractional with exponent", input: `6.626e-34`, - kind: ast.Float, + kind: Float, }, { desc: "float underscores", input: `224_617.445_991_228`, - kind: ast.Float, + kind: Float, }, { desc: "inf", input: `inf`, - kind: ast.Float, + kind: Float, }, { desc: "inf negative", input: `-inf`, - kind: ast.Float, + kind: Float, }, { desc: "inf positive", input: `+inf`, - kind: ast.Float, + kind: Float, }, { desc: "nan", input: `nan`, - kind: ast.Float, + kind: Float, }, { desc: "nan negative", input: `-nan`, - kind: ast.Float, + kind: Float, }, { desc: "nan positive", input: `+nan`, - kind: ast.Float, + kind: Float, }, } for _, e := range examples { e := e t.Run(e.desc, func(t *testing.T) { - p := parser{} + p := Parser{} p.Reset([]byte(`A = ` + e.input)) p.NextExpression() err := p.Error() @@ -147,10 +146,10 @@ func TestParser_AST_Numbers(t *testing.T) { require.NoError(t, err) expected := astNode{ - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ {Kind: e.kind, Data: []byte(e.input)}, - {Kind: ast.Key, Data: []byte(`A`)}, + {Kind: Key, Data: []byte(`A`)}, }, } compareNode(t, expected, p.Expression()) @@ -161,13 +160,13 @@ func TestParser_AST_Numbers(t *testing.T) { type ( astNode struct { - Kind ast.Kind + Kind Kind Data []byte Children []astNode } ) -func compareNode(t *testing.T, e astNode, n *ast.Node) { +func compareNode(t *testing.T, e astNode, n *Node) { t.Helper() require.Equal(t, e.Kind, n.Kind) require.Equal(t, e.Data, n.Data) @@ -175,7 +174,7 @@ func compareNode(t *testing.T, e astNode, n *ast.Node) { compareIterator(t, e.Children, n.Children()) } -func compareIterator(t *testing.T, expected []astNode, actual ast.Iterator) { +func compareIterator(t *testing.T, expected []astNode, actual Iterator) { t.Helper() idx := 0 @@ -209,14 +208,14 @@ func TestParser_AST(t *testing.T) { desc: "simple string assignment", input: `A = "hello"`, ast: astNode{ - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ { - Kind: ast.String, + Kind: String, Data: []byte(`hello`), }, { - Kind: ast.Key, + Kind: Key, Data: []byte(`A`), }, }, @@ -226,14 +225,14 @@ func TestParser_AST(t *testing.T) { desc: "simple bool assignment", input: `A = true`, ast: astNode{ - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ { - Kind: ast.Bool, + Kind: Bool, Data: []byte(`true`), }, { - Kind: ast.Key, + Kind: Key, Data: []byte(`A`), }, }, @@ -243,24 +242,24 @@ func TestParser_AST(t *testing.T) { desc: "array of strings", input: `A = ["hello", ["world", "again"]]`, ast: astNode{ - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ { - Kind: ast.Array, + Kind: Array, Children: []astNode{ { - Kind: ast.String, + Kind: String, Data: []byte(`hello`), }, { - Kind: ast.Array, + Kind: Array, Children: []astNode{ { - Kind: ast.String, + Kind: String, Data: []byte(`world`), }, { - Kind: ast.String, + Kind: String, Data: []byte(`again`), }, }, @@ -268,7 +267,7 @@ func TestParser_AST(t *testing.T) { }, }, { - Kind: ast.Key, + Kind: Key, Data: []byte(`A`), }, }, @@ -278,23 +277,23 @@ func TestParser_AST(t *testing.T) { desc: "array of arrays of strings", input: `A = ["hello", "world"]`, ast: astNode{ - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ { - Kind: ast.Array, + Kind: Array, Children: []astNode{ { - Kind: ast.String, + Kind: String, Data: []byte(`hello`), }, { - Kind: ast.String, + Kind: String, Data: []byte(`world`), }, }, }, { - Kind: ast.Key, + Kind: Key, Data: []byte(`A`), }, }, @@ -304,29 +303,29 @@ func TestParser_AST(t *testing.T) { desc: "inline table", input: `name = { first = "Tom", last = "Preston-Werner" }`, ast: astNode{ - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ { - Kind: ast.InlineTable, + Kind: InlineTable, Children: []astNode{ { - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ - {Kind: ast.String, Data: []byte(`Tom`)}, - {Kind: ast.Key, Data: []byte(`first`)}, + {Kind: String, Data: []byte(`Tom`)}, + {Kind: Key, Data: []byte(`first`)}, }, }, { - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ - {Kind: ast.String, Data: []byte(`Preston-Werner`)}, - {Kind: ast.Key, Data: []byte(`last`)}, + {Kind: String, Data: []byte(`Preston-Werner`)}, + {Kind: Key, Data: []byte(`last`)}, }, }, }, }, { - Kind: ast.Key, + Kind: Key, Data: []byte(`name`), }, }, @@ -337,7 +336,7 @@ func TestParser_AST(t *testing.T) { for _, e := range examples { e := e t.Run(e.desc, func(t *testing.T) { - p := parser{} + p := Parser{} p.Reset([]byte(e.input)) p.NextExpression() err := p.Error() @@ -352,7 +351,7 @@ func TestParser_AST(t *testing.T) { } func BenchmarkParseBasicStringWithUnicode(b *testing.B) { - p := &parser{} + p := &Parser{} b.Run("4", func(b *testing.B) { input := []byte(`"\u1234\u5678\u9ABC\u1234\u5678\u9ABC"`) b.ReportAllocs() @@ -374,7 +373,7 @@ func BenchmarkParseBasicStringWithUnicode(b *testing.B) { } func BenchmarkParseBasicStringsEasy(b *testing.B) { - p := &parser{} + p := &Parser{} for _, size := range []int{1, 4, 8, 16, 21} { b.Run(strconv.Itoa(size), func(b *testing.B) { @@ -394,40 +393,40 @@ func TestParser_AST_DateTimes(t *testing.T) { examples := []struct { desc string input string - kind ast.Kind + kind Kind err bool }{ { desc: "offset-date-time with delim 'T' and UTC offset", input: `2021-07-21T12:08:05Z`, - kind: ast.DateTime, + kind: DateTime, }, { desc: "offset-date-time with space delim and +8hours offset", input: `2021-07-21 12:08:05+08:00`, - kind: ast.DateTime, + kind: DateTime, }, { desc: "local-date-time with nano second", input: `2021-07-21T12:08:05.666666666`, - kind: ast.LocalDateTime, + kind: LocalDateTime, }, { desc: "local-date-time", input: `2021-07-21T12:08:05`, - kind: ast.LocalDateTime, + kind: LocalDateTime, }, { desc: "local-date", input: `2021-07-21`, - kind: ast.LocalDate, + kind: LocalDate, }, } for _, e := range examples { e := e t.Run(e.desc, func(t *testing.T) { - p := parser{} + p := Parser{} p.Reset([]byte(`A = ` + e.input)) p.NextExpression() err := p.Error() @@ -437,10 +436,10 @@ func TestParser_AST_DateTimes(t *testing.T) { require.NoError(t, err) expected := astNode{ - Kind: ast.KeyValue, + Kind: KeyValue, Children: []astNode{ {Kind: e.kind, Data: []byte(e.input)}, - {Kind: ast.Key, Data: []byte(`A`)}, + {Kind: Key, Data: []byte(`A`)}, }, } compareNode(t, expected, p.Expression()) @@ -448,3 +447,26 @@ func TestParser_AST_DateTimes(t *testing.T) { }) } } + +func ExampleParser() { + doc := ` + hello = "world" + value = 42 + ` + p := Parser{} + p.Reset([]byte(doc)) + for p.NextExpression() { + e := p.Expression() + fmt.Printf("Expression: %s\n", e.Kind) + value := e.Value() + it := e.Key() + k := it.Node() // shortcut: we know there is no dotted key in the example + fmt.Printf("%s -> (%s) %s\n", k.Data, value.Kind, value.Data) + } + + // Output: + // Expression: KeyValue + // hello -> (String) world + // Expression: KeyValue + // value -> (Integer) 42 +} diff --git a/scanner.go b/unstable/scanner.go similarity index 79% rename from scanner.go rename to unstable/scanner.go index bb445fab..af22ebbe 100644 --- a/scanner.go +++ b/unstable/scanner.go @@ -1,4 +1,6 @@ -package toml +package unstable + +import "github.com/pelletier/go-toml/v2/internal/characters" func scanFollows(b []byte, pattern string) bool { n := len(pattern) @@ -54,16 +56,16 @@ func scanLiteralString(b []byte) ([]byte, []byte, error) { case '\'': return b[:i+1], b[i+1:], nil case '\n', '\r': - return nil, nil, newDecodeError(b[i:i+1], "literal strings cannot have new lines") + return nil, nil, NewParserError(b[i:i+1], "literal strings cannot have new lines") } - size := utf8ValidNext(b[i:]) + size := characters.Utf8ValidNext(b[i:]) if size == 0 { - return nil, nil, newDecodeError(b[i:i+1], "invalid character") + return nil, nil, NewParserError(b[i:i+1], "invalid character") } i += size } - return nil, nil, newDecodeError(b[len(b):], "unterminated literal string") + return nil, nil, NewParserError(b[len(b):], "unterminated literal string") } func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) { @@ -98,39 +100,39 @@ func scanMultilineLiteralString(b []byte) ([]byte, []byte, error) { i++ if i < len(b) && b[i] == '\'' { - return nil, nil, newDecodeError(b[i-3:i+1], "''' not allowed in multiline literal string") + return nil, nil, NewParserError(b[i-3:i+1], "''' not allowed in multiline literal string") } return b[:i], b[i:], nil } case '\r': if len(b) < i+2 { - return nil, nil, newDecodeError(b[len(b):], `need a \n after \r`) + return nil, nil, NewParserError(b[len(b):], `need a \n after \r`) } if b[i+1] != '\n' { - return nil, nil, newDecodeError(b[i:i+2], `need a \n after \r`) + return nil, nil, NewParserError(b[i:i+2], `need a \n after \r`) } i += 2 // skip the \n continue } - size := utf8ValidNext(b[i:]) + size := characters.Utf8ValidNext(b[i:]) if size == 0 { - return nil, nil, newDecodeError(b[i:i+1], "invalid character") + return nil, nil, NewParserError(b[i:i+1], "invalid character") } i += size } - return nil, nil, newDecodeError(b[len(b):], `multiline literal string not terminated by '''`) + return nil, nil, NewParserError(b[len(b):], `multiline literal string not terminated by '''`) } func scanWindowsNewline(b []byte) ([]byte, []byte, error) { const lenCRLF = 2 if len(b) < lenCRLF { - return nil, nil, newDecodeError(b, "windows new line expected") + return nil, nil, NewParserError(b, "windows new line expected") } if b[1] != '\n' { - return nil, nil, newDecodeError(b, `windows new line should be \r\n`) + return nil, nil, NewParserError(b, `windows new line should be \r\n`) } return b[:lenCRLF], b[lenCRLF:], nil @@ -165,11 +167,11 @@ func scanComment(b []byte) ([]byte, []byte, error) { if i+1 < len(b) && b[i+1] == '\n' { return b[:i+1], b[i+1:], nil } - return nil, nil, newDecodeError(b[i:i+1], "invalid character in comment") + return nil, nil, NewParserError(b[i:i+1], "invalid character in comment") } - size := utf8ValidNext(b[i:]) + size := characters.Utf8ValidNext(b[i:]) if size == 0 { - return nil, nil, newDecodeError(b[i:i+1], "invalid character in comment") + return nil, nil, NewParserError(b[i:i+1], "invalid character in comment") } i += size @@ -192,17 +194,17 @@ func scanBasicString(b []byte) ([]byte, bool, []byte, error) { case '"': return b[:i+1], escaped, b[i+1:], nil case '\n', '\r': - return nil, escaped, nil, newDecodeError(b[i:i+1], "basic strings cannot have new lines") + return nil, escaped, nil, NewParserError(b[i:i+1], "basic strings cannot have new lines") case '\\': if len(b) < i+2 { - return nil, escaped, nil, newDecodeError(b[i:i+1], "need a character after \\") + return nil, escaped, nil, NewParserError(b[i:i+1], "need a character after \\") } escaped = true i++ // skip the next character } } - return nil, escaped, nil, newDecodeError(b[len(b):], `basic string not terminated by "`) + return nil, escaped, nil, NewParserError(b[len(b):], `basic string not terminated by "`) } func scanMultilineBasicString(b []byte) ([]byte, bool, []byte, error) { @@ -243,27 +245,27 @@ func scanMultilineBasicString(b []byte) ([]byte, bool, []byte, error) { i++ if i < len(b) && b[i] == '"' { - return nil, escaped, nil, newDecodeError(b[i-3:i+1], `""" not allowed in multiline basic string`) + return nil, escaped, nil, NewParserError(b[i-3:i+1], `""" not allowed in multiline basic string`) } return b[:i], escaped, b[i:], nil } case '\\': if len(b) < i+2 { - return nil, escaped, nil, newDecodeError(b[len(b):], "need a character after \\") + return nil, escaped, nil, NewParserError(b[len(b):], "need a character after \\") } escaped = true i++ // skip the next character case '\r': if len(b) < i+2 { - return nil, escaped, nil, newDecodeError(b[len(b):], `need a \n after \r`) + return nil, escaped, nil, NewParserError(b[len(b):], `need a \n after \r`) } if b[i+1] != '\n' { - return nil, escaped, nil, newDecodeError(b[i:i+2], `need a \n after \r`) + return nil, escaped, nil, NewParserError(b[i:i+2], `need a \n after \r`) } i++ // skip the \n } } - return nil, escaped, nil, newDecodeError(b[len(b):], `multiline basic string not terminated by """`) + return nil, escaped, nil, NewParserError(b[len(b):], `multiline basic string not terminated by """`) }