Use newer AppendEncode and AppenDecode methods

arshal_default.go

@@ -5,7 +5,6 @@
 package json
 
 import (
-	"bytes"
 	"encoding/base32"
 	"encoding/base64"
 	"encoding/hex"
@@ -215,47 +214,35 @@ func makeStringArshaler(t reflect.Type) *arshaler {
 	return &fncs
 }
 
+// Use strict adherence to RFC 4648.
+// It is easier to loosen these restrictions later on than the reverse.
+// In particular, we reject the presence of '\r or '\n' characters
+// and reject padding that contain non-zero bits.
 var (
-	encodeBase16        = func(dst, src []byte) { hex.Encode(dst, src) }
-	encodeBase32        = base32.StdEncoding.Encode
-	encodeBase32Hex     = base32.HexEncoding.Encode
-	encodeBase64        = base64.StdEncoding.Encode
-	encodeBase64URL     = base64.URLEncoding.Encode
-	encodedLenBase16    = hex.EncodedLen
-	encodedLenBase32    = base32.StdEncoding.EncodedLen
-	encodedLenBase32Hex = base32.HexEncoding.EncodedLen
-	encodedLenBase64    = base64.StdEncoding.EncodedLen
-	encodedLenBase64URL = base64.URLEncoding.EncodedLen
-	decodeBase16        = hex.Decode
-	decodeBase32        = base32.StdEncoding.Decode
-	decodeBase32Hex     = base32.HexEncoding.Decode
-	decodeBase64        = base64.StdEncoding.Decode
-	decodeBase64URL     = base64.URLEncoding.Decode
-	decodedLenBase16    = hex.DecodedLen
-	decodedLenBase32    = base32.StdEncoding.WithPadding(base32.NoPadding).DecodedLen
-	decodedLenBase32Hex = base32.HexEncoding.WithPadding(base32.NoPadding).DecodedLen
-	decodedLenBase64    = base64.StdEncoding.WithPadding(base64.NoPadding).DecodedLen
-	decodedLenBase64URL = base64.URLEncoding.WithPadding(base64.NoPadding).DecodedLen
+	base64StdEncoding = base64.StdEncoding.WithIgnored("").RejectNonZeroPadding(true)
+	base64URLEncoding = base64.URLEncoding.WithIgnored("").RejectNonZeroPadding(true)
+	base32StdEncoding = base32.StdEncoding.WithIgnored("").RejectNonZeroPadding(true)
+	base32HexEncoding = base32.HexEncoding.WithIgnored("").RejectNonZeroPadding(true)
 )
 
 func makeBytesArshaler(t reflect.Type, fncs *arshaler) *arshaler {
 	// NOTE: This handles both []byte and [N]byte.
 	marshalArray := fncs.marshal
 	fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error {
 		xe := export.Encoder(enc)
-		encode, encodedLen := encodeBase64, encodedLenBase64
+		appendEncode := base64StdEncoding.AppendEncode
 		if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() {
 			switch mo.Format {
 			case "base64":
-				encode, encodedLen = encodeBase64, encodedLenBase64
+				appendEncode = base64StdEncoding.AppendEncode
 			case "base64url":
-				encode, encodedLen = encodeBase64URL, encodedLenBase64URL
+				appendEncode = base64URLEncoding.AppendEncode
 			case "base32":
-				encode, encodedLen = encodeBase32, encodedLenBase32
+				appendEncode = base32StdEncoding.AppendEncode
 			case "base32hex":
-				encode, encodedLen = encodeBase32Hex, encodedLenBase32Hex
+				appendEncode = base32HexEncoding.AppendEncode
 			case "base16", "hex":
-				encode, encodedLen = encodeBase16, encodedLenBase16
+				appendEncode = hex.AppendEncode
 			case "array":
 				mo.Format = ""
 				return marshalArray(enc, va, mo)
@@ -269,35 +256,26 @@ func makeBytesArshaler(t reflect.Type, fncs *arshaler) *arshaler {
 			// TODO: Provide a "emitempty" format override?
 			return enc.WriteToken(jsontext.Null)
 		}
-		val := enc.UnusedBuffer()
-		b := va.Bytes()
-		n := len(`"`) + encodedLen(len(b)) + len(`"`)
-		if cap(val) < n {
-			val = make([]byte, n)
-		} else {
-			val = val[:n]
-		}
-		val[0] = '"'
-		encode(val[len(`"`):len(val)-len(`"`)], b)
-		val[len(val)-1] = '"'
-		return enc.WriteValue(val)
+		return xe.AppendRaw('"', true, func(b []byte) ([]byte, error) {
+			return appendEncode(b, va.Bytes()), nil
+		})
 	}
 	unmarshalArray := fncs.unmarshal
 	fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error {
 		xd := export.Decoder(dec)
-		decode, decodedLen, encodedLen := decodeBase64, decodedLenBase64, encodedLenBase64
+		appendDecode := base64StdEncoding.AppendDecode
 		if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() {
 			switch uo.Format {
 			case "base64":
-				decode, decodedLen, encodedLen = decodeBase64, decodedLenBase64, encodedLenBase64
+				appendDecode = base64StdEncoding.AppendDecode
 			case "base64url":
-				decode, decodedLen, encodedLen = decodeBase64URL, decodedLenBase64URL, encodedLenBase64URL
+				appendDecode = base64URLEncoding.AppendDecode
 			case "base32":
-				decode, decodedLen, encodedLen = decodeBase32, decodedLenBase32, encodedLenBase32
+				appendDecode = base32StdEncoding.AppendDecode
 			case "base32hex":
-				decode, decodedLen, encodedLen = decodeBase32Hex, decodedLenBase32Hex, encodedLenBase32Hex
+				appendDecode = base32HexEncoding.AppendDecode
 			case "base16", "hex":
-				decode, decodedLen, encodedLen = decodeBase16, decodedLenBase16, encodedLenBase16
+				appendDecode = hex.AppendDecode
 			case "array":
 				uo.Format = ""
 				return unmarshalArray(dec, va, uo)
@@ -319,46 +297,25 @@ func makeBytesArshaler(t reflect.Type, fncs *arshaler) *arshaler {
 			return nil
 		case '"':
 			val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim())
-
-			// For base64 and base32, decodedLen computes the maximum output size
-			// when given the original input size. To compute the exact size,
-			// adjust the input size by excluding trailing padding characters.
-			// This is unnecessary for base16, but also harmless.
-			n := len(val)
-			for n > 0 && val[n-1] == '=' {
-				n--
-			}
-			n = decodedLen(n)
-			b := va.Bytes()
-			if va.Kind() == reflect.Array {
-				if n != len(b) {
-					err := fmt.Errorf("decoded base64 length of %d mismatches array length of %d", n, len(b))
-					return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t, Err: err}
+			bi := va.Bytes()
+			var bo []byte
+			bo, err = appendDecode(bi[:0], val)
+			if va.Kind() == reflect.Slice {
+				if bo == nil {
+					bo = []byte{}
 				}
+				va.SetBytes(bo)
 			} else {
-				if b == nil || cap(b) < n {
-					b = make([]byte, n)
-				} else {
-					b = b[:n]
+				copy(bi, bo)
+				if err == nil && len(bi) != len(bo) {
+					err = fmt.Errorf("decoded base64 length of %d mismatches array length of %d", len(bo), va.Len())
 				}
 			}
-			n2, err := decode(b, val)
-			if err == nil && len(val) != encodedLen(n2) {
-				// TODO(https://go.dev/issue/53845): RFC 4648, section 3.3,
-				// specifies that non-alphabet characters must be rejected.
-				// Unfortunately, the "base32" and "base64" packages allow
-				// '\r' and '\n' characters by default.
-				err = errors.New("illegal data at input byte " + strconv.Itoa(bytes.IndexAny(val, "\r\n")))
-			}
-			if err != nil {
-				return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t, Err: err}
-			}
-			if va.Kind() == reflect.Slice {
-				va.SetBytes(b)
+			if err == nil {
+				return nil
 			}
-			return nil
 		}
-		return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t}
+		return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t, Err: err}
 	}
 	return fncs
 }

arshal_test.go

@@ -4376,7 +4376,7 @@ func TestUnmarshal(t *testing.T) {
 		inVal: new([0]byte),
 		want:  addr([0]byte{}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array0ByteType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 0), []byte("A"))
+			_, err := base64StdEncoding.Decode(make([]byte, 0), []byte("A"))
 			return err
 		}()},
 	}, {
@@ -4396,7 +4396,7 @@ func TestUnmarshal(t *testing.T) {
 		inVal: new([1]byte),
 		want:  addr([1]byte{}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array1ByteType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 1), []byte("$$=="))
+			_, err := base64StdEncoding.Decode(make([]byte, 1), []byte("$$=="))
 			return err
 		}()},
 	}, {
@@ -4409,7 +4409,7 @@ func TestUnmarshal(t *testing.T) {
 		name:    jsontest.Name("Bytes/ByteArray1/Overflow"),
 		inBuf:   `"AQI="`,
 		inVal:   new([1]byte),
-		want:    addr([1]byte{}),
+		want:    addr([1]byte{1}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array1ByteType, Err: errors.New("decoded base64 length of 2 mismatches array length of 1")},
 	}, {
 		name:  jsontest.Name("Bytes/ByteArray2/Valid"),
@@ -4422,20 +4422,20 @@ func TestUnmarshal(t *testing.T) {
 		inVal: new([2]byte),
 		want:  addr([2]byte{}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array2ByteType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 2), []byte("$$$="))
+			_, err := base64StdEncoding.Decode(make([]byte, 2), []byte("$$$="))
 			return err
 		}()},
 	}, {
 		name:    jsontest.Name("Bytes/ByteArray2/Underflow"),
 		inBuf:   `"AQ=="`,
 		inVal:   new([2]byte),
-		want:    addr([2]byte{}),
+		want:    addr([2]byte{1}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array2ByteType, Err: errors.New("decoded base64 length of 1 mismatches array length of 2")},
 	}, {
 		name:    jsontest.Name("Bytes/ByteArray2/Overflow"),
 		inBuf:   `"AQID"`,
 		inVal:   new([2]byte),
-		want:    addr([2]byte{}),
+		want:    addr([2]byte{1, 2}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array2ByteType, Err: errors.New("decoded base64 length of 3 mismatches array length of 2")},
 	}, {
 		name:  jsontest.Name("Bytes/ByteArray3/Valid"),
@@ -4448,20 +4448,20 @@ func TestUnmarshal(t *testing.T) {
 		inVal: new([3]byte),
 		want:  addr([3]byte{}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array3ByteType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 3), []byte("$$$$"))
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("$$$$"))
 			return err
 		}()},
 	}, {
 		name:    jsontest.Name("Bytes/ByteArray3/Underflow"),
 		inBuf:   `"AQI="`,
 		inVal:   new([3]byte),
-		want:    addr([3]byte{}),
+		want:    addr([3]byte{1, 2}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array3ByteType, Err: errors.New("decoded base64 length of 2 mismatches array length of 3")},
 	}, {
 		name:    jsontest.Name("Bytes/ByteArray3/Overflow"),
 		inBuf:   `"AQIDAQ=="`,
 		inVal:   new([3]byte),
-		want:    addr([3]byte{}),
+		want:    addr([3]byte{1, 2, 3}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array3ByteType, Err: errors.New("decoded base64 length of 4 mismatches array length of 3")},
 	}, {
 		name:  jsontest.Name("Bytes/ByteArray4/Valid"),
@@ -4474,20 +4474,20 @@ func TestUnmarshal(t *testing.T) {
 		inVal: new([4]byte),
 		want:  addr([4]byte{}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array4ByteType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 4), []byte("$$$$$$=="))
+			_, err := base64StdEncoding.Decode(make([]byte, 4), []byte("$$$$$$=="))
 			return err
 		}()},
 	}, {
 		name:    jsontest.Name("Bytes/ByteArray4/Underflow"),
 		inBuf:   `"AQID"`,
 		inVal:   new([4]byte),
-		want:    addr([4]byte{}),
+		want:    addr([4]byte{1, 2, 3}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array4ByteType, Err: errors.New("decoded base64 length of 3 mismatches array length of 4")},
 	}, {
 		name:    jsontest.Name("Bytes/ByteArray4/Overflow"),
 		inBuf:   `"AQIDBAU="`,
 		inVal:   new([4]byte),
-		want:    addr([4]byte{}),
+		want:    addr([4]byte{1, 2, 3, 4}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: array4ByteType, Err: errors.New("decoded base64 length of 5 mismatches array length of 4")},
 	}, {
 		// NOTE: []namedByte is not assignable to []byte,
@@ -4497,35 +4497,86 @@ func TestUnmarshal(t *testing.T) {
 		inVal: new([5]namedByte),
 		want:  addr([5]namedByte{'h', 'e', 'l', 'l', 'o'}),
 	}, {
-		name:  jsontest.Name("Bytes/Valid/Denormalized"),
+		// Per RFC 4648, it is up to the implementation to choose
+		// whether to allow or reject non-zero padding bits.
+		name:  jsontest.Name("Bytes/Invalid/Denormalized"),
 		inBuf: `"AR=="`,
 		inVal: new([]byte),
+		want:  addr([]byte{}),
+		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("AR=="))
+			return err
+		}()},
+	}, {
+		name:  jsontest.Name("Bytes/Invalid/LineFeedInDatum"),
+		inBuf: `"A\nQ=="`,
+		inVal: new([]byte),
+		want:  addr([]byte{}),
+		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("A\nQ=="))
+			return err
+		}()},
+	}, {
+		name:  jsontest.Name("Bytes/Invalid/LineFeedInPadding"),
+		inBuf: `"AQ=\n="`,
+		inVal: new([]byte),
+		want:  addr([]byte{}),
+		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("AQ=\n="))
+			return err
+		}()},
+	}, {
+		name:  jsontest.Name("Bytes/Invalid/LineFeedAfter"),
+		inBuf: `"AQ==\n"`,
+		inVal: new([]byte),
 		want:  addr([]byte{1}),
+		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("AQ==\n"))
+			return err
+		}()},
 	}, {
 		name:  jsontest.Name("Bytes/Invalid/Unpadded1"),
 		inBuf: `"AQ="`,
-		inVal: addr([]byte("nochange")),
-		want:  addr([]byte("nochange")),
+		inVal: addr([]byte("initial")),
+		want:  addr([]byte("")),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 0), []byte("AQ="))
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("AQ="))
 			return err
 		}()},
 	}, {
 		name:  jsontest.Name("Bytes/Invalid/Unpadded2"),
 		inBuf: `"AQ"`,
-		inVal: addr([]byte("nochange")),
-		want:  addr([]byte("nochange")),
+		inVal: addr([]byte("initial")),
+		want:  addr([]byte("")),
+		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("AQ"))
+			return err
+		}()},
+	}, {
+		name:  jsontest.Name("Bytes/Invalid/ExtraPadded"),
+		inBuf: `"AQ==="`,
+		inVal: addr([]byte("initial")),
+		want:  addr([]byte{1}),
+		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("AQ==="))
+			return err
+		}()},
+	}, {
+		name:  jsontest.Name("Bytes/Invalid/Concatenation"),
+		inBuf: `"AQ==AQ=="`,
+		inVal: addr([]byte("initial")),
+		want:  addr([]byte{1}),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 0), []byte("AQ"))
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("AQ==AQ=="))
 			return err
 		}()},
 	}, {
 		name:  jsontest.Name("Bytes/Invalid/Character"),
 		inBuf: `"@@@@"`,
-		inVal: addr([]byte("nochange")),
-		want:  addr([]byte("nochange")),
+		inVal: addr([]byte("initial")),
+		want:  addr([]byte("")),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 3), []byte("@@@@"))
+			_, err := base64StdEncoding.Decode(make([]byte, 3), []byte("@@@@"))
 			return err
 		}()},
 	}, {
@@ -5825,15 +5876,15 @@ func TestUnmarshal(t *testing.T) {
 		inBuf: `{"Base64": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"}`,
 		inVal: new(structFormatBytes),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
-			_, err := base64.StdEncoding.Decode(make([]byte, 48), []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"))
+			_, err := base64StdEncoding.Decode(make([]byte, 48), []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"))
 			return err
 		}()},
 	}, {
 		name:  jsontest.Name("Structs/Format/Bytes/Invalid/Base64URL/WrongAlphabet"),
 		inBuf: `{"Base64URL": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"}`,
 		inVal: new(structFormatBytes),
 		wantErr: &SemanticError{action: "unmarshal", JSONKind: '"', GoType: bytesType, Err: func() error {
-			_, err := base64.URLEncoding.Decode(make([]byte, 48), []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"))
+			_, err := base64URLEncoding.Decode(make([]byte, 48), []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"))
 			return err
 		}()},
 	}, {