Support binary integers: 0b1101 literals and int("0b1101", 0) (google#52

)

* Support binary integers: 0b1101 literals and int("0b1101", 0)

* Specification and tests for binary integers

Adding documentation for bnary integers in spec.md.
Adding tests for syntax/scan_test.go (for the scanner) and testdata/int.sky for the int(x) function.

* Correct some test cases in syntax/scan_test.go

* Correct a test case in testdata/int.sky

doc/spec.md

@@ -294,6 +294,7 @@ has string, integer, and floating-point literals.
 123                             # decimal int
 0x7f                            # hexadecimal int
 0755                            # octal int
+0b1011                          # binary int
 
 0.0     0.       .0             # float
 1e10    1e+10    1e-10
@@ -307,10 +308,11 @@ r'hello'     r"hello"           # raw string literal
 Integer and floating-point literal tokens are defined by the following grammar:
 
 ```grammar {.good}
-int         = decimal_lit | octal_lit | hex_lit .
+int         = decimal_lit | octal_lit | hex_lit | binary_lit .
 decimal_lit = ('1' … '9') {decimal_digit} .
 octal_lit   = '0' {octal_digit} .
 hex_lit     = '0' ('x'|'X') hex_digit {hex_digit} .
+binary_lit  = '0' ('b'|'B') binary_digit {binary_digit} .
 
 float     = decimals '.' [decimals] [exponent]
           | decimals exponent
@@ -320,8 +322,10 @@ decimals  = decimal_digit {decimal_digit} .
 exponent  = ('e'|'E') ['+'|'-'] decimals .
 
 decimal_digit = '0' … '9' .
+
 octal_digit   = '0' … '7' .
 hex_digit     = '0' … '9' | 'A' … 'F' | 'a' … 'f' .
+binary_digit  = '0' | '1' .
 ```
 
 TODO: define string_lit, indent, outdent, semicolon, newline, eof

library.go

@@ -589,6 +589,8 @@ func int_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error)
 						hasbase = 8
 					case 'x', 'X':
 						hasbase = 16
+					case 'b', 'B':
+						hasbase = 2
 					}
 
 					if hasbase != 0 && b != 0 {

syntax/scan.go

@@ -742,7 +742,7 @@ func (sc *scanner) scanNumber(val *tokenValue, c rune) Token {
 		}
 		fraction = true
 	} else if c == '0' {
-		// hex, octal, or float
+		// hex, octal, binary or float
 		sc.readRune()
 		c = sc.peekRune()
 
@@ -770,6 +770,17 @@ func (sc *scanner) scanNumber(val *tokenValue, c rune) Token {
 				sc.readRune()
 				c = sc.peekRune()
 			}
+		} else if c == 'b' || c == 'B' {
+			// binary
+			sc.readRune()
+			c = sc.peekRune()
+			if !isbdigit(c) {
+				sc.error(sc.pos, "invalid binary literal")
+			}
+			for isbdigit(c) {
+				sc.readRune()
+				c = sc.peekRune()
+			}
 		} else {
 			// float (or obsolete octal "0755")
 			allzeros, octal := true, true
@@ -853,6 +864,8 @@ func (sc *scanner) scanNumber(val *tokenValue, c rune) Token {
 		s := val.raw
 		if len(s) > 2 && s[0] == '0' && (s[1] == 'o' || s[1] == 'O') {
 			val.int, err = strconv.ParseInt(s[2:], 8, 64)
+		} else if len(s) > 2 && s[0] == '0' && (s[1] == 'b' || s[1] == 'B') {
+			val.int, err = strconv.ParseInt(s[2:], 2, 64)
 		} else {
 			val.int, err = strconv.ParseInt(s, 0, 64)
 		}
@@ -878,6 +891,7 @@ func isIdentStart(c rune) bool {
 func isdigit(c rune) bool  { return '0' <= c && c <= '9' }
 func isodigit(c rune) bool { return '0' <= c && c <= '7' }
 func isxdigit(c rune) bool { return isdigit(c) || 'A' <= c && c <= 'F' || 'a' <= c && c <= 'f' }
+func isbdigit(c rune) bool { return '0' == c || c == '1' }
 
 // keywordToken records the special tokens for
 // strings that should not be treated as ordinary identifiers.

syntax/scan_test.go

@@ -160,6 +160,13 @@ pass`, "pass newline pass EOF"}, // consecutive newlines are consolidated
 		{"0XG", `foo.sky:1:1: invalid hex literal`},
 		{"0xA.", `10 . EOF`},
 		{"0xA.e1", `10 . e1 EOF`},
+		// binary
+		{"0b1010", `10 EOF`},
+		{"0B111101", `61 EOF`},
+		{"0b3", `foo.sky:1:3: invalid binary literal`},
+		{"0b1010201", `10 201 EOF`},
+		{"0b1010.01", `10 1.000000e-02 EOF`},
+		{"0b0000", `0 EOF`},
 		// octal
 		{"0o123", `83 EOF`},
 		{"0o12834", `10 834 EOF`},

testdata/int.sky

@@ -93,10 +93,15 @@ assert.eq(int("12", 16), 18)
 assert.eq(int("-12", 16), -18)
 assert.eq(int("0x12", 16), 18)
 assert.eq(int("-0x12", 16), -18)
+assert.eq(int("1010", 2), 10)
+assert.eq(int("111111101", 2), 509)
+assert.eq(int("0b0101", 0), 5)
+assert.eq(int("0b00000", 0), 0)
 assert.fails(lambda: int("0x123", 8), "invalid literal.*base 8")
 assert.fails(lambda: int("-0x123", 8), "invalid literal.*base 8")
 assert.fails(lambda: int("0o123", 16), "invalid literal.*base 16")
 assert.fails(lambda: int("-0o123", 16), "invalid literal.*base 16")
+assert.fails(lambda: int("0x110", 2), "invalid literal.*base 2")
 # int from string, auto detect base
 assert.eq(int("123", 0), 123)
 assert.eq(int("+123", 0), +123)