skylark: range(...) no longer materializes the sequence as a list (go…

…ogle#25)

* skylark: range(...) no longer materializes the sequence as a list

Skylark now follows the Python3, not Python2, semantics for range.

Also: fix an off-by-one error.

* range: allow membership tests 'x in range(n)'

Change-Id: I0dffe70510f723c00da4153bcc0a17478cceb986

* range: allow floating point numbers for x in 'x in range(y)'

- use ConvertToInt to convert x to an integer.
- change ConvertToInt not to accept a bool.  This affects string interpolation.
  e.g. "%d" % True now (correctly) results in an error.
- rename ConvertToInt to NumberToInt

Change-Id: Ie9e9d40d993c227b834bb720e5dbe3714032836a

* range: doc tweaks

Change-Id: I3f242d38a5969fa228167b72be6cb78aa4677eb6

README.md

@@ -75,7 +75,7 @@ Interact with the read-eval-print loop (REPL):
 ```
 $ ./skylark
 >>> def fibonacci(n):
-...    res = range(n)
+...    res = list(range(n))
 ...    for i in res[2:]:
 ...        res[i] = res[i-2] + res[i-1]
 ...    return res

doc/spec.md

@@ -332,7 +332,7 @@ TODO: define string_lit, indent, outdent, semicolon, newline, eof
 
 ## Data types
 
-The following eleven data types are known to the interpreter:
+These are the main data types built in to the interpreter:
 
 ```shell
 NoneType           # the type of None
@@ -348,6 +348,9 @@ function           # a function implemented in Skylark
 builtin            # a function or method implemented by the interpreter or host application
 ```
 
+Some functions, such as the iteration methods of `string`, or the
+`range` function, return instances of special-purpose types that don't
+appear in this list.
 Additional data types may be defined by the host application into
 which the interpreter is embedded, and those data types may
 participate in basic operations of the language such as arithmetic,
@@ -1964,7 +1967,7 @@ c       string          x (string must encode a single Unicode code point)
 ```
 
 It is an error if the argument does not have the type required by the
-conversion specifier.
+conversion specifier.  A Boolean argument is not considered a number.
 
 Examples:
 
@@ -2992,7 +2995,7 @@ print(1, "hi", x=3)	# "1 hi x=3\n"
 
 ### range
 
-`range` returns a new list of integers drawn from the specified interval and stride.
+`range` returns an immutable sequence of integers defined by the specified interval and stride.
 
 ```python
 range(stop)                             # equivalent to range(0, stop)
@@ -3006,14 +3009,36 @@ With two arguments, `range(start, stop)` returns only integers not less than `st
 
 With three arguments, `range(start, stop, step)` returns integers
 formed by successively adding `step` to `start` until the value meets or passes `stop`.
+A call to `range` fails if the value of `step` is zero.
+
+A call to `range` does not materialize the entire sequence, but
+returns a fixed-size value of type `"range"` that represents the
+parameters that define the sequence.
+The `range` value is iterable and may be indexed efficiently.
 
 ```python
-range(10)                               # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-range(3, 10)                            # [3, 4, 5, 6, 7, 8, 9]
-range(3, 10, 2)                         # [3, 5, 7, 9]
-range(10, 3, -2)                        # [10, 8, 6, 4]
+list(range(10))                         # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+list(range(3, 10))                      # [3, 4, 5, 6, 7, 8, 9]
+list(range(3, 10, 2))                   # [3, 5, 7, 9]
+list(range(10, 3, -2))                  # [10, 8, 6, 4]
 ```
 
+The `len` function applied to a `range` value returns its length.
+The truth value of a `range` value is `True` if its length is non-zero.
+
+Range values are comparable: two `range` values compare equal if they
+denote the same sequence of integers, even if they were created using
+different parameters.
+
+Range values are not hashable.  <!-- should they be? -->
+
+The `str` function applied to a `range` value yields a string of the
+form `range(10)`, `range(1, 10)`, or `range(1, 10, 2)`.
+
+The `x in y` operator, where `y` is a range, reports whether `x` is equal to
+some member of the sequence `y`; the operation fails unless `x` is a
+number.
+
 ### repr
 
 `repr(x)` formats its argument as a string.

eval.go

@@ -1163,6 +1163,12 @@ func Binary(op syntax.Token, x, y Value) (Value, error) {
 				return nil, fmt.Errorf("'in <string>' requires string as left operand, not %s", x.Type())
 			}
 			return Bool(strings.Contains(string(y), string(needle))), nil
+		case rangeValue:
+			i, err := NumberToInt(x)
+			if err != nil {
+				return nil, fmt.Errorf("'in <range>' requires integer as left operand, not %s", x.Type())
+			}
+			return Bool(y.contains(i)), nil
 		}
 
 	case syntax.PIPE:
@@ -1934,7 +1940,7 @@ func interpolate(format string, x Value) (Value, error) {
 				writeValue(&buf, arg, path)
 			}
 		case 'd', 'i', 'o', 'x', 'X':
-			i, err := ConvertToInt(arg)
+			i, err := NumberToInt(arg)
 			if err != nil {
 				return nil, fmt.Errorf("%%%c format requires integer: %v", c, err)
 			}

int.go

@@ -179,17 +179,11 @@ func AsInt32(x Value) (int, error) {
 	return 0, fmt.Errorf("%s out of range", i)
 }
 
-// ConvertToInt converts x to an integer value.  An int is returned
-// unchanged, a bool becomes 0 or 1, a float is truncated towards
-// zero. ConvertToInt reports an error for all other values.
-func ConvertToInt(x Value) (Int, error) {
+// NumberToInt converts a number x to an integer value.
+// An int is returned unchanged, a float is truncated towards zero.
+// NumberToInt reports an error for all other values.
+func NumberToInt(x Value) (Int, error) {
 	switch x := x.(type) {
-	case Bool:
-		if x {
-			return one, nil
-		} else {
-			return zero, nil
-		}
 	case Int:
 		return x, nil
 	case Float:

library.go

@@ -665,7 +665,16 @@ func int_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error)
 	if base != nil {
 		return nil, fmt.Errorf("int: can't convert non-string with explicit base")
 	}
-	i, err := ConvertToInt(x)
+
+	if b, ok := x.(Bool); ok {
+		if b {
+			return one, nil
+		} else {
+			return zero, nil
+		}
+	}
+
+	i, err := NumberToInt(x)
 	if err != nil {
 		return nil, fmt.Errorf("int: %s", err)
 	}
@@ -834,34 +843,109 @@ func range_(thread *Thread, fn *Builtin, args Tuple, kwargs []Tuple) (Value, err
 	if err := UnpackPositionalArgs("range", args, kwargs, 1, &start, &stop, &step); err != nil {
 		return nil, err
 	}
-	list := new(List)
+
+	// TODO(adonovan): analyze overflow/underflows cases for 32-bit implementations.
+
+	var n int
 	switch len(args) {
 	case 1:
 		// range(stop)
 		start, stop = 0, start
 		fallthrough
 	case 2:
 		// range(start, stop)
-		for i := start; i < stop; i += step {
-			list.elems = append(list.elems, MakeInt(i))
+		if stop > start {
+			n = stop - start
 		}
 	case 3:
 		// range(start, stop, step)
-		if step == 0 {
-			return nil, fmt.Errorf("range: step argument must not be zero")
-		}
-		if step > 0 {
-			for i := start; i < stop; i += step {
-				list.elems = append(list.elems, MakeInt(i))
+		switch {
+		case step > 0:
+			if stop > start {
+				n = (stop-1-start)/step + 1
 			}
-		} else {
-			for i := start; i >= stop; i += step {
-				list.elems = append(list.elems, MakeInt(i))
+		case step < 0:
+			if start > stop {
+				n = (start-1-stop)/-step + 1
 			}
+		default:
+			return nil, fmt.Errorf("range: step argument must not be zero")
 		}
 	}
-	return list, nil
+
+	return rangeValue{start: start, stop: stop, step: step, len: n}, nil
+}
+
+// A rangeValue is a comparable, immutable, indexable sequence of integers
+// defined by the three parameters to a range(...) call.
+// Invariant: step != 0.
+type rangeValue struct{ start, stop, step, len int }
+
+var (
+	_ Indexable  = rangeValue{}
+	_ Sequence   = rangeValue{}
+	_ Comparable = rangeValue{}
+)
+
+func (r rangeValue) Len() int          { return r.len }
+func (r rangeValue) Index(i int) Value { return MakeInt(r.start + i*r.step) }
+func (r rangeValue) Iterate() Iterator { return &rangeIterator{r, 0} }
+func (r rangeValue) Freeze()           {} // immutable
+func (r rangeValue) String() string {
+	if r.step != 1 {
+		return fmt.Sprintf("range(%d, %d, %d)", r.start, r.stop, r.step)
+	} else if r.start != 0 {
+		return fmt.Sprintf("range(%d, %d)", r.start, r.stop)
+	} else {
+		return fmt.Sprintf("range(%d)", r.stop)
+	}
+}
+func (r rangeValue) Type() string          { return "range" }
+func (r rangeValue) Truth() Bool           { return r.len > 0 }
+func (r rangeValue) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: range") }
+
+func (x rangeValue) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) {
+	y := y_.(rangeValue)
+	switch op {
+	case syntax.EQL:
+		return rangeEqual(x, y), nil
+	case syntax.NEQ:
+		return !rangeEqual(x, y), nil
+	default:
+		return false, fmt.Errorf("%s %s %s not implemented", x.Type(), op, y.Type())
+	}
+}
+
+func rangeEqual(x, y rangeValue) bool {
+	// Two ranges compare equal if they denote the same sequence.
+	return x.len == y.len &&
+		(x.len == 0 || x.start == y.start && x.step == y.step)
+}
+
+func (r rangeValue) contains(x Int) bool {
+	x32, err := AsInt32(x)
+	if err != nil {
+		return false // out of range
+	}
+	delta := x32 - r.start
+	quo, rem := delta/r.step, delta%r.step
+	return rem == 0 && 0 <= quo && quo < r.len
+}
+
+type rangeIterator struct {
+	r rangeValue
+	i int
+}
+
+func (it *rangeIterator) Next(p *Value) bool {
+	if it.i < it.r.len {
+		*p = it.r.Index(it.i)
+		it.i++
+		return true
+	}
+	return false
 }
+func (*rangeIterator) Done() {}
 
 // https://github.com/google/skylark/blob/master/doc/spec.md#repr
 func repr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) {

testdata/builtins.sky

@@ -66,16 +66,40 @@ assert.eq(dict({1:2, 3:4}), {1: 2, 3: 4})
 assert.eq(dict({1:2, 3:4}.items()), {1: 2, 3: 4})
 
 # range
-assert.eq(range(5), [0, 1, 2, 3, 4])
-assert.eq(range(-5), [])
-assert.eq(range(2, 5), [2, 3, 4])
-assert.eq(range(5, 2), [])
-assert.eq(range(-2, -5), [])
-assert.eq(range(-5, -2), [-5, -4, -3])
-assert.eq(range(2, 10, 3), [2, 5, 8])
-assert.eq(range(10, 2, -3), [10, 7, 4])
-assert.eq(range(-2, -10, -3), [-2, -5, -8])
-assert.eq(range(-10, -2, 3), [-10, -7, -4])
+assert.eq("range", type(range(10)))
+assert.eq("range(10)", str(range(0, 10, 1)))
+assert.eq("range(1, 10)", str(range(1, 10)))
+assert.eq("range(0, 10, -1)", str(range(0, 10, -1)))
+assert.fails(lambda: {range(10): 10}, "unhashable: range")
+assert.true(bool(range(1, 2)))
+assert.true(not(range(2, 1))) # an empty range is false
+assert.eq([x*x for x in range(5)], [0, 1, 4, 9, 16])
+assert.eq(list(range(5)), [0, 1, 2, 3, 4])
+assert.eq(list(range(-5)), [])
+assert.eq(list(range(2, 5)), [2, 3, 4])
+assert.eq(list(range(5, 2)), [])
+assert.eq(list(range(-2, -5)), [])
+assert.eq(list(range(-5, -2)), [-5, -4, -3])
+assert.eq(list(range(2, 10, 3)), [2, 5, 8])
+assert.eq(list(range(10, 2, -3)), [10, 7, 4])
+assert.eq(list(range(-2, -10, -3)), [-2, -5, -8])
+assert.eq(list(range(-10, -2, 3)), [-10, -7, -4])
+assert.eq(list(range(10, 2, -1)), [10, 9, 8, 7, 6, 5, 4, 3])
+assert.fails(lambda: range(3000000000), "3000000000 out of range") # signed 32-bit values only
+assert.eq(len(range(0x7fffffff)), 0x7fffffff) # O(1)
+# Two ranges compare equal if they denote the same sequence:
+assert.eq(range(0), range(2, 1, 3))       # []
+assert.eq(range(0, 3, 2), range(0, 4, 2)) # [0, 2]
+assert.ne(range(1, 10), range(2, 10))
+assert.fails(lambda: range(0) < range(0), "range < range not implemented")
+# <number> in <range>
+assert.contains(range(3), 1)
+assert.contains(range(3), 2.0)    # acts like 2
+assert.fails(lambda: True in range(3), "requires integer.*not bool") # bools aren't numbers
+assert.fails(lambda: "one" in range(10), "requires integer.*not string")
+assert.true(4 not in range(4))
+assert.true(1e15 not in range(4)) # too big for int32
+assert.true(1e100 not in range(4)) # too big for int64
 
 # list
 assert.eq(list("abc".split_bytes()), ["a", "b", "c"])

testdata/int.sky

@@ -150,4 +150,5 @@ assert.eq(' '.join(["%i" % x for x in nums]), "-95 -1 0 1 95")
 assert.eq(' '.join(["%x" % x for x in nums]), "-5f -1 0 1 5f")
 assert.eq(' '.join(["%X" % x for x in nums]), "-5F -1 0 1 5F")
 assert.eq("%o %x %d" % (123, 123, 123), "173 7b 123")
-assert.eq("%o %x %d" % (123.1, 123.1, True), "173 7b 1") # non-int operands are acceptable
+assert.eq("%o %x %d" % (123.1, 123.1, 123.1), "173 7b 123") # non-int operands are acceptable
+assert.fails(lambda: "%d" % True, "cannot convert bool to int")

testdata/list.sky

@@ -148,7 +148,7 @@ assert.eq(x5a, [1, 2, 3, "a", "b", "c", True, False])
 
 # list.insert
 def insert_at(index):
-  x = range(3)
+  x = list(range(3))
   x.insert(index, 42)
   return x
 assert.eq(insert_at(-99), [42, 0, 1, 2])