From 33a8a4c5ea12fbe80b5c2ddff4583c0af36325ed Mon Sep 17 00:00:00 2001
From: Martin Atkins <mart@degeneration.co.uk>
Date: Sat, 27 May 2017 09:15:07 -0700
Subject: [PATCH] tests for AllTokens

---
 textseg/all_tokens_test.go | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 textseg/all_tokens_test.go

diff --git a/textseg/all_tokens_test.go b/textseg/all_tokens_test.go
new file mode 100644
index 0000000..e26a68a
--- /dev/null
+++ b/textseg/all_tokens_test.go
@@ -0,0 +1,68 @@
+package textseg
+
+import (
+	"bufio"
+	"reflect"
+	"testing"
+)
+
+func TestAllTokens(t *testing.T) {
+	tests := []struct {
+		input string
+		want  []string
+	}{
+		{
+			``,
+			[]string{},
+		},
+		{
+			`hello`,
+			[]string{
+				`hello`,
+			},
+		},
+		{
+			`hello world`,
+			[]string{
+				`hello`,
+				`world`,
+			},
+		},
+		{
+			`hello worldly world`,
+			[]string{
+				`hello`,
+				`worldly`,
+				`world`,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.input, func(t *testing.T) {
+			gotBytes, err := AllTokens([]byte(test.input), bufio.ScanWords)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+
+			got := make([]string, len(gotBytes))
+			for i, buf := range gotBytes {
+				got[i] = string(buf)
+			}
+
+			if !reflect.DeepEqual(got, test.want) {
+				wantBytes := make([][]byte, len(test.want))
+				for i, str := range test.want {
+					wantBytes[i] = []byte(str)
+				}
+
+				t.Errorf(
+					"wrong result\ninput: %s\ngot:   %s\nwant:  %s",
+					formatBytes([]byte(test.input)),
+					formatByteRanges(gotBytes),
+					formatByteRanges(wantBytes),
+				)
+			}
+		})
+	}
+}