Function for scanning UTF-8 sequences

This is pretty redundant given the built-in functionality in Go, but
sometimes it's convenient to have everything expressed as a
bufio.SplitFunc to get general behavior across any set of tokens, like
with the helper functions in this package.

textseg/utf8_seqs.go

@@ -0,0 +1,19 @@
+package textseg
+
+import "unicode/utf8"
+
+// ScanGraphemeClusters is a split function for bufio.Scanner that splits
+// on UTF8 sequence boundaries.
+//
+// This is included largely for completeness, since this behavior is already
+// built in to Go when ranging over a string.
+func ScanUTF8Sequences(data []byte, atEOF bool) (int, []byte, error) {
+	if len(data) == 0 {
+		return 0, nil, nil
+	}
+	r, seqLen := utf8.DecodeRune(data)
+	if r == utf8.RuneError && !atEOF {
+		return 0, nil, nil
+	}
+	return seqLen, data[:seqLen], nil
+}