10/10 would recommend

codegen/lex/keywords.tmpl

@@ -4,6 +4,10 @@ package lex
 
 type Keyword int
 
+func (K Keyword) Discard() bool {
+    return false
+}
+
 var _ Token = (Keyword)(0)
 
 type Mapper func(rune) (Mapper, Keyword)

lex/identifiers.go

@@ -7,7 +7,11 @@ import (
 
 type Identifier string
 
-var _ Token = (*Identifier)(nil)
+func (I Identifier) Discard() bool {
+	return false
+}
+
+var _ Token = Identifier("")
 
 type IdentifierBuilder struct {
 	builder strings.Builder
@@ -37,9 +41,7 @@ func (I *IdentifierBuilder) Finish() Token {
 		return nil
 	}
 
-	return Identifier(
-		I.builder.String(),
-	)
+	return Identifier(I.builder.String())
 }
 
 var _ Builder = (*IdentifierBuilder)(nil)

lex/lex.go

@@ -5,6 +5,7 @@ import (
 )
 
 type Token interface {
+	Discard() bool
 }
 
 type BuilderState int
@@ -55,13 +56,69 @@ func (L *Lexer) Rewind() {
 	L.useBuf = true
 }
 
-func (L *Lexer) Next() Token {
-	keywordBuilder := NewKeywordBuilder()
-	for next, ok := L.NextCharacter(); ok; next, ok = L.NextCharacter() {
-		if keywordBuilder.Append(next) == Failed {
+type compoundBuilder struct {
+	builders []Builder
+	failed   []Builder
+}
+
+func newCompoundBuilder(builders ...Builder) compoundBuilder {
+	return compoundBuilder{
+		builders: builders,
+	}
+}
+
+// append the character to all active builders
+// returns true if the builder is finished
+func (c *compoundBuilder) append(char rune) bool {
+	c.failed = c.failed[:0]
+	ok := c.builders
+	c.builders = c.builders[:0]
+	for _, builder := range ok {
+		if builder.Append(char) == Failed {
+			c.failed = append(c.failed, builder)
+		} else {
+			c.builders = append(c.builders, builder)
+		}
+	}
+	return len(c.builders) == 0
+}
+
+func (c *compoundBuilder) finish() Token {
+	if len(c.builders) != 0 {
+		return c.builders[0].Finish()
+	}
+	if len(c.failed) == 0 {
+		return nil
+	}
+	return c.failed[0].Finish()
+}
+
+func (L *Lexer) next() Token {
+	builder := newCompoundBuilder(NewKeywordBuilder(), NewWhitespaceBuilder())
+	for {
+		next, ok := L.NextCharacter()
+		if !ok {
+			break
+		}
+
+		if builder.append(next) {
 			L.Rewind()
-			return keywordBuilder.Finish()
+			break
+		}
+	}
+	return builder.finish()
+}
+
+func (L *Lexer) Next() Token {
+	var token Token
+	for {
+		token = L.next()
+		if token == nil {
+			return nil
+		}
+		if !token.Discard() {
+			break
 		}
 	}
-	return keywordBuilder.Finish()
+	return token
 }

lex/whitespace.go

+package lex
+
+import "unicode"
+
+type Whitespace struct{}
+
+func (W Whitespace) Discard() bool {
+	return true
+}
+
+var _ Token = Whitespace{}
+
+type WhitespaceBuilder struct{}
+
+func NewWhitespaceBuilder() WhitespaceBuilder {
+	return WhitespaceBuilder{}
+}
+
+func (w WhitespaceBuilder) Append(char rune) BuilderState {
+	if unicode.IsSpace(char) {
+		return Ok
+	}
+
+	return Failed
+}
+
+func (w WhitespaceBuilder) Finish() Token {
+	return Whitespace{}
+}
+
+var _ Builder = WhitespaceBuilder{}

lib_test.go

@@ -4,5 +4,5 @@ import "testing"
 
 func TestParser_Lex(t *testing.T) {
 	parser := NewParser()
-	t.Errorf("%#v", parser.Lex("select;||/;"))
+	t.Errorf("%#v", parser.Lex("select  ;||/;"))
 }