Refactor the lexer API

_example/even-lexer/main.go

@@ -11,28 +11,26 @@ func main() {
  p := prompt.New(
  executor,
  completer,
- prompt.OptionSetLexer(lexer),
+ prompt.OptionSetLexer(prompt.NewEagerLexer(lexer)),
  )
 
  p.Run()
 }
 
-func lexer(line string) []prompt.LexerElement {
- var elements []prompt.LexerElement
+func lexer(line string) []prompt.Token {
+ var elements []prompt.Token
 
  strArr := strings.Split(line, "")
 
- for k, v := range strArr {
- element := prompt.LexerElement{
- Text: v,
- }
-
+ for i, value := range strArr {
+ var color prompt.Color
  // every even char must be green.
- if k%2 == 0 {
- element.Color = prompt.Green
+ if i%2 == 0 {
+ color = prompt.Green
  } else {
- element.Color = prompt.White
+ color = prompt.White
  }
+ element := prompt.NewSimpleToken(color, value)
 
  elements = append(elements, element)
  }

lexer.go

@@ -1,34 +1,80 @@
 package prompt
 
-// LexerFunc is a callback from render.
-type LexerFunc = func(line string) []LexerElement
+// Lexer is a streaming lexer that takes in a piece of text
+// and streams tokens with the Next() method
+type Lexer interface {
+ Init(string) // Reset the lexer's state and initialise it with the given input.
+ // Next returns the next Token and a bool flag
+ // which is false when the end of input has been reached.
+ Next() (Token, bool)
+}
 
-// LexerElement is a element of lexer.
-type LexerElement struct {
- Color Color
- Text string
+// Token is a single unit of text returned by a Lexer.
+type Token interface {
+ Color() Color
+ Lexeme() string // original string that matches this token
 }
 
-// Lexer is a struct with lexer param and function.
-type Lexer struct {
- IsEnabled bool
- fn LexerFunc
+// SimpleToken as the default implementation of Token.
+type SimpleToken struct {
+ color Color
+ lexeme string
 }
 
-// NewLexer returns new Lexer.
-func NewLexer() *Lexer {
- return &Lexer{
- IsEnabled: false,
+// Create a new SimpleToken.
+func NewSimpleToken(color Color, lexeme string) *SimpleToken {
+ return &SimpleToken{
+ color: color,
+ lexeme: lexeme,
  }
 }
 
-// SetLexerFunction in lexer struct.
-func (l *Lexer) SetLexerFunction(fn LexerFunc) {
- l.IsEnabled = true
- l.fn = fn
+// Retrieve the color of this token.
+func (t *SimpleToken) Color() Color {
+ return t.color
+}
+
+// Retrieve the text that this token represents.
+func (t *SimpleToken) Lexeme() string {
+ return t.lexeme
 }
 
-// Process line with a custom function.
-func (l *Lexer) Process(line string) []LexerElement {
- return l.fn(line)
+// LexerFunc is a function implementing
+// a simple lexer that receives a string
+// and returns a complete slice of Tokens.
+type LexerFunc func(string) []Token
+
+// EagerLexer is a wrapper around LexerFunc that
+// transforms an eager lexer which produces an
+// array with all tokens at once into a streaming
+// lexer compatible with go-prompt.
+type EagerLexer struct {
+ lexFunc LexerFunc
+ tokens []Token
+ currentIndex int
+}
+
+// Create a new EagerLexer.
+func NewEagerLexer(fn LexerFunc) *EagerLexer {
+ return &EagerLexer{
+ lexFunc: fn,
+ }
+}
+
+// Initialise the lexer with the given input.
+func (l *EagerLexer) Init(input string) {
+ l.tokens = l.lexFunc(input)
+ l.currentIndex = 0
+}
+
+// Return the next token and true if the operation
+// was successful.
+func (l *EagerLexer) Next() (Token, bool) {
+ if l.currentIndex >= len(l.tokens) {
+ return nil, false
+ }
+
+ result := l.tokens[l.currentIndex]
+ l.currentIndex++
+ return result, true
 }

option.go

@@ -267,9 +267,9 @@ func OptionSetExitCheckerOnInput(fn ExitChecker) Option {
 }
 
 // OptionSetLexer set lexer function and enable it.
-func OptionSetLexer(fn LexerFunc) Option {
+func OptionSetLexer(lex Lexer) Option {
  return func(p *Prompt) error {
- p.lexer.SetLexerFunction(fn)
+ p.lexer = lex
  return nil
  }
 }
@@ -305,7 +305,6 @@ func New(executor Executor, completer Completer, opts ...Option) *Prompt {
  buf: NewBuffer(),
  executor: executor,
  history: NewHistory(),
- lexer: NewLexer(),
  completion: NewCompletionManager(completer, 6),
  keyBindMode: EmacsKeyBind, // All the above assume that bash is running in the default Emacs setting
  }

prompt.go

@@ -32,7 +32,7 @@ type Prompt struct {
  renderer *Render
  executor Executor
  history *History
- lexer *Lexer
+ lexer Lexer
  completion *CompletionManager
  keyBindings []KeyBind
  ASCIICodeBindings []ASCIICodeBind

render.go

@@ -1,6 +1,7 @@
 package prompt
 
 import (
+ "fmt"
  "runtime"
  "strings"
 
@@ -172,7 +173,7 @@ func (r *Render) renderCompletion(buf *Buffer, completions *CompletionManager) {
 }
 
 // Render renders to the console.
-func (r *Render) Render(buffer *Buffer, completion *CompletionManager, lexer *Lexer) {
+func (r *Render) Render(buffer *Buffer, completion *CompletionManager, lexer Lexer) {
  // In situations where a pseudo tty is allocated (e.g. within a docker container),
  // window size via TIOCGWINSZ is not immediately available and will result in 0,0 dimensions.
  if r.col == 0 {
@@ -200,17 +201,8 @@ func (r *Render) Render(buffer *Buffer, completion *CompletionManager, lexer *Le
 
  r.renderPrefix()
 
- if lexer.IsEnabled {
- processed := lexer.Process(line)
- var s = line
-
- for _, v := range processed {
- a := strings.SplitAfter(s, v.Text)
- s = strings.TrimPrefix(s, a[0])
-
- r.out.SetColor(v.Color, r.inputBGColor, false)
- r.out.WriteStr(a[0])
- }
+ if lexer != nil {
+ r.lex(lexer, line)
  } else {
  r.out.SetColor(r.inputTextColor, r.inputBGColor, false)
  r.out.WriteStr(line)
@@ -235,18 +227,8 @@ func (r *Render) Render(buffer *Buffer, completion *CompletionManager, lexer *Le
 
  rest := buffer.Document().TextAfterCursor()
 
- if lexer.IsEnabled {
- processed := lexer.Process(rest)
-
- var s = rest
-
- for _, v := range processed {
- a := strings.SplitAfter(s, v.Text)
- s = strings.TrimPrefix(s, a[0])
-
- r.out.SetColor(v.Color, r.inputBGColor, false)
- r.out.WriteStr(a[0])
- }
+ if lexer != nil {
+ r.lex(lexer, rest)
  } else {
  r.out.WriteStr(rest)
  }
@@ -261,26 +243,37 @@ func (r *Render) Render(buffer *Buffer, completion *CompletionManager, lexer *Le
  r.previousCursor = cursor
 }
 
+// lex processes the given input with the given lexer
+// and writes the result
+func (r *Render) lex(lexer Lexer, input string) {
+ lexer.Init(input)
+ s := input
+
+ for {
+ token, ok := lexer.Next()
+ if !ok {
+ break
+ }
+
+ a := strings.SplitAfter(s, token.Lexeme())
+ s = strings.TrimPrefix(s, a[0])
+
+ fmt.Printf("%#v\n", token)
+ r.out.SetColor(token.Color(), r.inputBGColor, false)
+ r.out.WriteStr(a[0])
+ }
+}
+
 // BreakLine to break line.
-func (r *Render) BreakLine(buffer *Buffer, lexer *Lexer) {
+func (r *Render) BreakLine(buffer *Buffer, lexer Lexer) {
  // Erasing and Render
  cursor := runewidth.StringWidth(buffer.Document().TextBeforeCursor()) + runewidth.StringWidth(r.getCurrentPrefix())
  r.clear(cursor)
 
  r.renderPrefix()
 
- if lexer.IsEnabled {
- processed := lexer.Process(buffer.Document().Text + "\n")
-
- var s = buffer.Document().Text + "\n"
-
- for _, v := range processed {
- a := strings.SplitAfter(s, v.Text)
- s = strings.TrimPrefix(s, a[0])
-
- r.out.SetColor(v.Color, r.inputBGColor, false)
- r.out.WriteStr(a[0])
- }
+ if lexer != nil {
+ r.lex(lexer, buffer.Document().Text+"\n")
  } else {
  r.out.SetColor(r.inputTextColor, r.inputBGColor, false)
  r.out.WriteStr(buffer.Document().Text + "\n")

render_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package prompt
@@ -96,8 +97,7 @@ func TestBreakLineCallback(t *testing.T) {
  col: 1,
  }
  b := NewBuffer()
- l := NewLexer()
- r.BreakLine(b, l)
+ r.BreakLine(b, nil)
 
  if i != 0 {
  t.Errorf("i should initially be 0, before applying a break line callback")
@@ -106,9 +106,9 @@ func TestBreakLineCallback(t *testing.T) {
  r.breakLineCallback = func(doc *Document) {
  i++
  }
- r.BreakLine(b, l)
- r.BreakLine(b, l)
- r.BreakLine(b, l)
+ r.BreakLine(b, nil)
+ r.BreakLine(b, nil)
+ r.BreakLine(b, nil)
 
  if i != 3 {
  t.Errorf("BreakLine callback not called, i should be 3")