Improve the lexer implementation

_example/even-lexer/main.go

@@ -12,8 +12,8 @@ import (
 func main() {
  p := prompt.New(
  executor,
- prompt.WithLexer(prompt.NewEagerLexer(wordLexer)),
  prompt.WithLexer(prompt.NewEagerLexer(charLexer)), // the last one overrides the other
+ prompt.WithLexer(prompt.NewEagerLexer(wordLexer)),
  )
 
  p.Run()
@@ -32,7 +32,7 @@ func charLexer(line string) []prompt.Token {
  color = prompt.White
  }
  lastByteIndex := strings.ByteNumber(i + utf8.RuneLen(value) - 1)
- element := prompt.NewSimpleToken(color, lastByteIndex)
+ element := prompt.NewSimpleToken(color, strings.ByteNumber(i), lastByteIndex)
 
  elements = append(elements, element)
  }
@@ -48,36 +48,47 @@ func wordLexer(line string) []prompt.Token {
 
  var elements []prompt.Token
  var currentByte strings.ByteNumber
+ var firstByte strings.ByteNumber
+ var firstCharSeen bool
  var wordIndex int
  var lastChar rune
 
  var color prompt.Color
  for i, char := range line {
  currentByte = strings.ByteNumber(i)
+ lastChar = char
  if unicode.IsSpace(char) {
+ if !firstCharSeen {
+ continue
+ }
  if wordIndex%2 == 0 {
  color = prompt.Green
  } else {
  color = prompt.White
  }
 
- element := prompt.NewSimpleToken(color, currentByte)
+ element := prompt.NewSimpleToken(color, firstByte, currentByte-1)
  elements = append(elements, element)
  wordIndex++
+ firstCharSeen = false
  continue
  }
- lastChar = char
+ if !firstCharSeen {
+ firstByte = strings.ByteNumber(i)
+ firstCharSeen = true
+ }
  }
  if !unicode.IsSpace(lastChar) {
  if wordIndex%2 == 0 {
  color = prompt.Green
  } else {
  color = prompt.White
  }
- element := prompt.NewSimpleToken(color, currentByte)
+ element := prompt.NewSimpleToken(color, firstByte, currentByte+strings.ByteNumber(utf8.RuneLen(lastChar))-1)
  elements = append(elements, element)
  }
 
+ prompt.Log("tokens: %#v", elements)
  return elements
 }
 

lexer.go

@@ -15,21 +15,24 @@ type Lexer interface {
 
 // Token is a single unit of text returned by a Lexer.
 type Token interface {
- Color() Color // Color of the token
- LastByteIndex() istrings.ByteNumber // Index of the last byte of this token
+ Color() Color // Color of the token
+ FirstByteIndex() istrings.ByteNumber // Index of the last byte of this token
+ LastByteIndex() istrings.ByteNumber // Index of the last byte of this token
 }
 
 // SimpleToken as the default implementation of Token.
 type SimpleToken struct {
- color Color
- lastByteIndex istrings.ByteNumber
+ color Color
+ lastByteIndex istrings.ByteNumber
+ firstByteIndex istrings.ByteNumber
 }
 
 // Create a new SimpleToken.
-func NewSimpleToken(color Color, index istrings.ByteNumber) *SimpleToken {
+func NewSimpleToken(color Color, firstIndex, lastIndex istrings.ByteNumber) *SimpleToken {
  return &SimpleToken{
- color: color,
- lastByteIndex: index,
+ color: color,
+ firstByteIndex: firstIndex,
+ lastByteIndex: lastIndex,
  }
 }
 
@@ -38,11 +41,16 @@ func (t *SimpleToken) Color() Color {
  return t.color
 }
 
-// Retrieve the text that this token represents.
+// The index of the last byte of the lexeme.
 func (t *SimpleToken) LastByteIndex() istrings.ByteNumber {
  return t.lastByteIndex
 }
 
+// The index of the first byte of the lexeme.
+func (t *SimpleToken) FirstByteIndex() istrings.ByteNumber {
+ return t.firstByteIndex
+}
+
 // LexerFunc is a function implementing
 // a simple lexer that receives a string
 // and returns a complete slice of Tokens.

lexer_test.go

@@ -70,7 +70,7 @@ func TestEagerLexerNext(t *testing.T) {
 func charLex(s string) []Token {
  var result []Token
  for i := range s {
- result = append(result, NewSimpleToken(0, istrings.ByteNumber(i)))
+ result = append(result, NewSimpleToken(0, istrings.ByteNumber(i), istrings.ByteNumber(i)))
  }
 
  return result
@@ -86,18 +86,18 @@ func TestEagerLexerInit(t *testing.T) {
  lexer: &EagerLexer{
  lexFunc: charLex,
  tokens: []Token{
- &SimpleToken{lastByteIndex: 2},
- &SimpleToken{lastByteIndex: 10},
+ &SimpleToken{firstByteIndex: 2, lastByteIndex: 2},
+ &SimpleToken{firstByteIndex: 10, lastByteIndex: 10},
  },
  currentIndex: 11,
  },
  input: "foo",
  want: &EagerLexer{
  lexFunc: charLex,
  tokens: []Token{
- &SimpleToken{lastByteIndex: 0},
- &SimpleToken{lastByteIndex: 1},
- &SimpleToken{lastByteIndex: 2},
+ &SimpleToken{firstByteIndex: 0, lastByteIndex: 0},
+ &SimpleToken{firstByteIndex: 1, lastByteIndex: 1},
+ &SimpleToken{firstByteIndex: 2, lastByteIndex: 2},
  },
  currentIndex: 0,
  },

renderer.go

@@ -363,49 +363,79 @@ func (r *Renderer) lex(lexer Lexer, input string, startLine int) {
 tokenLoop:
  for {
  token, ok := lexer.Next()
- if !ok {
+ var currentFirstByteIndex istrings.ByteNumber
+ var currentLastByteIndex istrings.ByteNumber
+ var tokenColor Color
+ var noToken bool
+ if ok {
+ currentFirstByteIndex = token.FirstByteIndex()
+ currentLastByteIndex = token.LastByteIndex()
+ tokenColor = token.Color()
+ } else if previousByteIndex == istrings.Len(input)-1 {
  break tokenLoop
+ } else {
+ currentFirstByteIndex = istrings.Len(input)
+ tokenColor = DefaultColor
+ noToken = true
  }
 
- currentByteIndex := token.LastByteIndex()
- text := input[previousByteIndex+1 : currentByteIndex+1]
- previousByteIndex = currentByteIndex
+ color := DefaultColor
+ text := input[previousByteIndex+1 : currentFirstByteIndex]
+ previousByteIndex = currentLastByteIndex
  lineBuffer = lineBuffer[:0]
-
- charLoop:
- for _, char := range text {
- if lineCharIndex >= col || char == '\n' {
- lineNumber++
- lineCharIndex = 0
- if lineNumber-1 < startLine {
+ interToken := true
+
+ repeatLoop:
+ for {
+
+ charLoop:
+ for _, char := range text {
+ if lineCharIndex >= col || char == '\n' {
+ lineNumber++
+ lineCharIndex = 0
+ if lineNumber-1 < startLine {
+ continue charLoop
+ }
+ if lineNumber >= endLine {
+ break tokenLoop
+ }
+ lineBuffer = append(lineBuffer, '\n')
+ r.writeColor(lineBuffer, color)
+ r.renderPrefix(multilinePrefix)
+ lineBuffer = lineBuffer[:0]
+ if char != '\n' {
+ size := utf8.EncodeRune(runeBuffer, char)
+ lineBuffer = append(lineBuffer, runeBuffer[:size]...)
+ lineCharIndex += istrings.GetRuneWidth(char)
+ }
  continue charLoop
  }
- if lineNumber >= endLine {
- break tokenLoop
- }
- lineBuffer = append(lineBuffer, '\n')
- r.writeColor(lineBuffer, token.Color())
- r.renderPrefix(multilinePrefix)
- lineBuffer = lineBuffer[:0]
- if char != '\n' {
- size := utf8.EncodeRune(runeBuffer, char)
- lineBuffer = append(lineBuffer, runeBuffer[:size]...)
- lineCharIndex += istrings.GetRuneWidth(char)
+
+ lineCharIndex += istrings.GetRuneWidth(char)
+ if lineNumber < startLine {
+ continue charLoop
  }
- continue charLoop
+ size := utf8.EncodeRune(runeBuffer, char)
+ lineBuffer = append(lineBuffer, runeBuffer[:size]...)
+ }
+ if len(lineBuffer) > 0 {
+ r.writeColor(lineBuffer, color)
  }
 
- lineCharIndex += istrings.GetRuneWidth(char)
- if lineNumber < startLine {
- continue charLoop
+ if !interToken {
+ break repeatLoop
  }
- size := utf8.EncodeRune(runeBuffer, char)
- lineBuffer = append(lineBuffer, runeBuffer[:size]...)
- }
- if len(lineBuffer) > 0 {
- r.writeColor(lineBuffer, token.Color())
+
+ if noToken {
+ break tokenLoop
+ }
+ color = tokenColor
+ text = input[currentFirstByteIndex : currentLastByteIndex+1]
+ lineBuffer = lineBuffer[:0]
+ interToken = false
  }
  }
+
 }
 
 // BreakLine to break line.