Add more formatting options for tokens

CHANGELOG.md

@@ -5,8 +5,36 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.2] - 25.07.2023
 
-## [1.0.0]
+[Diff](https://github.com/elk-language/go-prompt/compare/v1.0.1...elk-language:go-prompt:v1.0.2)
+
+### Added
+
+- `prompt.Token` has new methods:
+  - `BackgroundColor() prompt.Color` - define the background color for the token
+  - `DisplayAttributes() []prompt.DisplayAttribute` - define the font eg. bold, italic, underline
+- `prompt.NewSimpleToken` has new options:
+  - `prompt.SimpleTokenWithColor(c Color) SimpleTokenOption`
+  - `prompt.SimpleTokenWithBackgroundColor(c Color) SimpleTokenOption`
+  - `prompt.SimpleTokenWithDisplayAttributes(attrs ...DisplayAttribute) SimpleTokenOption`
+- `prompt.Writer` has new methods:
+  - `prompt.SetDisplayAttributes(fg, bg Color, attrs ...DisplayAttribute)`
+
+### Changed
+
+- change the signature of `prompt.NewSimpleToken` from `func NewSimpleToken(color Color, firstIndex, lastIndex istrings.ByteNumber) *SimpleToken` to `func NewSimpleToken(firstIndex, lastIndex istrings.ByteNumber, opts ...SimpleTokenOption) *SimpleToken`
+
+## [1.0.1] - 25.07.2023
+
+[Diff](https://github.com/elk-language/go-prompt/compare/v1.0.0...elk-language:go-prompt:v1.0.1)
+
+### Added
+
+- `prompt.Token` has a new method `FirstByteIndex() strings.ByteNumber`
+
+
+## [1.0.0] - 25.07.2023
 
 [Diff](https://github.com/elk-language/go-prompt/compare/v0.2.6...elk-language:go-prompt:v1.0.0)
 

_example/even-lexer/main.go

@@ -32,7 +32,11 @@ func charLexer(line string) []prompt.Token {
 			color = prompt.White
 		}
 		lastByteIndex := strings.ByteNumber(i + utf8.RuneLen(value) - 1)
-		element := prompt.NewSimpleToken(color, strings.ByteNumber(i), lastByteIndex)
+		element := prompt.NewSimpleToken(
+			strings.ByteNumber(i),
+			lastByteIndex,
+			prompt.SimpleTokenWithColor(color),
+		)
 
 		elements = append(elements, element)
 	}
@@ -67,7 +71,11 @@ func wordLexer(line string) []prompt.Token {
 				color = prompt.White
 			}
 
-			element := prompt.NewSimpleToken(color, firstByte, currentByte-1)
+			element := prompt.NewSimpleToken(
+				firstByte,
+				currentByte-1,
+				prompt.SimpleTokenWithColor(color),
+			)
 			elements = append(elements, element)
 			wordIndex++
 			firstCharSeen = false
@@ -84,11 +92,14 @@ func wordLexer(line string) []prompt.Token {
 		} else {
 			color = prompt.White
 		}
-		element := prompt.NewSimpleToken(color, firstByte, currentByte+strings.ByteNumber(utf8.RuneLen(lastChar))-1)
+		element := prompt.NewSimpleToken(
+			firstByte,
+			currentByte+strings.ByteNumber(utf8.RuneLen(lastChar))-1,
+			prompt.SimpleTokenWithColor(color),
+		)
 		elements = append(elements, element)
 	}
 
-	prompt.Log("tokens: %#v", elements)
 	return elements
 }
 

buffer.go

@@ -9,19 +9,28 @@ import (
 
 // Buffer emulates the console buffer.
 type Buffer struct {
-	workingLines   []string // The working lines. Similar to history
-	workingIndex   int      // index of the current line
-	startLine      int      // Line number of the first visible line in the terminal (0-indexed)
-	cursorPosition istrings.RuneNumber
-	cacheDocument  *Document
-	lastKeyStroke  Key
+	workingLines    []string // The working lines. Similar to history
+	workingIndex    int      // index of the current line
+	startLine       int      // Line number of the first visible line in the terminal (0-indexed)
+	cursorPosition  istrings.RuneNumber
+	cacheDocument   *Document
+	preferredColumn istrings.RuneNumber // Remember the original column for the next up/down movement.
+	lastKeyStroke   Key
 }
 
 // Text returns string of the current line.
 func (b *Buffer) Text() string {
 	return b.workingLines[b.workingIndex]
 }
 
+func (b *Buffer) resetPreferredColumn() {
+	b.preferredColumn = -1
+}
+
+func (b *Buffer) updatePreferredColumn() {
+	b.preferredColumn = b.Document().CursorPositionCol()
+}
+
 // Document method to return document instance from the current text and cursor position.
 func (b *Buffer) Document() (d *Document) {
 	if b.cacheDocument == nil ||
@@ -80,16 +89,17 @@ func (b *Buffer) insertText(text string, columns istrings.Width, rows int, overw
 
 	if moveCursor {
 		b.cursorPosition += istrings.RuneCount(text)
-		b.RecalculateStartLine(columns, rows)
+		b.recalculateStartLine(columns, rows)
+		b.updatePreferredColumn()
 	}
 }
 
-func (b *Buffer) ResetStartLine() {
+func (b *Buffer) resetStartLine() {
 	b.startLine = 0
 }
 
 // Calculates the startLine once again and returns true when it's been changed.
-func (b *Buffer) RecalculateStartLine(columns istrings.Width, rows int) bool {
+func (b *Buffer) recalculateStartLine(columns istrings.Width, rows int) bool {
 	origStartLine := b.startLine
 	pos := b.DisplayCursorPosition(columns)
 	if pos.Y > b.startLine+rows-1 {
@@ -110,7 +120,8 @@ func (b *Buffer) RecalculateStartLine(columns istrings.Width, rows int) bool {
 func (b *Buffer) setText(text string, col istrings.Width, row int) {
 	debug.Assert(b.cursorPosition <= istrings.RuneCount(text), "length of input should be shorter than cursor position")
 	b.workingLines[b.workingIndex] = text
-	b.RecalculateStartLine(col, row)
+	b.recalculateStartLine(col, row)
+	b.resetPreferredColumn()
 }
 
 // Set cursor position. Return whether it changed.
@@ -126,41 +137,42 @@ func (b *Buffer) setDocument(d *Document, columns istrings.Width, rows int) {
 	b.cacheDocument = d
 	b.setCursorPosition(d.cursorPosition) // Call before setText because setText check the relation between cursorPosition and line length.
 	b.setText(d.Text, columns, rows)
-	b.RecalculateStartLine(columns, rows)
+	b.recalculateStartLine(columns, rows)
+	b.resetPreferredColumn()
 }
 
 // Move to the left on the current line.
 // Returns true when the view should be rerendered.
 func (b *Buffer) CursorLeft(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
 	l := b.Document().GetCursorLeftPosition(count)
 	b.cursorPosition += l
-	return b.RecalculateStartLine(columns, rows)
+	b.updatePreferredColumn()
+	return b.recalculateStartLine(columns, rows)
 }
 
 // Move to the right on the current line.
 // Returns true when the view should be rerendered.
 func (b *Buffer) CursorRight(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
 	l := b.Document().GetCursorRightPosition(count)
 	b.cursorPosition += l
-	return b.RecalculateStartLine(columns, rows)
+	b.updatePreferredColumn()
+	return b.recalculateStartLine(columns, rows)
 }
 
 // CursorUp move cursor to the previous line.
 // (for multi-line edit).
 // Returns true when the view should be rerendered.
 func (b *Buffer) CursorUp(count int, columns istrings.Width, rows int) bool {
-	orig := b.Document().CursorPositionCol()
-	b.cursorPosition += b.Document().GetCursorUpPosition(count, orig)
-	return b.RecalculateStartLine(columns, rows)
+	b.cursorPosition += b.Document().GetCursorUpPosition(count, b.preferredColumn)
+	return b.recalculateStartLine(columns, rows)
 }
 
 // CursorDown move cursor to the next line.
 // (for multi-line edit).
 // Returns true when the view should be rerendered.
 func (b *Buffer) CursorDown(count int, columns istrings.Width, rows int) bool {
-	orig := b.Document().CursorPositionCol()
-	b.cursorPosition += b.Document().GetCursorDownPosition(count, orig)
-	return b.RecalculateStartLine(columns, rows)
+	b.cursorPosition += b.Document().GetCursorDownPosition(count, b.preferredColumn)
+	return b.recalculateStartLine(columns, rows)
 }
 
 // DeleteBeforeCursor delete specified number of characters before cursor and return the deleted text.
@@ -179,7 +191,8 @@ func (b *Buffer) DeleteBeforeCursor(count istrings.RuneNumber, columns istrings.
 			cursorPosition: b.cursorPosition - istrings.RuneNumber(len([]rune(deleted))),
 		}, columns, rows)
 	}
-	b.RecalculateStartLine(columns, rows)
+	b.recalculateStartLine(columns, rows)
+	b.updatePreferredColumn()
 	return
 }
 
@@ -206,7 +219,6 @@ func (b *Buffer) Delete(count istrings.RuneNumber, col istrings.Width, row int)
 		)
 
 		deleted := string(deletedRunes)
-		b.RecalculateStartLine(col, row)
 		return deleted
 	}
 
@@ -243,9 +255,10 @@ func (b *Buffer) SwapCharactersBeforeCursor(col istrings.Width, row int) {
 // NewBuffer is constructor of Buffer struct.
 func NewBuffer() (b *Buffer) {
 	b = &Buffer{
-		workingLines: []string{""},
-		workingIndex: 0,
-		startLine:    0,
+		workingLines:    []string{""},
+		workingIndex:    0,
+		startLine:       0,
+		preferredColumn: -1,
 	}
 	return
 }

lexer.go

@@ -15,32 +15,71 @@ type Lexer interface {
 
 // Token is a single unit of text returned by a Lexer.
 type Token interface {
-	Color() Color                        // Color of the token
+	Color() Color // Color of the token's text
+	BackgroundColor() Color
+	DisplayAttributes() []DisplayAttribute
 	FirstByteIndex() istrings.ByteNumber // Index of the last byte of this token
 	LastByteIndex() istrings.ByteNumber  // Index of the last byte of this token
 }
 
 // SimpleToken as the default implementation of Token.
 type SimpleToken struct {
-	color          Color
-	lastByteIndex  istrings.ByteNumber
-	firstByteIndex istrings.ByteNumber
+	color             Color
+	backgroundColor   Color
+	displayAttributes []DisplayAttribute
+	lastByteIndex     istrings.ByteNumber
+	firstByteIndex    istrings.ByteNumber
+}
+
+type SimpleTokenOption func(*SimpleToken)
+
+func SimpleTokenWithColor(c Color) SimpleTokenOption {
+	return func(t *SimpleToken) {
+		t.color = c
+	}
+}
+
+func SimpleTokenWithBackgroundColor(c Color) SimpleTokenOption {
+	return func(t *SimpleToken) {
+		t.backgroundColor = c
+	}
+}
+
+func SimpleTokenWithDisplayAttributes(attrs ...DisplayAttribute) SimpleTokenOption {
+	return func(t *SimpleToken) {
+		t.displayAttributes = attrs
+	}
 }
 
 // Create a new SimpleToken.
-func NewSimpleToken(color Color, firstIndex, lastIndex istrings.ByteNumber) *SimpleToken {
-	return &SimpleToken{
-		color:          color,
+func NewSimpleToken(firstIndex, lastIndex istrings.ByteNumber, opts ...SimpleTokenOption) *SimpleToken {
+	t := &SimpleToken{
 		firstByteIndex: firstIndex,
 		lastByteIndex:  lastIndex,
 	}
+
+	for _, opt := range opts {
+		opt(t)
+	}
+
+	return t
 }
 
-// Retrieve the color of this token.
+// Retrieve the text color of this token.
 func (t *SimpleToken) Color() Color {
 	return t.color
 }
 
+// Retrieve the background color of this token.
+func (t *SimpleToken) BackgroundColor() Color {
+	return t.backgroundColor
+}
+
+// Retrieve the display attributes of this token eg. bold, underline.
+func (t *SimpleToken) DisplayAttributes() []DisplayAttribute {
+	return t.displayAttributes
+}
+
 // The index of the last byte of the lexeme.
 func (t *SimpleToken) LastByteIndex() istrings.ByteNumber {
 	return t.lastByteIndex

lexer_test.go

@@ -70,7 +70,7 @@ func TestEagerLexerNext(t *testing.T) {
 func charLex(s string) []Token {
 	var result []Token
 	for i := range s {
-		result = append(result, NewSimpleToken(0, istrings.ByteNumber(i), istrings.ByteNumber(i)))
+		result = append(result, NewSimpleToken(istrings.ByteNumber(i), istrings.ByteNumber(i)))
 	}
 
 	return result

prompt.go

@@ -125,8 +125,8 @@ func (p *Prompt) Run() {
 			}
 		case w := <-winSizeCh:
 			p.renderer.UpdateWinSize(w)
-			p.Buffer.ResetStartLine()
-			p.Buffer.RecalculateStartLine(p.renderer.UserInputColumns(), int(p.renderer.row))
+			p.Buffer.resetStartLine()
+			p.Buffer.recalculateStartLine(p.renderer.UserInputColumns(), int(p.renderer.row))
 			p.renderer.Render(p.Buffer, p.completion, p.lexer)
 		case code := <-exitCh:
 			p.renderer.BreakLine(p.Buffer, p.lexer)