Merge pull request #15 from elk-language/feature/support_grapheme_clu…

…sters

Fix horizontal cursor movements when there are grapheme clusters

CHANGELOG.md

@@ -5,6 +5,60 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.0] - 28.07.2023
+
+[Diff](https://github.com/elk-language/go-prompt/compare/v1.0.3...elk-language:go-prompt:v1.1.0)
+
+### Fixed
+- Fix cursor movement for text with grapheme clusters like 🇵🇱, 🙆🏿‍♂️
+
+### Added
+- Add `strings.GraphemeNumber`, a type that represents the amount of grapheme clusters in a string (or an offset of a grapheme cluster in a string)
+  - `type strings.GraphemeNumber int`
+- `func strings.GraphemeCountInString(text string) strings.GraphemeNumber`
+- `func strings.RuneCountInString(s string) strings.RuneNumber`
+- `func strings.RuneIndexNthGrapheme(text string, n strings.GraphemeNumber) strings.RuneNumber`
+- `func strings.RuneIndexNthColumn(text string, n strings.Width) strings.RuneNumber`
+- `func (*prompt.Document).GetCursorLeftPositionRunes(count strings.RuneNumber) strings.RuneNumber`
+- `func (*prompt.Document).GetCursorRightPositionRunes(count strings.RuneNumber) strings.RuneNumber`
+- `func (*prompt.Document).LastLineIndentLevel(indentSize int) int`
+- `func (*prompt.Document).LastLineIndentSpaces() int`
+- `func (*prompt.Buffer).DeleteRunes(count strings.RuneNumber, col strings.Width, row int) string`
+- `func (*prompt.Buffer).DeleteBeforeCursorRunes(count strings.RuneNumber, col strings.Width, row int) string`
+
+### Changed
+- Change signatures:
+  - `strings.RuneCount`
+    - from `func strings.RuneCount(s string) strings.RuneNumber`
+    - to `func strings.RuneCount(b []byte) strings.RuneNumber`
+  - `prompt.ExecuteOnEnterCallback`
+    - from `func(input string, indentSize int) (indent int, execute bool)`
+    - to `func(buffer *prompt.Buffer, indentSize int) (indent int, execute bool)`
+  - `(*prompt.Document).CursorPositionCol`
+    - from `func (*prompt.Document).CursorPositionCol() (col strings.RuneNumber)`
+    - to `func (*prompt.Document).CursorPositionCol() (col strings.Width)`
+  - `(*prompt.Document).GetCursorRightPosition`
+    - from `func (*prompt.Document).GetCursorRightPosition(count strings.RuneNumber) strings.RuneNumber`
+    - to `func (*prompt.Document).GetCursorRightPosition(count strings.Width) strings.RuneNumber`
+  - `(*prompt.Document).GetCursorLeftPosition`
+    - from `func (*prompt.Document).GetCursorLeftPosition(count strings.RuneNumber) strings.RuneNumber`
+    - to `func (*prompt.Document).GetCursorLeftPosition(count strings.Width) strings.RuneNumber`
+  - `(*prompt.Document).GetCursorUpPosition`
+    - from `func (*prompt.Document).GetCursorUpPosition(count int, preferredColumn strings.RuneNumber) strings.RuneNumber`
+    - to `func (*prompt.Document).GetCursorUpPosition(count int, preferredColumn strings.Width) strings.RuneNumber`
+  - `(*prompt.Document).GetCursorDownPosition`
+    - from `func (*prompt.Document).GetCursorDownPosition(count int, preferredColumn strings.RuneNumber) strings.RuneNumber`
+    - to `func (*prompt.Document).GetCursorDownPosition(count int, preferredColumn strings.Width) strings.RuneNumber`
+  - `(*prompt.Document).TranslateRowColToIndex`
+    - from `func (*prompt.Document).TranslateRowColToIndex(row int, column strings.RuneNumber) strings.RuneNumber`
+    - to `func (*prompt.Document).TranslateRowColToIndex(row int, column strings.Width) strings.RuneNumber`
+  - `(*prompt.Buffer).Delete`
+    - from `func (*Buffer).Delete(count istrings.RuneNumber, col istrings.Width, row int) string`
+    - to `func (*Buffer).Delete(count istrings.GraphemeNumber, col istrings.Width, row int) string`
+  - `(*prompt.Buffer).DeleteBeforeCursor`
+    - from `func (*Buffer).DeleteBeforeCursor(count istrings.RuneNumber, col istrings.Width, row int) string`
+    - to `func (*Buffer).DeleteBeforeCursor(count istrings.GraphemeNumber, col istrings.Width, row int) string`
+
 ## [1.0.3] - 25.07.2023
 
 [Diff](https://github.com/elk-language/go-prompt/compare/v1.0.2...elk-language:go-prompt:v1.0.3)

_example/automatic-indenter/main.go

@@ -18,7 +18,8 @@ func main() {
 	p.Run()
 }
 
-func ExecuteOnEnter(input string, indentSize int) (int, bool) {
+func ExecuteOnEnter(buffer *prompt.Buffer, indentSize int) (int, bool) {
+	input := buffer.Text()
 	lines := strings.SplitAfter(input, "\n")
 	var spaces int
 	if len(lines) > 0 {

_example/bang-executor/main.go

@@ -17,7 +17,8 @@ func main() {
 	p.Run()
 }
 
-func ExecuteOnEnter(input string, indentSize int) (int, bool) {
+func ExecuteOnEnter(buffer *prompt.Buffer, indentSize int) (int, bool) {
+	input := buffer.Text()
 	char, _ := utf8.DecodeLastRuneInString(input)
 	return 0, char == '!'
 }

_example/http-prompt/main.go

@@ -164,7 +164,7 @@ func completer(in prompt.Document) ([]prompt.Suggest, istrings.RuneNumber, istri
 	if w == "" {
 		return []prompt.Suggest{}, 0, 0
 	}
-	startIndex := endIndex - istrings.RuneCount(w)
+	startIndex := endIndex - istrings.RuneCountInString(w)
 	return prompt.FilterHasPrefix(suggestions, w, true), startIndex, endIndex
 }
 

_example/live-prefix/main.go

@@ -21,7 +21,7 @@ func executor(in string) {
 func completer(in prompt.Document) ([]prompt.Suggest, istrings.RuneNumber, istrings.RuneNumber) {
 	endIndex := in.CurrentRuneIndex()
 	w := in.GetWordBeforeCursor()
-	startIndex := endIndex - istrings.RuneCount(w)
+	startIndex := endIndex - istrings.RuneCountInString(w)
 
 	s := []prompt.Suggest{
 		{Text: "users", Description: "Store the username and age"},

_example/simple-echo/cjk-cyrillic/main.go

@@ -4,20 +4,25 @@ import (
 	"fmt"
 
 	prompt "github.com/elk-language/go-prompt"
+	pstrings "github.com/elk-language/go-prompt/strings"
 )
 
 func executor(in string) {
 	fmt.Println("Your input: " + in)
 }
 
-func completer(in prompt.Document) []prompt.Suggest {
+func completer(in prompt.Document) ([]prompt.Suggest, pstrings.RuneNumber, pstrings.RuneNumber) {
 	s := []prompt.Suggest{
 		{Text: "こんにちは", Description: "'こんにちは' means 'Hello' in Japanese"},
 		{Text: "감사합니다", Description: "'안녕하세요' means 'Hello' in Korean."},
 		{Text: "您好", Description: "'您好' means 'Hello' in Chinese."},
 		{Text: "Добрый день", Description: "'Добрый день' means 'Hello' in Russian."},
 	}
-	return prompt.FilterHasPrefix(s, in.GetWordBeforeCursor(), true)
+	endIndex := in.CurrentRuneIndex()
+	w := in.GetWordBeforeCursor()
+	startIndex := endIndex - pstrings.RuneCountInString(w)
+
+	return prompt.FilterHasPrefix(s, w, true), startIndex, endIndex
 }
 
 func main() {

_example/simple-echo/main.go

@@ -4,16 +4,21 @@ import (
 	"fmt"
 
 	prompt "github.com/elk-language/go-prompt"
+	pstrings "github.com/elk-language/go-prompt/strings"
 )
 
-func completer(in prompt.Document) []prompt.Suggest {
+func completer(in prompt.Document) ([]prompt.Suggest, pstrings.RuneNumber, pstrings.RuneNumber) {
 	s := []prompt.Suggest{
 		{Text: "users", Description: "Store the username and age"},
 		{Text: "articles", Description: "Store the article text posted by user"},
 		{Text: "comments", Description: "Store the text commented to articles"},
 		{Text: "groups", Description: "Combine users with specific rules"},
 	}
-	return prompt.FilterHasPrefix(s, in.GetWordBeforeCursor(), true)
+	endIndex := in.CurrentRuneIndex()
+	w := in.GetWordBeforeCursor()
+	startIndex := endIndex - pstrings.RuneCountInString(w)
+
+	return prompt.FilterHasPrefix(s, w, true), startIndex, endIndex
 }
 
 func main() {
@@ -22,7 +27,6 @@ func main() {
 		prompt.WithTitle("sql-prompt"),
 		prompt.WithHistory([]string{"SELECT * FROM users;"}),
 		prompt.WithPrefixTextColor(prompt.Yellow),
-		prompt.WithPreviewSuggestionTextColor(prompt.Blue),
 		prompt.WithSelectedSuggestionBGColor(prompt.LightGray),
 		prompt.WithSuggestionBGColor(prompt.DarkGray),
 		prompt.WithCompleter(completer),

buffer.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/elk-language/go-prompt/debug"
 	istrings "github.com/elk-language/go-prompt/strings"
+	"golang.org/x/exp/utf8string"
 )
 
 // Buffer emulates the console buffer.
@@ -14,7 +15,7 @@ type Buffer struct {
 	startLine       int      // Line number of the first visible line in the terminal (0-indexed)
 	cursorPosition  istrings.RuneNumber
 	cacheDocument   *Document
-	preferredColumn istrings.RuneNumber // Remember the original column for the next up/down movement.
+	preferredColumn istrings.Width // Remember the original column for the next up/down movement.
 	lastKeyStroke   Key
 }
 
@@ -69,13 +70,13 @@ func (b *Buffer) insertText(text string, columns istrings.Width, rows int, overw
 	if overwrite {
 		overwritten := string(currentTextRunes[cursor:])
 		if len(overwritten) >= int(cursor)+len(text) {
-			overwritten = string(currentTextRunes[cursor : cursor+istrings.RuneCount(text)])
+			overwritten = string(currentTextRunes[cursor : cursor+istrings.RuneCountInString(text)])
 		}
 		if i := strings.IndexAny(overwritten, "\n"); i != -1 {
 			overwritten = overwritten[:i]
 		}
 		b.setText(
-			string(currentTextRunes[:cursor])+text+string(currentTextRunes[cursor+istrings.RuneCount(overwritten):]),
+			string(currentTextRunes[:cursor])+text+string(currentTextRunes[cursor+istrings.RuneCountInString(overwritten):]),
 			columns,
 			rows,
 		)
@@ -88,7 +89,7 @@ func (b *Buffer) insertText(text string, columns istrings.Width, rows int, overw
 	}
 
 	if moveCursor {
-		b.cursorPosition += istrings.RuneCount(text)
+		b.cursorPosition += istrings.RuneCountInString(text)
 		b.recalculateStartLine(columns, rows)
 		b.updatePreferredColumn()
 	}
@@ -118,7 +119,7 @@ func (b *Buffer) recalculateStartLine(columns istrings.Width, rows int) bool {
 // (When doing this, make sure that the cursor_position is valid for this text.
 // text/cursor_position should be consistent at any time, otherwise set a Document instead.)
 func (b *Buffer) setText(text string, col istrings.Width, row int) {
-	debug.Assert(b.cursorPosition <= istrings.RuneCount(text), "length of input should be shorter than cursor position")
+	debug.Assert(b.cursorPosition <= istrings.RuneCountInString(text), "length of input should be shorter than cursor position")
 	b.workingLines[b.workingIndex] = text
 	b.recalculateStartLine(col, row)
 	b.resetPreferredColumn()
@@ -141,20 +142,32 @@ func (b *Buffer) setDocument(d *Document, columns istrings.Width, rows int) {
 	b.resetPreferredColumn()
 }
 
-// Move to the left on the current line.
+// Move to the left on the current line by the given amount of graphemes.
 // Returns true when the view should be rerendered.
-func (b *Buffer) CursorLeft(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
-	l := b.Document().GetCursorLeftPosition(count)
-	b.cursorPosition += l
-	b.updatePreferredColumn()
-	return b.recalculateStartLine(columns, rows)
+func (b *Buffer) CursorLeft(count istrings.GraphemeNumber, columns istrings.Width, rows int) bool {
+	return b.cursorHorizontalMove(b.Document().GetCursorLeftPosition(count), columns, rows)
+}
+
+// Move to the left on the current line by the given amount of runes.
+// Returns true when the view should be rerendered.
+func (b *Buffer) CursorLeftRunes(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
+	return b.cursorHorizontalMove(b.Document().GetCursorLeftPositionRunes(count), columns, rows)
+}
+
+// Move to the right on the current line by the given amount of graphemes.
+// Returns true when the view should be rerendered.
+func (b *Buffer) CursorRight(count istrings.GraphemeNumber, columns istrings.Width, rows int) bool {
+	return b.cursorHorizontalMove(b.Document().GetCursorRightPosition(count), columns, rows)
 }
 
-// Move to the right on the current line.
+// Move to the right on the current line by the given amount of runes.
 // Returns true when the view should be rerendered.
-func (b *Buffer) CursorRight(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
-	l := b.Document().GetCursorRightPosition(count)
-	b.cursorPosition += l
+func (b *Buffer) CursorRightRunes(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
+	return b.cursorHorizontalMove(b.Document().GetCursorRightPositionRunes(count), columns, rows)
+}
+
+func (b *Buffer) cursorHorizontalMove(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
+	b.cursorPosition += count
 	b.updatePreferredColumn()
 	return b.recalculateStartLine(columns, rows)
 }
@@ -175,22 +188,59 @@ func (b *Buffer) CursorDown(count int, columns istrings.Width, rows int) bool {
 	return b.recalculateStartLine(columns, rows)
 }
 
-// DeleteBeforeCursor delete specified number of characters before cursor and return the deleted text.
-func (b *Buffer) DeleteBeforeCursor(count istrings.RuneNumber, columns istrings.Width, rows int) (deleted string) {
+// DeleteBeforeCursor delete specified number of graphemes before the cursor and returns the deleted text.
+func (b *Buffer) DeleteBeforeCursor(count istrings.GraphemeNumber, columns istrings.Width, rows int) string {
+	debug.Assert(count >= 0, "count should be positive")
+	if b.cursorPosition < 0 {
+		return ""
+	}
+
+	var deleted string
+
+	textUtf8 := utf8string.NewString(b.Text())
+	textBeforeCursor := textUtf8.Slice(0, int(b.cursorPosition))
+	graphemeLength := istrings.GraphemeCountInString(textBeforeCursor)
+
+	start := istrings.RuneIndexNthGrapheme(textBeforeCursor, graphemeLength-count)
+	if start < 0 {
+		start = 0
+	}
+	deleted = textUtf8.Slice(int(start), int(b.cursorPosition))
+	b.setDocument(
+		&Document{
+			Text:           textUtf8.Slice(0, int(start)) + textUtf8.Slice(int(b.cursorPosition), textUtf8.RuneCount()),
+			cursorPosition: b.cursorPosition - istrings.RuneCountInString(deleted),
+		},
+		columns,
+		rows,
+	)
+
+	b.recalculateStartLine(columns, rows)
+	b.updatePreferredColumn()
+	return deleted
+}
+
+// Deletes the specified number of runes before the cursor and returns the deleted text.
+func (b *Buffer) DeleteBeforeCursorRunes(count istrings.RuneNumber, columns istrings.Width, rows int) (deleted string) {
 	debug.Assert(count >= 0, "count should be positive")
+	if b.cursorPosition <= 0 {
+		return ""
+	}
 	r := []rune(b.Text())
 
-	if b.cursorPosition > 0 {
-		start := b.cursorPosition - count
-		if start < 0 {
-			start = 0
-		}
-		deleted = string(r[start:b.cursorPosition])
-		b.setDocument(&Document{
+	start := b.cursorPosition - count
+	if start < 0 {
+		start = 0
+	}
+	deleted = string(r[start:b.cursorPosition])
+	b.setDocument(
+		&Document{
 			Text:           string(r[:start]) + string(r[b.cursorPosition:]),
 			cursorPosition: b.cursorPosition - istrings.RuneNumber(len([]rune(deleted))),
-		}, columns, rows)
-	}
+		},
+		columns,
+		rows,
+	)
 	b.recalculateStartLine(columns, rows)
 	b.updatePreferredColumn()
 	return
@@ -205,8 +255,30 @@ func (b *Buffer) NewLine(columns istrings.Width, rows int, copyMargin bool) {
 	}
 }
 
-// Delete specified number of characters and Return the deleted text.
-func (b *Buffer) Delete(count istrings.RuneNumber, col istrings.Width, row int) string {
+// Deletes the specified number of graphemes and returns the deleted text.
+func (b *Buffer) Delete(count istrings.GraphemeNumber, col istrings.Width, row int) string {
+	textUtf8 := utf8string.NewString(b.Text())
+	if b.cursorPosition >= istrings.RuneCountInString(b.Text()) {
+		return ""
+	}
+
+	textAfterCursor := b.Document().TextAfterCursor()
+	textAfterCursorUtf8 := utf8string.NewString(textAfterCursor)
+
+	deletedRunes := textAfterCursorUtf8.Slice(0, int(istrings.RuneIndexNthGrapheme(textAfterCursor, count)))
+
+	b.setText(
+		textUtf8.Slice(0, int(b.cursorPosition))+textUtf8.Slice(int(b.cursorPosition)+int(istrings.RuneCountInString(deletedRunes)), textUtf8.RuneCount()),
+		col,
+		row,
+	)
+
+	deleted := string(deletedRunes)
+	return deleted
+}
+
+// Deletes the specified number of runes and returns the deleted text.
+func (b *Buffer) DeleteRunes(count istrings.RuneNumber, col istrings.Width, row int) string {
 	r := []rune(b.Text())
 	if b.cursorPosition < istrings.RuneNumber(len(r)) {
 		textAfterCursor := b.Document().TextAfterCursor()