Merge pull request #15 from elk-language/feature/support_grapheme_clu…

…sters

Fix horizontal cursor movements when there are grapheme clusters

CHANGELOG.md

@@ -5,6 +5,60 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.0] - 28.07.2023
+
+[Diff](https://github.com/elk-language/go-prompt/compare/v1.0.3...elk-language:go-prompt:v1.1.0)
+
+### Fixed
+- Fix cursor movement for text with grapheme clusters like 🇵🇱, 🙆🏿‍♂️
+
+### Added
+- Add `strings.GraphemeNumber`, a type that represents the amount of grapheme clusters in a string (or an offset of a grapheme cluster in a string)
+ - `type strings.GraphemeNumber int`
+- `func strings.GraphemeCountInString(text string) strings.GraphemeNumber`
+- `func strings.RuneCountInString(s string) strings.RuneNumber`
+- `func strings.RuneIndexNthGrapheme(text string, n strings.GraphemeNumber) strings.RuneNumber`
+- `func strings.RuneIndexNthColumn(text string, n strings.Width) strings.RuneNumber`
+- `func (*prompt.Document).GetCursorLeftPositionRunes(count strings.RuneNumber) strings.RuneNumber`
+- `func (*prompt.Document).GetCursorRightPositionRunes(count strings.RuneNumber) strings.RuneNumber`
+- `func (*prompt.Document).LastLineIndentLevel(indentSize int) int`
+- `func (*prompt.Document).LastLineIndentSpaces() int`
+- `func (*prompt.Buffer).DeleteRunes(count strings.RuneNumber, col strings.Width, row int) string`
+- `func (*prompt.Buffer).DeleteBeforeCursorRunes(count strings.RuneNumber, col strings.Width, row int) string`
+
+### Changed
+- Change signatures:
+ - `strings.RuneCount`
+ - from `func strings.RuneCount(s string) strings.RuneNumber`
+ - to `func strings.RuneCount(b []byte) strings.RuneNumber`
+ - `prompt.ExecuteOnEnterCallback`
+ - from `func(input string, indentSize int) (indent int, execute bool)`
+ - to `func(buffer *prompt.Buffer, indentSize int) (indent int, execute bool)`
+ - `(*prompt.Document).CursorPositionCol`
+ - from `func (*prompt.Document).CursorPositionCol() (col strings.RuneNumber)`
+ - to `func (*prompt.Document).CursorPositionCol() (col strings.Width)`
+ - `(*prompt.Document).GetCursorRightPosition`
+ - from `func (*prompt.Document).GetCursorRightPosition(count strings.RuneNumber) strings.RuneNumber`
+ - to `func (*prompt.Document).GetCursorRightPosition(count strings.Width) strings.RuneNumber`
+ - `(*prompt.Document).GetCursorLeftPosition`
+ - from `func (*prompt.Document).GetCursorLeftPosition(count strings.RuneNumber) strings.RuneNumber`
+ - to `func (*prompt.Document).GetCursorLeftPosition(count strings.Width) strings.RuneNumber`
+ - `(*prompt.Document).GetCursorUpPosition`
+ - from `func (*prompt.Document).GetCursorUpPosition(count int, preferredColumn strings.RuneNumber) strings.RuneNumber`
+ - to `func (*prompt.Document).GetCursorUpPosition(count int, preferredColumn strings.Width) strings.RuneNumber`
+ - `(*prompt.Document).GetCursorDownPosition`
+ - from `func (*prompt.Document).GetCursorDownPosition(count int, preferredColumn strings.RuneNumber) strings.RuneNumber`
+ - to `func (*prompt.Document).GetCursorDownPosition(count int, preferredColumn strings.Width) strings.RuneNumber`
+ - `(*prompt.Document).TranslateRowColToIndex`
+ - from `func (*prompt.Document).TranslateRowColToIndex(row int, column strings.RuneNumber) strings.RuneNumber`
+ - to `func (*prompt.Document).TranslateRowColToIndex(row int, column strings.Width) strings.RuneNumber`
+ - `(*prompt.Buffer).Delete`
+ - from `func (*Buffer).Delete(count istrings.RuneNumber, col istrings.Width, row int) string`
+ - to `func (*Buffer).Delete(count istrings.GraphemeNumber, col istrings.Width, row int) string`
+ - `(*prompt.Buffer).DeleteBeforeCursor`
+ - from `func (*Buffer).DeleteBeforeCursor(count istrings.RuneNumber, col istrings.Width, row int) string`
+ - to `func (*Buffer).DeleteBeforeCursor(count istrings.GraphemeNumber, col istrings.Width, row int) string`
+
 ## [1.0.3] - 25.07.2023
 
 [Diff](https://github.com/elk-language/go-prompt/compare/v1.0.2...elk-language:go-prompt:v1.0.3)

_example/automatic-indenter/main.go

@@ -18,7 +18,8 @@ func main() {
  p.Run()
 }
 
-func ExecuteOnEnter(input string, indentSize int) (int, bool) {
+func ExecuteOnEnter(buffer *prompt.Buffer, indentSize int) (int, bool) {
+ input := buffer.Text()
  lines := strings.SplitAfter(input, "\n")
  var spaces int
  if len(lines) > 0 {

_example/bang-executor/main.go

@@ -17,7 +17,8 @@ func main() {
  p.Run()
 }
 
-func ExecuteOnEnter(input string, indentSize int) (int, bool) {
+func ExecuteOnEnter(buffer *prompt.Buffer, indentSize int) (int, bool) {
+ input := buffer.Text()
  char, _ := utf8.DecodeLastRuneInString(input)
  return 0, char == '!'
 }

_example/http-prompt/main.go

@@ -164,7 +164,7 @@ func completer(in prompt.Document) ([]prompt.Suggest, istrings.RuneNumber, istri
  if w == "" {
  return []prompt.Suggest{}, 0, 0
  }
- startIndex := endIndex - istrings.RuneCount(w)
+ startIndex := endIndex - istrings.RuneCountInString(w)
  return prompt.FilterHasPrefix(suggestions, w, true), startIndex, endIndex
 }
 

_example/live-prefix/main.go

@@ -21,7 +21,7 @@ func executor(in string) {
 func completer(in prompt.Document) ([]prompt.Suggest, istrings.RuneNumber, istrings.RuneNumber) {
  endIndex := in.CurrentRuneIndex()
  w := in.GetWordBeforeCursor()
- startIndex := endIndex - istrings.RuneCount(w)
+ startIndex := endIndex - istrings.RuneCountInString(w)
 
  s := []prompt.Suggest{
  {Text: "users", Description: "Store the username and age"},

_example/simple-echo/cjk-cyrillic/main.go

@@ -4,20 +4,25 @@ import (
  "fmt"
 
  prompt "github.com/elk-language/go-prompt"
+ pstrings "github.com/elk-language/go-prompt/strings"
 )
 
 func executor(in string) {
  fmt.Println("Your input: " + in)
 }
 
-func completer(in prompt.Document) []prompt.Suggest {
+func completer(in prompt.Document) ([]prompt.Suggest, pstrings.RuneNumber, pstrings.RuneNumber) {
  s := []prompt.Suggest{
  {Text: "こんにちは", Description: "'こんにちは' means 'Hello' in Japanese"},
  {Text: "감사합니다", Description: "'안녕하세요' means 'Hello' in Korean."},
  {Text: "您好", Description: "'您好' means 'Hello' in Chinese."},
  {Text: "Добрый день", Description: "'Добрый день' means 'Hello' in Russian."},
  }
- return prompt.FilterHasPrefix(s, in.GetWordBeforeCursor(), true)
+ endIndex := in.CurrentRuneIndex()
+ w := in.GetWordBeforeCursor()
+ startIndex := endIndex - pstrings.RuneCountInString(w)
+
+ return prompt.FilterHasPrefix(s, w, true), startIndex, endIndex
 }
 
 func main() {

_example/simple-echo/main.go

@@ -4,16 +4,21 @@ import (
  "fmt"
 
  prompt "github.com/elk-language/go-prompt"
+ pstrings "github.com/elk-language/go-prompt/strings"
 )
 
-func completer(in prompt.Document) []prompt.Suggest {
+func completer(in prompt.Document) ([]prompt.Suggest, pstrings.RuneNumber, pstrings.RuneNumber) {
  s := []prompt.Suggest{
  {Text: "users", Description: "Store the username and age"},
  {Text: "articles", Description: "Store the article text posted by user"},
  {Text: "comments", Description: "Store the text commented to articles"},
  {Text: "groups", Description: "Combine users with specific rules"},
  }
- return prompt.FilterHasPrefix(s, in.GetWordBeforeCursor(), true)
+ endIndex := in.CurrentRuneIndex()
+ w := in.GetWordBeforeCursor()
+ startIndex := endIndex - pstrings.RuneCountInString(w)
+
+ return prompt.FilterHasPrefix(s, w, true), startIndex, endIndex
 }
 
 func main() {
@@ -22,7 +27,6 @@ func main() {
  prompt.WithTitle("sql-prompt"),
  prompt.WithHistory([]string{"SELECT * FROM users;"}),
  prompt.WithPrefixTextColor(prompt.Yellow),
- prompt.WithPreviewSuggestionTextColor(prompt.Blue),
  prompt.WithSelectedSuggestionBGColor(prompt.LightGray),
  prompt.WithSuggestionBGColor(prompt.DarkGray),
  prompt.WithCompleter(completer),

buffer.go

@@ -5,6 +5,7 @@ import (
 
  "github.com/elk-language/go-prompt/debug"
  istrings "github.com/elk-language/go-prompt/strings"
+ "golang.org/x/exp/utf8string"
 )
 
 // Buffer emulates the console buffer.
@@ -14,7 +15,7 @@ type Buffer struct {
  startLine int // Line number of the first visible line in the terminal (0-indexed)
  cursorPosition istrings.RuneNumber
  cacheDocument *Document
- preferredColumn istrings.RuneNumber // Remember the original column for the next up/down movement.
+ preferredColumn istrings.Width // Remember the original column for the next up/down movement.
  lastKeyStroke Key
 }
 
@@ -69,13 +70,13 @@ func (b *Buffer) insertText(text string, columns istrings.Width, rows int, overw
  if overwrite {
  overwritten := string(currentTextRunes[cursor:])
  if len(overwritten) >= int(cursor)+len(text) {
- overwritten = string(currentTextRunes[cursor : cursor+istrings.RuneCount(text)])
+ overwritten = string(currentTextRunes[cursor : cursor+istrings.RuneCountInString(text)])
  }
  if i := strings.IndexAny(overwritten, "\n"); i != -1 {
  overwritten = overwritten[:i]
  }
  b.setText(
- string(currentTextRunes[:cursor])+text+string(currentTextRunes[cursor+istrings.RuneCount(overwritten):]),
+ string(currentTextRunes[:cursor])+text+string(currentTextRunes[cursor+istrings.RuneCountInString(overwritten):]),
  columns,
  rows,
  )
@@ -88,7 +89,7 @@ func (b *Buffer) insertText(text string, columns istrings.Width, rows int, overw
  }
 
  if moveCursor {
- b.cursorPosition += istrings.RuneCount(text)
+ b.cursorPosition += istrings.RuneCountInString(text)
  b.recalculateStartLine(columns, rows)
  b.updatePreferredColumn()
  }
@@ -118,7 +119,7 @@ func (b *Buffer) recalculateStartLine(columns istrings.Width, rows int) bool {
 // (When doing this, make sure that the cursor_position is valid for this text.
 // text/cursor_position should be consistent at any time, otherwise set a Document instead.)
 func (b *Buffer) setText(text string, col istrings.Width, row int) {
- debug.Assert(b.cursorPosition <= istrings.RuneCount(text), "length of input should be shorter than cursor position")
+ debug.Assert(b.cursorPosition <= istrings.RuneCountInString(text), "length of input should be shorter than cursor position")
  b.workingLines[b.workingIndex] = text
  b.recalculateStartLine(col, row)
  b.resetPreferredColumn()
@@ -141,20 +142,32 @@ func (b *Buffer) setDocument(d *Document, columns istrings.Width, rows int) {
  b.resetPreferredColumn()
 }
 
-// Move to the left on the current line.
+// Move to the left on the current line by the given amount of graphemes.
 // Returns true when the view should be rerendered.
-func (b *Buffer) CursorLeft(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
- l := b.Document().GetCursorLeftPosition(count)
- b.cursorPosition += l
- b.updatePreferredColumn()
- return b.recalculateStartLine(columns, rows)
+func (b *Buffer) CursorLeft(count istrings.GraphemeNumber, columns istrings.Width, rows int) bool {
+ return b.cursorHorizontalMove(b.Document().GetCursorLeftPosition(count), columns, rows)
+}
+
+// Move to the left on the current line by the given amount of runes.
+// Returns true when the view should be rerendered.
+func (b *Buffer) CursorLeftRunes(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
+ return b.cursorHorizontalMove(b.Document().GetCursorLeftPositionRunes(count), columns, rows)
+}
+
+// Move to the right on the current line by the given amount of graphemes.
+// Returns true when the view should be rerendered.
+func (b *Buffer) CursorRight(count istrings.GraphemeNumber, columns istrings.Width, rows int) bool {
+ return b.cursorHorizontalMove(b.Document().GetCursorRightPosition(count), columns, rows)
 }
 
-// Move to the right on the current line.
+// Move to the right on the current line by the given amount of runes.
 // Returns true when the view should be rerendered.
-func (b *Buffer) CursorRight(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
- l := b.Document().GetCursorRightPosition(count)
- b.cursorPosition += l
+func (b *Buffer) CursorRightRunes(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
+ return b.cursorHorizontalMove(b.Document().GetCursorRightPositionRunes(count), columns, rows)
+}
+
+func (b *Buffer) cursorHorizontalMove(count istrings.RuneNumber, columns istrings.Width, rows int) bool {
+ b.cursorPosition += count
  b.updatePreferredColumn()
  return b.recalculateStartLine(columns, rows)
 }
@@ -175,22 +188,59 @@ func (b *Buffer) CursorDown(count int, columns istrings.Width, rows int) bool {
  return b.recalculateStartLine(columns, rows)
 }
 
-// DeleteBeforeCursor delete specified number of characters before cursor and return the deleted text.
-func (b *Buffer) DeleteBeforeCursor(count istrings.RuneNumber, columns istrings.Width, rows int) (deleted string) {
+// DeleteBeforeCursor delete specified number of graphemes before the cursor and returns the deleted text.
+func (b *Buffer) DeleteBeforeCursor(count istrings.GraphemeNumber, columns istrings.Width, rows int) string {
+ debug.Assert(count >= 0, "count should be positive")
+ if b.cursorPosition < 0 {
+ return ""
+ }
+
+ var deleted string
+
+ textUtf8 := utf8string.NewString(b.Text())
+ textBeforeCursor := textUtf8.Slice(0, int(b.cursorPosition))
+ graphemeLength := istrings.GraphemeCountInString(textBeforeCursor)
+
+ start := istrings.RuneIndexNthGrapheme(textBeforeCursor, graphemeLength-count)
+ if start < 0 {
+ start = 0
+ }
+ deleted = textUtf8.Slice(int(start), int(b.cursorPosition))
+ b.setDocument(
+ &Document{
+ Text: textUtf8.Slice(0, int(start)) + textUtf8.Slice(int(b.cursorPosition), textUtf8.RuneCount()),
+ cursorPosition: b.cursorPosition - istrings.RuneCountInString(deleted),
+ },
+ columns,
+ rows,
+ )
+
+ b.recalculateStartLine(columns, rows)
+ b.updatePreferredColumn()
+ return deleted
+}
+
+// Deletes the specified number of runes before the cursor and returns the deleted text.
+func (b *Buffer) DeleteBeforeCursorRunes(count istrings.RuneNumber, columns istrings.Width, rows int) (deleted string) {
  debug.Assert(count >= 0, "count should be positive")
+ if b.cursorPosition <= 0 {
+ return ""
+ }
  r := []rune(b.Text())
 
- if b.cursorPosition > 0 {
-  start := b.cursorPosition - count
- if start < 0 {
-  start = 0
-  }
-  deleted = string(r[start:b.cursorPosition])
- b.setDocument(&Document{
+ start := b.cursorPosition - count
+ if start < 0 {
+ start = 0
+ }
+ deleted = string(r[start:b.cursorPosition])
+ b.setDocument(
+ &Document{
  Text: string(r[:start]) + string(r[b.cursorPosition:]),
  cursorPosition: b.cursorPosition - istrings.RuneNumber(len([]rune(deleted))),
- }, columns, rows)
- }
+ },
+ columns,
+ rows,
+ )
  b.recalculateStartLine(columns, rows)
  b.updatePreferredColumn()
  return
@@ -205,8 +255,30 @@ func (b *Buffer) NewLine(columns istrings.Width, rows int, copyMargin bool) {
  }
 }
 
-// Delete specified number of characters and Return the deleted text.
-func (b *Buffer) Delete(count istrings.RuneNumber, col istrings.Width, row int) string {
+// Deletes the specified number of graphemes and returns the deleted text.
+func (b *Buffer) Delete(count istrings.GraphemeNumber, col istrings.Width, row int) string {
+ textUtf8 := utf8string.NewString(b.Text())
+ if b.cursorPosition >= istrings.RuneCountInString(b.Text()) {
+ return ""
+ }
+
+ textAfterCursor := b.Document().TextAfterCursor()
+ textAfterCursorUtf8 := utf8string.NewString(textAfterCursor)
+
+ deletedRunes := textAfterCursorUtf8.Slice(0, int(istrings.RuneIndexNthGrapheme(textAfterCursor, count)))
+
+ b.setText(
+ textUtf8.Slice(0, int(b.cursorPosition))+textUtf8.Slice(int(b.cursorPosition)+int(istrings.RuneCountInString(deletedRunes)), textUtf8.RuneCount()),
+ col,
+ row,
+ )
+
+ deleted := string(deletedRunes)
+ return deleted
+}
+
+// Deletes the specified number of runes and returns the deleted text.
+func (b *Buffer) DeleteRunes(count istrings.RuneNumber, col istrings.Width, row int) string {
  r := []rune(b.Text())
  if b.cursorPosition < istrings.RuneNumber(len(r)) {
  textAfterCursor := b.Document().TextAfterCursor()