Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Reduce memory allocations #18

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Reduce memory allocations #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ed6073d
Use pools in formatter
nineinchnick Mar 21, 2021
268ec57
Repeat with fewer writes
nineinchnick Mar 21, 2021
3294773
Reuse buffer for default encoder
nineinchnick Mar 28, 2021
e5ca7d3
Reuse rowStyles
nineinchnick Mar 28, 2021
ec8fe94
Reuse value slices in formatter
nineinchnick Mar 28, 2021
21cf5ea
Treat filler as runes, not bytes
nineinchnick Mar 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 93 additions & 46 deletions encode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ type TableEncoder struct {
inline bool
// lineStyle is the table line style.
lineStyle LineStyle
// rowStyles is the table of row style.
rowStyles rowStyles
// formatter handles formatting values prior to output.
formatter Formatter
// skipHeader allows to skip drawing header
Expand Down Expand Up @@ -72,6 +74,12 @@ type TableEncoder struct {
scanCount int
// w is the undelying writer
w *bufio.Writer

// scanBuffer for scanning row into
scanBuffer []interface{}

// valsBuffer for prefetching a set of rows to calculate max column width
valsBuffer [][]*Value
}

// NewTableEncoder creates a new table encoder using the provided options.
Expand All @@ -94,6 +102,11 @@ func NewTableEncoder(resultSet ResultSet, opts ...Option) (Encoder, error) {
return nil, err
}
}
enc.rowStyles.Top = enc.lineToRowStyle(enc.lineStyle.Top)
enc.rowStyles.Mid = enc.lineToRowStyle(enc.lineStyle.Mid)
enc.rowStyles.Row = enc.lineToRowStyle(enc.lineStyle.Row)
enc.rowStyles.Wrap = enc.lineToRowStyle(enc.lineStyle.Wrap)
enc.rowStyles.End = enc.lineToRowStyle(enc.lineStyle.End)
// check linestyle runes
// TODO: this check should be removed
for _, l := range [][4]rune{
Expand Down Expand Up @@ -143,6 +156,7 @@ func (enc *TableEncoder) Encode(w io.Writer) error {
if err != nil {
return err
}
enc.initBuffers()
var cmd *exec.Cmd
var cmdBuf io.WriteCloser
for {
Expand All @@ -159,7 +173,9 @@ func (enc *TableEncoder) Encode(w io.Writer) error {
enc.calcWidth(vals)
if enc.minExpandWidth != 0 && enc.tableWidth() >= enc.minExpandWidth {
t := *enc
t.formatter = NewEscapeFormatter()
if f, ok := t.formatter.(*EscapeFormatter); ok {
f.Configure(WithHeaderAlign(AlignLeft))
}
exp := ExpandedEncoder{
TableEncoder: t,
}
Expand Down Expand Up @@ -199,7 +215,7 @@ func (enc *TableEncoder) Encode(w io.Writer) error {
}
// draw end border
if enc.border >= 2 {
enc.divider(enc.rowStyle(enc.lineStyle.End))
enc.divider(&enc.rowStyles.End)
}
}
// add summary
Expand Down Expand Up @@ -235,17 +251,35 @@ func checkErr(err error, cmd *exec.Cmd) error {
return err
}

func (enc *TableEncoder) initBuffers() {
// create buffers for scanning rows and prefetching records
bufSize := enc.count
if bufSize == 0 {
bufSize = 100
}
enc.valsBuffer = make([][]*Value, 0, bufSize)
enc.scanBuffer = make([]interface{}, len(enc.headers))
for i := 0; i < len(enc.headers); i++ {
enc.scanBuffer[i] = new(interface{})
}
}

func (enc *TableEncoder) encodeVals(vals [][]*Value) error {
rs := enc.rowStyle(enc.lineStyle.Row)
rs := enc.rowStyles.Row
// print buffered vals
for i := 0; i < len(vals); i++ {
enc.row(vals[i], rs)
enc.row(vals[i], &rs)
if i+1%1000 == 0 {
// check error every 1k rows
if err := enc.w.Flush(); err != nil {
return err
}
}
for _, v := range vals[i] {
if v != nil {
enc.formatter.Free(v)
}
}
}
return nil
}
Expand All @@ -272,35 +306,27 @@ func (enc *TableEncoder) EncodeAll(w io.Writer) error {
// nextResults reads the next enc.count values,
// or all values if enc.count = 0
func (enc *TableEncoder) nextResults() ([][]*Value, error) {
var vals [][]*Value
if enc.count != 0 {
vals = make([][]*Value, 0, enc.count)
}
// set up storage for results
r := make([]interface{}, len(enc.headers))
for i := 0; i < len(enc.headers); i++ {
r[i] = new(interface{})
}
enc.valsBuffer = enc.valsBuffer[:0]
// read to count (or all)
var i int
for enc.resultSet.Next() {
v, err := enc.scanAndFormat(r)
v, err := enc.scanAndFormat(enc.scanBuffer)
if err != nil {
return vals, err
return enc.valsBuffer, err
}
vals, i = append(vals, v), i+1
enc.valsBuffer, i = append(enc.valsBuffer, v), i+1
// read by batches of enc.count rows
if enc.count != 0 && i%enc.count == 0 {
break
}
}
return vals, nil
return enc.valsBuffer, nil
}

func (enc *TableEncoder) calcWidth(vals [][]*Value) {
// calc offsets and widths for this batch of rows
var offset int
rs := enc.rowStyle(enc.lineStyle.Row)
rs := enc.rowStyles.Row
offset += runewidth.StringWidth(string(rs.left))
for i, h := range enc.headers {
if i != 0 {
Expand All @@ -327,32 +353,32 @@ func (enc *TableEncoder) calcWidth(vals [][]*Value) {
}

func (enc *TableEncoder) header() {
rs := enc.rowStyle(enc.lineStyle.Row)
rs := enc.rowStyles.Row
if enc.title != nil && enc.title.Width != 0 {
maxWidth := ((enc.tableWidth() - enc.title.Width) / 2) + enc.title.Width
enc.writeAligned(enc.title.Buf, rs.filler, AlignRight, maxWidth-enc.title.Width)
enc.writeAligned(enc.title.Buf, &rs, AlignRight, maxWidth-enc.title.Width)
enc.w.Write(enc.newline)
}
// draw top border
if enc.border >= 2 && !enc.inline {
enc.divider(enc.rowStyle(enc.lineStyle.Top))
enc.divider(&enc.rowStyles.Top)
}
// draw the header row with top border style
if enc.inline {
rs = enc.rowStyle(enc.lineStyle.Top)
rs = enc.rowStyles.Top
}
// write header
enc.row(enc.headers, rs)
enc.row(enc.headers, &rs)
if !enc.inline {
// draw mid divider
enc.divider(enc.rowStyle(enc.lineStyle.Mid))
enc.divider(&enc.rowStyles.Mid)
}
}

// rowStyle returns the left, right and midle borders.
// It also profides the filler string, and indicates
// if this style uses a wrapping indicator.
func (enc TableEncoder) rowStyle(r [4]rune) rowStyle {
func (enc TableEncoder) lineToRowStyle(r [4]rune) rowStyle {
var left, right, middle, spacer, filler string
spacer = strings.Repeat(string(r[1]), runewidth.RuneWidth(enc.lineStyle.Row[1]))
filler = string(r[1])
Expand All @@ -378,7 +404,8 @@ func (enc TableEncoder) rowStyle(r [4]rune) rowStyle {
wrapper: []byte(string(enc.lineStyle.Wrap[1])),
middle: []byte(middle),
right: []byte(right + string(enc.newline)),
filler: []byte(filler),
filler: bytes.Repeat([]byte(filler), 8),
fillerWidth: len([]byte(filler)),
hasWrapping: runewidth.RuneWidth(enc.lineStyle.Row[1]) > 0,
}
}
Expand All @@ -396,15 +423,15 @@ func (enc *TableEncoder) scanAndFormat(vals []interface{}) ([]*Value, error) {
}

// divider draws a divider.
func (enc *TableEncoder) divider(rs rowStyle) {
func (enc *TableEncoder) divider(rs *rowStyle) {
// left
enc.w.Write(rs.left)
for i, width := range enc.maxWidths {
// column
enc.w.Write(bytes.Repeat(rs.filler, width))
rs.filler = repeat(enc.w, rs.filler, rs.fillerWidth*width)
// line feed indicator
if rs.hasWrapping && enc.border >= 1 {
enc.w.Write(rs.filler)
enc.w.Write(rs.filler[:rs.fillerWidth])
}
// middle separator
if i != len(enc.maxWidths)-1 {
Expand All @@ -417,7 +444,7 @@ func (enc *TableEncoder) divider(rs rowStyle) {

// tableWidth calculates total table width.
func (enc *TableEncoder) tableWidth() int {
rs := enc.rowStyle(enc.lineStyle.Mid)
rs := enc.rowStyles.Mid
width := runewidth.StringWidth(string(rs.left)) + runewidth.StringWidth(string(rs.right))
for i, w := range enc.maxWidths {
width += w
Expand Down Expand Up @@ -471,7 +498,7 @@ func (enc *TableEncoder) tableHeight(rows [][]*Value) int {
}

// row draws the a table row.
func (enc *TableEncoder) row(vals []*Value, rs rowStyle) {
func (enc *TableEncoder) row(vals []*Value, rs *rowStyle) {
var l int
for {
// left
Expand Down Expand Up @@ -503,18 +530,18 @@ func (enc *TableEncoder) row(vals []*Value, rs rowStyle) {
if enc.border <= 1 && i == len(vals)-1 && (!rs.hasWrapping || l >= len(v.Newlines)) {
padding = 0
}
enc.writeAligned(v.Buf[start:end], rs.filler, v.Align, padding)
enc.writeAligned(v.Buf[start:end], rs, v.Align, padding)
} else {
if enc.border > 1 || i != len(vals)-1 {
enc.w.Write(bytes.Repeat(rs.filler, enc.maxWidths[i]))
rs.filler = repeat(enc.w, rs.filler, rs.fillerWidth*enc.maxWidths[i])
}
}
// write newline wrap value
if rs.hasWrapping {
if l < len(v.Newlines) {
enc.w.Write(rs.wrapper)
} else {
enc.w.Write(rs.filler)
enc.w.Write(rs.filler[:rs.fillerWidth])
}
}
remaining = remaining || l < len(v.Newlines)
Expand All @@ -533,7 +560,7 @@ func (enc *TableEncoder) row(vals []*Value, rs rowStyle) {
}
}

func (enc *TableEncoder) writeAligned(b, filler []byte, a Align, padding int) {
func (enc *TableEncoder) writeAligned(b []byte, rs *rowStyle, a Align, padding int) {
// calc padding
paddingLeft := 0
paddingRight := 0
Expand All @@ -550,13 +577,13 @@ func (enc *TableEncoder) writeAligned(b, filler []byte, a Align, padding int) {
}
// add padding left
if paddingLeft > 0 {
enc.w.Write(bytes.Repeat(filler, paddingLeft))
rs.filler = repeat(enc.w, rs.filler, rs.fillerWidth*paddingLeft)
}
// write
enc.w.Write(b)
// add padding right
if paddingRight > 0 {
enc.w.Write(bytes.Repeat(filler, paddingRight))
rs.filler = repeat(enc.w, rs.filler, rs.fillerWidth*paddingRight)
}
}

Expand Down Expand Up @@ -587,9 +614,14 @@ func (enc *TableEncoder) summarize(w io.Writer) error {
// rowStyle is the row style for a row, as arrays of bytes to print.
type rowStyle struct {
left, right, middle, filler, wrapper []byte
fillerWidth int
hasWrapping bool
}

type rowStyles struct {
Top, Mid, Row, Wrap, End rowStyle
}

// ExpandedEncoder is a buffered, lookahead expanded table encoder for result sets.
type ExpandedEncoder struct {
TableEncoder
Expand All @@ -602,7 +634,9 @@ func NewExpandedEncoder(resultSet ResultSet, opts ...Option) (Encoder, error) {
return nil, err
}
t := tableEnc.(*TableEncoder)
t.formatter = NewEscapeFormatter()
if f, ok := t.formatter.(*EscapeFormatter); ok {
f.Configure(WithHeaderAlign(AlignLeft))
}
if !t.isCustomSummary {
t.summary = nil
}
Expand Down Expand Up @@ -636,6 +670,7 @@ func (enc *ExpandedEncoder) Encode(w io.Writer) error {
if err != nil {
return err
}
enc.initBuffers()
var cmd *exec.Cmd
var cmdBuf io.WriteCloser
wroteTitle := enc.skipHeader
Expand Down Expand Up @@ -685,7 +720,7 @@ func (enc *ExpandedEncoder) Encode(w io.Writer) error {
}

func (enc *ExpandedEncoder) encodeVals(vals [][]*Value) error {
rs := enc.rowStyle(enc.lineStyle.Row)
rs := enc.rowStyles.Row
// print buffered vals
for i := 0; i < len(vals); i++ {
enc.record(i, vals[i], rs)
Expand All @@ -695,10 +730,15 @@ func (enc *ExpandedEncoder) encodeVals(vals [][]*Value) error {
return err
}
}
for _, v := range vals[i] {
if v != nil {
enc.formatter.Free(v)
}
}
}
// draw end border
if enc.border >= 2 && enc.scanCount != 0 {
enc.divider(enc.rowStyle(enc.lineStyle.End))
enc.divider(&enc.rowStyles.End)
}
return nil
}
Expand All @@ -723,7 +763,7 @@ func (enc *ExpandedEncoder) EncodeAll(w io.Writer) error {
}

func (enc *ExpandedEncoder) calcWidth(vals [][]*Value) {
rs := enc.rowStyle(enc.lineStyle.Row)
rs := enc.rowStyles.Row
offset := runewidth.StringWidth(string(rs.left))
enc.offsets[0] = offset
// first column is always the column name
Expand Down Expand Up @@ -782,27 +822,27 @@ func (enc *ExpandedEncoder) record(i int, vals []*Value, rs rowStyle) {
headerRS := rs
header := enc.recordHeader(i)
if enc.border != 0 {
headerRS = enc.rowStyle(enc.lineStyle.Top)
headerRS = enc.rowStyles.Top
if i != 0 {
headerRS = enc.rowStyle(enc.lineStyle.Mid)
headerRS = enc.rowStyles.Mid
}
}
enc.w.Write(headerRS.left)
enc.w.WriteString(header)
padding := enc.maxWidths[0] + enc.maxWidths[1] + runewidth.StringWidth(string(headerRS.middle))*2 - len(header) - 1
if padding > 0 {
enc.w.Write(bytes.Repeat(headerRS.filler, padding))
headerRS.filler = repeat(enc.w, headerRS.filler, rs.fillerWidth*padding)
}
// write newline wrap value
enc.w.Write(headerRS.filler)
enc.w.Write(headerRS.filler[:rs.fillerWidth])
enc.w.Write(headerRS.right)
}
// write each value with column name in first col
for j, v := range vals {
if v != nil {
v.Align = AlignLeft
}
enc.row([]*Value{enc.headers[j], v}, rs)
enc.row([]*Value{enc.headers[j], v}, &rs)
}
}

Expand Down Expand Up @@ -933,6 +973,10 @@ func (enc *JSONEncoder) Encode(w io.Writer) error {
return err
}
}

if v != enc.empty {
enc.formatter.Free(v)
}
}
if _, err = w.Write(cls); err != nil {
return err
Expand Down Expand Up @@ -1110,6 +1154,9 @@ func (enc *UnalignedEncoder) Encode(w io.Writer) error {
if _, err := w.Write(buf); err != nil {
return err
}
if v != enc.empty {
enc.formatter.Free(v)
}
}
if _, err := w.Write(enc.newline); err != nil {
return err
Expand Down
Loading