From 206c162f4d319b69c189a98afb044407a991308d Mon Sep 17 00:00:00 2001 From: Vladimir Skipor Date: Sun, 18 Feb 2018 15:33:42 +0300 Subject: [PATCH] GH-85: DataSource basic implementations --- core/coretest/source.go | 58 +++++++++++++++++++++++++++++ core/datasink/{buffer.go => std.go} | 4 +- core/datasource/file.go | 45 ++++++++++++++++++++++ core/datasource/file_test.go | 25 +++++++++++++ core/datasource/std.go | 46 +++++++++++++++++++++++ lib/ioutil2/closer.go | 11 ++++++ lib/testutil2/afero.go | 11 ++++++ lib/testutil2/matchers.go | 2 +- 8 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 core/coretest/source.go rename core/datasink/{buffer.go => std.go} (89%) create mode 100644 core/datasource/file.go create mode 100644 core/datasource/file_test.go create mode 100644 core/datasource/std.go create mode 100644 lib/ioutil2/closer.go diff --git a/core/coretest/source.go b/core/coretest/source.go new file mode 100644 index 000000000..8bf6f7c2e --- /dev/null +++ b/core/coretest/source.go @@ -0,0 +1,58 @@ +// Copyright (c) 2018 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package coretest + +import ( + "io" + "io/ioutil" + "os" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/yandex/pandora/core" + "github.com/yandex/pandora/lib/testutil2" +) + +func AssertSourceEqualStdStream(t *testing.T, expectedPtr **os.File, getSource func() core.DataSource) { + temp, err := ioutil.TempFile("", "") + require.NoError(t, err) + + backup := *expectedPtr + defer func() { + *expectedPtr = backup + }() + *expectedPtr = temp + const testdata = "abcd" + _, err = io.WriteString(temp, testdata) + require.NoError(t, err) + + rc, err := getSource().OpenSource() + require.NoError(t, err) + + err = rc.Close() + require.NoError(t, err, "std stream should not be closed") + + temp.Seek(0, io.SeekStart) + data, err := ioutil.ReadAll(temp) + assert.Equal(t, testdata, string(data)) +} + +func AssertSourceEqualFile(t *testing.T, fs afero.Fs, filename string, source core.DataSource) { + const testdata = "abcd" + afero.WriteFile(fs, filename, []byte(testdata), 644) + + rc, err := source.OpenSource() + require.NoError(t, err) + + data := testutil2.ReadString(t, rc) + err = rc.Close() + require.NoError(t, err) + + assert.Equal(t, testdata, data) +} diff --git a/core/datasink/buffer.go b/core/datasink/std.go similarity index 89% rename from core/datasink/buffer.go rename to core/datasink/std.go index 9ab79d497..cfae62f7d 100644 --- a/core/datasink/buffer.go +++ b/core/datasink/std.go @@ -10,10 +10,12 @@ import ( "io" "github.com/yandex/pandora/core" + "github.com/yandex/pandora/lib/ioutil2" ) type Buffer struct { bytes.Buffer + ioutil2.NopCloser } var _ core.DataSink = &Buffer{} @@ -22,8 +24,6 @@ func (b *Buffer) OpenSink() (wc io.WriteCloser, err error) { return b, nil } -func (b *Buffer) Close() error { return nil } - func NewBuffer() *Buffer { return &Buffer{} } diff --git a/core/datasource/file.go b/core/datasource/file.go new file mode 100644 index 000000000..cefefa94f --- /dev/null +++ b/core/datasource/file.go @@ -0,0 +1,45 @@ +// Copyright (c) 2018 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package datasource + +import ( + "io" + "os" + + "github.com/spf13/afero" + + "github.com/yandex/pandora/core" +) + +// TODO(skipor): auto unzip +type FileConfig struct { + Path string `config:"path" validate:"required"` +} + +func NewFile(fs afero.Fs, conf FileConfig) core.DataSource { + return &fileSource{afero.Afero{fs}, conf} +} + +type fileSource struct { + fs afero.Afero + conf FileConfig +} + +func (s *fileSource) OpenSource() (wc io.ReadCloser, err error) { + return s.fs.Open(s.conf.Path) +} + +func NewStdin() core.DataSource { + return hideCloseFileSource{os.Stdin} +} + +type hideCloseFileSource struct{ afero.File } + +func (f hideCloseFileSource) OpenSource() (wc io.ReadCloser, err error) { + return f, nil +} + +func (f hideCloseFileSource) Close() error { return nil } diff --git a/core/datasource/file_test.go b/core/datasource/file_test.go new file mode 100644 index 000000000..714655a27 --- /dev/null +++ b/core/datasource/file_test.go @@ -0,0 +1,25 @@ +// Copyright (c) 2018 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package datasource + +import ( + "os" + "testing" + + "github.com/spf13/afero" + "github.com/yandex/pandora/core/coretest" +) + +func TestFileSource(t *testing.T) { + const filename = "/xxx/yyy" + fs := afero.NewMemMapFs() + source := NewFile(fs, FileConfig{Path: filename}) + coretest.AssertSourceEqualFile(t, fs, filename, source) +} + +func TestStdin(t *testing.T) { + coretest.AssertSourceEqualStdStream(t, &os.Stdout, NewStdin) +} diff --git a/core/datasource/std.go b/core/datasource/std.go new file mode 100644 index 000000000..db6ef39c1 --- /dev/null +++ b/core/datasource/std.go @@ -0,0 +1,46 @@ +// Copyright (c) 2018 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package datasource + +import ( + "bytes" + "io" + "io/ioutil" + + "github.com/yandex/pandora/core" + "github.com/yandex/pandora/lib/ioutil2" +) + +func NewBuffer(buf *bytes.Buffer) core.DataSource { + return buffer{Buffer: buf} +} + +type buffer struct { + *bytes.Buffer + ioutil2.NopCloser +} + +func (b buffer) OpenSource() (wc io.ReadCloser, err error) { + return b, nil +} + +// NewReader returns dummy core.DataSource that returns it on OpenSource call, wrapping it +// ioutil.NopCloser if r is not io.Closer. +// NONE(skipor): such wrapping hide +func NewReader(r io.Reader) core.DataSource { + return &reader{r} +} + +type reader struct { + source io.Reader +} + +func (r *reader) OpenSource() (rc io.ReadCloser, err error) { + if rc, ok := r.source.(io.ReadCloser); ok { + return rc, nil + } + return ioutil.NopCloser(r.source), nil +} diff --git a/lib/ioutil2/closer.go b/lib/ioutil2/closer.go new file mode 100644 index 000000000..3fcfea9be --- /dev/null +++ b/lib/ioutil2/closer.go @@ -0,0 +1,11 @@ +// Copyright (c) 2018 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package ioutil2 + +// NopCloser may be embedded to any struct to implement io.Closer doing nothing on closer. +type NopCloser struct{} + +func (NopCloser) Close() error { return nil } diff --git a/lib/testutil2/afero.go b/lib/testutil2/afero.go index 6c0f09735..b06494a56 100644 --- a/lib/testutil2/afero.go +++ b/lib/testutil2/afero.go @@ -6,12 +6,22 @@ package testutil2 import ( + "io" + "io/ioutil" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func ReadString(t TestingT, r io.Reader) string { + data, err := ioutil.ReadAll(r) + require.NoError(t, err) + return string(data) +} + func ReadFileString(t TestingT, fs afero.Fs, name string) string { + getHelper(t).Helper() data, err := afero.ReadFile(fs, name) require.NoError(t, err) return string(data) @@ -19,6 +29,7 @@ func ReadFileString(t TestingT, fs afero.Fs, name string) string { } func AssertFileEqual(t TestingT, fs afero.Fs, name string, expected string) { + getHelper(t).Helper() actual := ReadFileString(t, fs, name) assert.Equal(t, expected, actual) } diff --git a/lib/testutil2/matchers.go b/lib/testutil2/matchers.go index 112249420..08ffcaa24 100644 --- a/lib/testutil2/matchers.go +++ b/lib/testutil2/matchers.go @@ -71,7 +71,7 @@ func RunFlaky(t *testing.T, test func(t TestingT)) { } // getHelper allows to call t.Helper() without breaking compatibility with go version < 1.9 -func getHelper(t *testing.T) helper { +func getHelper(t TestingT) helper { var tInterface interface{} = t if h, ok := tInterface.(helper); ok { return h