HTTP scenario acceptance test

fc2291d352a2cac83737de6c0c734caf49379847

.mapping.json

@@ -53,7 +53,6 @@
   "components/guns/http/trace.go":"load/projects/pandora/components/guns/http/trace.go",
   "components/guns/http/wrapper.go":"load/projects/pandora/components/guns/http/wrapper.go",
   "components/guns/http_scenario/ammo.go":"load/projects/pandora/components/guns/http_scenario/ammo.go",
-  "components/guns/http_scenario/client.go":"load/projects/pandora/components/guns/http_scenario/client.go",
   "components/guns/http_scenario/gun.go":"load/projects/pandora/components/guns/http_scenario/gun.go",
   "components/guns/http_scenario/gun_test.go":"load/projects/pandora/components/guns/http_scenario/gun_test.go",
   "components/guns/http_scenario/import.go":"load/projects/pandora/components/guns/http_scenario/import.go",
@@ -360,6 +359,7 @@
   "tests/acceptance/config_model.go":"load/projects/pandora/tests/acceptance/config_model.go",
   "tests/acceptance/connect_test.go":"load/projects/pandora/tests/acceptance/connect_test.go",
   "tests/acceptance/grpc_test.go":"load/projects/pandora/tests/acceptance/grpc_test.go",
+  "tests/acceptance/http_scenario_test.go":"load/projects/pandora/tests/acceptance/http_scenario_test.go",
   "tests/acceptance/http_test.go":"load/projects/pandora/tests/acceptance/http_test.go",
   "tests/acceptance/testdata/connect/connect-check-limit.yaml":"load/projects/pandora/tests/acceptance/testdata/connect/connect-check-limit.yaml",
   "tests/acceptance/testdata/connect/connect-check-passes.yaml":"load/projects/pandora/tests/acceptance/testdata/connect/connect-check-passes.yaml",
@@ -376,6 +376,10 @@
   "tests/acceptance/testdata/http/https.yaml":"load/projects/pandora/tests/acceptance/testdata/http/https.yaml",
   "tests/acceptance/testdata/http/payload.uri":"load/projects/pandora/tests/acceptance/testdata/http/payload.uri",
   "tests/acceptance/testdata/http/payload5.uri":"load/projects/pandora/tests/acceptance/testdata/http/payload5.uri",
+  "tests/acceptance/testdata/http_scenario/filter.json":"load/projects/pandora/tests/acceptance/testdata/http_scenario/filter.json",
+  "tests/acceptance/testdata/http_scenario/http_payload.hcl":"load/projects/pandora/tests/acceptance/testdata/http_scenario/http_payload.hcl",
+  "tests/acceptance/testdata/http_scenario/scenario.yaml":"load/projects/pandora/tests/acceptance/testdata/http_scenario/scenario.yaml",
+  "tests/acceptance/testdata/http_scenario/users.csv":"load/projects/pandora/tests/acceptance/testdata/http_scenario/users.csv",
   "tests/grpc_scenario/main_test.go":"load/projects/pandora/tests/grpc_scenario/main_test.go",
   "tests/grpc_scenario/testdata/filter.json":"load/projects/pandora/tests/grpc_scenario/testdata/filter.json",
   "tests/grpc_scenario/testdata/grpc_payload.hcl":"load/projects/pandora/tests/grpc_scenario/testdata/grpc_payload.hcl",

components/guns/grpc/scenario/ammo.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
+	"github.com/yandex/pandora/components/providers/scenario"
 )
 
 type SourceStorage interface {
@@ -22,6 +23,15 @@ func (a *Scenario) SetID(id uint64) {
 	a.id = id
 }
 
+func (a *Scenario) Clone() scenario.ProvAmmo {
+	return &Scenario{
+		Calls:           a.Calls,
+		Name:            a.Name,
+		MinWaitingTime:  a.MinWaitingTime,
+		VariableStorage: a.VariableStorage,
+	}
+}
+
 type Call struct {
 	Name           string
 	Preprocessors  []Preprocessor

components/guns/http/base.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/httptrace"
 	"net/http/httputil"
@@ -15,6 +16,7 @@ import (
 	"github.com/yandex/pandora/core"
 	"github.com/yandex/pandora/core/aggregator/netsample"
 	"github.com/yandex/pandora/core/warmup"
+	"github.com/yandex/pandora/lib/netutil"
 	"go.uber.org/zap"
 )
 
@@ -304,3 +306,34 @@ func GetBody(req *http.Request) []byte {
 	return nil
 
 }
+
+// DNS resolve optimisation.
+// When DNSCache turned off - do nothing extra, host will be resolved on every shoot.
+// When using resolved target, don't use DNS caching logic - it is useless.
+// If we can resolve accessible target addr - use it as target, not use caching.
+// Otherwise just use DNS cache - we should not fail shooting, we should try to
+// connect on every shoot. DNS cache will save resolved addr after first successful connect.
+func PreResolveTargetAddr(clientConf *ClientConfig, target string) (string, error) {
+	if !clientConf.Dialer.DNSCache {
+		return target, nil
+	}
+	if endpointIsResolved(target) {
+		clientConf.Dialer.DNSCache = false
+		return target, nil
+	}
+	resolved, err := netutil.LookupReachable(target, clientConf.Dialer.Timeout)
+	if err != nil {
+		zap.L().Warn("DNS target pre resolve failed", zap.String("target", target), zap.Error(err))
+		return target, err
+	}
+	clientConf.Dialer.DNSCache = false
+	return resolved, nil
+}
+
+func endpointIsResolved(endpoint string) bool {
+	host, _, err := net.SplitHostPort(endpoint)
+	if err != nil {
+		return false
+	}
+	return net.ParseIP(host) != nil
+}

components/guns/http/client.go

@@ -27,7 +27,7 @@ type ClientConfig struct {
 	ConnectSSL bool            `config:"connect-ssl"` // Defines if tunnel encrypted.
 }
 
-type clientConstructor func(clientConfig ClientConfig, target string) Client
+type ClientConstructor func(clientConfig ClientConfig, target string) Client
 
 func DefaultClientConfig() ClientConfig {
 	return ClientConfig{
@@ -130,7 +130,7 @@ func NewHTTP2Transport(conf TransportConfig, dial netutil.DialerFunc, target str
 	return tr
 }
 
-func newClient(tr *http.Transport, redirect bool) Client {
+func NewRedirectingClient(tr *http.Transport, redirect bool) Client {
 	if redirect {
 		return redirectClient{&http.Client{Transport: tr}}
 	}
@@ -173,6 +173,20 @@ func (c *panicOnHTTP1Client) Do(req *http.Request) (*http.Response, error) {
 	return res, nil
 }
 
+func WrapClientHostResolving(client Client, cfg HTTPGunConfig, targetResolved string) Client {
+	hostname := getHostWithoutPort(cfg.Target)
+	scheme := "http"
+	if cfg.SSL {
+		scheme = "https"
+	}
+	return &httpDecoratedClient{
+		client:         client,
+		scheme:         scheme,
+		hostname:       hostname,
+		targetResolved: targetResolved,
+	}
+}
+
 type httpDecoratedClient struct {
 	client         Client
 	scheme         string

components/guns/http/connect.go

@@ -54,10 +54,10 @@ func newConnectClient(conf ClientConfig, target string) Client {
 			NewDialer(conf.Dialer),
 		),
 		target)
-	return newClient(transport, conf.Redirect)
+	return NewRedirectingClient(transport, conf.Redirect)
 }
 
-var _ clientConstructor = newConnectClient
+var _ ClientConstructor = newConnectClient
 
 func newConnectDialFunc(target string, connectSSL bool, dialer netutil.Dialer) netutil.DialerFunc {
 	return func(ctx context.Context, network, address string) (conn net.Conn, err error) {

components/guns/http/http.go

@@ -18,11 +18,11 @@ func NewHTTP1Gun(conf HTTPGunConfig, answLog *zap.Logger, targetResolved string)
 
 func HTTP1ClientConstructor(clientConfig ClientConfig, target string) Client {
 	transport := NewTransport(clientConfig.Transport, NewDialer(clientConfig.Dialer).DialContext, target)
-	client := newClient(transport, clientConfig.Redirect)
+	client := NewRedirectingClient(transport, clientConfig.Redirect)
 	return client
 }
 
-var _ clientConstructor = HTTP1ClientConstructor
+var _ ClientConstructor = HTTP1ClientConstructor
 
 // NewHTTP2Gun return simple HTTP/2 gun that can shoot sequentially through one connection.
 func NewHTTP2Gun(conf HTTPGunConfig, answLog *zap.Logger, targetResolved string) (*BaseGun, error) {
@@ -35,25 +35,16 @@ func NewHTTP2Gun(conf HTTPGunConfig, answLog *zap.Logger, targetResolved string)
 
 func HTTP2ClientConstructor(clientConfig ClientConfig, target string) Client {
 	transport := NewHTTP2Transport(clientConfig.Transport, NewDialer(clientConfig.Dialer).DialContext, target)
-	client := newClient(transport, clientConfig.Redirect)
+	client := NewRedirectingClient(transport, clientConfig.Redirect)
 	// Will panic and cancel shooting whet target doesn't support HTTP/2.
 	return &panicOnHTTP1Client{Client: client}
 }
 
-var _ clientConstructor = HTTP2ClientConstructor
+var _ ClientConstructor = HTTP2ClientConstructor
 
-func newHTTPGun(clientConstructor clientConstructor, cfg HTTPGunConfig, answLog *zap.Logger, targetResolved string) *BaseGun {
-	scheme := "http"
-	if cfg.SSL {
-		scheme = "https"
-	}
+func newHTTPGun(clientConstructor ClientConstructor, cfg HTTPGunConfig, answLog *zap.Logger, targetResolved string) *BaseGun {
 	client := clientConstructor(cfg.Client, cfg.Target)
-	wrappedClient := &httpDecoratedClient{
-		client:         client,
-		hostname:       getHostWithoutPort(cfg.Target),
-		targetResolved: targetResolved,
-		scheme:         scheme,
-	}
+	wrappedClient := WrapClientHostResolving(client, cfg, targetResolved)
 	return &BaseGun{
 		Config: cfg.Base,
 		OnClose: func() error {

components/guns/http_scenario/ammo.go

@@ -4,6 +4,8 @@ import (
 	"io"
 	"net/http"
 	"time"
+
+	"github.com/yandex/pandora/components/providers/scenario"
 )
 
 type SourceStorage interface {
@@ -22,6 +24,15 @@ func (a *Scenario) SetID(id uint64) {
 	a.ID = id
 }
 
+func (a *Scenario) Clone() scenario.ProvAmmo {
+	return &Scenario{
+		Requests:        a.Requests,
+		Name:            a.Name,
+		MinWaitingTime:  a.MinWaitingTime,
+		VariableStorage: a.VariableStorage,
+	}
+}
+
 type Request struct {
 	Method         string
 	Headers        map[string]string

components/guns/http_scenario/gun.go

@@ -39,7 +39,7 @@ type BaseGun struct {
 	scheme         string
 	hostname       string
 	targetResolved string
-	client         Client
+	client         phttp.Client
 }
 
 var _ Gun = (*BaseGun)(nil)

components/guns/http_scenario/gun_test.go

@@ -29,7 +29,7 @@ func TestBaseGun_shoot(t *testing.T) {
 		scheme         string
 		hostname       string
 		targetResolved string
-		client         Client
+		client         phttp.Client
 	}
 
 	tests := []struct {

components/guns/http_scenario/import.go

@@ -1,16 +1,12 @@
 package httpscenario
 
 import (
-	"net"
-
 	"github.com/spf13/afero"
 	phttp "github.com/yandex/pandora/components/guns/http"
 	"github.com/yandex/pandora/core"
 	"github.com/yandex/pandora/core/aggregator/netsample"
 	"github.com/yandex/pandora/core/register"
 	"github.com/yandex/pandora/lib/answlog"
-	"github.com/yandex/pandora/lib/netutil"
-	"go.uber.org/zap"
 )
 
 func WrapGun(g Gun) core.Gun {
@@ -34,7 +30,7 @@ func (g *gunWrapper) Bind(a core.Aggregator, deps core.GunDeps) error {
 
 func Import(fs afero.Fs) {
 	register.Gun("http/scenario", func(conf phttp.HTTPGunConfig) func() core.Gun {
-		targetResolved, _ := PreResolveTargetAddr(&conf.Client, conf.Target)
+		targetResolved, _ := phttp.PreResolveTargetAddr(&conf.Client, conf.Target)
 		answLog := answlog.Init(conf.Base.AnswLog.Path, conf.Base.AnswLog.Enabled)
 		return func() core.Gun {
 			gun := NewHTTPGun(conf, answLog, targetResolved)
@@ -43,42 +39,11 @@ func Import(fs afero.Fs) {
 	}, phttp.DefaultHTTPGunConfig)
 
 	register.Gun("http2/scenario", func(conf phttp.HTTPGunConfig) func() (core.Gun, error) {
-		targetResolved, _ := PreResolveTargetAddr(&conf.Client, conf.Target)
+		targetResolved, _ := phttp.PreResolveTargetAddr(&conf.Client, conf.Target)
 		answLog := answlog.Init(conf.Base.AnswLog.Path, conf.Base.AnswLog.Enabled)
 		return func() (core.Gun, error) {
 			gun, err := NewHTTP2Gun(conf, answLog, targetResolved)
 			return WrapGun(gun), err
 		}
 	}, phttp.DefaultHTTP2GunConfig)
 }
-
-// DNS resolve optimisation.
-// When DNSCache turned off - do nothing extra, host will be resolved on every shoot.
-// When using resolved target, don't use DNS caching logic - it is useless.
-// If we can resolve accessible target addr - use it as target, not use caching.
-// Otherwise just use DNS cache - we should not fail shooting, we should try to
-// connect on every shoot. DNS cache will save resolved addr after first successful connect.
-func PreResolveTargetAddr(clientConf *phttp.ClientConfig, target string) (string, error) {
-	if !clientConf.Dialer.DNSCache {
-		return target, nil
-	}
-	if endpointIsResolved(target) {
-		clientConf.Dialer.DNSCache = false
-		return target, nil
-	}
-	resolved, err := netutil.LookupReachable(target, clientConf.Dialer.Timeout)
-	if err != nil {
-		zap.L().Warn("DNS target pre resolve failed", zap.String("target", target), zap.Error(err))
-		return target, err
-	}
-	clientConf.Dialer.DNSCache = false
-	return resolved, nil
-}
-
-func endpointIsResolved(endpoint string) bool {
-	host, _, err := net.SplitHostPort(endpoint)
-	if err != nil {
-		return false
-	}
-	return net.ParseIP(host) != nil
-}