diff --git a/karate-core/pom.xml b/karate-core/pom.xml index 65bde396e..287965631 100644 --- a/karate-core/pom.xml +++ b/karate-core/pom.xml @@ -50,22 +50,10 @@ - org.apache.httpcomponents - httpclient - 4.5.14 - - - commons-logging - commons-logging - - + org.apache.httpcomponents.client5 + httpclient5 + 5.3 - - - commons-codec - commons-codec - 1.16.0 - ch.qos.logback logback-classic @@ -135,6 +123,13 @@ ${junit5.version} test + + org.mockito + mockito-core + 5.5.0 + test + + diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 3b2833964..2d90d4e1a 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -24,20 +24,20 @@ package com.intuit.karate.http; import com.intuit.karate.Constants; -import com.intuit.karate.FileUtils; import com.intuit.karate.Logger; import com.intuit.karate.core.Config; import com.intuit.karate.core.ScenarioEngine; + import io.netty.handler.codec.http.cookie.ClientCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; import java.io.IOException; -import java.io.InputStream; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Proxy; import java.net.ProxySelector; import java.net.SocketAddress; import java.net.URI; +import java.net.URISyntaxException; import java.security.KeyStore; import java.util.ArrayList; import java.util.Collections; @@ -47,47 +47,57 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; + import javax.net.ssl.SSLContext; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpException; -import org.apache.http.HttpHost; -import org.apache.http.HttpMessage; -import org.apache.http.HttpRequestInterceptor; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.NTCredentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.CookieStore; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.config.AuthSchemes; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.EntityBuilder; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.RequestBuilder; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.config.SocketConfig; -import org.apache.http.conn.ssl.LenientSslConnectionSocketFactory; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustAllStrategy; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.CookieSpecProvider; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.LaxRedirectStrategy; -import org.apache.http.impl.conn.SystemDefaultRoutePlanner; -import org.apache.http.impl.cookie.DefaultCookieSpec; -import org.apache.http.protocol.HttpContext; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; + +import org.apache.hc.client5.http.ClientProtocolException; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.cookie.BasicCookieStore; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.CookieSpecFactory; +import org.apache.hc.client5.http.cookie.CookieStore; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.entity.EntityBuilder; +import org.apache.hc.client5.http.impl.DefaultRedirectStrategy; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.cookie.CookieSpecBase; +import org.apache.hc.client5.http.impl.cookie.RFC6265StrictSpec; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner; +import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; +import org.apache.hc.client5.http.routing.HttpRoutePlanner; +import org.apache.hc.client5.http.ssl.LenientSslConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.TrustAllStrategy; +import org.apache.hc.client5.http.ssl.TrustSelfSignedStrategy; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.EntityDetails; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpMessage; +import org.apache.hc.core5.http.HttpRequestInterceptor; +import org.apache.hc.core5.http.config.Registry; +import org.apache.hc.core5.http.config.RegistryBuilder; +import org.apache.hc.core5.http.io.SocketConfig; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.pool.PoolConcurrencyPolicy; +import org.apache.hc.core5.pool.PoolReusePolicy; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.ssl.SSLContexts; /** * @@ -102,12 +112,19 @@ public class ApacheHttpClient implements HttpClient, HttpRequestInterceptor { private HttpClientBuilder clientBuilder; private CookieStore cookieStore; - public static class LenientCookieSpec extends DefaultCookieSpec { + // Not sure what the rationale was behind this class. + // But the httpclient4 ApacheHttpClient, based on DefaultCookieSpec, supported: + // - set-cookie2 which is now deprecated https://stackoverflow.com/questions/9462180/difference-between-set-cookie2-and-set-cookie + // - "netscape style cookies" and versioned cookies... whatever that was, I'm asusming its not widely used any more + // - other than that, it defaulted to a RFC2965Strict Spec. + // So as part of the httpclient5 migration, we directly default to RFC6265StrictSpec + public static class LenientCookieSpec extends CookieSpecBase { static final String KARATE = "karate"; + final RFC6265StrictSpec strict = new RFC6265StrictSpec(); + public LenientCookieSpec() { - super(new String[]{"EEE, dd-MMM-yy HH:mm:ss z", "EEE, dd MMM yyyy HH:mm:ss Z"}, false); } @Override @@ -120,12 +137,22 @@ public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieE // do nothing } - public static Registry registry() { - CookieSpecProvider specProvider = (HttpContext hc) -> new LenientCookieSpec(); - return RegistryBuilder.create() - .register(KARATE, specProvider).build(); + + @Override + public List parse(Header header, CookieOrigin origin) throws MalformedCookieException { + return strict.parse(header, origin); } + @Override + public List
formatCookies(List cookies) { + return strict.formatCookies(cookies); + } + + public static Registry registry() { + CookieSpecFactory specProvider = (HttpContext hc) -> new LenientCookieSpec(); + return RegistryBuilder.create() + .register(KARATE, specProvider).build(); + } } public ApacheHttpClient(ScenarioEngine engine) { @@ -136,17 +163,22 @@ public ApacheHttpClient(ScenarioEngine engine) { } private void configure(Config config) { + PoolingHttpClientConnectionManagerBuilder connectionManagerBuilder = PoolingHttpClientConnectionManagerBuilder.create(); + clientBuilder = HttpClientBuilder.create(); + if (config.isHttpRetryEnabled()) { - clientBuilder.setRetryHandler(new CustomHttpRequestRetryHandler(logger)); + clientBuilder.setRetryStrategy(new CustomHttpRequestRetryHandler(logger)); } else { clientBuilder.disableAutomaticRetries(); } if (!config.isFollowRedirects()) { clientBuilder.disableRedirectHandling(); - } else { // support redirect on POST by default - clientBuilder.setRedirectStrategy(LaxRedirectStrategy.INSTANCE); + } else { + clientBuilder.setRedirectStrategy(DefaultRedirectStrategy.INSTANCE); + // httpclient4 was using LaxRedirectStrategy.INSTANCE as it supported redirect on POST methods. + // httpclient5 seems to be status code based, not method based, so default strategy should be fine. } cookieStore = new BasicCookieStore(); clientBuilder.setDefaultCookieStore(cookieStore); @@ -181,71 +213,109 @@ private void configure(Config config) { } else { socketFactory = new LenientSslConnectionSocketFactory(sslContext, new NoopHostnameVerifier()); } - clientBuilder.setSSLSocketFactory(socketFactory); + connectionManagerBuilder.setSSLSocketFactory(socketFactory); } catch (Exception e) { logger.error("ssl context init failed: {}", e.getMessage()); throw new RuntimeException(e); } } + connectionManagerBuilder + .setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT) + .setConnPoolPolicy(PoolReusePolicy.LIFO) + .setDefaultConnectionConfig(ConnectionConfig.custom() + .setSocketTimeout(config.getReadTimeout(), TimeUnit.MILLISECONDS) + .setConnectTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); RequestConfig.Builder configBuilder = RequestConfig.custom() - .setCookieSpec(LenientCookieSpec.KARATE) - .setConnectTimeout(config.getConnectTimeout()) - .setSocketTimeout(config.getReadTimeout()); - if (config.getLocalAddress() != null) { + .setCookieSpec(LenientCookieSpec.KARATE); + if (config.isNtlmEnabled()) { + //No longer supported since 5.3. See https://hc.apache.org/httpcomponents-client-5.3.x/current/httpclient5/apidocs/index.html?org/apache/hc/client5/http/auth/NTCredentials.html + throw new UnsupportedOperationException("NTLM authentication is not supported any more. Please consider using Basic or Bearer authentication with TLS instead."); + } + connectionManagerBuilder.setDefaultSocketConfig(SocketConfig.custom() + .setSoTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); + + connManager = connectionManagerBuilder.build(); + clientBuilder.setRoutePlanner(buildRoutePlanner(config)) + .setDefaultRequestConfig(configBuilder.build()) + // set shared flag to true so that we can close the client. + //ConnectionManager won't be closed automatically by Apache, it is now our responsability to do so. + // See comments in https://github.com/karatelabs/karate/pull/2471 + .setConnectionManagerShared(true) + .setConnectionManager(connManager) + // Not sure about this. With the default reuseStrategy, ProxyServerTest fails with a SocketConnection(client.feature#11). + // Could not work out the exact reason. But the same SocketHandler was being used for the first two calls and was failing the second time. + // By setting a no reuse strategy, the connections are closed and the test passes. + // Impact on performance to be checked. + .setConnectionReuseStrategy((req, resp, ctx) -> false) + .addRequestInterceptorLast(this); + } + + // Differences with httpclient4 implementation: + // - RequestBuilder.setLocalAddress does not exist any more, so instead, RoutePlanner.determineLocalAddress is overridden + // - clientBuilder.setProxy does not exist any more. + // Instead, the new RoutePlanner exposes determineProxy and determineLocalAddress methods that may be overridden. + // Karate actually uses two flavors of RoutePlanner's which both implement those methods: + // - one that leverages ProxySelector when the nonProxyHosts property is specified + // - a default one in all other cases, whether a proxy is specified or not. + + protected HttpRoutePlanner buildRoutePlanner(Config config) { + + // Handle localAddress. + // From a Karate perspective, localAddress is primarily designed to be used with Gatling and is not related to proxy. + // However, in apache client 5, it is handled by the RoutePlanner too. + InetAddress localAddress = null; + if (config.getLocalAddress() != null) { try { - InetAddress localAddress = InetAddress.getByName(config.getLocalAddress()); - configBuilder.setLocalAddress(localAddress); + localAddress = InetAddress.getByName(config.getLocalAddress()); } catch (Exception e) { logger.warn("failed to resolve local address: {} - {}", config.getLocalAddress(), e.getMessage()); } } - if (config.isNtlmEnabled()) { - List authSchemes = new ArrayList<>(); - authSchemes.add(AuthSchemes.NTLM); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - NTCredentials ntCredentials = new NTCredentials( - config.getNtlmUsername(), config.getNtlmPassword(), config.getNtlmWorkstation(), config.getNtlmDomain()); - credentialsProvider.setCredentials(AuthScope.ANY, ntCredentials); - clientBuilder.setDefaultCredentialsProvider(credentialsProvider); - configBuilder.setTargetPreferredAuthSchemes(authSchemes); - } - clientBuilder.setDefaultRequestConfig(configBuilder.build()); - SocketConfig.Builder socketBuilder = SocketConfig.custom().setSoTimeout(config.getConnectTimeout()); - clientBuilder.setDefaultSocketConfig(socketBuilder.build()); + HttpHost proxy; if (config.getProxyUri() != null) { + URI proxyUri; try { - URI proxyUri = new URIBuilder(config.getProxyUri()).build(); - clientBuilder.setProxy(new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme())); - if (config.getProxyUsername() != null && config.getProxyPassword() != null) { - CredentialsProvider credsProvider = new BasicCredentialsProvider(); - credsProvider.setCredentials( - new AuthScope(proxyUri.getHost(), proxyUri.getPort()), - new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); - clientBuilder.setDefaultCredentialsProvider(credsProvider); - } - if (config.getNonProxyHosts() != null) { - ProxySelector proxySelector = new ProxySelector() { - private final List proxyExceptions = config.getNonProxyHosts(); - - @Override - public List select(URI uri) { - return Collections.singletonList(proxyExceptions.contains(uri.getHost()) - ? Proxy.NO_PROXY - : new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))); - } - - @Override - public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { - logger.info("connect failed to uri: {}", uri, ioe); - } - }; - clientBuilder.setRoutePlanner(new SystemDefaultRoutePlanner(proxySelector)); - } - } catch (Exception e) { + proxyUri = new URIBuilder(config.getProxyUri()).build(); + } catch (URISyntaxException e) { throw new RuntimeException(e); } + + // Manage proxy authenticator. + // Unfortunately, default credentials are part of the clientBuilder, not routePlanner, so there's a side effect on clientBuilder here. + if (config.getProxyUsername() != null && config.getProxyPassword() != null) { + BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(proxyUri.getHost(), proxyUri.getPort()), + new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword().toCharArray())); + clientBuilder.setDefaultCredentialsProvider(credsProvider); + } + + if (config.getNonProxyHosts() != null) { + // Create ProxySelector and its associated route planner + ProxySelector proxySelector = new ProxySelector() { + + @Override + public List select(URI uri) { + return Collections.singletonList(proxyUri == null || config.getNonProxyHosts().contains(uri.getHost()) + ? Proxy.NO_PROXY + : new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))); + } + + @Override + public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { + logger.info("connect failed to uri: {}", uri, ioe); + } + }; + return new ProxySelectorRoutePlanner(proxySelector, localAddress); + } else { + // use simple proxy + proxy = new HttpHost(proxyUri.getScheme(), proxyUri.getHost(), proxyUri.getPort()); + } + } else { + // NO proxy at all + proxy = null; } - clientBuilder.addInterceptorLast(this); + return new ProxyableRoutePlanner(proxy, localAddress); } @Override @@ -259,11 +329,12 @@ public Config getConfig() { } private HttpRequest request; + private PoolingHttpClientConnectionManager connManager; @Override public Response invoke(HttpRequest request) { this.request = request; - RequestBuilder requestBuilder = RequestBuilder.create(request.getMethod()).setUri(request.getUrl()); + ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.create(request.getMethod()).setUri(request.getUrl()); if (request.getBody() != null) { EntityBuilder entityBuilder = EntityBuilder.create().setBinary(request.getBody()); List transferEncoding = request.getHeaderValues(HttpConstants.HDR_TRANSFER_ENCODING); @@ -276,7 +347,7 @@ public Response invoke(HttpRequest request) { entityBuilder.chunked(); } if (te.contains("gzip")) { - entityBuilder.gzipCompress(); + entityBuilder.gzipCompressed(); } } request.removeHeader(HttpConstants.HDR_TRANSFER_ENCODING); @@ -285,20 +356,12 @@ public Response invoke(HttpRequest request) { } if (request.getHeaders() != null) { request.getHeaders().forEach((k, vals) -> vals.forEach(v -> requestBuilder.addHeader(k, v))); - } - CloseableHttpResponse httpResponse; - byte[] bytes; + } try (CloseableHttpClient client = clientBuilder.build()) { - httpResponse = client.execute(requestBuilder.build()); - HttpEntity responseEntity = httpResponse.getEntity(); - if (responseEntity == null || responseEntity.getContent() == null) { - bytes = Constants.ZERO_BYTES; - } else { - InputStream is = responseEntity.getContent(); - bytes = FileUtils.toBytes(is); - } + Response response = client.execute(requestBuilder.build(), this::buildResponse); request.setEndTime(System.currentTimeMillis()); - httpResponse.close(); + httpLogger.logResponse(getConfig(), request, response); + return response; } catch (Exception e) { if (e instanceof ClientProtocolException && e.getCause() != null) { // better error message throw new RuntimeException(e.getCause()); @@ -306,14 +369,19 @@ public Response invoke(HttpRequest request) { throw new RuntimeException(e); } } - int statusCode = httpResponse.getStatusLine().getStatusCode(); + } + + private Response buildResponse(ClassicHttpResponse httpResponse) throws IOException{ + HttpEntity entity = httpResponse.getEntity(); + byte[] bytes = entity != null ? EntityUtils.toByteArray(entity) : Constants.ZERO_BYTES; + int statusCode = httpResponse.getCode(); Map> headers = toHeaders(httpResponse); List storedCookies = cookieStore.getCookies(); Header[] requestCookieHeaders = httpResponse.getHeaders(HttpConstants.HDR_SET_COOKIE); // edge case where the apache client // auto-followed a redirect where cookies were involved - List mergedCookieValues = new ArrayList(requestCookieHeaders.length); - Set alreadyMerged = new HashSet(requestCookieHeaders.length); + List mergedCookieValues = new ArrayList<>(requestCookieHeaders.length); + Set alreadyMerged = new HashSet<>(requestCookieHeaders.length); for (Header ch : requestCookieHeaders) { String requestCookieValue = ch.getValue(); io.netty.handler.codec.http.cookie.Cookie c = ClientCookieDecoder.LAX.decode(requestCookieValue); @@ -326,7 +394,7 @@ public Response invoke(HttpRequest request) { if (alreadyMerged.contains(name)) { continue; } - Map map = new HashMap(); + Map map = new HashMap<>(); map.put(Cookies.NAME, name); map.put(Cookies.VALUE, c.getValue()); map.put(Cookies.DOMAIN, c.getDomain()); @@ -347,19 +415,19 @@ public Response invoke(HttpRequest request) { } @Override - public void process(org.apache.http.HttpRequest hr, HttpContext hc) throws HttpException, IOException { + public void process(org.apache.hc.core5.http.HttpRequest hr, EntityDetails entity, HttpContext context) throws HttpException, IOException { request.setHeaders(toHeaders(hr)); httpLogger.logRequest(getConfig(), request); request.setStartTime(System.currentTimeMillis()); } private static Map> toHeaders(HttpMessage msg) { - Header[] headers = msg.getAllHeaders(); - Map> map = new LinkedHashMap(headers.length); + Header[] headers = msg.getHeaders(); + Map> map = new LinkedHashMap<>(headers.length); for (Header outer : headers) { String name = outer.getName(); Header[] inner = msg.getHeaders(name); - List list = new ArrayList(inner.length); + List list = new ArrayList<>(inner.length); for (Header h : inner) { list.add(h.getValue()); } @@ -368,4 +436,57 @@ private static Map> toHeaders(HttpMessage msg) { return map; } + public void close() { + connManager.close(); + } + + /** + * Extends SystemDefaultRoutePlanner to add support for localAddress. + * To be used when nonProxyHosts are specified + */ + private static class ProxySelectorRoutePlanner extends SystemDefaultRoutePlanner { + + private final InetAddress localAddress; + + public ProxySelectorRoutePlanner(ProxySelector proxySelector, InetAddress localAddress) { + super(proxySelector); + this.localAddress = localAddress; + } + + protected InetAddress determineLocalAddress( + final HttpHost firstHop, + final HttpContext context) throws HttpException { + return localAddress; + } + } + + /** + * Default Route planner that supports localAddress. + * May be used with or without a Proxy, but not with a ProxySelector. + */ + private static class ProxyableRoutePlanner extends DefaultRoutePlanner { + + private HttpHost proxy; + private InetAddress localAddress; + + public ProxyableRoutePlanner(HttpHost proxy, InetAddress localAddress) { + super(null); + this.proxy = proxy; + this.localAddress = localAddress; + } + + @Override + protected HttpHost determineProxy( + final HttpHost target, + final HttpContext context) throws HttpException { + return proxy; + } + + @Override + protected InetAddress determineLocalAddress( + final HttpHost firstHop, + final HttpContext context) throws HttpException { + return localAddress; + } + } } diff --git a/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java index 174c54057..738fb0ae1 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java @@ -134,4 +134,9 @@ public HttpResponse execute(com.linecorp.armeria.client.HttpClient delegate, Cli return delegate.execute(ctx, req); } + @Override + public void close() { + // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471 + } + } diff --git a/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java b/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java index 2157cf10c..cf9b81911 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java +++ b/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java @@ -2,9 +2,12 @@ import java.io.IOException; -import org.apache.http.NoHttpResponseException; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.protocol.HttpContext; +import org.apache.hc.client5.http.HttpRequestRetryStrategy; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.NoHttpResponseException; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.util.TimeValue; import com.intuit.karate.Logger; @@ -13,7 +16,7 @@ * This is usually the case when there is steal connection. The retry cause that * the connection is renewed and the second call will succeed. */ -public class CustomHttpRequestRetryHandler implements HttpRequestRetryHandler +public class CustomHttpRequestRetryHandler implements HttpRequestRetryStrategy { private final Logger logger; @@ -22,9 +25,7 @@ public CustomHttpRequestRetryHandler(Logger logger) this.logger = logger; } - @Override - public boolean retryRequest(IOException exception, int executionCount, HttpContext context) - { + private boolean shouldRetry(IOException exception, int executionCount) { if (exception instanceof NoHttpResponseException && executionCount < 1) { logger.error("Thrown an NoHttpResponseException retry..."); @@ -36,4 +37,19 @@ public boolean retryRequest(IOException exception, int executionCount, HttpConte return false; } } + + @Override + public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, HttpContext context) { + return shouldRetry(exception, executionCount); + } + + @Override + public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) { + return false; + } + + @Override + public TimeValue getRetryInterval(HttpResponse response, int execCount, HttpContext context) { + return TimeValue.ofSeconds(1); // NOt sure what the interval was in httpclient4 ... Sticking with the default value of the default http5 implementation. + } } \ No newline at end of file diff --git a/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java index c6f19dfa3..f96a96353 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java @@ -37,4 +37,5 @@ public interface HttpClient { Response invoke(HttpRequest request); + void close(); } diff --git a/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java b/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java index 98f6937ac..f8989d1c3 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java +++ b/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java @@ -51,7 +51,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import org.apache.http.client.utils.URIBuilder; +import org.apache.hc.core5.net.URIBuilder; import org.graalvm.polyglot.Value; import org.graalvm.polyglot.proxy.ProxyObject; import org.slf4j.Logger; diff --git a/karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java b/karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java similarity index 95% rename from karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java rename to karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java index 0441f6a2d..f42a80839 100644 --- a/karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java +++ b/karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java @@ -21,13 +21,14 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -package org.apache.http.conn.ssl; +package org.apache.hc.client5.http.ssl; import java.io.IOException; import java.net.Socket; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; -import org.apache.http.protocol.HttpContext; + +import org.apache.hc.core5.http.protocol.HttpContext; /** * in a separate package just for log level config consistency diff --git a/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java b/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java index e3bcac9d2..eeb67757e 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java +++ b/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java @@ -24,6 +24,11 @@ public Config getConfig() { @Override public Response invoke(HttpRequest request) { throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void close() { + // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471 } } diff --git a/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java b/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java index a09eebd90..4c2af4185 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java +++ b/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java @@ -401,9 +401,10 @@ void testTypeConv() { run("type-conv.feature"); } - @Test - void testConfigureNtlmAuthentication() { - run("ntlm-authentication.feature"); - } + // NTLM not supported in apache client 5.3 + // @Test + // void testConfigureNtlmAuthentication() { + // run("ntlm-authentication.feature"); + // } } diff --git a/karate-core/src/test/java/com/intuit/karate/core/MockClient.java b/karate-core/src/test/java/com/intuit/karate/core/MockClient.java index fbef24633..1b3cedaed 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/MockClient.java +++ b/karate-core/src/test/java/com/intuit/karate/core/MockClient.java @@ -33,4 +33,9 @@ public Response invoke(HttpRequest request) { return handler.handle(request.toRequest()); } + @Override + public void close() { + // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471 + } + } diff --git a/karate-core/src/test/java/com/intuit/karate/core/ntlm-authentication.feature b/karate-core/src/test/java/com/intuit/karate/core/ntlm-authentication.feature deleted file mode 100644 index c89cc701e..000000000 --- a/karate-core/src/test/java/com/intuit/karate/core/ntlm-authentication.feature +++ /dev/null @@ -1,10 +0,0 @@ -Feature: ntlm authentication - - Scenario: various ways to configure ntlm authentication - * configure ntlmAuth = { username: 'admin', password: 'secret', domain: 'my.domain', workstation: 'my-pc' } - * configure ntlmAuth = { username: 'admin', password: 'secret' } - * configure ntlmAuth = null - * eval - """ - karate.configure('ntlmAuth', { username: 'admin', password: 'secret' }) - """ diff --git a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java index 1ab3c9169..5eb56ffab 100644 --- a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java +++ b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java @@ -10,17 +10,20 @@ import java.nio.charset.StandardCharsets; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpUriRequest; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.io.entity.StringEntity; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -81,15 +84,18 @@ int http(HttpUriRequest request) throws Exception { SSLContext sc = SSLContext.getInstance("SSL"); sc.init(null, new TrustManager[]{LenientTrustManager.INSTANCE}, null); CloseableHttpClient client = HttpClients.custom() - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(sc) + .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create() + .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create() + .setSslContext(sc) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build()) + .build()) .setProxy(new HttpHost("localhost", proxy.getPort())) .build(); HttpResponse response = client.execute(request); - InputStream is = response.getEntity().getContent(); - String responseString = FileUtils.toString(is); + String responseString = response.getReasonPhrase(); logger.debug("response: {}", responseString); - return response.getStatusLine().getStatusCode(); + return response.getCode(); } } diff --git a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java index 171ba094a..43a383187 100644 --- a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java +++ b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java @@ -7,16 +7,17 @@ import com.intuit.karate.core.MockServer; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpUriRequest; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.io.entity.StringEntity; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -81,10 +82,9 @@ static int http(HttpUriRequest request) throws Exception { .setProxy(new HttpHost("localhost", proxy.getPort())) .build(); HttpResponse response = client.execute(request); - InputStream is = response.getEntity().getContent(); - String responseString = FileUtils.toString(is); + String responseString = response.getReasonPhrase(); logger.debug("response: {}", responseString); - return response.getStatusLine().getStatusCode(); + return response.getCode(); } } diff --git a/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java b/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java new file mode 100644 index 000000000..1b8991aaa --- /dev/null +++ b/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java @@ -0,0 +1,91 @@ +package com.intuit.karate.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import com.intuit.karate.core.Config; +import com.intuit.karate.core.ScenarioEngine; +import com.intuit.karate.core.Variable; + +class ApacheHttpServerTest { + + private ScenarioEngine engine; + private Config config; + private HttpHost host; + private HttpContext context; + + private ApacheHttpClient client; + + @BeforeEach + void configure() { + engine = mock(ScenarioEngine.class); + config = new Config(); + Mockito.when(engine.getConfig()).thenReturn(config); + host = new HttpHost("foo.com"); + context = mock(HttpContext.class); + + client = new ApacheHttpClient(engine); + } + + @Test + void noProxy() { + HttpRoute route = determineRoute(host); + Assertions.assertNull(route.getProxyHost()); + assertNull(route.getLocalAddress()); + } + + @Test + void proxy() { + config.configure("proxy", new Variable("http://proxy:80")); + HttpRoute route = determineRoute(host); + assertEquals("http://proxy:80", route.getProxyHost().toURI()); + } + + @Test + void nonProxyHosts() { + Map proxyConfiguration = new HashMap<>(); + proxyConfiguration.put("uri", "http://proxy:80"); + proxyConfiguration.put("nonProxyHosts", Collections.singletonList("foo.com")); + config.configure("proxy", new Variable(proxyConfiguration)); + + HttpRoute nonProxiedRoute = determineRoute(host); + assertNull(nonProxiedRoute.getProxyHost()); + + HttpRoute proxiedRoute = determineRoute(new HttpHost("bar.com")); + assertEquals("http://proxy:80", proxiedRoute.getProxyHost().toURI()); + } + + // From a Karate perspective, localAddress is primarily designed to be used with Gatling and is not related to proxy. + // However, in apache client 5, it is handled by the RoutePlanner. + @Test + void localAddress() { + config.configure("localAddress", new Variable("localhost")); + + HttpRoute route = determineRoute(host); + + assertNull(route.getProxyHost()); + assertEquals("localhost", route.getLocalAddress().getHostName()); + } + + private HttpRoute determineRoute(HttpHost host) { + try { + return client.buildRoutePlanner(config).determineRoute(host, context); + } catch (HttpException e) { + throw new RuntimeException(e); + } + } +} diff --git a/karate-demo/pom.xml b/karate-demo/pom.xml index 40bb062a6..e474f2421 100644 --- a/karate-demo/pom.xml +++ b/karate-demo/pom.xml @@ -20,6 +20,12 @@ + + org.apache.httpcomponents.client5 + httpclient5 + + 5.3 + org.springframework.boot spring-boot-dependencies