Weekly Sync 2024-08-23

.bleep

@@ -1 +1 @@
-d2eaddcd278a00f25792bf06f046b39aa321abe3
+b01a9bc71ff892b2fdbb47f6bb3f9eac88907435

pingora-core/src/connectors/l4.rs

@@ -34,7 +34,7 @@ pub trait Connect: std::fmt::Debug {
 }
 
 /// Establish a connection (l4) to the given peer using its settings and an optional bind address.
-pub async fn connect<P>(peer: &P, bind_to: Option<InetSocketAddr>) -> Result<Stream>
+pub(crate) async fn connect<P>(peer: &P, bind_to: Option<InetSocketAddr>) -> Result<Stream>
 where
  P: Peer + Send + Sync,
 {

pingora-core/src/connectors/mod.rs

@@ -15,7 +15,7 @@
 //! Connecting to servers
 
 pub mod http;
-mod l4;
+pub mod l4;
 mod offload;
 mod tls;
 

pingora-core/src/lib.rs

@@ -58,6 +58,9 @@ pub use pingora_boringssl as tls;
 #[cfg(all(not(feature = "boringssl"), feature = "openssl"))]
 pub use pingora_openssl as tls;
 
+#[cfg(all(not(feature = "boringssl"), not(feature = "openssl")))]
+pub mod tls;
+
 pub mod prelude {
  pub use crate::server::configuration::Opt;
  pub use crate::server::Server;

pingora-core/src/protocols/http/compression/mod.rs

@@ -19,7 +19,6 @@
 use super::HttpTask;
 
 use bytes::Bytes;
-use http::header::ACCEPT_RANGES;
 use log::warn;
 use pingora_error::{ErrorType, Result};
 use pingora_http::{RequestHeader, ResponseHeader};
@@ -210,6 +209,17 @@ impl ResponseCompressionCtx {
  return;
  }
 
+ if depends_on_accept_encoding(
+ resp,
+ levels.iter().any(|level| *level != 0),
+ decompress_enable,
+ ) {
+ // The response depends on the Accept-Encoding header, make sure to indicate it
+ // in the Vary response header.
+ // https://www.rfc-editor.org/rfc/rfc9110#name-vary
+ add_vary_header(resp, &http::header::ACCEPT_ENCODING);
+ }
+
  let action = decide_action(resp, accept_encoding);
  let encoder = match action {
  Action::Noop => None,
@@ -428,6 +438,47 @@ fn test_accept_encoding_req_header() {
  assert_eq!(ac_list[1], Algorithm::Gzip);
 }
 
+// test whether the response depends on Accept-Encoding header
+fn depends_on_accept_encoding(
+ resp: &ResponseHeader,
+ compress_enabled: bool,
+ decompress_enabled: &[bool],
+) -> bool {
+ use http::header::CONTENT_ENCODING;
+
+ (decompress_enabled.iter().any(|enabled| *enabled)
+ && resp.headers.get(CONTENT_ENCODING).is_some())
+ || (compress_enabled && compressible(resp))
+}
+
+#[test]
+fn test_decide_on_accept_encoding() {
+ let mut resp = ResponseHeader::build(200, None).unwrap();
+ resp.insert_header("content-length", "50").unwrap();
+ resp.insert_header("content-type", "text/html").unwrap();
+ resp.insert_header("content-encoding", "gzip").unwrap();
+
+ // enabled
+ assert!(depends_on_accept_encoding(&resp, false, &[true]));
+
+ // decompress disabled => disabled
+ assert!(!depends_on_accept_encoding(&resp, false, &[false]));
+
+ // no content-encoding => disabled
+ resp.remove_header("content-encoding");
+ assert!(!depends_on_accept_encoding(&resp, false, &[true]));
+
+ // compress enabled and compressible response => enabled
+ assert!(depends_on_accept_encoding(&resp, true, &[false]));
+
+ // compress disabled and compressible response => disabled
+ assert!(!depends_on_accept_encoding(&resp, false, &[false]));
+
+ // compress enabled and not compressible response => disabled
+ resp.insert_header("content-type", "text/html+zip").unwrap();
+ assert!(!depends_on_accept_encoding(&resp, true, &[false]));
+}
+
 // filter response header to see if (de)compression is needed
 fn decide_action(resp: &ResponseHeader, accept_encoding: &[Algorithm]) -> Action {
  use http::header::CONTENT_ENCODING;
@@ -580,14 +631,104 @@ fn compressible(resp: &ResponseHeader) -> bool {
  }
 }
 
+// add Vary header with the specified value or extend an existing Vary header value
+fn add_vary_header(resp: &mut ResponseHeader, value: &http::header::HeaderName) {
+ use http::header::{HeaderValue, VARY};
+
+ let already_present = resp.headers.get_all(VARY).iter().any(|existing| {
+ existing
+ .as_bytes()
+ .split(|b| *b == b',')
+ .map(|mut v| {
+ // This is equivalent to slice.trim_ascii() which is unstable
+ while let [first, rest @ ..] = v {
+ if first.is_ascii_whitespace() {
+ v = rest;
+ } else {
+ break;
+ }
+ }
+ while let [rest @ .., last] = v {
+ if last.is_ascii_whitespace() {
+ v = rest;
+ } else {
+ break;
+ }
+ }
+ v
+ })
+ .any(|v| v == b"*" || v.eq_ignore_ascii_case(value.as_ref()))
+ });
+
+ if !already_present {
+ resp.append_header(&VARY, HeaderValue::from_name(value.clone()))
+ .unwrap();
+ }
+}
+
+#[test]
+fn test_add_vary_header() {
+ let mut header = ResponseHeader::build(200, None).unwrap();
+ add_vary_header(&mut header, &http::header::ACCEPT_ENCODING);
+ assert_eq!(
+ header
+ .headers
+ .get_all("Vary")
+ .into_iter()
+ .collect::<Vec<_>>(),
+ vec!["accept-encoding"]
+ );
+
+ let mut header = ResponseHeader::build(200, None).unwrap();
+ header.insert_header("Vary", "Accept-Language").unwrap();
+ add_vary_header(&mut header, &http::header::ACCEPT_ENCODING);
+ assert_eq!(
+ header
+ .headers
+ .get_all("Vary")
+ .into_iter()
+ .collect::<Vec<_>>(),
+ vec!["Accept-Language", "accept-encoding"]
+ );
+
+ let mut header = ResponseHeader::build(200, None).unwrap();
+ header
+ .insert_header("Vary", "Accept-Language, Accept-Encoding")
+ .unwrap();
+ add_vary_header(&mut header, &http::header::ACCEPT_ENCODING);
+ assert_eq!(
+ header
+ .headers
+ .get_all("Vary")
+ .into_iter()
+ .collect::<Vec<_>>(),
+ vec!["Accept-Language, Accept-Encoding"]
+ );
+
+ let mut header = ResponseHeader::build(200, None).unwrap();
+ header.insert_header("Vary", "*").unwrap();
+ add_vary_header(&mut header, &http::header::ACCEPT_ENCODING);
+ assert_eq!(
+ header
+ .headers
+ .get_all("Vary")
+ .into_iter()
+ .collect::<Vec<_>>(),
+ vec!["*"]
+ );
+}
+
 fn adjust_response_header(resp: &mut ResponseHeader, action: &Action) {
- use http::header::{HeaderValue, CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
+ use http::header::{
+ HeaderValue, ACCEPT_RANGES, CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING,
+ };
 
  fn set_stream_headers(resp: &mut ResponseHeader) {
  // because the transcoding is streamed, content length is not known ahead
  resp.remove_header(&CONTENT_LENGTH);
  // remove Accept-Ranges header because range requests will no longer work
  resp.remove_header(&ACCEPT_RANGES);
+
  // we stream body now TODO: chunked is for h1 only
  resp.insert_header(&TRANSFER_ENCODING, HeaderValue::from_static("chunked"))
  .unwrap();
@@ -616,6 +757,7 @@ fn test_adjust_response_header() {
  let mut header = ResponseHeader::build(200, None).unwrap();
  header.insert_header("content-length", "20").unwrap();
  header.insert_header("content-encoding", "gzip").unwrap();
+ header.insert_header("accept-ranges", "bytes").unwrap();
  adjust_response_header(&mut header, &Noop);
  assert_eq!(
  header.headers.get("content-encoding").unwrap().as_bytes(),
@@ -631,13 +773,15 @@ fn test_adjust_response_header() {
  let mut header = ResponseHeader::build(200, None).unwrap();
  header.insert_header("content-length", "20").unwrap();
  header.insert_header("content-encoding", "gzip").unwrap();
+ header.insert_header("accept-ranges", "bytes").unwrap();
  adjust_response_header(&mut header, &Decompress(Gzip));
  assert!(header.headers.get("content-encoding").is_none());
  assert!(header.headers.get("content-length").is_none());
  assert_eq!(
  header.headers.get("transfer-encoding").unwrap().as_bytes(),
  b"chunked"
  );
+ assert!(header.headers.get("accept-ranges").is_none());
 
  // compress
  let mut header = ResponseHeader::build(200, None).unwrap();