Merge branch 'scylladb:master' into helpers-for-scylla-17443

.github/workflows/python-lint.yaml

@@ -0,0 +1,16 @@
+name: Python format
+
+on: [push, pull_request]
+
+jobs:
+  python-format:
+    name: Enforce python format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/[email protected]
+        with:
+            version: "24.8.0"
+            src: ./scripts
+            # override options so that we can specify only specific files for now
+            options: "--check --diff --include=.*addr2line.*"

CMakeLists.txt

@@ -582,6 +582,7 @@ add_library (seastar
   include/seastar/core/shared_ptr_debug_helper.hh
   include/seastar/core/shared_ptr_incomplete.hh
   include/seastar/core/simple-stream.hh
+  include/seastar/core/signal.hh
   include/seastar/core/slab.hh
   include/seastar/core/sleep.hh
   include/seastar/core/sstring.hh
@@ -715,6 +716,7 @@ add_library (seastar
   src/core/sharded.cc
   src/core/scollectd.cc
   src/core/scollectd-impl.hh
+  src/core/signal.cc
   src/core/systemwide_memory_barrier.cc
   src/core/smp.cc
   src/core/sstring.cc
@@ -778,6 +780,10 @@ add_library (seastar
   src/websocket/server.cc
   )
 
+# We disable _FORTIFY_SOURCE because it generates false positives with longjmp() (src/core/thread.cc)
+set_source_files_properties(src/core/thread.cc
+  PROPERTIES COMPILE_FLAGS -U_FORTIFY_SOURCE)
+
 add_library (Seastar::seastar ALIAS seastar)
 
 add_dependencies (seastar
@@ -815,6 +821,14 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     -Wno-error=deprecated-declarations)
 endif ()
 
+if (CMAKE_CXX_STANDARD GREATER_EQUAL 23)
+  include (CheckP2582R1)
+  if (Cxx_Compiler_IMPLEMENTS_P2581R1)
+    target_compile_definitions (seastar
+      PUBLIC SEASTAR_P2581R1)
+  endif ()
+endif ()
+
 if (BUILD_SHARED_LIBS)
   # use initial-exec TLS, as it puts the TLS variables in the static TLS space
   # instead of allocating them using malloc. otherwise intercepting mallocs and
@@ -877,7 +891,7 @@ if (condition)
   if (NOT Sanitizers_FOUND)
     message (FATAL_ERROR "Sanitizers not found!")
   endif ()
-  set (Seastar_Sanitizers_OPTIONS ${Sanitizers_COMPILER_OPTIONS})
+  set (Seastar_Sanitizers_OPTIONS ${Sanitizers_COMPILE_OPTIONS})
   target_link_libraries (seastar
     PUBLIC
       $<${condition}:Sanitizers::address>
@@ -898,12 +912,6 @@ include (CTest)
 #
 # To disable -Werror, pass -Wno-error to Seastar_CXX_FLAGS.
 #
-# We disable _FORTIFY_SOURCE because it generates false positives with longjmp() (src/core/thread.cc)
-#
-
-target_compile_options (seastar
-  PUBLIC
-    -U_FORTIFY_SOURCE)
 
 target_compile_definitions(seastar
   PUBLIC

README.md

@@ -164,13 +164,16 @@ The documentation is available on the [web](http://docs.seastar.io/master/index.
 
 Resources
 ---------
-Ask questions and post patches on the development mailing list. Subscription
-information and archives are available [here](https://groups.google.com/forum/#!forum/seastar-dev),
-or just send an email to [email protected].
 
-Information can be found on the main [project website](http://seastar.io).
+* Seasatar Development Mailing List: Discuss challenges, propose improvements with
+  sending code contributions (patches), and get help from experienced developers.
+  Subscribe or browse archives: [here](https://groups.google.com/forum/#!forum/seastar-dev)
+  (or email [email protected]).
+* GitHub Discussions: For more casual conversations and quick questions, consider
+  using the Seastar project's [discussions on Github](https://github.com/scylladb/seastar/discussions).
+* Issue Tracker: File bug reports on the project's [issue tracker](https://github.com/scylladb/seastar/issues).
 
-File bug reports on the project [issue tracker](https://github.com/scylladb/seastar/issues).
+Learn more about Seastar on the main [project website](http://seastar.io).
 
 The Native TCP/IP Stack
 -----------------------
@@ -199,3 +202,4 @@ Projects using Seastar
 * [redpanda](https://vectorized.io/): A Kafka replacement for mission critical systems
 * [Scylla](https://github.com/scylladb/scylla): A fast and reliable NoSQL data store compatible with Cassandra and DynamoDB
 * [smf](https://github.com/smfrpc/smf): The fastest RPC in the West
+* [Ceph - Crimson](https://github.com/ceph/ceph): Next-generation OSD (Object Storage Daemon) implementation based on the Seastar framework

apps/httpd/main.cc

@@ -73,8 +73,6 @@ void set_routes(routes& r) {
 }
 
 int main(int ac, char** av) {
-    httpd::http_server_control prometheus_server;
-    prometheus::config pctx;
     app_template app;
 
     app.add_options()("port", bpo::value<uint16_t>()->default_value(10000), "HTTP Server port");

apps/io_tester/io_tester.cc

@@ -219,6 +219,7 @@ class shard_config {
 struct shard_info {
     unsigned parallelism = 0;
     unsigned rps = 0;
+    unsigned batch = 1;
     unsigned limit = std::numeric_limits<unsigned>::max();
     unsigned shares = 10;
     std::string sched_class = "";
@@ -326,49 +327,44 @@ class class_data {
         }
     }
 
-    future<> issue_requests_in_parallel(std::chrono::steady_clock::time_point stop, unsigned parallelism) {
-        return parallel_for_each(boost::irange(0u, parallelism), [this, stop] (auto dummy) mutable {
+    future<> issue_request(char* buf, io_intent* intent, std::chrono::steady_clock::time_point start, std::chrono::steady_clock::time_point stop) {
+        return issue_request(buf, intent).then([this, start, stop] (auto size) {
+            auto now = std::chrono::steady_clock::now();
+            if (now < stop) {
+                this->add_result(size, std::chrono::duration_cast<std::chrono::microseconds>(now - start));
+            }
+            return make_ready_future<>();
+        });
+    }
+
+    future<> issue_requests_in_parallel(std::chrono::steady_clock::time_point stop) {
+        return parallel_for_each(boost::irange(0u, parallelism()), [this, stop] (auto dummy) mutable {
             auto bufptr = allocate_aligned_buffer<char>(this->req_size(), _alignment);
             auto buf = bufptr.get();
             return do_until([this, stop] { return std::chrono::steady_clock::now() > stop || requests() > limit(); }, [this, buf, stop] () mutable {
                 auto start = std::chrono::steady_clock::now();
-                return issue_request(buf, nullptr).then([this, start, stop] (auto size) {
-                    auto now = std::chrono::steady_clock::now();
-                    if (now < stop) {
-                        this->add_result(size, std::chrono::duration_cast<std::chrono::microseconds>(now - start));
-                    }
+                return issue_request(buf, nullptr, start, stop).then([this] {
                     return think();
                 });
             }).finally([bufptr = std::move(bufptr)] {});
         });
     }
 
-    future<> issue_requests_at_rate(std::chrono::steady_clock::time_point stop, unsigned rps, unsigned parallelism) {
-        return do_with(io_intent{}, 0u, [this, stop, rps, parallelism] (io_intent& intent, unsigned& in_flight) {
-            return parallel_for_each(boost::irange(0u, parallelism), [this, stop, rps, &intent, &in_flight, parallelism] (auto dummy) mutable {
+    future<> issue_requests_at_rate(std::chrono::steady_clock::time_point stop) {
+        return do_with(io_intent{}, 0u, [this, stop] (io_intent& intent, unsigned& in_flight) {
+            return parallel_for_each(boost::irange(0u, parallelism()), [this, stop, &intent, &in_flight] (auto dummy) mutable {
                 auto bufptr = allocate_aligned_buffer<char>(this->req_size(), _alignment);
                 auto buf = bufptr.get();
-                auto pause = std::chrono::duration_cast<std::chrono::microseconds>(1s) / rps;
+                auto pause = std::chrono::duration_cast<std::chrono::microseconds>(1s) / rps();
                 auto pause_dist = _config.options.pause_fn(pause);
-                return seastar::sleep((pause / parallelism) * dummy).then([this, buf, stop, pause = pause_dist.get(), &intent, &in_flight] () mutable {
+                return seastar::sleep((pause / parallelism()) * dummy).then([this, buf, stop, pause = pause_dist.get(), &intent, &in_flight] () mutable {
                     return do_until([this, stop] { return std::chrono::steady_clock::now() > stop || requests() > limit(); }, [this, buf, stop, pause, &intent, &in_flight] () mutable {
                         auto start = std::chrono::steady_clock::now();
                         in_flight++;
-                        return issue_request(buf, &intent).then_wrapped([this, start, pause, stop, &in_flight] (auto size_f) {
-                            size_t size;
-                            try {
-                                size = size_f.get();
-                            } catch (...) {
-                                // cancelled
-                                in_flight--;
-                                return make_ready_future<>();
-                            }
-
+                        return parallel_for_each(boost::irange(0u, batch()), [this, buf, &intent, start, stop] (auto dummy) {
+                            return issue_request(buf, &intent, start, stop);
+                        }).then([this, start, pause] {
                             auto now = std::chrono::steady_clock::now();
-                            if (now < stop) {
-                                this->add_result(size, std::chrono::duration_cast<std::chrono::microseconds>(now - start));
-                            }
-                            in_flight--;
                             auto p = pause->template get_as<std::chrono::microseconds>();
                             auto next = start + p;
 
@@ -378,12 +374,16 @@ class class_data {
                                 // probably the system cannot keep-up with this rate
                                 return make_ready_future<>();
                             }
+                        }).handle_exception_type([] (const cancelled_error&) {
+                            // expected
+                        }).finally([&in_flight] {
+                            in_flight--;
                         });
                     });
-                }).then([&intent, &in_flight] {
-                    intent.cancel();
-                    return do_until([&in_flight] { return in_flight == 0; }, [] { return seastar::sleep(100ms /* ¯\_(ツ)_/¯ */); });
                 }).finally([bufptr = std::move(bufptr), pause = std::move(pause_dist)] {});
+            }).then([&intent, &in_flight] {
+                intent.cancel();
+                return do_until([&in_flight] { return in_flight == 0; }, [] { return seastar::sleep(100ms /* ¯\_(ツ)_/¯ */); });
             });
         });
     }
@@ -393,9 +393,9 @@ class class_data {
         _start = std::chrono::steady_clock::now();
         return with_scheduling_group(_sg, [this, stop] {
             if (rps() == 0) {
-                return issue_requests_in_parallel(stop, parallelism());
+                return issue_requests_in_parallel(stop);
             } else {
-                return issue_requests_at_rate(stop, rps(), parallelism());
+                return issue_requests_at_rate(stop);
             }
         }).then([this] {
             _total_duration = std::chrono::steady_clock::now() - _start;
@@ -477,6 +477,10 @@ class class_data {
         return _config.shard_info.rps;
     }
 
+    unsigned batch() const {
+        return _config.shard_info.batch;
+    }
+
     unsigned limit() const noexcept {
         return _config.shard_info.limit;
     }
@@ -651,6 +655,7 @@ class io_class_data : public class_data {
         emit_one_metrics(out, "io_queue_starvation_time_sec");
         emit_one_metrics(out, "io_queue_consumption");
         emit_one_metrics(out, "io_queue_adjusted_consumption");
+        emit_one_metrics(out, "io_queue_activations");
     }
 
 public:
@@ -922,6 +927,9 @@ struct convert<shard_info> {
         if (node["rps"]) {
             sl.rps = node["rps"].as<unsigned>();
         }
+        if (node["batch"]) {
+            sl.batch = node["batch"].as<unsigned>();
+        }
         if (node["limit"]) {
             sl.limit = node["limit"].as<unsigned>();
         }
@@ -965,7 +973,7 @@ struct convert<options> {
             } else if (st == "steady") {
                 op.sleep_fn = timer_sleep<std::chrono::steady_clock>;
             } else {
-                throw std::runtime_error(format("Unknown sleep_type {}", st));
+                throw std::runtime_error(seastar::format("Unknown sleep_type {}", st));
             }
         }
         if (node["pause_distribution"]) {
@@ -975,7 +983,7 @@ struct convert<options> {
             } else if (pd == "poisson") {
                 op.pause_fn = make_poisson_pause;
             } else {
-                throw std::runtime_error(format("Unknown pause_distribution {}", pd));
+                throw std::runtime_error(seastar::format("Unknown pause_distribution {}", pd));
             }
         }
         return true;