Support RSS trimming for idle bthread stacks

config_brpc.sh

@@ -685,8 +685,9 @@ else
     GTEST_HDR=$(find_dir_of_header_or_die gtest/gtest.h)
     append_to_output_libs $GTEST_LIB "    "
     append_to_output_headers $GTEST_HDR "    "
-    append_to_output_linkings $GTEST_LIB gtest "    "
+    # gtest_main depends on gtest, including when all tests are compiled out.
     append_to_output_linkings $GTEST_LIB gtest_main "    "
+    append_to_output_linkings $GTEST_LIB gtest "    "
 fi
 append_to_output "endif"
 

src/bthread/stack.cpp

@@ -20,11 +20,12 @@
 // Date: Sun Sep  7 22:37:39 CST 2014
 
 #include <unistd.h>                               // getpagesize
-#include <sys/mman.h>                             // mmap, munmap, mprotect
+#include <sys/mman.h>                             // mmap, munmap, mprotect, madvise
 #include <algorithm>                              // std::max
 #include <stdlib.h>                               // posix_memalign
 #include "butil/macros.h"                          // BAIDU_CASSERT
-#include "butil/memory/singleton_on_pthread_once.h"
+#include "butil/reloadable_flags.h"
+#include "butil/threading/platform_thread.h"
 #include "butil/third_party/dynamic_annotations/dynamic_annotations.h" // RunningOnValgrind
 #include "butil/third_party/valgrind/valgrind.h"   // VALGRIND_STACK_REGISTER
 #include "bvar/passive_status.h"
@@ -37,6 +38,14 @@ DEFINE_int32(stack_size_large, 8388608, "size of large stacks");
 DEFINE_int32(guard_page_size, 4096, "size of guard page, allocate stacks by malloc if it's 0(not recommended)");
 DEFINE_int32(tc_stack_small, 32, "maximum small stacks cached by each thread");
 DEFINE_int32(tc_stack_normal, 8, "maximum normal stacks cached by each thread");
+DEFINE_int32(bthread_stack_trim_idle_s, 0,
+             "Release memory of stacks idle in the global pool for this many "
+             "seconds, 0 disables (Linux only, unsupported under ASan)");
+BUTIL_VALIDATE_GFLAG(bthread_stack_trim_idle_s, butil::NonNegativeInteger);
+DEFINE_int32(bthread_stack_trim_batch_size, 64,
+             "Object budget per idle stack trimming batch; whole chunks may "
+             "exceed it by up to 63 stacks (must be positive)");
+BUTIL_VALIDATE_GFLAG(bthread_stack_trim_batch_size, butil::PositiveInteger);
 
 namespace bthread {
 
@@ -150,4 +159,117 @@ int* SmallStackClass::stack_size_flag = &FLAGS_stack_size_small;
 int* NormalStackClass::stack_size_flag = &FLAGS_stack_size_normal;
 int* LargeStackClass::stack_size_flag = &FLAGS_stack_size_large;
 
+#if defined(BTHREAD_STACK_TRIM_SUPPORTED)
+
+#if defined(UNIT_TEST)
+// Replace only when no other thread is trimming stacks.
+int (*g_stack_madvise)(void*, size_t, int) = ::madvise;
+#endif
+
+namespace internal {
+
+static butil::static_atomic<int64_t> s_trimmed_count = BUTIL_STATIC_ATOMIC_INIT(0);
+static int64_t get_trimmed_count(void*) {
+    return s_trimmed_count.load(butil::memory_order_relaxed);
+}
+static bvar::PassiveStatus<int64_t> bvar_trimmed_count(
+    "bthread_stack_trimmed_count", get_trimmed_count, nullptr);
+
+template <typename StackClass>
+static size_t trim_idle_stacks(int64_t idle_us) {
+    typedef typename StackFactory<StackClass>::Wrapper Wrapper;
+    size_t released = 0;
+    int error = 0;
+    auto fn = [&released, &error](Wrapper* w) noexcept {
+        StackStorage& s = w->storage;
+        // malloc owns unguarded stacks. Already processed stacks can return
+        // to the free list when a borrowed chunk is flushed from TLS.
+        if (w->context == nullptr || s.guardsize == 0) {
+            return;
+        }
+        void* low = (char*)s.bottom - s.stacksize;
+#if defined(UNIT_TEST)
+        int rc = g_stack_madvise(low, s.stacksize, MADV_DONTNEED);
+#else
+        int rc = madvise(low, s.stacksize, MADV_DONTNEED);
+#endif
+        // An error can follow partial reclamation across VMAs. Never resume
+        // the old continuation, even on failure; get_stack rebuilds it.
+        w->context = nullptr;
+        if (rc == 0) {
+            ++released;
+        } else {
+            error = errno;
+        }
+    };
+    size_t n = butil::process_idle_objects<Wrapper>(
+        idle_us, FLAGS_bthread_stack_trim_batch_size, fn);
+    if (released != 0) {
+        s_trimmed_count.fetch_add(released, butil::memory_order_relaxed);
+    }
+    if (error != 0) {
+        errno = error;
+        PLOG_EVERY_SECOND(ERROR) << "Fail to madvise idle stacks";
+    }
+    return n;
+}
+
+template <typename StackClass>
+static size_t trim_one_batch() {
+    // Runtime changes take effect at the next batch, including after sleeping.
+    int idle_s = FLAGS_bthread_stack_trim_idle_s;
+    if (idle_s <= 0) {
+        return 0;
+    }
+    size_t n = trim_idle_stacks<StackClass>(
+        static_cast<int64_t>(idle_s) * 1000000);
+    if (n != 0) {
+        // Spread madvise work across batches when the idle backlog is large.
+        butil::PlatformThread::Sleep(butil::TimeDelta::FromMilliseconds(10));
+    }
+    return n;
+}
+
+static void* stack_trim_thread(void*) {
+    butil::PlatformThread::SetNameSimple("bthread_trim");
+    for (;;) {
+        size_t n = trim_one_batch<SmallStackClass>();
+        n += trim_one_batch<NormalStackClass>();
+        n += trim_one_batch<LargeStackClass>();
+        if (n == 0) {
+            butil::PlatformThread::Sleep(butil::TimeDelta::FromSeconds(1));
+        }
+    }
+    return nullptr;
+}
+
+void init_stack_trimming() {
+    static pthread_once_t once = PTHREAD_ONCE_INIT;
+    pthread_once(&once, [] {
+        pthread_attr_t attr;
+        pthread_attr_init(&attr);
+        pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+        pthread_t tid;
+        int rc = pthread_create(&tid, &attr, stack_trim_thread, nullptr);
+        pthread_attr_destroy(&attr);
+        if (rc != 0) {
+            LOG(ERROR) << "Fail to start stack trimmer: " << berror(rc);
+        }
+    });
+}
+
+#if defined(UNIT_TEST)
+// Synchronous test entries must not overlap background reclamation.
+size_t trim_idle_stacks_for_test(int64_t idle_us) {
+    size_t n = trim_idle_stacks<SmallStackClass>(idle_us);
+    n += trim_idle_stacks<NormalStackClass>(idle_us);
+    n += trim_idle_stacks<LargeStackClass>(idle_us);
+    return n;
+}
+#endif
+
+}  // namespace internal
+
+#endif  // BTHREAD_STACK_TRIM_SUPPORTED
+
 }  // namespace bthread

src/bthread/stack.h

@@ -26,8 +26,15 @@
 #include <gflags/gflags.h>          // DECLARE_int32
 #include "bthread/types.h"
 #include "bthread/context.h"        // bthread_fcontext_t
+#include "butil/build_config.h"
+#include "butil/compiler_specific.h"
 #include "butil/object_pool.h"
 
+// Discarding the continuation would leak ASan's saved fake stack.
+#if defined(OS_LINUX) && !defined(BUTIL_USE_ASAN)
+#define BTHREAD_STACK_TRIM_SUPPORTED
+#endif
+
 namespace bthread {
 
 struct StackStorage {
@@ -53,6 +60,13 @@ int allocate_stack_storage(StackStorage* s, int stacksize, int guardsize);
 // corresponding allocate_stack_storage() otherwise behavior is undefined.
 void deallocate_stack_storage(StackStorage* s);
 
+#if defined(BTHREAD_STACK_TRIM_SUPPORTED)
+namespace internal {
+// Called from TaskControl::init, never during command-line parsing.
+void init_stack_trimming();
+}
+#endif
+
 enum StackType {
     STACK_TYPE_MAIN = 0,
     STACK_TYPE_PTHREAD = BTHREAD_STACKTYPE_PTHREAD,

src/bthread/stack_inl.h

@@ -136,7 +136,8 @@ template <typename StackClass> struct StackFactory {
             BTHREAD_ASAN_POISON_MEMORY_REGION(storage);
         }
         ~Wrapper() {
-            if (context) {
+            // A trimmed stack has no context but still owns its mapping.
+            if (storage.bottom) {
                 context = nullptr;
                 // Unpoison to avoid affecting other allocator.
                 BTHREAD_ASAN_UNPOISON_MEMORY_REGION(storage);
@@ -147,9 +148,18 @@ template <typename StackClass> struct StackFactory {
     };
 
     static ContextualStack* get_stack(void (*entry)(intptr_t)) {
-        ContextualStack* cs = butil::get_object<Wrapper>(entry);
+        Wrapper* cs = butil::get_object<Wrapper>(entry);
+        if (cs == nullptr) {
+            return nullptr;
+        }
         // Marks stack as addressable.
         BTHREAD_ASAN_UNPOISON_MEMORY_REGION(cs->storage);
+        // Reclamation may discard the saved continuation, even on failure.
+        // The mapping is kept, so restarting at entry needs no allocation.
+        if (cs->context == nullptr) {
+            cs->context = bthread_make_fcontext(
+                cs->storage.bottom, cs->storage.stacksize, entry);
+        }
         return cs;
     }
 

src/bthread/task_control.cpp

@@ -283,6 +283,9 @@ int TaskControl::init(int concurrency) {
         LOG(ERROR) << "Fail to get global_timer_thread";
         return -1;
     }
+#if defined(BTHREAD_STACK_TRIM_SUPPORTED)
+    internal::init_stack_trimming();
+#endif
 
 #ifdef BRPC_BTHREAD_TRACER
     if (!_task_tracer.Init()) {

src/butil/object_pool.h

@@ -23,6 +23,7 @@
 #define BUTIL_OBJECT_POOL_H
 
 #include <cstddef>                       // size_t
+#include <cstdint>
 #include "butil/type_traits.h"
 
 // ObjectPool is a derivative class of ResourcePool to allocate and
@@ -109,6 +110,23 @@ template <typename T> ObjectPoolInfo describe_objects() {
     return ObjectPool<T>::singleton()->describe_objects();
 }
 
+// Process objects cached globally for at least idle_us (monotonic time).
+// Exclusively borrow whole chunks until at least max_items objects are selected
+// or no eligible chunks remain, then invoke fn outside the pool lock. The count
+// may exceed max_items by at most ObjectPool<T>::FREE_CHUNK_NITEM - 1 objects.
+// Use SIZE_MAX to process all eligible global chunks in one call.
+// fn must be noexcept, preserve each object's lifetime/address,
+// and leave it reusable. Borrowers restore any resources released by fn.
+// Processed chunks become cold and are reused only after the regular global
+// free list is empty. They are not processed again until borrowed and returned
+// to the global free list.
+// Return the number of objects passed to fn, regardless of its effects.
+// TLS caches are untouched. Finish all calls before clearing a pool in tests.
+template <typename T, typename Fn>
+inline size_t process_idle_objects(int64_t idle_us, size_t max_items, Fn& fn) {
+    return ObjectPool<T>::singleton()->process_idle_objects(idle_us, max_items, fn);
+}
+
 }  // namespace butil
 
 #endif  // BUTIL_OBJECT_POOL_H