Small FlatMap optimization

src/brpc/builtin/hotspots_service.cpp

@@ -68,7 +68,6 @@ static DisplayType StringToDisplayType(const std::string& val) {
     static std::once_flag flag;
     std::call_once(flag, []() {
         display_type_map = new butil::CaseIgnoredFlatMap<DisplayType>;
-        display_type_map->init(10);
         (*display_type_map)["dot"] = DisplayType::kDot;
 #if defined(OS_LINUX)
         (*display_type_map)["flame"] = DisplayType::kFlameGraph;

src/brpc/controller.h

@@ -264,7 +264,6 @@ friend void policy::ProcessThriftRequest(InputMessageBase*);
     UserFieldsMap* request_user_fields() {
         if (!_request_user_fields) {
             _request_user_fields = new UserFieldsMap;
-            _request_user_fields->init(29);
         }
         return _request_user_fields;
     }
@@ -274,7 +273,6 @@ friend void policy::ProcessThriftRequest(InputMessageBase*);
     UserFieldsMap* response_user_fields() {
         if (!_response_user_fields) {
             _response_user_fields = new UserFieldsMap;
-            _response_user_fields->init(29);
         }
         return _response_user_fields;
     }

src/brpc/extension.h

@@ -49,8 +49,7 @@ class Extension {
 
 private:
 friend class butil::GetLeakySingleton<Extension<T> >;
-    Extension();
-    ~Extension();
+    Extension() = default;
     butil::CaseIgnoredFlatMap<T*> _instance_map;
     butil::Mutex _map_mutex;
 };

src/brpc/extension_inl.h

@@ -29,15 +29,6 @@ Extension<T>* Extension<T>::instance() {
     return butil::get_leaky_singleton<Extension<T> >();
 }
 
-template <typename T>
-Extension<T>::Extension() {
-    _instance_map.init(29);
-}
-
-template <typename T>
-Extension<T>::~Extension() {
-}
-
 template <typename T>
 int Extension<T>::Register(const std::string& name, T* instance) {
     if (NULL == instance) {

src/brpc/kvmap.h

@@ -39,14 +39,20 @@ class KVMap {
     // Get value of a key(case-sensitive)
     // Return pointer to the value, NULL on not found.
     const std::string* Get(const char* key) const { return _entries.seek(key); }
-    const std::string* Get(const std::string& key) const { return _entries.seek(key); }
+    const std::string* Get(const std::string& key) const {
+        return _entries.seek(key);
+    }
 
     // Set value of a key
-    void Set(const std::string& key, const std::string& value) { GetOrAdd(key) = value; }
-    void Set(const std::string& key, const char* value) { GetOrAdd(key) = value; }
+    void Set(const std::string& key, const std::string& value) {
+        _entries[key] = value;
+    }
+    void Set(const std::string& key, const char* value) { _entries[key] = value; }
     // Convert other types to string as well
     template <typename T>
-    void Set(const std::string& key, const T& value) { GetOrAdd(key) = std::to_string(value); }
+    void Set(const std::string& key, const T& value) {
+        _entries[key] = std::to_string(value);
+    }
 
     // Remove a key
     void Remove(const char* key) { _entries.erase(key); }
@@ -60,12 +66,6 @@ class KVMap {
     size_t Count() const { return _entries.size(); }
 
 private:
-    std::string& GetOrAdd(const std::string& key) {
-        if (!_entries.initialized()) {
-            _entries.init(29);
-        }
-        return _entries[key];
-    }
 
     Map _entries;
 };

src/brpc/uri.cpp

@@ -72,9 +72,6 @@ static void ParseQueries(URI::QueryMap& query_map, const std::string &query) {
     }
     for (QuerySplitter sp(query.c_str()); sp; ++sp) {
         if (!sp.key().empty()) {
-            if (!query_map.initialized()) {
-                query_map.init(URI::QUERY_MAP_INITIAL_BUCKET);
-            }
             std::string key(sp.key().data(), sp.key().size());
             std::string value(sp.value().data(), sp.value().size());
             query_map[key] = value;
@@ -347,9 +344,6 @@ void URI::PrintWithoutHost(std::ostream& os) const {
 }
 
 void URI::InitializeQueryMap() const {
-    if (!_query_map.initialized()) {
-        CHECK_EQ(0, _query_map.init(QUERY_MAP_INITIAL_BUCKET));
-    }
     ParseQueries(_query_map, _query);
     _query_was_modified = false;
     _initialized_query_map = true;

src/brpc/uri.h

@@ -51,7 +51,6 @@ namespace brpc {
 //                                               interpretable as extension
 class URI {
 public:
-    static const size_t QUERY_MAP_INITIAL_BUCKET = 16;
     typedef butil::FlatMap<std::string, std::string> QueryMap;
     typedef QueryMap::const_iterator QueryIterator;
 

src/bthread/butex.cpp

@@ -360,7 +360,6 @@ int butex_wake_n(void* arg, size_t n, bool nosignal) {
         return nwakeup;
     }
     butil::FlatMap<bthread_tag_t, TaskGroup*> nwakeups;
-    nwakeups.init(FLAGS_task_group_ntags);
     // We will exchange with first waiter in the end.
     ButexBthreadWaiter* next = static_cast<ButexBthreadWaiter*>(
         bthread_waiters.head()->value());
@@ -440,7 +439,6 @@ int butex_wake_except(void* arg, bthread_t excluded_bthread) {
         return nwakeup;
     }
     butil::FlatMap<bthread_tag_t, TaskGroup*> nwakeups;
-    nwakeups.init(FLAGS_task_group_ntags);
     do {
         // pop reversely
         ButexBthreadWaiter* w = static_cast<ButexBthreadWaiter*>(bthread_waiters.tail()->value());

src/butil/containers/flat_map.h

@@ -94,6 +94,7 @@
 #define BUTIL_FLAT_MAP_H
 
 #include <stdint.h>
+#include <cstddef>
 #include <functional>
 #include <iostream>                               // std::ostream
 #include <type_traits>                            // std::aligned_storage
@@ -121,6 +122,14 @@ struct BucketInfo {
     double average_length;
 };
 
+#ifndef BRPC_FLATMAP_DEFAULT_NBUCKET
+#ifdef FLAT_MAP_ROUND_BUCKET_BY_USE_NEXT_PRIME
+#define BRPC_FLATMAP_DEFAULT_NBUCKET 29U
+#else
+#define BRPC_FLATMAP_DEFAULT_NBUCKET 16U
+#endif
+#endif // BRPC_FLATMAP_DEFAULT_NBUCKET
+
 // NOTE: Objects stored in FlatMap MUST be copyable.
 template <typename _K, typename _T,
           // Compute hash code from key.
@@ -148,6 +157,9 @@ class FlatMap {
         FlatMapIterator<FlatMap, const value_type> >::type const_iterator;
     typedef _Hash hasher;
     typedef _Equal key_equal;
+    typedef typename conditional<_Multi, true_type, false_type>::type multi_type;
+    static constexpr size_t DEFAULT_NBUCKET = BRPC_FLATMAP_DEFAULT_NBUCKET;
+
     struct PositionHint {
         size_t nbucket;
         size_t offset;
@@ -158,13 +170,13 @@ class FlatMap {
     explicit FlatMap(const hasher& hashfn = hasher(),
                      const key_equal& eql = key_equal(),
                      const allocator_type& alloc = allocator_type());
-    ~FlatMap();
     FlatMap(const FlatMap& rhs);
+    ~FlatMap();
+
     FlatMap& operator=(const FlatMap& rhs);
     void swap(FlatMap & rhs);
 
-    // Must be called to initialize this map, otherwise insert/operator[]
-    // crashes, and seek/erase fails.
+    // FlatMap will be automatically initialized, so no need to call this function.
     // `nbucket' is the initial number of buckets. `load_factor' is the 
     // maximum value of size()*100/nbucket, if the value is reached, nbucket
     // will be doubled and all items stored will be rehashed which is costly.
@@ -216,7 +228,7 @@ class FlatMap {
     // Resize this map. This is optional because resizing will be triggered by
     // insert() or operator[] if there're too many items.
     // Returns successful or not.
-    bool resize(size_t nbucket);
+    bool resize(size_t new_nbucket);
 
     // Iterators
     iterator begin();
@@ -254,7 +266,7 @@ class FlatMap {
     void save_iterator(const const_iterator&, PositionHint*) const;
     const_iterator restore_iterator(const PositionHint&) const;
 
-    // True if init() was successfully called.
+    // Always returns true.
     bool initialized() const { return _buckets != NULL; }
 
     bool empty() const { return _size == 0; }
@@ -282,14 +294,28 @@ class FlatMap {
         const Element& element() const {
             return *element_;
         }
+
+        void swap(Bucket& rhs) {
+            if (!is_valid() && !rhs.is_valid()) {
+                return;
+            } else if (is_valid() && !rhs.is_valid()) {
+                rhs.element_.Init(std::move(element()));
+                element().~Element();
+            } else if (!is_valid() && rhs.is_valid()) {
+                element_.Init(std::move(rhs.element()));
+                rhs.element().~Element();
+            } else {
+                element().swap(rhs.element());
+            }
+            std::swap(next, rhs.next);
+        }
+
         Bucket *next;
 
     private:
         ManualConstructor<Element> element_;
     };
 
-    allocator_type& get_allocator() { return _pool.get_allocator(); }
-
 private:
 template <typename _Map, typename _Element> friend class FlatMapIterator;
 template <typename _Map, typename _Element> friend class SparseFlatMapIterator;
@@ -304,7 +330,8 @@ template <typename _Map, typename _Element> friend class SparseFlatMapIterator;
         size_t nbucket;
     };
 
-    NewBucketsInfo new_buckets_and_thumbnail(size_t size, size_t new_nbucket);
+    NewBucketsInfo new_buckets_and_thumbnail(
+        size_t size, size_t new_nbucket, bool ignore_same_nbucket);
 
     // For `_Multi=true'.
     // Insert a new default-constructed associated with |key| always.
@@ -314,6 +341,9 @@ template <typename _Map, typename _Element> friend class SparseFlatMapIterator;
     template<bool Multi = _Multi>
     typename std::enable_if<Multi, mapped_type&>::type operator[](const key_type& key);
 
+    allocator_type& get_allocator() { return _pool.get_allocator(); }
+    allocator_type get_allocator() const { return _pool.get_allocator(); }
+
     // True if buckets need to be resized before holding `size' elements.
     bool is_too_crowded(size_t size) const {
         return is_too_crowded(size, _nbucket, _load_factor);
@@ -322,8 +352,21 @@ template <typename _Map, typename _Element> friend class SparseFlatMapIterator;
         return size * 100 >= nbucket * load_factor;
     }
 
-    static void init_buckets_and_thumbnail(
-        Bucket* buckets, uint64_t* thumbnail, size_t nbucket) {
+    void init_load_factor(u_int load_factor) {
+        if (_is_default_load_factor) {
+            _is_default_load_factor = false;
+            _load_factor = load_factor;
+        }
+    }
+
+    // True if using default buckets which is for small map optimization.
+    bool is_default_buckets() const {
+        return _buckets == (Bucket*)(&_default_buckets_spaces);
+    }
+
+    static void init_buckets_and_thumbnail(Bucket* buckets,
+                                           uint64_t* thumbnail,
+                                           size_t nbucket) {
         for (size_t i = 0; i < nbucket; ++i) {
             buckets[i].set_invalid();
         }
@@ -332,12 +375,20 @@ template <typename _Map, typename _Element> friend class SparseFlatMapIterator;
             bit_array_clear(thumbnail, nbucket);
         }
     }
-
+
+    static const size_t default_nthumbnail = BIT_ARRAY_LEN(DEFAULT_NBUCKET);
+    // Small map optimization.
+    typedef typename std::aligned_storage<sizeof(Bucket), alignof(Bucket)>::type
+            buckets_spaces_type;
+    // Note: need an extra bucket to let iterator know where buckets end.
+    buckets_spaces_type _default_buckets_spaces[DEFAULT_NBUCKET + 1];
+    uint64_t _default_thumbnail[default_nthumbnail];
     size_t _size;
     size_t _nbucket;
     Bucket* _buckets;
     uint64_t* _thumbnail;
     u_int _load_factor;
+    bool _is_default_load_factor;
     hasher _hashfn;
     key_equal _eql;
     SingleThreadedPool<sizeof(Bucket), 1024, 16, allocator_type> _pool;
@@ -348,7 +399,8 @@ template <typename _K, typename _T,
           typename _Equal = DefaultEqualTo<_K>,
           bool _Sparse = false,
           typename _Alloc = PtAllocator>
-using MultiFlatMap = FlatMap<_K, _T, _Hash, _Equal, _Sparse, _Alloc, true>;
+using MultiFlatMap = FlatMap<
+    _K, _T, _Hash, _Equal, _Sparse, _Alloc, true>;
 
 template <typename _K,
           typename _Hash = DefaultHasher<_K>,
@@ -357,7 +409,8 @@ template <typename _K,
           typename _Alloc = PtAllocator>
 class FlatSet {
 public:
-    typedef FlatMap<_K, FlatMapVoid, _Hash, _Equal, _Sparse, _Alloc> Map;
+    typedef FlatMap<_K, FlatMapVoid, _Hash, _Equal, _Sparse,
+                    _Alloc, false> Map;
     typedef typename Map::key_type key_type;
     typedef typename Map::value_type value_type;
     typedef typename Map::Bucket Bucket;
@@ -408,15 +461,18 @@ class FlatSet {
 
 template <typename _K, typename _T,
           typename _Hash = DefaultHasher<_K>,
-          typename _Equal = DefaultEqualTo<_K> >
-class SparseFlatMap : public FlatMap<_K, _T, _Hash, _Equal, true> {
-};
+          typename _Equal = DefaultEqualTo<_K>,
+          typename _Alloc = PtAllocator,
+          bool _Multi = false>
+class SparseFlatMap : public FlatMap<
+    _K, _T, _Hash, _Equal, true, _Alloc, _Multi> {};
 
 template <typename _K,
           typename _Hash = DefaultHasher<_K>,
-          typename _Equal = DefaultEqualTo<_K> >
-class SparseFlatSet : public FlatSet<_K, _Hash, _Equal, true> {
-};
+          typename _Equal = DefaultEqualTo<_K>,
+          typename _Alloc = PtAllocator>
+class SparseFlatSet : public FlatSet<
+    _K, _Hash, _Equal, true, _Alloc> {};
 
 // Implement FlatMapElement
 template <typename K, typename T>
@@ -431,6 +487,13 @@ class FlatMapElement {
     // POD) which is wrong generally.
     explicit FlatMapElement(const K& k) : _key(k), _value(T()) {}
     //                                             ^^^^^^^^^^^
+
+    FlatMapElement(const FlatMapElement& rhs)
+        : _key(rhs._key), _value(rhs._value) {}
+
+    FlatMapElement(FlatMapElement&& rhs) noexcept
+        : _key(std::move(rhs._key)), _value(std::move(rhs._value)) {}
+
     const K& first_ref() const { return _key; }
     T& second_ref() { return _value; }
     T&& second_movable_ref() { return std::move(_value); }
@@ -442,16 +505,25 @@ class FlatMapElement {
     inline static T&& second_movable_ref_from_value(value_type& v)
     { return std::move(v.second); }
 
+    void swap(FlatMapElement& rhs) {
+        std::swap(_key, rhs._key);
+        std::swap(_value, rhs._value);
+    }
+
 private:
-    const K _key;
+    K _key;
     T _value;
 };
 
 template <typename K>
 class FlatMapElement<K, FlatMapVoid> {
 public:
     typedef const K value_type;
+    // See the comment in the above FlatMapElement.
     explicit FlatMapElement(const K& k) : _key(k) {}
+    FlatMapElement(const FlatMapElement& rhs) : _key(rhs._key) {}
+    FlatMapElement(FlatMapElement&& rhs) noexcept : _key(std::move(rhs._key)) {}
+
     const K& first_ref() const { return _key; }
     FlatMapVoid& second_ref() { return second_ref_from_value(_key); }
     FlatMapVoid& second_movable_ref() { return second_ref(); }
@@ -470,8 +542,7 @@ class FlatMapElement<K, FlatMapVoid> {
 
 // Implement DefaultHasher and DefaultEqualTo
 template <typename K>
-struct DefaultHasher : public BUTIL_HASH_NAMESPACE::hash<K> {
-};
+struct DefaultHasher : public BUTIL_HASH_NAMESPACE::hash<K> {};
 
 template <>
 struct DefaultHasher<std::string> {