// Copyright 2019 The TCMalloc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This file documents extensions supported by TCMalloc. These extensions
// provide hooks for both surfacing telemetric data about TCMalloc's usage and
// tuning the internal implementation of TCMalloc. The internal implementation
// functions use weak linkage, allowing an application to link against the
// extensions without always linking against TCMalloc.
//
// Many of these APIs are also supported when built with sanitizers.

#ifndef TCMALLOC_MALLOC_EXTENSION_H_
#define TCMALLOC_MALLOC_EXTENSION_H_

#include <cstddef>
#include <cstdint>
#include <limits>
#include <map>
#include <memory>
#include <new>
#include <optional>
#include <string>
#include <utility>

#include "third_party/base/censushandle.h"  // tcmalloc:google3-only(b/197108627)
#include "third_party/abseil-cpp/absl/base/attributes.h"
// tcmalloc:google3-begin(b/143069684)
#include "third_party/abseil-cpp/absl/base/internal/malloc_hook.h"
// tcmalloc:google3-end
#include "third_party/abseil-cpp/absl/base/macros.h"
#include "third_party/abseil-cpp/absl/functional/function_ref.h"
#include "third_party/abseil-cpp/absl/strings/numbers.h"
#include "third_party/abseil-cpp/absl/strings/str_cat.h"
#include "third_party/abseil-cpp/absl/strings/string_view.h"
#include "third_party/abseil-cpp/absl/time/time.h"
#include "third_party/abseil-cpp/absl/types/optional.h"
#include "third_party/abseil-cpp/absl/types/span.h"

// Not all versions of Abseil provide this macro.
// TODO(b/323943471): Remove on upgrading to version that provides the macro.
#ifndef ABSL_DEPRECATE_AND_INLINE
#define ABSL_DEPRECATE_AND_INLINE()
#endif

// Indicates how frequently accessed the allocation is expected to be.
// 0   - The allocation is rarely accessed.
// ...
// 255 - The allocation is accessed very frequently.
enum class __hot_cold_t : uint8_t;

// TODO(ckennelly): Lifetimes

namespace tcmalloc {

// Alias to the newer type in the global namespace, so that existing code works
// as is.
    using hot_cold_t = __hot_cold_t;

    constexpr hot_cold_t kDefaultMinHotAccessHint =
    static_cast<tcmalloc::hot_cold_t>(2);

}  // namespace tcmalloc

inline bool AbslParseFlag(absl::string_view text, tcmalloc::hot_cold_t* hotness,
                          std::string* /* error */) {
    uint32_t value;
    if (!absl::SimpleAtoi(text, &value)) {
        return false;
    }
    // hot_cold_t is a uint8_t, so make sure the flag is within the allowable
    // range before casting.
    if (value > std::numeric_limits<uint8_t>::max()) {
        return false;
    }
    *hotness = static_cast<tcmalloc::hot_cold_t>(value);
    return true;
}

inline std::string AbslUnparseFlag(tcmalloc::hot_cold_t hotness) {
    return absl::StrCat(hotness);
}

namespace tcmalloc {
    namespace tcmalloc_internal {
        class AllocationProfilingTokenAccessor;
        class AllocationProfilingTokenBase;
        class ProfileAccessor;
        class ProfileBase;
    }  // namespace tcmalloc_internal

    enum class ProfileType {
        // Approximation of current heap usage
        kHeap,

        // Fragmentation report
        kFragmentation,

        // Sample of objects that were live at a recent peak of total heap usage. The
        // specifics of when exactly this profile is collected are subject to change.
        kPeakHeap,

        // Sample of objects allocated from the start of allocation profiling until
        // the profile was terminated with Stop().
        kAllocations,

        // Lifetimes of sampled objects that are live during the profiling session.
        kLifetimes,

        // Only present to prevent switch statements without a default clause so that
        // we can extend this enumeration without breaking code.
        kDoNotUse,
    };

    class Profile final {
    public:
        Profile() = default;
        Profile(Profile&&) = default;
        Profile(const Profile&) = delete;

        ~Profile();

        Profile& operator=(Profile&&) = default;
        Profile& operator=(const Profile&) = delete;

        struct Sample {
            static constexpr int kMaxStackDepth = 64;

            int64_t sum;
            // The reported count of samples, with possible rounding up for unsample.
            // A given sample typically corresponds to some allocated objects, and the
            // number of objects is the quotient of weight (number of bytes requested
            // between previous and current samples) divided by the requested size.
            int64_t count;

            size_t requested_size;
            size_t requested_alignment;
            size_t allocated_size;

            // tcmalloc:google3-begin(b/197108627)
            CensusHandle census_handle;
            // tcmalloc:google3-end

            // tcmalloc:google3-begin(b/143069684)
            MallocHook::AllocHandle alloc_handle;
            // tcmalloc:google3-end

            // Return whether the allocation was returned with
            // tcmalloc_size_returning_operator_new or its variants.
            bool requested_size_returning;

            // tcmalloc:google3-begin(context)
            // TODO(b/265065949): Remove this field when we can use `census_handle` to
            // tell if the context is present. `census_handle` is available when Census
            // memory accounting is enabled. For now, all C++ binaries have enabled it.
            // However, for mixed binaries such as C++/Java (e.g. GWS) and C++/Go
            // (e.g. Raffia) it is disabled because the Java implementation does not
            // exist and the Go implementation has non-trivial overhead. We can remove
            // the field when mixed binaries enable Census memory accounting.
            enum class Context : uint8_t {
                Absent,
                Present,
            };

            Context context;
            // tcmalloc:google3-end

            enum class Access : uint8_t {
                Hot,
                Cold,

                // Only present to prevent switch statements without a default clause so
                // that we can extend this enumeration without breaking code.
                kDoNotUse,
            };
            hot_cold_t access_hint;
            Access access_allocated;

            // Whether this sample captures allocations where the deallocation event
            // was not observed. Thus the measurements are censored in the statistical
            // sense, see https://en.wikipedia.org/wiki/Censoring_(statistics)#Types.
            bool is_censored = false;

            // Provide the status of GWP-ASAN guarding for a given sample.
            enum class GuardedStatus : int8_t {
                // Conditions which represent why a sample was not guarded:
                //
                // The requested_size of the allocation sample is larger than the
                // available pages which are guardable.
                LargerThanOnePage = -1,
                // By flag, the guarding of samples has been disabled.
                Disabled = -2,
                // Too many guards have been placed, any further guards will cause
                // unexpected load on binary.
                RateLimited = -3,
                // The requested_size of the allocation sample is too small (= 0) to be
                // guarded.
                TooSmall = -4,
                // Too many samples are already guarded.
                NoAvailableSlots = -5,
                // Perhaps the only true error, when the mprotect call fails.
                MProtectFailed = -6,
                // Used in an improved guarding selection algorithm.
                Filtered = -7,
                // An unexpected state, which represents that branch for selection was
                // missed.
                Unknown = -100,
                // When guarding is not even considered on a sample.
                NotAttempted = 0,
                // The following values do not represent final states, but rather intent
                // based on the applied algorithm for selecting guarded samples:
                //
                // Request guard: may still not be guarded for other reasons (see
                //    above)
                Requested = 1,
                // Unused.
                Required = 2,
                // The result when a sample is actually guarded by GWP-ASAN.
                Guarded = 10,
            };
            GuardedStatus guarded_status = GuardedStatus::Unknown;

            // How the memory was allocated (new/malloc/etc.).
            enum class AllocationType : uint8_t {
                New,
                Malloc,
                AlignedMalloc,
            };

            AllocationType type;

            // tcmalloc:google3-begin(static)
            int static_initialization_depth;
            // tcmalloc:google3-end

            int depth;
            void* stack[kMaxStackDepth];

            // The following vars are used by the lifetime (deallocation) profiler.
            uint64_t profile_id;

            // Timestamp of allocation.
            absl::Time allocation_time;

            // Aggregated lifetime statistics per callstack.
            absl::Duration avg_lifetime;
            absl::Duration stddev_lifetime;
            absl::Duration min_lifetime;
            absl::Duration max_lifetime;

            // For the *_matched vars below we use true = "same", false = "different".
            // When the value is unavailable the profile contains "none". For
            // right-censored observations, CPU and thread matched values are "none".
            std::optional<bool> allocator_deallocator_physical_cpu_matched;
            std::optional<bool> allocator_deallocator_virtual_cpu_matched;
            std::optional<bool> allocator_deallocator_l3_matched;
            std::optional<bool> allocator_deallocator_numa_matched;
            std::optional<bool> allocator_deallocator_thread_matched;
            // tcmalloc:google3-begin(context)
            std::optional<bool> allocator_deallocator_context_matched;
            // tcmalloc:google3-end

            // The start address of the sampled allocation, used to calculate the
            // residency info for the objects represented by this sampled allocation.
            void* span_start_address;
        };

        void Iterate(absl::FunctionRef<void(const Sample&)> f) const;

        ProfileType Type() const;

        // Time stamp when the profile collection started.  Returns std::nullopt if
        // this is not available.
        std::optional<absl::Time> StartTime() const;

        // The duration the profile was collected for.  For instantaneous profiles
        // (heap, peakheap, etc.), this returns absl::ZeroDuration().
        absl::Duration Duration() const;

    private:
        explicit Profile(std::unique_ptr<const tcmalloc_internal::ProfileBase>);

        std::unique_ptr<const tcmalloc_internal::ProfileBase> impl_;
        friend class tcmalloc_internal::ProfileAccessor;
    };

// tcmalloc:google3-begin(coroner)
    class MallocExtensionAsanWriter;
// tcmalloc:google3-end

    class AddressRegion {
    public:
        AddressRegion() {}
        virtual ~AddressRegion();

        // Allocates at least size bytes of memory from this region, aligned with
        // alignment.  Returns a pair containing a pointer to the start the allocated
        // memory and the actual size allocated.  Returns {nullptr, 0} on failure.
        //
        // Alloc must return memory located within the address range given in the call
        // to AddressRegionFactory::Create that created this AddressRegion.
        virtual std::pair<void*, size_t> Alloc(size_t size, size_t alignment) = 0;
    };

// Interface to a pluggable address region allocator.
    class AddressRegionFactory {
    public:
        enum class UsageHint {
            kNormal,                // Normal usage.
            kInfrequentAllocation,  // TCMalloc allocates from these regions less
            // frequently than normal regions.
            kInfrequent ABSL_DEPRECATED("Use kInfrequentAllocation") =
            kInfrequentAllocation,
            kMetadata,          // Metadata for TCMalloc not returned via new/malloc.
            kInfrequentAccess,  // TCMalloc places cold allocations in these regions.
            // Usage of the below implies numa_aware is enabled. tcmalloc will mbind the
            // address region to the hinted socket, but also passes the hint in case
            // mbind is not sufficient (e.g. when dealing with pre-faulted memory).
            kNormalNumaAwareS0,  // Normal usage intended for NUMA S0 under numa_aware.
            kNormalNumaAwareS1,  // Normal usage intended for NUMA S1 under numa_aware.
        };

        constexpr AddressRegionFactory() = default;
        virtual ~AddressRegionFactory() = default;

        // Returns an AddressRegion with the specified start address and size.  hint
        // indicates how the caller intends to use the returned region (helpful for
        // deciding which regions to remap with hugepages, which regions should have
        // pages prefaulted, etc.).  The returned AddressRegion must never be deleted.
        //
        // The caller must have reserved size bytes of address space starting at
        // start_addr with mmap(PROT_NONE) prior to calling this function (so it is
        // safe for Create() to mmap(MAP_FIXED) over the specified address range).
        // start_addr and size are always page-aligned.
        virtual AddressRegion* Create(void* start_addr, size_t size,
                                      UsageHint hint) = 0;

        // Gets a human-readable description of the current state of the allocator.
        //
        // The state is stored in the provided buffer.  The number of bytes used (or
        // would have been required, had the buffer been of sufficient size) is
        // returned.
        virtual size_t GetStats(absl::Span<char> buffer);

        // Gets a description of the current state of the allocator in pbtxt format.
        //
        // The state is stored in the provided buffer.  The number of bytes used (or
        // would have been required, had the buffer been of sufficient size) is
        // returned.
        virtual size_t GetStatsInPbtxt(absl::Span<char> buffer);

        // Returns the total number of bytes allocated by MallocInternal().
        static size_t InternalBytesAllocated();

    protected:
        // Dynamically allocates memory for use by AddressRegionFactory.  Particularly
        // useful for creating AddressRegions inside Create().
        //
        // This memory is never freed, so allocate sparingly.
        static void* MallocInternal(size_t size);
    };

    class MallocExtension final {
    public:
        // Gets a human readable description of the current state of the malloc data
        // structures.
        //
        // See g3doc/third_party/tcmalloc/g3doc/stats.md for how to interpret these
        // statistics.
        static std::string GetStats();

        // -------------------------------------------------------------------
        // Control operations for getting malloc implementation specific parameters.
        // Some currently useful properties:
        //
        // generic
        // -------
        // "generic.current_allocated_bytes"
        //      Number of bytes currently allocated by application
        //
        // "generic.heap_size"
        //      Number of bytes in the heap ==
        //            current_allocated_bytes +
        //            fragmentation +
        //            freed (but not released to OS) memory regions
        //
        // tcmalloc
        // --------
        // "tcmalloc.max_total_thread_cache_bytes"
        //      Upper limit on total number of bytes stored across all
        //      per-thread caches.  Default: 16MB.
        //
        // "tcmalloc.current_total_thread_cache_bytes"
        //      Number of bytes used across all thread caches.
        //
        // "tcmalloc.pageheap_free_bytes"
        //      Number of bytes in free, mapped pages in page heap.  These
        //      bytes can be used to fulfill allocation requests.  They
        //      always count towards virtual memory usage, and unless the
        //      underlying memory is swapped out by the OS, they also count
        //      towards physical memory usage.
        //
        // "tcmalloc.pageheap_unmapped_bytes"
        //      Number of bytes in free, unmapped pages in page heap.
        //      These are bytes that have been released back to the OS,
        //      possibly by one of the MallocExtension "Release" calls.
        //      They can be used to fulfill allocation requests, but
        //      typically incur a page fault.  They always count towards
        //      virtual memory usage, and depending on the OS, typically
        //      do not count towards physical memory usage.
        //
        //  "tcmalloc.per_cpu_caches_active"
        //      Whether tcmalloc is using per-CPU caches (1 or 0 respectively).
        // -------------------------------------------------------------------

        // Gets the named property's value or a nullopt if the property is not valid.
        static std::optional<size_t> GetNumericProperty(absl::string_view property);

        // Marks the current thread as "idle".  This function may optionally be called
        // by threads as a hint to the malloc implementation that any thread-specific
        // resources should be released.  Note: this may be an expensive function, so
        // it should not be called too often.
        //
        // Also, if the code that calls this function will go to sleep for a while, it
        // should take care to not allocate anything between the call to this function
        // and the beginning of the sleep.
        static void MarkThreadIdle();

        // Marks the current thread as "busy".  This function should be called after
        // MarkThreadIdle() if the thread will now do more work.  If this method is
        // not called, performance may suffer.
        static void MarkThreadBusy();

        // Attempts to free any resources associated with cpu <cpu> (in the sense of
        // only being usable from that CPU.)  Returns the number of bytes previously
        // assigned to "cpu" that were freed.  Safe to call from any processor, not
        // just <cpu>.
        static size_t ReleaseCpuMemory(int cpu);

        // Gets the region factory used by the malloc extension instance. Returns null
        // for malloc implementations that do not support pluggable region factories.
        static AddressRegionFactory* GetRegionFactory();

        // Sets the region factory to the specified.
        //
        // Users could register their own region factories by doing:
        //   factory = new MyOwnRegionFactory();
        //   MallocExtension::SetRegionFactory(factory);
        //
        // It's up to users whether to fall back (recommended) to the default region
        // factory (use GetRegionFactory() above) or not. The caller is responsible to
        // any necessary locking.
        static void SetRegionFactory(AddressRegionFactory* a);

        // Tries to release at least num_bytes of free memory back to the OS for
        // reuse.
        //
        // Depending on the state of the malloc implementation, more than num_bytes of
        // memory may be released to the OS.
        //
        // This request may not be completely honored if:
        // * The underlying malloc implementation does not support releasing memory to
        //   the OS.
        // * There are not at least num_bytes of free memory cached, or free memory is
        //   fragmented in ways that keep it from being returned to the OS.
        //
        // Returning memory to the OS can hurt performance in two ways:
        // * Parts of huge pages may be free and returning them to the OS requires
        //   breaking up the huge page they are located on.  This can slow accesses to
        //   still-allocated memory due to increased TLB pressure for the working set.
        // * If the memory is ultimately needed again, pages will need to be faulted
        //   back in.
        static void ReleaseMemoryToSystem(size_t num_bytes);

        enum class LimitKind { kSoft, kHard };

        // Make a best effort attempt to prevent more than limit bytes of memory
        // from being allocated by the system. In particular, if satisfying a given
        // malloc call would require passing this limit, release as much memory to
        // the OS as needed to stay under it if possible.
        //
        // If limit_kind == kHard, crash if returning memory is unable to get below
        // the limit.
        static size_t GetMemoryLimit(LimitKind limit_kind);
        static void SetMemoryLimit(size_t limit, LimitKind limit_kind);

        // Gets the sampling interval.  Returns a value < 0 if unknown.
        static int64_t GetProfileSamplingInterval();
        // Sets the sampling interval for heap profiles.  TCMalloc samples
        // approximately every interval bytes allocated.
        static void SetProfileSamplingInterval(int64_t interval);

        // Gets the guarded sampling rate.  Returns a value < 0 if unknown.
        static int64_t GetGuardedSamplingInterval();
        // Sets the guarded sampling interval for sampled allocations.  TCMalloc
        // samples approximately every interval bytes allocated, subject to
        // implementation limitations in GWP-ASan.
        //
        // Guarded samples provide probabilistic protections against buffer underflow,
        // overflow, and use-after-free when GWP-ASan is active (via calling
        // ActivateGuardedSampling).
        static void SetGuardedSamplingInterval(int64_t interval);

        // The old names to get and set profile sampling intervals used "rate" to
        // refer to intervals. Use of the below is deprecated to avoid confusion.
        ABSL_DEPRECATE_AND_INLINE()
        static int64_t GetProfileSamplingRate() {
            return GetProfileSamplingInterval();
        }
        ABSL_DEPRECATE_AND_INLINE()
        static void SetProfileSamplingRate(int64_t rate) {
            SetProfileSamplingInterval(rate);
        }
        ABSL_DEPRECATE_AND_INLINE()
        static int64_t GetGuardedSamplingRate() {
            return GetGuardedSamplingInterval();
        }
        ABSL_DEPRECATE_AND_INLINE()
        static void SetGuardedSamplingRate(int64_t rate) {
            SetGuardedSamplingInterval(rate);
        }

        // Switches TCMalloc to guard sampled allocations for underflow, overflow, and
        // use-after-free according to the guarded sample parameter value.
        static void ActivateGuardedSampling();

        // Gets whether TCMalloc is using per-CPU caches.
        static bool PerCpuCachesActive();

        // Gets the current maximum cache size per CPU cache.
        static int32_t GetMaxPerCpuCacheSize();
        // Sets the maximum cache size per CPU cache.  This is a per-core limit.
        static void SetMaxPerCpuCacheSize(int32_t value);

        // Gets the current maximum thread cache.
        static int64_t GetMaxTotalThreadCacheBytes();
        // Sets the maximum thread cache size.  This is a whole-process limit.
        static void SetMaxTotalThreadCacheBytes(int64_t value);

        // Enables or disables background processes.
        static bool GetBackgroundProcessActionsEnabled();
        static void SetBackgroundProcessActionsEnabled(bool value);

        // Gets and sets background process sleep time. This controls the interval
        // granularity at which the actions are invoked.
        static absl::Duration GetBackgroundProcessSleepInterval();
        static void SetBackgroundProcessSleepInterval(absl::Duration value);

        // Gets and sets intervals used for finding recent demand peak, short-term
        // demand fluctuation, and long-term demand trend. Zero duration means not
        // considering corresponding demand history for delayed subrelease. Delayed
        // subrelease is disabled if all intervals are zero.
        static absl::Duration GetSkipSubreleaseInterval();
        static void SetSkipSubreleaseInterval(absl::Duration value);
        static absl::Duration GetSkipSubreleaseShortInterval();
        static void SetSkipSubreleaseShortInterval(absl::Duration value);
        static absl::Duration GetSkipSubreleaseLongInterval();
        static void SetSkipSubreleaseLongInterval(absl::Duration value);

        // Gets and sets intervals used for finding the recent short-term demand
        // fluctuation and long-term demand trend in HugeCache. Zero duration means
        // not considering corresponding demand history for delayed (demand-based)
        // hugepage release. The feature is disabled if both intervals are zero.
        static absl::Duration GetCacheDemandReleaseShortInterval();
        static void SetCacheDemandReleaseShortInterval(absl::Duration value);
        static absl::Duration GetCacheDemandReleaseLongInterval();
        static void SetCacheDemandReleaseLongInterval(absl::Duration value);

        // tcmalloc:google3-begin(memfs malloc)
        // Parameters for the hugetlbfs-backed allocator
        struct MemfsMallocConfig final {
            // If no path is specified, the hugetlbfs allocator is inactive.
            std::optional<std::string> path;

            // Declare the ctor and dtor explicitly so that it is put in the
            // google_malloc section.
            MemfsMallocConfig() = default;
            ~MemfsMallocConfig() = default;
        };

        // Gets the current parameters for the hugetlbfs allocator.
        static MemfsMallocConfig GetMemfsMallocConfig();

        // Switches TCMalloc to using the hugetlbfs allocator with the specified
        // configuration.
        static void ActivateMemfsMalloc(const MemfsMallocConfig& config);
        // tcmalloc:google3-end

        // Returns the estimated number of bytes that will be allocated for a request
        // of "size" bytes.  This is an estimate: an allocation of "size" bytes may
        // reserve more bytes, but will never reserve fewer.
        static size_t GetEstimatedAllocatedSize(size_t size);

        // Returns the actual number N of bytes reserved by tcmalloc for the pointer
        // p.  This number may be equal to or greater than the number of bytes
        // requested when p was allocated.
        //
        // This function is just useful for statistics collection.  The client must
        // *not* read or write from the extra bytes that are indicated by this call.
        //
        // Example, suppose the client gets memory by calling
        //    p = malloc(10)
        // and GetAllocatedSize(p) returns 16.  The client must only use the first 10
        // bytes p[0..9], and not attempt to read or write p[10..15].
        //
        // p must have been allocated by TCMalloc and must not be an interior pointer
        // -- that is, must be exactly the pointer returned to by malloc() et al., not
        // some offset from that -- and should not have been freed yet.  p may be
        // null.
        static std::optional<size_t> GetAllocatedSize(const void* p);

        // Returns
        // * kOwned if TCMalloc allocated the memory pointed to by p, or
        // * kNotOwned if allocated elsewhere or p is null.
        //
        // REQUIRES: p must be a value returned from a previous call to malloc(),
        // calloc(), realloc(), memalign(), posix_memalign(), valloc(), pvalloc(),
        // new, or new[], and must refer to memory that is currently allocated (so,
        // for instance, you should not pass in a pointer after having called free()
        // on it).
        enum class Ownership { kUnknown = 0, kOwned, kNotOwned };
        static Ownership GetOwnership(const void* p);

        // Type used by GetProperties.  See comment on GetProperties.
        struct Property {
            size_t value;
        };

        // Returns detailed statistics about the state of TCMalloc.  The map is keyed
        // by the name of the statistic.
        //
        // Common across malloc implementations:
        //  generic.bytes_in_use_by_app  -- Bytes currently in use by application
        //  generic.physical_memory_used -- Overall (including malloc internals)
        //  generic.virtual_memory_used  -- Overall (including malloc internals)
        //
        // Tcmalloc specific properties
        //  tcmalloc.cpu_free            -- Bytes in per-cpu free-lists
        //  tcmalloc.thread_cache_free   -- Bytes in per-thread free-lists
        //  tcmalloc.transfer_cache      -- Bytes in cross-thread transfer caches
        //  tcmalloc.central_cache_free  -- Bytes in central cache
        //  tcmalloc.page_heap_free      -- Bytes in page heap
        //  tcmalloc.page_heap_unmapped  -- Bytes in page heap (no backing phys. mem)
        //  tcmalloc.metadata_bytes      -- Used by internal data structures
        //  tcmalloc.thread_cache_count  -- Number of thread caches in use
        //  tcmalloc.experiment.NAME     -- Experiment NAME is running if 1
        static std::map<std::string, Property> GetProperties();

        static Profile SnapshotCurrent(tcmalloc::ProfileType type);

        // AllocationProfilingToken tracks an active profiling session started with
        // StartAllocationProfiling.  Profiling continues until Stop() is called.
        class AllocationProfilingToken {
        public:
            AllocationProfilingToken() = default;
            AllocationProfilingToken(AllocationProfilingToken&&) = default;
            AllocationProfilingToken(const AllocationProfilingToken&) = delete;
            ~AllocationProfilingToken();

            AllocationProfilingToken& operator=(AllocationProfilingToken&&) = default;
            AllocationProfilingToken& operator=(const AllocationProfilingToken&) =
            delete;

            // Finish the recording started by the corresponding call to
            // StartAllocationProfile, and return samples of calls to each function.  If
            // it is called more than once, subsequent calls will return an empty
            // profile.
            Profile Stop() &&;

        private:
            explicit AllocationProfilingToken(
                    std::unique_ptr<tcmalloc_internal::AllocationProfilingTokenBase>);

            std::unique_ptr<tcmalloc_internal::AllocationProfilingTokenBase> impl_;
            friend class tcmalloc_internal::AllocationProfilingTokenAccessor;
        };

        // Start recording a sample of allocation and deallocation calls.  Returns
        // null if the implementation does not support profiling.
        static AllocationProfilingToken StartAllocationProfiling();

        // Start recording lifetimes of objects live during this profiling
        // session. Returns null if the implementation does not support profiling.
        static AllocationProfilingToken StartLifetimeProfiling();

        // Runs housekeeping actions for the allocator off of the main allocation path
        // of new/delete.  As of 2020, this includes:
        // * Inspecting the current CPU mask and releasing memory from inaccessible
        //   CPUs.
        // * Releasing GetBackgroundReleaseRate() bytes per second from the page
        //   heap, if that many bytes are free, via ReleaseMemoryToSystem().
        //
        // When linked against TCMalloc, this method does not return.
        static void ProcessBackgroundActions();

        // Return true if ProcessBackgroundActions should be called on this platform.
        // Not all platforms need/support background actions. As of 2021 this
        // includes Apple and Emscripten.
        static bool NeedsProcessBackgroundActions();

        // Specifies a rate in bytes per second.
        //
        // The enum is used to provide strong-typing for the value.
        enum class BytesPerSecond : size_t {};

        // Gets the current release rate (in bytes per second) from the page heap.
        // Zero inhibits the release path.
        static BytesPerSecond GetBackgroundReleaseRate();
        // Specifies the release rate from the page heap.  ProcessBackgroundActions
        // must be called for this to be operative.
        static void SetBackgroundReleaseRate(BytesPerSecond rate);
    };

// tcmalloc:google3-begin(coroner)
    class MallocExtensionAsanWriter {
    public:
        virtual ~MallocExtensionAsanWriter() {}

        // Argument for WriteErrorType. Should match error values tracked by
        // GuardedPageAllocator::ErrorType.
        enum class ErrorType {
            kUseAfterFree,
            kUseAfterFreeRead,
            kUseAfterFreeWrite,
            kBufferUnderflow,
            kBufferUnderflowRead,
            kBufferUnderflowWrite,
            kBufferOverflow,
            kBufferOverflowRead,
            kBufferOverflowWrite,
            kDoubleFree,
            kBufferOverflowOnDealloc,
            kSizedDeleteMismatch,
            kUnknown,
        };

        // Record that an ASAN error happened with the provided error.
        virtual void WriteErrorType(ErrorType error_type) = 0;

        // Record an ASAN debug string with additional information about the error.
        virtual void WriteDebugString(absl::string_view message) = 0;

        enum class StackType {
            // Stack corresponds to the location of the ASAN allocation.
            kAllocation = 0,
            // Stack corresponds to the location of the ASAN deallocation.
            kDeallocation,
        };

        // Store the provided stack for the ASAN error.
        virtual void WriteStack(StackType type, void* const stack[], int depth) = 0;
    };

// If the provided signal corresponds to an ASAN error, write ASAN crash
// metadata to the provided writer. `ptr` is the address of the faulting
// instruction from the caught signal. May be unimplemented for some variants of
// malloc.
    extern "C" void MallocExtension_ExtractGWPAsan(
            const void* ptr, const void* context, MallocExtensionAsanWriter* writer);
// tcmalloc:google3-end

}  // namespace tcmalloc

// The nallocx function allocates no memory, but it performs the same size
// computation as the malloc function, and returns the real size of the
// allocation that would result from the equivalent malloc function call.
// Default weak implementation returns size unchanged, but tcmalloc overrides it
// and returns rounded up size. See the following link for details:
// http://www.unix.com/man-page/freebsd/3/nallocx/
// NOTE: prefer using tcmalloc_size_returning_operator_new over nallocx.
// tcmalloc_size_returning_operator_new is more efficienct and provides tcmalloc
// with better telemetry.
extern "C" size_t nallocx(size_t size, int flags) noexcept;

// The sdallocx function deallocates memory allocated by malloc or memalign.  It
// takes a size parameter to pass the original allocation size.
//
// The default weak implementation calls free(), but TCMalloc overrides it and
// uses the size to improve deallocation performance.
extern "C" void sdallocx(void* ptr, size_t size, int flags) noexcept;

#if !defined(__STDC_VERSION_STDLIB_H__) || __STDC_VERSION_STDLIB_H__ < 202311L
// Frees ptr allocated with malloc(size) introduced in C23.
extern "C" void free_sized(void* ptr, size_t size);

// Frees ptr allocated with aligned_alloc/posix_memalign with the specified size
// and alignment introduced in C23.
extern "C" void free_aligned_sized(void* ptr, size_t alignment, size_t size);
#endif

// Define __sized_ptr_t in the global namespace so that it can be named by the
// __size_returning_new implementations defined in tcmalloc.cc.
// tcmalloc:google3-begin(b/283854704)
#if defined(__has_include) && __has_include(<__google3/new.h>)

// Use the __sized_ptr_t defined in __google3/new.h extension.
using __sized_ptr_t = std::__google3::__sized_ptr_t;

#else  // __has_include(<__google3/new.h>)

struct __sized_ptr_t {
    void* p;
    size_t n;
};

#endif  // __has_include(<__google3/new.h>)

// tcmalloc:oss-replace-begin
// struct __sized_ptr_t {
//   void* p;
//   size_t n;
// };
// tcmalloc:oss-replace-end

namespace tcmalloc {
// sized_ptr_t constains pointer / capacity information as returned
// by `tcmalloc_size_returning_operator_new()`.
// See `tcmalloc_size_returning_operator_new()` for more information.
    using sized_ptr_t = __sized_ptr_t;
}  // namespace tcmalloc

// Allocates memory of at least the requested size.
//
// Returns a `sized_ptr_t` struct holding the allocated pointer, and the
// capacity of the allocated memory, which may be larger than the requested
// size.
//
// The returned pointer follows the alignment requirements of the standard new
// operator. This function will terminate on failure, except for the APIs
// accepting the std::nothrow parameter which will return {nullptr, 0} on
// failure.
//
// The returned pointer must be freed calling the matching ::operator delete.
//
// If a sized operator delete operator is invoked, then the 'size' parameter
// passed to delete must be greater or equal to the original requested size, and
// less than or equal to the capacity of the allocated memory as returned by the
// `tcmalloc_size_returning_operator_new` method.
//
// If neither the original size or capacity is known, then the non-sized
// operator delete can be invoked, however, this should be avoided, as this is
// substantially less efficient.
//
// The default weak implementation allocates the memory using the corresponding
// (matching) ::operator new(size_t, ...).
//
// This is a prototype API for the extension to C++ "size feedback in operator
// new" proposal:
// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p0901r5.html
extern "C" {
// The following declarations provide an alternative spelling which should be
// used so that the compiler can identify these as allocator functions.
__sized_ptr_t __size_returning_new(size_t size);
__sized_ptr_t __size_returning_new_hot_cold(size_t, __hot_cold_t);
__sized_ptr_t __size_returning_new_aligned(size_t, std::align_val_t);
__sized_ptr_t __size_returning_new_aligned_hot_cold(size_t, std::align_val_t,
                                                    __hot_cold_t);

// tcmalloc:google3-begin(b/283856455)
// Experiment for using __size_returning_new in libc++.
__sized_ptr_t __size_returning_new_experiment(size_t size);
__sized_ptr_t __size_returning_new_aligned_experiment(size_t, std::align_val_t);
// tcmalloc:google3-end

ABSL_DEPRECATE_AND_INLINE()
inline __sized_ptr_t tcmalloc_size_returning_operator_new(size_t size) {
    return __size_returning_new(size);
}
__sized_ptr_t tcmalloc_size_returning_operator_new_nothrow(
        size_t size) noexcept;
ABSL_DEPRECATE_AND_INLINE()
inline __sized_ptr_t tcmalloc_size_returning_operator_new_hot_cold(
        size_t size, tcmalloc::hot_cold_t hot_cold) {
    return __size_returning_new_hot_cold(size, hot_cold);
}
__sized_ptr_t tcmalloc_size_returning_operator_new_hot_cold_nothrow(
        size_t size, tcmalloc::hot_cold_t hot_cold) noexcept;

#if defined(__cpp_aligned_new)

// Identical to `tcmalloc_size_returning_operator_new` except that the returned
// memory is aligned according to the `alignment` argument.
ABSL_DEPRECATE_AND_INLINE()
inline __sized_ptr_t tcmalloc_size_returning_operator_new_aligned(
    size_t size, std::align_val_t alignment) {
  return __size_returning_new_aligned(size, alignment);
}
__sized_ptr_t tcmalloc_size_returning_operator_new_aligned_nothrow(
    size_t size, std::align_val_t alignment) noexcept;
ABSL_DEPRECATE_AND_INLINE()
inline __sized_ptr_t tcmalloc_size_returning_operator_new_aligned_hot_cold(
    size_t size, std::align_val_t alignment, tcmalloc::hot_cold_t hot_cold) {
  return __size_returning_new_aligned_hot_cold(size, alignment, hot_cold);
}
__sized_ptr_t tcmalloc_size_returning_operator_new_aligned_hot_cold_nothrow(
    size_t size, std::align_val_t alignment,
    tcmalloc::hot_cold_t hot_cold) noexcept;

#endif  // __cpp_aligned_new

}  // extern "C"

void* operator new(size_t size, tcmalloc::hot_cold_t hot_cold) noexcept(false);
void* operator new(size_t size, const std::nothrow_t&,
                   tcmalloc::hot_cold_t hot_cold) noexcept;
void* operator new[](size_t size,
                     tcmalloc::hot_cold_t hot_cold) noexcept(false);
void* operator new[](size_t size, const std::nothrow_t&,
                     tcmalloc::hot_cold_t hot_cold) noexcept;

#ifdef __cpp_aligned_new
void* operator new(size_t size, std::align_val_t alignment,
                   tcmalloc::hot_cold_t hot_cold) noexcept(false);
void* operator new(size_t size, std::align_val_t alignment,
                   const std::nothrow_t&,
                   tcmalloc::hot_cold_t hot_cold) noexcept;
void* operator new[](size_t size, std::align_val_t alignment,
                     tcmalloc::hot_cold_t hot_cold) noexcept(false);
void* operator new[](size_t size, std::align_val_t alignment,
                     const std::nothrow_t&,
                     tcmalloc::hot_cold_t hot_cold) noexcept;
#endif  // __cpp_aligned_new

#ifndef MALLOCX_LG_ALIGN
#define MALLOCX_LG_ALIGN(la) (la)
#endif

namespace tcmalloc {
    namespace tcmalloc_internal {

// AllocationProfilingTokenBase tracks an on-going profiling session of sampled
// allocations.  The session ends when Stop() is called.
//
// This decouples the implementation details (of TCMalloc) from the interface,
// allowing non-TCMalloc allocators (such as libc and sanitizers) to be provided
// while allowing the library to compile and link.
        class AllocationProfilingTokenBase {
        public:
            // Explicitly declare the ctor to put it in the google_malloc section.
            AllocationProfilingTokenBase() = default;

            virtual ~AllocationProfilingTokenBase() = default;

            // Finish recording started during construction of this object.
            //
            // After the first call, Stop() will return an empty profile.
            virtual Profile Stop() && = 0;
        };

// ProfileBase contains a profile of allocations.
//
// This decouples the implementation details (of TCMalloc) from the interface,
// allowing non-TCMalloc allocators (such as libc and sanitizers) to be provided
// while allowing the library to compile and link.
        class ProfileBase {
        public:
            virtual ~ProfileBase() = default;

            // For each sample in the profile, Iterate invokes the callback f on the
            // sample.
            virtual void Iterate(
                    absl::FunctionRef<void(const Profile::Sample&)> f) const = 0;

            // The type of profile (live objects, allocated, etc.).
            virtual ProfileType Type() const = 0;

            virtual std::optional<absl::Time> StartTime() const = 0;

            // The duration the profile was collected for.  For instantaneous profiles
            // (heap, peakheap, etc.), this returns absl::ZeroDuration().
            virtual absl::Duration Duration() const = 0;
        };

        enum class MadvisePreference {
            kNever = 0x0,
            kDontNeed = 0x1,
            kFreeAndDontNeed = 0x3,
            kFreeOnly = 0x2,
        };

        inline bool AbslParseFlag(absl::string_view text, MadvisePreference* preference,
                                  std::string* /* error */) {
            if (text == "NEVER") {
                *preference = MadvisePreference::kNever;
                return true;
            } else if (text == "DONTNEED") {
                *preference = MadvisePreference::kDontNeed;
                return true;
            } else if (text == "FREE_AND_DONTNEED") {
                *preference = MadvisePreference::kFreeAndDontNeed;
                return true;
            } else if (text == "FREE_ONLY") {
                *preference = MadvisePreference::kFreeOnly;
                return true;
            } else {
                return false;
            }
        }

        inline std::string AbslUnparseFlag(MadvisePreference preference) {
            switch (preference) {
                case MadvisePreference::kNever:
                    return "NEVER";
                case MadvisePreference::kDontNeed:
                    return "DONTNEED";
                case MadvisePreference::kFreeAndDontNeed:
                    return "FREE_AND_DONTNEED";
                case MadvisePreference::kFreeOnly:
                    return "FREE_ONLY";
            }

            ABSL_UNREACHABLE();
            return "";
        }

    }  // namespace tcmalloc_internal
}  // namespace tcmalloc

#endif  // THIRD_PARTY_TCMALLOC_MALLOC_EXTENSION_H_