diff --git a/tcmalloc/BUILD b/tcmalloc/BUILD index 4efbe3c5b..44fa78d5e 100644 --- a/tcmalloc/BUILD +++ b/tcmalloc/BUILD @@ -83,6 +83,7 @@ tcmalloc_deps = [ "//tcmalloc/internal:declarations", "//tcmalloc/internal:linked_list", "//tcmalloc/internal:logging", + "//tcmalloc/internal:memory_tag", "//tcmalloc/internal:optimization", "//tcmalloc/internal:percpu", "//tcmalloc/internal:sampled_allocation", @@ -296,6 +297,7 @@ create_tcmalloc_libraries( "//tcmalloc/internal:linked_list", "//tcmalloc/internal:logging", "//tcmalloc/internal:memory_stats", + "//tcmalloc/internal:memory_tag", "//tcmalloc/internal:mincore", "//tcmalloc/internal:mismatched_delete_state", "//tcmalloc/internal:numa", diff --git a/tcmalloc/common.cc b/tcmalloc/common.cc index fbc8f5858..155893724 100644 --- a/tcmalloc/common.cc +++ b/tcmalloc/common.cc @@ -14,7 +14,6 @@ #include "tcmalloc/common.h" -#include "absl/strings/string_view.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/optimization.h" @@ -22,25 +21,6 @@ GOOGLE_MALLOC_SECTION_BEGIN namespace tcmalloc { namespace tcmalloc_internal { -absl::string_view MemoryTagToLabel(MemoryTag tag) { - switch (tag) { - case MemoryTag::kNormal: - return "NORMAL"; - case MemoryTag::kNormalP1: - return "NORMAL_P1"; - case MemoryTag::kSampled: - return "SAMPLED"; - case MemoryTag::kSelSan: - return "SELSAN"; - case MemoryTag::kCold: - return "COLD"; - case MemoryTag::kMetadata: - return "METADATA"; - } - - ASSUME(false); -} - // This only provides correct answer for TCMalloc-allocated memory, // and may give a false positive for non-allocated block. extern "C" bool TCMalloc_Internal_PossiblyCold(const void* ptr) { diff --git a/tcmalloc/common.h b/tcmalloc/common.h index ae8d6f462..0fbc30da3 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -32,6 +32,7 @@ #include "absl/strings/string_view.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/logging.h" +#include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/internal/optimization.h" #include "tcmalloc/malloc_extension.h" @@ -167,21 +168,6 @@ inline constexpr size_t kDefaultProfileSamplingInterval = 1 << 21; #error "Unsupported TCMALLOC_PAGE_SHIFT value!" #endif -#ifdef TCMALLOC_INTERNAL_SELSAN -inline constexpr bool kSelSanPresent = true; -#else -inline constexpr bool kSelSanPresent = false; -#endif - -// Sanitizers constrain the memory layout which causes problems with the -// enlarged tags required to represent NUMA partitions and for SelSan. -#if defined(ABSL_HAVE_MEMORY_SANITIZER) || defined(ABSL_HAVE_THREAD_SANITIZER) -static_assert(!kSelSanPresent, "MSan/TSan are incompatible with SelSan."); -inline constexpr bool kSanitizerAddressSpace = true; -#else -inline constexpr bool kSanitizerAddressSpace = false; -#endif - // Disable NUMA awareness under Sanitizers to avoid failing to mmap memory. #if defined(TCMALLOC_INTERNAL_NUMA_AWARE) inline constexpr size_t kNumaPartitions = kSanitizerAddressSpace ? 1 : 2; @@ -233,66 +219,8 @@ inline constexpr int kMaxOverages = 3; // scavenging code will shrink it down when its contents are not in use. inline constexpr size_t kMaxDynamicFreeListLength = 8192; -enum class MemoryTag : uint8_t { - // Sampled, infrequently allocated - kSampled = 0x0, - // Normal memory, NUMA partition 0 - kNormalP0 = kSanitizerAddressSpace ? 0x1 : 0x4, - // Normal memory, NUMA partition 1 - kNormalP1 = kSanitizerAddressSpace ? 0xff : 0x6, - // Normal memory - kNormal = kNormalP0, - // Cold - kCold = 0x2, - // Metadata - kMetadata = 0x3, - // SelSan sampled spans, kept separately because we need to quickly - // distinguish them from the rest during delete and they also consume - // shadow memory. 0xfe is an arbitrary value that shouldn't be used. - kSelSan = kSelSanPresent ? 0x1 : 0xfe, -}; - -inline constexpr uintptr_t kTagShift = std::min(kAddressBits - 4, 42); -inline constexpr uintptr_t kTagMask = - uintptr_t{kSanitizerAddressSpace ? 0x3 : 0x7} << kTagShift; - -inline MemoryTag GetMemoryTag(const void* ptr) { - return static_cast((reinterpret_cast(ptr) & kTagMask) >> - kTagShift); -} - -inline bool IsNormalMemory(const void* ptr) { - // This is slightly faster than checking kNormalP0/P1 separetly. - static_assert((static_cast(MemoryTag::kNormalP0) & - (static_cast(MemoryTag::kSampled) | - static_cast(MemoryTag::kCold))) == 0); - bool res = (static_cast(GetMemoryTag(ptr)) & - static_cast(MemoryTag::kNormal)) != 0; - TC_ASSERT(res == (GetMemoryTag(ptr) == MemoryTag::kNormalP0 || - GetMemoryTag(ptr) == MemoryTag::kNormalP1), - "ptr=%p res=%d tag=%d", ptr, res, - static_cast(GetMemoryTag(ptr))); - return res; -} - -inline bool IsSelSanMemory(const void* ptr) { - // This is a faster way to check for SelSan memory provided we already know - // it's not a normal memory, and assuming it's not kMetadata (both assumptions - // are checked by the assert below). A straightforward comparison with kSelSan - // leads to extraction/check of 2 bits (these use 2 8-byte immediates); - // this check can be done with a single BT instruction. - // kSelSanPresent part allows to optimize away branches in non SelSan build. - bool res = - kSelSanPresent && (static_cast(GetMemoryTag(ptr)) & - static_cast(MemoryTag::kSelSan)) != 0; - TC_ASSERT_EQ(res, GetMemoryTag(ptr) == MemoryTag::kSelSan); - return res; -} - inline constexpr bool ColdFeatureActive() { return kHasExpandedClasses; } -absl::string_view MemoryTagToLabel(MemoryTag tag); - inline constexpr bool IsExpandedSizeClass(unsigned size_class) { return kHasExpandedClasses && (size_class >= kExpandedClassesStart); } diff --git a/tcmalloc/internal/BUILD b/tcmalloc/internal/BUILD index c20b97cc5..d87ffe72a 100644 --- a/tcmalloc/internal/BUILD +++ b/tcmalloc/internal/BUILD @@ -330,6 +330,21 @@ cc_test( ], ) +cc_library( + name = "memory_tag", + srcs = ["memory_tag.cc"], + hdrs = ["memory_tag.h"], + visibility = [ + "//tcmalloc:__subpackages__", + ], + deps = [ + ":config", + ":logging", + ":optimization", + "@com_google_absl//absl/strings:string_view", + ], +) + cc_library( name = "mincore", srcs = ["mincore.cc"], diff --git a/tcmalloc/internal/config.h b/tcmalloc/internal/config.h index 5b8dac4c9..5d2e9fce7 100644 --- a/tcmalloc/internal/config.h +++ b/tcmalloc/internal/config.h @@ -137,6 +137,21 @@ inline constexpr int kAddressBits = 48; inline constexpr int kAddressBits = 8 * sizeof(void*); #endif +#ifdef TCMALLOC_INTERNAL_SELSAN +inline constexpr bool kSelSanPresent = true; +#else +inline constexpr bool kSelSanPresent = false; +#endif + +// Sanitizers constrain the memory layout which causes problems with the +// enlarged tags required to represent NUMA partitions and for SelSan. +#if defined(ABSL_HAVE_MEMORY_SANITIZER) || defined(ABSL_HAVE_THREAD_SANITIZER) +static_assert(!kSelSanPresent, "MSan/TSan are incompatible with SelSan."); +inline constexpr bool kSanitizerAddressSpace = true; +#else +inline constexpr bool kSanitizerAddressSpace = false; +#endif + #if defined(__x86_64__) // x86 has 2 MiB huge pages static constexpr size_t kHugePageShift = 21; diff --git a/tcmalloc/internal/memory_tag.cc b/tcmalloc/internal/memory_tag.cc new file mode 100644 index 000000000..30d873ba5 --- /dev/null +++ b/tcmalloc/internal/memory_tag.cc @@ -0,0 +1,44 @@ +// Copyright 2024 The TCMalloc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tcmalloc/internal/memory_tag.h" + +#include "absl/strings/string_view.h" +#include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/optimization.h" + +GOOGLE_MALLOC_SECTION_BEGIN +namespace tcmalloc::tcmalloc_internal { + +absl::string_view MemoryTagToLabel(MemoryTag tag) { + switch (tag) { + case MemoryTag::kNormal: + return "NORMAL"; + case MemoryTag::kNormalP1: + return "NORMAL_P1"; + case MemoryTag::kSampled: + return "SAMPLED"; + case MemoryTag::kSelSan: + return "SELSAN"; + case MemoryTag::kCold: + return "COLD"; + case MemoryTag::kMetadata: + return "METADATA"; + } + + ASSUME(false); +} + +} // namespace tcmalloc::tcmalloc_internal +GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/internal/memory_tag.h b/tcmalloc/internal/memory_tag.h new file mode 100644 index 000000000..f91d26c0c --- /dev/null +++ b/tcmalloc/internal/memory_tag.h @@ -0,0 +1,89 @@ +// Copyright 2024 The TCMalloc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TCMALLOC_INTERNAL_MEMORY_TAG_H_ +#define TCMALLOC_INTERNAL_MEMORY_TAG_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/logging.h" + +GOOGLE_MALLOC_SECTION_BEGIN +namespace tcmalloc::tcmalloc_internal { + +enum class MemoryTag : uint8_t { + // Sampled, infrequently allocated + kSampled = 0x0, + // Normal memory, NUMA partition 0 + kNormalP0 = kSanitizerAddressSpace ? 0x1 : 0x4, + // Normal memory, NUMA partition 1 + kNormalP1 = kSanitizerAddressSpace ? 0xff : 0x6, + // Normal memory + kNormal = kNormalP0, + // Cold + kCold = 0x2, + // Metadata + kMetadata = 0x3, + // SelSan sampled spans, kept separately because we need to quickly + // distinguish them from the rest during delete and they also consume + // shadow memory. 0xfe is an arbitrary value that shouldn't be used. + kSelSan = kSelSanPresent ? 0x1 : 0xfe, +}; + +inline constexpr uintptr_t kTagShift = std::min(kAddressBits - 4, 42); +inline constexpr uintptr_t kTagMask = + uintptr_t{kSanitizerAddressSpace ? 0x3 : 0x7} << kTagShift; + +inline MemoryTag GetMemoryTag(const void* ptr) { + return static_cast((reinterpret_cast(ptr) & kTagMask) >> + kTagShift); +} + +inline bool IsNormalMemory(const void* ptr) { + // This is slightly faster than checking kNormalP0/P1 separetly. + static_assert((static_cast(MemoryTag::kNormalP0) & + (static_cast(MemoryTag::kSampled) | + static_cast(MemoryTag::kCold))) == 0); + bool res = (static_cast(GetMemoryTag(ptr)) & + static_cast(MemoryTag::kNormal)) != 0; + TC_ASSERT(res == (GetMemoryTag(ptr) == MemoryTag::kNormalP0 || + GetMemoryTag(ptr) == MemoryTag::kNormalP1), + "ptr=%p res=%d tag=%d", ptr, res, + static_cast(GetMemoryTag(ptr))); + return res; +} + +inline bool IsSelSanMemory(const void* ptr) { + // This is a faster way to check for SelSan memory provided we already know + // it's not a normal memory, and assuming it's not kMetadata (both assumptions + // are checked by the assert below). A straightforward comparison with kSelSan + // leads to extraction/check of 2 bits (these use 2 8-byte immediates); + // this check can be done with a single BT instruction. + // kSelSanPresent part allows to optimize away branches in non SelSan build. + bool res = + kSelSanPresent && (static_cast(GetMemoryTag(ptr)) & + static_cast(MemoryTag::kSelSan)) != 0; + TC_ASSERT_EQ(res, GetMemoryTag(ptr) == MemoryTag::kSelSan); + return res; +} + +absl::string_view MemoryTagToLabel(MemoryTag tag); + +} // namespace tcmalloc::tcmalloc_internal +GOOGLE_MALLOC_SECTION_END + +#endif // TCMALLOC_INTERNAL_MEMORY_TAG_H_ diff --git a/tcmalloc/selsan/BUILD b/tcmalloc/selsan/BUILD index 0e6d3127c..4c94450c6 100644 --- a/tcmalloc/selsan/BUILD +++ b/tcmalloc/selsan/BUILD @@ -44,6 +44,7 @@ cc_library( cc_test( name = "shadow_test", srcs = ["shadow_test.cc"], + tags = ["nosan"], deps = [ ":selsan", "@com_google_googletest//:gtest_main", diff --git a/tcmalloc/system-alloc.h b/tcmalloc/system-alloc.h index 83fdfcc25..929d53fbd 100644 --- a/tcmalloc/system-alloc.h +++ b/tcmalloc/system-alloc.h @@ -21,8 +21,8 @@ #include #include "absl/base/attributes.h" -#include "tcmalloc/common.h" #include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/malloc_extension.h" GOOGLE_MALLOC_SECTION_BEGIN