diff --git a/.github/workflows/topling-jni.yml b/.github/workflows/topling-jni.yml new file mode 100644 index 0000000000..3079515167 --- /dev/null +++ b/.github/workflows/topling-jni.yml @@ -0,0 +1,129 @@ +# TODO: How to cache make files / speed up build progress here? +name: "build topling-jni" + +on: + workflow_dispatch: + inputs: + repository_url: + required: true + default: 'topling/toplingdb' + repository_branch: + required: false + default: 'sideplugin-7.10.0-2022-12-21-bec42648' + test: + required: false + type: boolean + description: test SideGetBenchmarks + default: false + deploy_maven: + required: false + type: boolean + description: publish to maven repo + default: true + +jobs: + build: + # refer https://github.com/actions/runner-images to get the details + runs-on: ubuntu-latest + env: + GCC_VER: "11.3" # TODO: better get from the 'gcc --version' + GITHUB_TOKEN: ${{ github.token }} + REP_URL: ${{ inputs.repository_url }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository_url }} + ref: ${{ inputs.repository_branch }} + fetch-depth: 1 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + server-id: github # Value of the distributionManagement/repository/id field of the pom.xml + settings-path: ${{ github.workspace }} # location for the settings.xml file + #- name: Cache Maven # Replace by setup-java now + # uses: actions/cache@v3 + # with: + # path: ~/.m2/repository + # key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + # restore-keys: ${{ runner.os }}-m2 + + - name: Init Env & Compile RocksDB + run: | + cat $GITHUB_WORKSPACE/settings.xml + sudo apt-get update -y && sudo apt-get install -y \ + libjemalloc-dev libaio-dev libgflags-dev zlib1g-dev \ + libbz2-dev libcurl4-gnutls-dev liburing-dev \ + libsnappy-dev libbz2-dev liblz4-dev libzstd-dev + + gcc --version + git submodule update --init --recursive + mkdir -p ~/.ssh && mkdir -p /opt/lib + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + # this step could take a long time? + make -j`nproc` DEBUG_LEVEL=0 shared_lib + sudo make install-shared PREFIX=/opt + ls -l /opt/lib + + - name: Compile RocksDBJava + run: | + echo $JAVA_HOME + make rocksdbjava -j`nproc` DEBUG_LEVEL=0 + + - name: Move to Local Maven Repo + run: | + cd java/target || exit + cp -v rocksdbjni-7.10.0-linux64.jar rocksdbjni-7.10.0-SNAPSHOT-linux64.jar + mvn install:install-file -ntp -Dfile=rocksdbjni-7.10.0-SNAPSHOT-linux64.jar \ + -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=7.10.0-SNAPSHOT -Dpackaging=jar + # TODO: why 'deploy' doesn't include install step here? if we only use deploy, will lack local jar + if ${{ inputs.deploy_maven }}; then + # TODO: what's the pom file for it? add with '-DpomFile=/xx/pom.xml' + mvn deploy:deploy-file -e -s $GITHUB_WORKSPACE/settings.xml \ + -DpomFile=$GITHUB_WORKSPACE/java/pom.xml.template \ + -Durl=https://maven.pkg.github.com/$REP_URL -DrepositoryId=github \ + -Dfile=rocksdbjni-7.10.0-SNAPSHOT-linux64.jar -DgroupId=org.rocksdb \ + -DartifactId=rocksdbjni -Dversion=7.10.0-SNAPSHOT -Dpackaging=jar + fi + + # for compile jmh.jar to test the performance + - name: Build SideGetBenchmarks with Maven + run: | + echo ${{ github.workspace }} && echo $GITHUB_WORKSPACE + pwd && ls -l + (cd java/jmh && ls -l && pwd) || exit + mvn clean package -e -ntp -f $GITHUB_WORKSPACE/java/jmh/pom.xml # -B in non-interactive (Batch) mode + + - name: Run SideGetBenchmarks & Check it + if: ${{ inputs.test }} + run: | + mkdir -p /dev/shm/db_bench_community + cd $GITHUB_WORKSPACE/java/jmh || exit + ls ../../sideplugin/rockside/src/topling/web + cp -v $GITHUB_WORKSPACE/sideplugin/rockside/src/topling/web/{style.css,index.html} /dev/shm/db_bench_community + echo $LD_LIBRARY_PATH + export LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH # for libterark-* + echo $LD_LIBRARY_PATH && ls -l /opt/lib + # Note: webserver should visit while running + export LD_PRELOAD=libterark-zbs-g++-11.3-r.so:libterark-fsa-g++-11.3-r.so:libjemalloc.so + java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar \ + -p keyCount=1000 -p keySize=128 -p valueSize=32768 \ + -p sideConf=$GITHUB_WORKSPACE/sideplugin/rockside/sample-conf/db_bench_community.yaml SideGetBenchmarks + + - name: Publish JAR to GitHub Packages + if: ${{ inputs.deploy_maven }} + run: | + cd $GITHUB_WORKSPACE/java/jmh || exit + ls -l $GITHUB_WORKSPACE && tail -15 pom.xml + mvn deploy -e -f $GITHUB_WORKSPACE/java/jmh/pom.xml -s $GITHUB_WORKSPACE/settings.xml \ + -DaltDeploymentRepository=github::default::https://maven.pkg.github.com/$REP_URL + #env: + # GITHUB_TOKEN: ${{ github.token }} diff --git a/.gitignore b/.gitignore index 8bd9fea598..09da7844d0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ rocksdb.pc *.dylib* *.gcda *.gcno +*.log *.o *.o.tmp *.so @@ -25,11 +26,13 @@ rocksdb.pc *.vcxproj *.vcxproj.filters *.sln +*.sst *.cmake .watchmanconfig CMakeCache.txt CMakeFiles/ build/ +build-ut/ ldb manifest_dump @@ -98,3 +101,8 @@ cmake-build-* third-party/folly/ .cache *.sublime-* +*_dbg +*_test + +generated-sources +target diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..791e51fd91 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "sideplugin/rockside"] + path = sideplugin/rockside + url = https://github.com/topling/rockside.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 49868be894..ff6f5007a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,6 +171,8 @@ else() endif() endif() +include_directories(sideplugin/rockside/src) + option(WITH_MD_LIBRARY "build with MD" ON) if(WIN32 AND MSVC) if(WITH_MD_LIBRARY) @@ -181,6 +183,11 @@ if(WIN32 AND MSVC) endif() if(MSVC) + if(MSVC_VERSION LESS 1926) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /experimental:preprocessor") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:preprocessor") + endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324") else() @@ -613,8 +620,68 @@ endif() find_package(Threads REQUIRED) # Main library source code +if (EXISTS ${PROJECT_SOURCE_DIR}/sideplugin/topling-rocks/CMakeFileList.txt) + message(STATUS "found ${PROJECT_SOURCE_DIR}/sideplugin/topling-rocks/CMakeFileList.txt") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + include(${PROJECT_SOURCE_DIR}/sideplugin/topling-rocks/CMakeFileList.txt) +else() + message(STATUS "not found ${PROJECT_SOURCE_DIR}/sideplugin/topling-rocks/CMakeFileList.txt") +endif() + +if (EXISTS ${PROJECT_SOURCE_DIR}/sideplugin/rockside/CMakeFileList.txt) + message(STATUS "found ${PROJECT_SOURCE_DIR}/sideplugin/rockside/CMakeFileList.txt") + include(${PROJECT_SOURCE_DIR}/sideplugin/rockside/CMakeFileList.txt) +else() + message(STATUS "not found ${PROJECT_SOURCE_DIR}/sideplugin/rockside/CMakeFileList.txt") +endif() + +set (cspp_memtab ${PROJECT_SOURCE_DIR}/sideplugin/cspp-memtable/cspp_memtable.cc) +if (EXISTS ${cspp_memtab}) + message(STATUS "found ${cspp_memtab}") + set (topling_rocks_src ${topling_rocks_src} ${cspp_memtab}) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_TOPLING_CSPP_MEMTABLE") +else() + message(STATUS "not found ${cspp_memtab}") +endif() + +set (cspp_wbwi ${PROJECT_SOURCE_DIR}/sideplugin/cspp-wbwi/cspp_wbwi.cc) +if (EXISTS ${cspp_wbwi}) + message(STATUS "found ${cspp_wbwi}") + set (topling_rocks_src ${topling_rocks_src} ${cspp_wbwi}) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_TOPLING_CSPP_WBWI") +else() + message(STATUS "not found ${cspp_wbwi}") +endif() + +FILE(GLOB topling_sst ${PROJECT_SOURCE_DIR}/sideplugin/topling-sst/src/table/*.cc) +if (EXISTS ${PROJECT_SOURCE_DIR}/sideplugin/topling-sst/src/table) + message(STATUS "found ${topling_sst}") + set (topling_rocks_src ${topling_rocks_src} ${topling_sst}) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_TOPLING_SST -Isideplugin/topling-sst/src") +else() + message(STATUS "not found ${topling_sst}") +endif() + +FILE(GLOB topling_zip_table_reader ${PROJECT_SOURCE_DIR}/sideplugin/topling-zip_table_reader/src/table/*.cc) +if (EXISTS ${PROJECT_SOURCE_DIR}/sideplugin/topling-zip_table_reader/src/table) + message(STATUS "found ${topling_zip_table_reader}") + set (topling_rocks_src ${topling_rocks_src} ${topling_zip_table_reader}) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Isideplugin/topling-zip_table_reader/src") +else() + message(STATUS "not found ${topling_zip_table_reader}") +endif() + +FILE(GLOB topling_dcompact ${PROJECT_SOURCE_DIR}/sideplugin/topling-dcompact/src/dcompact/*.cc) +if (EXISTS ${PROJECT_SOURCE_DIR}/sideplugin/topling-dcompact/src/dcompact) + message(STATUS "found ${topling_dcompact}") + set (topling_rocks_src ${topling_rocks_src} ${topling_dcompact}) +else() + message(STATUS "not found ${topling_dcompact}") +endif() set(SOURCES + ${rockside_src} + ${topling_rocks_src} cache/cache.cc cache/cache_entry_roles.cc cache/cache_key.cc @@ -646,6 +713,7 @@ set(SOURCES db/builder.cc db/c.cc db/column_family.cc + db/compaction/compaction_executor.cc db/compaction/compaction.cc db/compaction/compaction_iterator.cc db/compaction/compaction_picker.cc diff --git a/INSTALL.md b/INSTALL.md index fb4651e4b8..716a53fb4b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,6 +32,15 @@ most processors made since roughly 2013. ## Dependencies +* ToplingDB dependencies + - [libcurl](https://curl.se/libcurl/) - libcurl is a free and easy-to-use client-side URL transfer library + * ToplingDB [dcompact](https://github.com/topling/topling-dcompact) use libcurl to submit compaction jobs to compaction service(dcompact_worker) + - [liburing](https://github.com/axboe/liburing) - the io_uring library, ToplingDB use it to optimize MultiGet + * ToplingDB adds `ReadOptions::async_queue_depth` for queue depth of io_uring + * When compiled to shared library, this is not needed - it's used in [topling-zip](https://github.com/topling/topling-zip) + - [libaio](https://pagure.io/libaio) - The Linux-native asynchronous I/O facility + * libaio is old linux async io, io_uring should be preferred than libaio + * You can link RocksDB with following compression libraries: - [zlib](http://www.zlib.net/) - a library for data compression. - [bzip2](http://www.bzip.org/) - a library for data compression. diff --git a/LICENSE.Apache b/LICENSE.Apache index d645695673..261eeb9e9f 100644 --- a/LICENSE.Apache +++ b/LICENSE.Apache @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/Makefile b/Makefile index 8829be9d85..96d53fb507 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,15 @@ MACHINE ?= $(shell uname -m) ARFLAGS = ${EXTRA_ARFLAGS} rs STRIPFLAGS = -S -x +# beg topling specific +DISABLE_WARNING_AS_ERROR=1 +LIB_MODE=shared +USE_RTTI=1 +ROCKSDB_USE_IO_URING=0 +ROCKSDB_DISABLE_TCMALLOC=1 +SKIP_FORMAT_BUCK_CHECKS=1 +# end topling specific + # Transform parallel LOG output into something more readable. perl_command = perl -n \ -e '@a=split("\t",$$_,-1); $$t=$$a[8];' \ @@ -74,6 +83,8 @@ else ifneq ($(filter jtest rocksdbjava%, $(MAKECMDGOALS)),) endif endif +$(info $$DEBUG_LEVEL is ${DEBUG_LEVEL}, MAKE_RESTARTS is [${MAKE_RESTARTS}]) + # LIB_MODE says whether or not to use/build "shared" or "static" libraries. # Mode "static" means to link against static libraries (.a) # Mode "shared" means to link against shared libraries (.so, .sl, .dylib, etc) @@ -99,11 +110,18 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ export COMPILE_WITH_UBSAN="$(COMPILE_WITH_UBSAN)"; \ export PORTABLE="$(PORTABLE)"; \ export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \ + export ROCKSDB_USE_IO_URING="$(ROCKSDB_USE_IO_URING)"; \ + export ROCKSDB_DISABLE_TCMALLOC="$(ROCKSDB_DISABLE_TCMALLOC)"; \ + export ROCKSDB_DISABLE_ZSTD=1; \ export USE_CLANG="$(USE_CLANG)"; \ export LIB_MODE="$(LIB_MODE)"; \ - export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ - export USE_FOLLY="$(USE_FOLLY)"; \ + export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \ + export USE_FOLLY="$(USE_FOLLY)"; \ "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) +ifneq (${.SHELLSTATUS},0) + $(error $(CURDIR)/build_tools/build_detect_platform failed with exit code ${.SHELLSTATUS}) +endif + # this file is generated by the previous line to set build flags and sources include make_config.mk @@ -117,12 +135,14 @@ OPT += $(OPTIMIZE_LEVEL) # compile with -O2 if debug level is not 2 ifneq ($(DEBUG_LEVEL), 2) +ifeq ($(WITH_FRAME_POINTER),1) OPT += -fno-omit-frame-pointer # Skip for archs that don't support -momit-leaf-frame-pointer ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1)) OPT += -momit-leaf-frame-pointer endif endif +endif ifeq (,$(shell $(CXX) -fsyntax-only -maltivec -xc /dev/null 2>&1)) CXXFLAGS += -DHAS_ALTIVEC @@ -209,6 +229,330 @@ endif #----------------------------------------------- include src.mk +# ROCKSDB_NO_DYNAMIC_EXTENSION makes dll load twice, disable it +CXXFLAGS += -DROCKSDB_NO_DYNAMIC_EXTENSION + +# civetweb show server stats +CXXFLAGS += -DUSE_SERVER_STATS=1 +CFLAGS += -DUSE_SERVER_STATS=1 + +# civetweb-v1.15 requires OPENSSL_API_1_1 or OPENSSL_API_1_0 +CXXFLAGS += -DOPENSSL_API_1_1=1 +CFLAGS += -DOPENSSL_API_1_1=1 + +ifneq ($(filter check_% check-% %_tests %_test %_test2 \ + watch-log format clean% tags% \ + package% install install-%, \ + $(MAKECMDGOALS)),) + UPDATE_REPO ?= 0 +endif + +ifeq (,$(wildcard sideplugin/rockside/3rdparty/rapidyaml)) + $(warning NotFound sideplugin/rockside/3rdparty/rapidyaml) + $(warning sideplugin/rockside is a submodule, auto init...) + IsCloneOK := $(shell \ + set -x -e; \ + git submodule update --init --recursive >&2; \ + echo $$?\ + ) + ifneq ("${IsCloneOK}","0") + $(error "IsCloneOK=${IsCloneOK} Error cloning rockside, stop!") + endif +else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell set -ex; git pull && git submodule update --init --recursive) + endif + endif +endif +EXTRA_LIB_SOURCES += sideplugin/rockside/src/topling/rapidyaml_all.cc +CXXFLAGS += -Isideplugin/rockside/3rdparty/rapidyaml \ + -Isideplugin/rockside/3rdparty/rapidyaml/src \ + -Isideplugin/rockside/3rdparty/rapidyaml/ext/c4core/src \ + -DSIDE_PLUGIN_WITH_YAML=1 + +# topling-core is topling private +ifneq (,$(wildcard sideplugin/topling-core)) + TOPLING_CORE_DIR := sideplugin/topling-core + CXXFLAGS += -DGITHUB_TOPLING_ZIP='"https://github.com/rockeet/topling-core"' +else + CXXFLAGS += -DGITHUB_TOPLING_ZIP='"https://github.com/topling/topling-zip"' + # topling-zip is topling public + ifeq (,$(wildcard sideplugin/topling-zip)) + $(warning sideplugin/topling-zip is not present, clone it from github...) + IsCloneOK := $(shell \ + set -x -e; \ + cd sideplugin; \ + git clone https://github.com/topling/topling-zip.git >&2; \ + cd topling-zip; \ + git submodule update --init --recursive >&2; \ + echo $$?\ + ) + ifneq ("${IsCloneOK}","0") + $(error "IsCloneOK=${IsCloneOK} Error cloning topling-zip, stop!") + endif + else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell set -ex; cd sideplugin/topling-zip && \ + git pull && git submodule update --init --recursive) + endif + endif + endif + TOPLING_CORE_DIR := sideplugin/topling-zip +endif + +COMPILER := $(shell set -e; tmpfile=`mktemp -u compiler-XXXXXX`; \ + ${CXX} ${TOPLING_CORE_DIR}/tools/configure/compiler.cpp -o $${tmpfile}.exe; \ + ./$${tmpfile}.exe && rm -f $${tmpfile}*) +UNAME_MachineSystem := $(shell uname -m -s | sed 's:[ /]:-:g') +WITH_BMI2 ?= $(shell bash ${TOPLING_CORE_DIR}/cpu_has_bmi2.sh) +BUILD_NAME := ${UNAME_MachineSystem}-${COMPILER}-bmi2-${WITH_BMI2} +BUILD_ROOT := build/${BUILD_NAME} +ifeq (${DEBUG_LEVEL}, 0) + BUILD_TYPE_SIG := r + OBJ_DIR := ${BUILD_ROOT}/rls +endif +ifeq (${DEBUG_LEVEL}, 1) + BUILD_TYPE_SIG := a + OBJ_DIR := ${BUILD_ROOT}/afr +endif +ifeq (${DEBUG_LEVEL}, 2) + BUILD_TYPE_SIG := d + OBJ_DIR := ${BUILD_ROOT}/dbg +endif +ifneq ($(filter auto_all_tests check check_0 watch-log gen_parallel_tests %_test %_test2, $(MAKECMDGOALS)),) + CXXFLAGS += -DROCKSDB_UNIT_TEST + CXXFLAGS += -DROCKSDB_DYNAMIC_CREATE_CF + CXXFLAGS += -DTOPLINGDB_WITH_TIMESTAMP + CXXFLAGS += -DTOPLINGDB_WITH_WIDE_COLUMNS + MAKE_UNIT_TEST := 1 + OBJ_DIR := $(subst build/,build-ut/,${OBJ_DIR}) +endif + +# 1. we define ROCKSDB_DISABLE_ZSTD=1 on build_detect_platform. +# 2. zstd lib is included in libterark-zbs +# 3. we alway use ZSTD +CXXFLAGS += -DZSTD \ + -I${TOPLING_CORE_DIR}/3rdparty/zstd/zstd \ + -I${TOPLING_CORE_DIR}/3rdparty/zstd/zstd/dictBuilder + +CXXFLAGS += \ + -I${TOPLING_CORE_DIR}/src \ + -I${TOPLING_CORE_DIR}/boost-include \ + -I${TOPLING_CORE_DIR}/3rdparty/zstd + +LDFLAGS += -L${TOPLING_CORE_DIR}/${BUILD_ROOT}/lib_shared \ + -lterark-{zbs,fsa,core}-${COMPILER}-${BUILD_TYPE_SIG} + +ifndef WITH_TOPLING_ROCKS + # auto check + ifeq (,$(wildcard sideplugin/topling-rocks)) + # topling specific: just for people who has permission to topling-rocks + dummy := $(shell set -e -x; \ + cd sideplugin; \ + git clone git@github.com:rockeet/topling-rocks; \ + cd topling-rocks; \ + git submodule update --init --recursive \ + ) + endif + ifeq (,$(wildcard sideplugin/topling-rocks)) + WITH_TOPLING_ROCKS := 0 + else + WITH_TOPLING_ROCKS := 1 + endif +endif + +ifeq (${WITH_TOPLING_ROCKS},1) +ifeq (,$(wildcard sideplugin/topling-rocks)) + # topling specific: just for people who has permission to topling-rocks + dummy := $(shell set -e -x; \ + cd sideplugin; \ + git clone git@github.com:rockeet/topling-rocks; \ + cd topling-rocks; \ + git submodule update --init --recursive \ + ) +else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell set -ex; cd sideplugin/topling-rocks && git pull) + endif + endif +endif +ifeq (,$(wildcard sideplugin/topling-rocks/src/table/top_zip_table_builder.cc)) + $(error WITH_TOPLING_ROCKS=1 but repo sideplugin/topling-rocks is broken) +endif +endif + +ifeq (,$(wildcard sideplugin/cspp-memtable)) + # topling specific: just for people who has permission to cspp-memtable + dummy := $(shell set -e -x; \ + cd sideplugin; \ + git clone https://github.com/topling/cspp-memtable; \ + cd cspp-memtable; \ + ) +else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell set -ex; cd sideplugin/cspp-memtable && git pull) + endif + endif +endif +ifeq (,$(wildcard sideplugin/cspp-wbwi)) + dummy := $(shell set -e -x; \ + cd sideplugin; \ + git clone https://github.com/topling/cspp-wbwi; \ + cd cspp-wbwi; \ + ) +else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell set -ex; cd sideplugin/cspp-wbwi && git pull) + endif + endif +endif + +ifneq (,$(wildcard sideplugin/cspp-memtable)) + # now we have cspp-memtable + CXXFLAGS += -DHAS_TOPLING_CSPP_MEMTABLE + CSPP_MEMTABLE_GIT_VER_SRC = ${BUILD_ROOT}/git-version-cspp_memtable.cc + EXTRA_LIB_SOURCES += sideplugin/cspp-memtable/cspp_memtable.cc \ + sideplugin/cspp-memtable/${CSPP_MEMTABLE_GIT_VER_SRC} +else + $(warning NotFound sideplugin/cspp-memtable, this is ok, only Topling CSPP MemTab is disabled) +endif + +ifneq (,$(wildcard sideplugin/cspp-wbwi)) + # now we have cspp-wbwi + CXXFLAGS += -DHAS_TOPLING_CSPP_WBWI + CSPP_WBWI_GIT_VER_SRC = ${BUILD_ROOT}/git-version-cspp_wbwi.cc + EXTRA_LIB_SOURCES += sideplugin/cspp-wbwi/cspp_wbwi.cc \ + sideplugin/cspp-wbwi/${CSPP_WBWI_GIT_VER_SRC} +else + $(warning NotFound sideplugin/cspp-wbwi, this is ok, only Topling CSPP WBWI(WriteBatchWithIndex) is disabled) +endif + +ifeq (,$(wildcard sideplugin/topling-sst/src/table)) + dummy := $(shell set -e -x; \ + cd sideplugin; \ + git clone https://github.com/topling/topling-sst; \ + cd topling-sst; \ + ) +else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell cd sideplugin/topling-sst && git pull) + endif + endif +endif +ifneq (,$(wildcard sideplugin/topling-sst/src/table)) + # now we have topling-sst + CXXFLAGS += -DHAS_TOPLING_SST -Isideplugin/topling-sst/src + TOPLING_SST_GIT_VER_SRC = ${BUILD_ROOT}/git-version-topling_sst.cc + EXTRA_LIB_SOURCES += $(wildcard sideplugin/topling-sst/src/table/*.cc) \ + sideplugin/topling-sst/${TOPLING_SST_GIT_VER_SRC} +else + $(warning NotFound sideplugin/topling-sst, this is ok, only Topling Open SST(s) are disabled) +endif + +ifeq (,$(wildcard sideplugin/topling-zip_table_reader/src/table)) + dummy := $(shell set -e -x; \ + cd sideplugin; \ + git clone https://github.com/topling/topling-zip_table_reader; \ + cd topling-zip_table_reader; \ + ) +else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell cd sideplugin/topling-zip_table_reader && git pull) + endif + endif +endif +ifneq (,$(wildcard sideplugin/topling-zip_table_reader/src/table)) + # now we have topling-zip_table_reader + CXXFLAGS += -DHAS_TOPLING_SST -Isideplugin/topling-zip_table_reader/src + TOPLING_ZIP_TABLE_READER_GIT_VER_SRC = ${BUILD_ROOT}/git-version-topling_zip_table_reader.cc + EXTRA_LIB_SOURCES += $(wildcard sideplugin/topling-zip_table_reader/src/table/*.cc) \ + sideplugin/topling-zip_table_reader/${TOPLING_ZIP_TABLE_READER_GIT_VER_SRC} +else + $(warning NotFound sideplugin/topling-zip_table_reader, this is ok, only Topling Open SST(s) are disabled) +endif + + +ifeq (,$(wildcard sideplugin/topling-dcompact/src/dcompact)) + dummy := $(shell set -e -x; \ + cd sideplugin; \ + git clone https://github.com/topling/topling-dcompact; \ + cd topling-dcompact; \ + ) +else + ifneq (${UPDATE_REPO},0) + ifeq (${MAKE_RESTARTS},) + dummy := $(shell cd sideplugin/topling-dcompact && git pull) + endif + endif +endif +ifneq (,$(wildcard sideplugin/topling-dcompact/src/dcompact)) + # now we have topling-dcompact + #CXXFLAGS += -Isideplugin/topling-dcompact/src + LDFLAGS += -lstdc++fs -lcurl + TOPLING_DCOMPACT_GIT_VER_SRC := ${BUILD_ROOT}/git-version-topling_dcompact.cc + EXTRA_LIB_SOURCES += $(wildcard sideplugin/topling-dcompact/src/dcompact/*.cc) \ + sideplugin/topling-dcompact/${TOPLING_DCOMPACT_GIT_VER_SRC} +else + $(warning NotFound sideplugin/topling-dcompact, this is ok, only topling-dcompact is disabled) +endif + +export LD_LIBRARY_PATH:=${TOPLING_CORE_DIR}/${BUILD_ROOT}/lib_shared:${LD_LIBRARY_PATH} +ifeq (${WITH_TOPLING_ROCKS},1) +ifneq (,$(wildcard sideplugin/topling-rocks)) + CXXFLAGS += -I sideplugin/topling-rocks/src + TOPLING_ROCKS_GIT_VER_SRC = ${BUILD_ROOT}/git-version-topling_rocks.cc + EXTRA_LIB_SOURCES += \ + $(wildcard sideplugin/topling-rocks/src/table/*.cc) \ + sideplugin/topling-rocks/${TOPLING_ROCKS_GIT_VER_SRC} +else + $(warning NotFound sideplugin/topling-rocks, this is ok, only ToplingZipTable is disabled) +endif +endif + +TOPLING_DCOMPACT_USE_ETCD := 0 +ifneq (,$(wildcard sideplugin/topling-dcompact/3rdparty/etcd-cpp-apiv3/build/src/libetcd-cpp-api.${PLATFORM_SHARED_EXT})) +ifneq (,$(wildcard sideplugin/topling-dcompact/3rdparty/etcd-cpp-apiv3/build/proto/gen/proto)) + CXXFLAGS += -I sideplugin/topling-dcompact/3rdparty/etcd-cpp-apiv3/build/proto/gen/proto \ + -I sideplugin/topling-dcompact/3rdparty/etcd-cpp-apiv3 + LDFLAGS += -L sideplugin/topling-dcompact/3rdparty/etcd-cpp-apiv3/build/src -letcd-cpp-api + export LD_LIBRARY_PATH:=${TOPLING_ROCKS_DIR}/3rdparty/etcd-cpp-apiv3/build/src:${LD_LIBRARY_PATH} + ifneq (,$(wildcard ../vcpkg/packages/grpc_x64-linux/include)) + CXXFLAGS += -I ../vcpkg/packages/grpc_x64-linux/include + else + $(error NotFound ../vcpkg/packages/grpc_x64-linux/include) + endif + ifneq (,$(wildcard ../vcpkg/packages/protobuf_x64-linux/include)) + CXXFLAGS += -I ../vcpkg/packages/protobuf_x64-linux/include + else + $(error NotFound ../vcpkg/packages/protobuf_x64-linux/include) + endif + ifneq (,$(wildcard ../vcpkg/packages/cpprestsdk_x64-linux/include)) + CXXFLAGS += -I ../vcpkg/packages/cpprestsdk_x64-linux/include + else + $(error NotFound ../vcpkg/packages/cpprestsdk_x64-linux/include) + endif + CXXFLAGS += -DTOPLING_DCOMPACT_USE_ETCD + TOPLING_DCOMPACT_USE_ETCD := 1 +endif +endif + +#ifeq (${TOPLING_DCOMPACT_USE_ETCD},0) +# $(warning NotFound etcd-cpp-apiv3, this is ok, only etcd is disabled) +#endif + +#export ROCKSDB_KICK_OUT_OPTIONS_FILE=1 + +# prepend EXTRA_LIB_SOURCES to LIB_SOURCES because +# EXTRA_LIB_SOURCES single file compiling is slow +LIB_SOURCES := ${EXTRA_LIB_SOURCES} ${LIB_SOURCES} + AM_DEFAULT_VERBOSITY ?= 0 AM_V_GEN = $(am__v_GEN_$(V)) @@ -240,7 +584,7 @@ am__v_AR_0 = @echo " AR " $@; am__v_AR_1 = AM_LINK = $(AM_V_CCLD)$(CXX) -L. $(patsubst lib%.a, -l%, $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^)) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -AM_SHARE = $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$@ -L. $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^) $(EXEC_LDFLAGS) $(LDFLAGS) -o $@ +AM_SHARE = $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$@ -L. $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^) $(EXTRA_SHARED_LIB_LIB) $(EXEC_LDFLAGS) $(LDFLAGS) -o $@ ROCKSDB_PLUGIN_MKS = $(foreach plugin, $(ROCKSDB_PLUGINS), plugin/$(plugin)/*.mk) include $(ROCKSDB_PLUGIN_MKS) @@ -267,6 +611,8 @@ ROCKSDB_PLUGIN_JNI_NATIVE_SOURCES = $(foreach plugin, $(ROCKSDB_PLUGINS), $(fore ALL_JNI_NATIVE_SOURCES = $(JNI_NATIVE_SOURCES) $(ROCKSDB_PLUGIN_JNI_NATIVE_SOURCES) ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS = $(foreach plugin, $(ROCKSDB_PLUGINS), -I./plugin/$(plugin)) +ALL_JNI_NATIVE_OBJECTS := $(patsubst %.cc, $(OBJ_DIR)/%.o, $(ALL_JNI_NATIVE_SOURCES)) + ifneq ($(strip $(ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES)),) LDFLAGS := $(LDFLAGS) $(shell pkg-config --libs $(ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES)) ifneq ($(.SHELLSTATUS),0) @@ -298,7 +644,7 @@ $(info $(shell $(CXX) --version)) endif missing_make_config_paths := $(shell \ - grep "\./\S*\|/\S*" -o $(CURDIR)/make_config.mk | \ + egrep "\.+/\S*|([a-z_]*)/\S*" -o $(CURDIR)/make_config.mk | \ while read path; \ do [ -e $$path ] || echo $$path; \ done | sort | uniq | grep -v "/DOES/NOT/EXIST") @@ -309,8 +655,10 @@ $(foreach path, $(missing_make_config_paths), \ ifeq ($(PLATFORM), OS_AIX) # no debug info else ifneq ($(PLATFORM), IOS) -CFLAGS += -g -CXXFLAGS += -g +# default disable dwarf +DBG_DWARF ?= +CFLAGS += ${DBG_DWARF} -g3 +CXXFLAGS += ${DBG_DWARF} -g3 else # no debug info for IOS, that will make our library big OPT += -DNDEBUG @@ -337,12 +685,20 @@ ifneq ($(MACHINE), arm64) # linking with jemalloc (as it won't be arm64-compatible) and remove some other options # set during platform detection DISABLE_JEMALLOC=1 -PLATFORM_CCFLAGS := $(filter-out -march=native, $(PLATFORM_CCFLAGS)) -PLATFORM_CXXFLAGS := $(filter-out -march=native, $(PLATFORM_CXXFLAGS)) +PLATFORM_CCFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CCFLAGS)) +PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CXXFLAGS)) endif endif endif +ifeq (${WITH_BMI2},1) + CPU_ARCH ?= -march=haswell +endif +ifdef CPU_ARCH + PLATFORM_CCFLAGS := ${CPU_ARCH} $(filter-out -march=native -DHAVE_AVX2, $(PLATFORM_CCFLAGS)) + PLATFORM_CXXFLAGS := ${CPU_ARCH} $(filter-out -march=native -DHAVE_AVX2, $(PLATFORM_CXXFLAGS)) +endif + # ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc. ifdef COMPILE_WITH_ASAN DISABLE_JEMALLOC=1 @@ -550,6 +906,9 @@ ifndef DISABLE_WARNING_AS_ERROR WARNING_FLAGS += -Werror endif +# topling specific WARNING_FLAGS +WARNING_FLAGS := -Wall -Wno-shadow +WARNING_FLAGS += -Wno-deprecated-builtins ifdef LUA_PATH @@ -582,6 +941,7 @@ ifeq ($(NO_THREEWAY_CRC32C), 1) endif CFLAGS += $(C_WARNING_FLAGS) $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) +CXXFLAGS += -Isideplugin/rockside/src CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers # Allow offsetof to work on non-standard layout types. Some compiler could @@ -591,10 +951,11 @@ CXXFLAGS += -Wno-invalid-offsetof LDFLAGS += $(PLATFORM_LDFLAGS) -LIB_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(LIB_SOURCES)) +LIB_OBJECTS := $(patsubst %.cc, $(OBJ_DIR)/%.o, $(LIB_SOURCES)) +LIB_OBJECTS := $(patsubst %.cpp,$(OBJ_DIR)/%.o, $(LIB_OBJECTS)) LIB_OBJECTS += $(patsubst %.cc, $(OBJ_DIR)/%.o, $(ROCKSDB_PLUGIN_SOURCES)) -ifeq ($(HAVE_POWER8),1) LIB_OBJECTS += $(patsubst %.c, $(OBJ_DIR)/%.o, $(LIB_SOURCES_C)) +ifeq ($(HAVE_POWER8),1) LIB_OBJECTS += $(patsubst %.S, $(OBJ_DIR)/%.o, $(LIB_SOURCES_ASM)) endif @@ -605,6 +966,9 @@ endif # range_tree is not compatible with non GNU libc on ppc64 # see https://jira.percona.com/browse/PS-7559 ifneq ($(PPC_LIBC_IS_GNU),0) + # topling: should move this line above and delete LIB_OBJECTS += .., add here for min-diff principle + # add to LIB_SOURCES to generate *.cc.d dependency rules + LIB_SOURCES += ${RANGE_TREE_SOURCES} LIB_OBJECTS += $(patsubst %.cc, $(OBJ_DIR)/%.o, $(RANGE_TREE_SOURCES)) endif @@ -634,6 +998,13 @@ PLUGIN_TESTS = $(patsubst %.cc, %, $(notdir $(ROCKSDB_PLUGIN_TESTS))) TESTS = $(patsubst %.cc, %, $(notdir $(TEST_MAIN_SOURCES))) TESTS += $(patsubst %.c, %, $(notdir $(TEST_MAIN_SOURCES_C))) TESTS += $(PLUGIN_TESTS) +ifeq (${MAKE_UNIT_TEST},1) + ifeq (cspp,$(patsubst cspp:%,cspp,${DefaultWBWIFactory})) + # cspp WBWI does not support txn with ts(timestamp) + $(warning "test with CSPP_WBWI, skip write_committed_transaction_ts_test") + TESTS := $(filter-out write_committed_transaction_ts_test,${TESTS}) + endif +endif # `make check-headers` to very that each header file includes its own # dependencies @@ -749,14 +1120,24 @@ MICROBENCHS = $(patsubst %.cc, %, $(notdir $(MICROBENCH_SOURCES))) ifeq ($(LIBNAME),) LIBNAME=librocksdb # we should only run rocksdb in production with DEBUG_LEVEL 0 -ifneq ($(DEBUG_LEVEL),0) +ifeq ($(DEBUG_LEVEL),2) LIBDEBUG=_debug + ifeq (${MAKE_UNIT_TEST},1) + LIBDEBUG=_debug_ut + endif +endif +ifeq ($(DEBUG_LEVEL),1) + LIBDEBUG=_debug_1 + ifeq (${MAKE_UNIT_TEST},1) + LIBDEBUG=_debug_ut_1 + endif endif endif STATIC_LIBRARY = ${LIBNAME}$(LIBDEBUG).a STATIC_TEST_LIBRARY = ${LIBNAME}_test$(LIBDEBUG).a STATIC_TOOLS_LIBRARY = ${LIBNAME}_tools$(LIBDEBUG).a STATIC_STRESS_LIBRARY = ${LIBNAME}_stress$(LIBDEBUG).a +#$(error LIBDEBUG = ${LIBDEBUG} PLATFORM_SHARED_VERSIONED=${PLATFORM_SHARED_VERSIONED}) ALL_STATIC_LIBS = $(STATIC_LIBRARY) $(STATIC_TEST_LIBRARY) $(STATIC_TOOLS_LIBRARY) $(STATIC_STRESS_LIBRARY) @@ -821,8 +1202,8 @@ default: all #----------------------------------------------- ifneq ($(PLATFORM_SHARED_EXT),) -ifneq ($(PLATFORM_SHARED_VERSIONED),true) SHARED1 = ${LIBNAME}$(LIBDEBUG).$(PLATFORM_SHARED_EXT) +ifneq ($(PLATFORM_SHARED_VERSIONED),true) SHARED2 = $(SHARED1) SHARED3 = $(SHARED1) SHARED4 = $(SHARED1) @@ -831,7 +1212,6 @@ else SHARED_MAJOR = $(ROCKSDB_MAJOR) SHARED_MINOR = $(ROCKSDB_MINOR) SHARED_PATCH = $(ROCKSDB_PATCH) -SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT) ifeq ($(PLATFORM), OS_MACOSX) SHARED_OSX = $(LIBNAME)$(LIBDEBUG).$(SHARED_MAJOR) SHARED2 = $(SHARED_OSX).$(PLATFORM_SHARED_EXT) @@ -852,7 +1232,7 @@ $(SHARED3): $(SHARED4) endif # PLATFORM_SHARED_VERSIONED $(SHARED4): $(LIB_OBJECTS) - $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(LIB_OBJECTS) $(LDFLAGS) -o $@ + $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(LIB_OBJECTS) $(EXTRA_SHARED_LIB_LIB) $(LDFLAGS) -o $@ endif # PLATFORM_SHARED_EXT .PHONY: check clean coverage ldb_tests package dbg gen-pc build_size \ @@ -866,7 +1246,11 @@ all_but_some_tests: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(ROCKSDB static_lib: $(STATIC_LIBRARY) +ifdef TOPLING_DCOMPACT_GIT_VER_SRC +shared_lib: $(SHARED) dcompact_worker +else shared_lib: $(SHARED) +endif stress_lib: $(STRESS_LIBRARY) @@ -1226,6 +1610,9 @@ clean-rocks: rm -f ${LIBNAME}*.so* ${LIBNAME}*.a rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(MICROBENCHS) rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report + rm -rf build build-ut + rm -rf sideplugin/topling-dcompact/tools/dcompact/build + +$(MAKE) -C ${TOPLING_CORE_DIR} clean $(FIND) . -name "*.[oda]" -exec rm -f {} \; $(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \; @@ -1306,6 +1693,14 @@ librocksdb_env_basic_test.a: $(OBJ_DIR)/env/env_basic_test.o $(LIB_OBJECTS) $(TE db_bench: $(OBJ_DIR)/tools/db_bench.o $(BENCH_OBJECTS) $(TESTUTIL) $(LIBRARY) $(AM_LINK) +ifeq (${DEBUG_LEVEL},2) +db_bench_dbg: $(OBJ_DIR)/tools/db_bench.o $(BENCH_OBJECTS) $(TESTUTIL) $(LIBRARY) + $(AM_LINK) +endif +ifeq (${DEBUG_LEVEL},0) +db_bench_rls: $(OBJ_DIR)/tools/db_bench.o $(BENCH_OBJECTS) $(TESTUTIL) $(LIBRARY) + $(AM_LINK) +endif trace_analyzer: $(OBJ_DIR)/tools/trace_analyzer.o $(ANALYZE_OBJECTS) $(TOOLS_LIBRARY) $(LIBRARY) $(AM_LINK) @@ -1348,7 +1743,7 @@ $(foreach test, $(ROCKSDB_PLUGIN_TESTS), $(eval $(call MakeTestRule, $(test)))) arena_test: $(OBJ_DIR)/memory/arena_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -memory_allocator_test: memory/memory_allocator_test.o $(TEST_LIBRARY) $(LIBRARY) +memory_allocator_test: $(OBJ_DIR)/memory/memory_allocator_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) autovector_test: $(OBJ_DIR)/util/autovector_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -1450,7 +1845,7 @@ db_wide_basic_test: $(OBJ_DIR)/db/wide/db_wide_basic_test.o $(TEST_LIBRARY) $(LI db_with_timestamp_basic_test: $(OBJ_DIR)/db/db_with_timestamp_basic_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -db_with_timestamp_compaction_test: db/db_with_timestamp_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) +db_with_timestamp_compaction_test: $(OBJ_DIR)/db/db_with_timestamp_compaction_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) db_encryption_test: $(OBJ_DIR)/db/db_encryption_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -1666,7 +2061,7 @@ random_access_file_reader_test: $(OBJ_DIR)/file/random_access_file_reader_test.o file_reader_writer_test: $(OBJ_DIR)/util/file_reader_writer_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -block_based_table_reader_test: table/block_based/block_based_table_reader_test.o $(TEST_LIBRARY) $(LIBRARY) +block_based_table_reader_test: $(OBJ_DIR)/table/block_based/block_based_table_reader_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) full_filter_block_test: $(OBJ_DIR)/table/block_based/full_filter_block_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -1684,7 +2079,7 @@ cleanable_test: $(OBJ_DIR)/table/cleanable_test.o $(TEST_LIBRARY) $(LIBRARY) table_test: $(OBJ_DIR)/table/table_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -block_fetcher_test: table/block_fetcher_test.o $(TEST_LIBRARY) $(LIBRARY) +block_fetcher_test: $(OBJ_DIR)/table/block_fetcher_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) block_test: $(OBJ_DIR)/table/block_based/block_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -1774,10 +2169,10 @@ thread_list_test: $(OBJ_DIR)/util/thread_list_test.o $(TEST_LIBRARY) $(LIBRARY) compact_files_test: $(OBJ_DIR)/db/compact_files_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -configurable_test: options/configurable_test.o $(TEST_LIBRARY) $(LIBRARY) +configurable_test: $(OBJ_DIR)/options/configurable_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -customizable_test: options/customizable_test.o $(TEST_LIBRARY) $(LIBRARY) +customizable_test: $(OBJ_DIR)/options/customizable_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) options_test: $(OBJ_DIR)/options/options_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -1831,7 +2226,7 @@ write_callback_test: $(OBJ_DIR)/db/write_callback_test.o $(TEST_LIBRARY) $(LIBRA heap_test: $(OBJ_DIR)/util/heap_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -point_lock_manager_test: utilities/transactions/lock/point/point_lock_manager_test.o $(TEST_LIBRARY) $(LIBRARY) +point_lock_manager_test: $(OBJ_DIR)/utilities/transactions/lock/point/point_lock_manager_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) transaction_test: $(OBJ_DIR)/utilities/transactions/transaction_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -1900,7 +2295,7 @@ blob_db_test: $(OBJ_DIR)/utilities/blob_db/blob_db_test.o $(TEST_LIBRARY) $(LIBR repeatable_thread_test: $(OBJ_DIR)/util/repeatable_thread_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) -range_locking_test: utilities/transactions/lock/range/range_locking_test.o $(TEST_LIBRARY) $(LIBRARY) +range_locking_test: $(OBJ_DIR)/utilities/transactions/lock/range/range_locking_test.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) range_tombstone_fragmenter_test: $(OBJ_DIR)/db/range_tombstone_fragmenter_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -1964,6 +2359,57 @@ io_tracer_parser_test: $(OBJ_DIR)/tools/io_tracer_parser_test.o $(OBJ_DIR)/tools $(AM_LINK) io_tracer_parser: $(OBJ_DIR)/tools/io_tracer_parser.o $(TOOLS_LIBRARY) $(LIBRARY) +#-------------------------------------------------- +ifndef ROCKSDB_USE_LIBRADOS + AUTO_ALL_EXCLUDE_SRC += utilities/env_librados_test.cc + AUTO_ALL_EXCLUDE_SRC += utilities/env_mirror_test.cc +endif + +AUTO_ALL_TESTS_SRC := $(shell find * -name '*_test.cc' -not -path 'java/*' -not -path '*/3rdparty/*') ${EXTRA_TESTS_SRC} +AUTO_ALL_TESTS_SRC := $(filter-out ${AUTO_ALL_EXCLUDE_SRC},${AUTO_ALL_TESTS_SRC}) +AUTO_ALL_TESTS_OBJ := $(addprefix $(OBJ_DIR)/,$(AUTO_ALL_TESTS_SRC:%.cc=%.o)) +AUTO_ALL_TESTS_EXE := $(AUTO_ALL_TESTS_OBJ:%.o=%) + +define LN_TEST_TARGET +t${DEBUG_LEVEL}/${1}: ${2} + mkdir -p $(dir $$@) && ln -sf `realpath ${2}` $$@ + +endef +#intentional one blank line above + +.PHONY: auto_all_tests +auto_all_tests: ${AUTO_ALL_TESTS_EXE} + +$(OBJ_DIR)/tools/%_test: $(OBJ_DIR)/tools/%_test.o \ + ${TOOLS_LIBRARY} $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +$(OBJ_DIR)/%_test: $(OBJ_DIR)/%_test.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +$(eval $(foreach test,${AUTO_ALL_TESTS_EXE},$(call LN_TEST_TARGET,$(notdir ${test}),${test}))) + +$(OBJ_DIR)/tools/db_bench_tool_test : \ +$(OBJ_DIR)/tools/db_bench_tool_test.o \ + ${BENCH_OBJECTS} $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +$(OBJ_DIR)/file/prefetch_test : \ +$(OBJ_DIR)/file/prefetch_test.o \ +$(OBJ_DIR)/tools/io_tracer_parser_tool.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +$(OBJ_DIR)/tools/trace_analyzer_test : \ +$(OBJ_DIR)/tools/trace_analyzer_test.o \ + ${ANALYZE_OBJECTS} ${TOOLS_LIBRARY} $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +$(OBJ_DIR)/tools/block_cache_analyzer/block_cache_trace_analyzer_test : \ +$(OBJ_DIR)/tools/block_cache_analyzer/block_cache_trace_analyzer_test.o \ +$(OBJ_DIR)/tools/block_cache_analyzer/block_cache_trace_analyzer.o $(TEST_LIBRARY) $(LIBRARY) + $(AM_LINK) + +$(OBJ_DIR)/%: $(OBJ_DIR)/%.o $(TEST_LIBRARY) $(LIBRARY) $(AM_LINK) db_blob_corruption_test: $(OBJ_DIR)/db/blob/db_blob_corruption_test.o $(TEST_LIBRARY) $(LIBRARY) @@ -2018,22 +2464,48 @@ install-headers: gen-pc install -d $(DESTDIR)/$(PREFIX)/include/rocksdb/`dirname $$header`; \ install -C -m 644 $$header $(DESTDIR)/$(PREFIX)/include/rocksdb/$$header; \ done + install -d $(DESTDIR)/$(PREFIX)/include/topling + install -C -m 644 sideplugin/rockside/src/topling/json.h $(DESTDIR)/$(PREFIX)/include/topling + install -C -m 644 sideplugin/rockside/src/topling/json_fwd.h $(DESTDIR)/$(PREFIX)/include/topling + install -C -m 644 sideplugin/rockside/src/topling/builtin_table_factory.h $(DESTDIR)/$(PREFIX)/include/topling + install -C -m 644 sideplugin/rockside/src/topling/side_plugin_repo.h $(DESTDIR)/$(PREFIX)/include/topling + install -C -m 644 sideplugin/rockside/src/topling/side_plugin_factory.h $(DESTDIR)/$(PREFIX)/include/topling + install -d $(DESTDIR)/$(PREFIX)/include/terark + install -d $(DESTDIR)/$(PREFIX)/include/terark/io + install -d $(DESTDIR)/$(PREFIX)/include/terark/succinct + install -d $(DESTDIR)/$(PREFIX)/include/terark/thread + install -d $(DESTDIR)/$(PREFIX)/include/terark/util + install -d $(DESTDIR)/$(PREFIX)/include/terark/fsa + install -d $(DESTDIR)/$(PREFIX)/include/terark/fsa/ppi + install -d $(DESTDIR)/$(PREFIX)/include/terark/zbs + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/*.hpp $(DESTDIR)/$(PREFIX)/include/terark + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/io/*.hpp $(DESTDIR)/$(PREFIX)/include/terark/io + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/succinct/*.hpp $(DESTDIR)/$(PREFIX)/include/terark/succinct + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/thread/*.hpp $(DESTDIR)/$(PREFIX)/include/terark/thread + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/util/*.hpp $(DESTDIR)/$(PREFIX)/include/terark/util + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/fsa/*.hpp $(DESTDIR)/$(PREFIX)/include/terark/fsa + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/fsa/*.inl $(DESTDIR)/$(PREFIX)/include/terark/fsa + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/fsa/ppi/*.hpp $(DESTDIR)/$(PREFIX)/include/terark/fsa/ppi + install -C -m 644 ${TOPLING_CORE_DIR}/src/terark/zbs/*.hpp $(DESTDIR)/$(PREFIX)/include/terark/zbs + cp -ar ${TOPLING_CORE_DIR}/boost-include/boost $(DESTDIR)/$(PREFIX)/include install -C -m 644 rocksdb.pc $(INSTALL_LIBDIR)/pkgconfig/rocksdb.pc -install-static: install-headers $(LIBRARY) +install-static: install-headers $(LIBRARY) static_lib install -d $(INSTALL_LIBDIR) install -C -m 755 $(LIBRARY) $(INSTALL_LIBDIR) + cp -a ${TOPLING_CORE_DIR}/${BUILD_ROOT}/lib_static/* $(INSTALL_LIBDIR) -install-shared: install-headers $(SHARED4) +install-shared: install-headers $(SHARED4) shared_lib install -d $(INSTALL_LIBDIR) install -C -m 755 $(SHARED4) $(INSTALL_LIBDIR) ln -fs $(SHARED4) $(INSTALL_LIBDIR)/$(SHARED3) ln -fs $(SHARED4) $(INSTALL_LIBDIR)/$(SHARED2) ln -fs $(SHARED4) $(INSTALL_LIBDIR)/$(SHARED1) + cp -a ${TOPLING_CORE_DIR}/${BUILD_ROOT}/lib_shared/* $(INSTALL_LIBDIR) + mkdir -p $(DESTDIR)$(PREFIX)/bin + cp -a sideplugin/topling-dcompact/tools/dcompact/${OBJ_DIR}/*.exe $(DESTDIR)$(PREFIX)/bin -# install static by default + install shared if it exists -install: install-static - [ -e $(SHARED4) ] && $(MAKE) install-shared || : +install: install-${LIB_MODE} # Generate the pkg-config file gen-pc: @@ -2046,7 +2518,7 @@ gen-pc: -echo 'Description: An embeddable persistent key-value store for fast storage' >> rocksdb.pc -echo Version: $(shell ./build_tools/version.sh full) >> rocksdb.pc -echo 'Libs: -L$${libdir} $(EXEC_LDFLAGS) -lrocksdb' >> rocksdb.pc - -echo 'Libs.private: $(PLATFORM_LDFLAGS)' >> rocksdb.pc + -echo 'Libs.private: -lterark-zbs-r -lterark-fsa-r -lterark-core-r $(PLATFORM_LDFLAGS)' >> rocksdb.pc -echo 'Cflags: -I$${includedir} $(PLATFORM_CXXFLAGS)' >> rocksdb.pc -echo 'Requires: $(subst ",,$(ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES))' >> rocksdb.pc @@ -2399,18 +2871,31 @@ rocksdbjavastaticnexusbundlejar: rocksdbjavageneratepom jl/%.o: %.cc $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ $(COVERAGEFLAGS) -rocksdbjava: $(LIB_OBJECTS) +${ALL_JNI_NATIVE_OBJECTS}: CXXFLAGS += -I./java/. -I./java/rocksjni $(JAVA_INCLUDE) $(ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS) +${ALL_JNI_NATIVE_OBJECTS}: rocksdbjava-header +rocksdbjava: $(LIB_OBJECTS) $(ALL_JNI_NATIVE_OBJECTS) ifeq ($(JAVA_HOME),) $(error JAVA_HOME is not set) endif - $(AM_V_GEN)cd java; $(MAKE) javalib; $(AM_V_at)rm -f ./java/target/$(ROCKSDBJNILIB) - $(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. -I./java/rocksjni $(JAVA_INCLUDE) $(ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_SOURCES) $(LIB_OBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS) + $(AM_V_at)$(CXX) $(CXXFLAGS) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_OBJECTS) $(LIB_OBJECTS) $(JAVA_LDFLAGS) $(LDFLAGS) + $(AM_V_at)cp -a ${TOPLING_CORE_DIR}/${BUILD_ROOT}/lib_shared/*${COMPILER}*-r.so java/target + $(AM_V_at)cp -a sideplugin/rockside/src/topling/web/{style.css,index.html} java/target +ifeq ($(STRIP_DEBUG_INFO),1) + $(AM_V_at)strip java/target/*.so +endif $(AM_V_at)cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md - $(AM_V_at)cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) + $(AM_V_at)cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) *.so + $(AM_V_at)cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) style.css index.html $(AM_V_at)cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class $(AM_V_at)openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 +rocksdbjava-header: +ifeq ($(JAVA_HOME),) + $(error JAVA_HOME is not set) +endif + $(AM_V_GEN)cd java; $(MAKE) javalib; + jclean: cd java;$(MAKE) clean; @@ -2531,7 +3016,16 @@ $(OBJ_DIR)/%.o: %.cpp $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS) $(OBJ_DIR)/%.o: %.c - $(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@ + $(AM_V_CC)mkdir -p $(@D) && $(CC) $(CFLAGS) -c $< -o $@ + +$(OBJ_DIR)/%.s: %.cc + $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -Wa,-adhln -fverbose-asm -masm=intel -S $< -o $@ $(COVERAGEFLAGS) + +$(OBJ_DIR)/%.s: %.cpp + $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -fverbose-asm -masm=intel -S $< -o $@ $(COVERAGEFLAGS) + +$(OBJ_DIR)/%.s: %.c + $(AM_V_CC)mkdir -p $(@D) && $(CC) $(CFLAGS) -fverbose-asm -masm=intel -S $< -o $@ endif # --------------------------------------------------------------------------- @@ -2539,8 +3033,9 @@ endif # --------------------------------------------------------------------------- # If skip dependencies is ON, skip including the dep files ifneq ($(SKIP_DEPENDS), 1) -DEPFILES = $(patsubst %.cc, $(OBJ_DIR)/%.cc.d, $(ALL_SOURCES)) -DEPFILES+ = $(patsubst %.c, $(OBJ_DIR)/%.c.d, $(LIB_SOURCES_C) $(TEST_MAIN_SOURCES_C)) +DEPFILES := $(patsubst %.cc, $(OBJ_DIR)/%.cc.d, $(ALL_SOURCES)) +DEPFILES := $(patsubst %.cpp,$(OBJ_DIR)/%.cpp.d,$(DEPFILES)) +DEPFILES += $(patsubst %.c, $(OBJ_DIR)/%.c.d, $(LIB_SOURCES_C) $(TEST_MAIN_SOURCES_C)) ifeq ($(USE_FOLLY_LITE),1) DEPFILES +=$(patsubst %.cpp, $(OBJ_DIR)/%.cpp.d, $(FOLLY_SOURCES)) endif @@ -2554,12 +3049,12 @@ endif $(OBJ_DIR)/%.cc.d: %.cc @mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \ -MM -MT'$@' -MT'$(<:.cc=.o)' -MT'$(<:%.cc=$(OBJ_DIR)/%.o)' \ - "$<" -o '$@' + "$<" -o '$@' $(OBJ_DIR)/%.cpp.d: %.cpp @mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \ -MM -MT'$@' -MT'$(<:.cpp=.o)' -MT'$(<:%.cpp=$(OBJ_DIR)/%.o)' \ - "$<" -o '$@' + "$<" -o '$@' ifeq ($(HAVE_POWER8),1) DEPFILES_C = $(patsubst %.c, $(OBJ_DIR)/%.c.d, $(LIB_SOURCES_C)) @@ -2587,6 +3082,70 @@ build_subset_tests: $(ROCKSDBTESTS_SUBSET) list_all_tests: echo "$(ROCKSDBTESTS_SUBSET)" +TOPLING_ZBS_TARGET := ${BUILD_ROOT}/lib_shared/libterark-zbs-${COMPILER}-${BUILD_TYPE_SIG}.${PLATFORM_SHARED_EXT} +${SHARED4}: ${TOPLING_CORE_DIR}/${TOPLING_ZBS_TARGET} +${TOPLING_CORE_DIR}/${TOPLING_ZBS_TARGET}: CXXFLAGS = +${TOPLING_CORE_DIR}/${TOPLING_ZBS_TARGET}: LDFLAGS = +${TOPLING_CORE_DIR}/${TOPLING_ZBS_TARGET}: + +make -C ${TOPLING_CORE_DIR} ${TOPLING_ZBS_TARGET} + +${STATIC_LIBRARY}: ${BUILD_ROOT}/lib_shared/libterark-zbs-${COMPILER}-${BUILD_TYPE_SIG}.a +${BUILD_ROOT}/lib_shared/libterark-zbs-${COMPILER}-${BUILD_TYPE_SIG}.a: + +make -C ${TOPLING_CORE_DIR} core fsa zbs + +ifeq (${WITH_TOPLING_ROCKS},1) +ifneq (,$(wildcard sideplugin/topling-rocks)) +sideplugin/topling-rocks/${TOPLING_ROCKS_GIT_VER_SRC}: \ + $(shell find sideplugin/topling-rocks/{src,tools} -name '*.cc' -o -name '*.h') + +make -C sideplugin/topling-rocks ${TOPLING_ROCKS_GIT_VER_SRC} +endif +endif + +ifneq (,$(wildcard sideplugin/cspp-memtable)) +sideplugin/cspp-memtable/${CSPP_MEMTABLE_GIT_VER_SRC}: \ + sideplugin/cspp-memtable/cspp_memtable.cc \ + sideplugin/cspp-memtable/Makefile + +make -C sideplugin/cspp-memtable ${CSPP_MEMTABLE_GIT_VER_SRC} +endif +ifneq (,$(wildcard sideplugin/cspp-wbwi)) +sideplugin/cspp-wbwi/${CSPP_WBWI_GIT_VER_SRC}: \ + sideplugin/cspp-wbwi/cspp_wbwi.cc \ + sideplugin/cspp-wbwi/Makefile + +make -C sideplugin/cspp-wbwi ${CSPP_WBWI_GIT_VER_SRC} +endif +ifneq (,$(wildcard sideplugin/topling-sst/src/table)) +sideplugin/topling-sst/${TOPLING_SST_GIT_VER_SRC}: \ + $(wildcard sideplugin/topling-sst/src/table/*.h) \ + $(wildcard sideplugin/topling-sst/src/table/*.cc) \ + sideplugin/topling-sst/Makefile + +make -C sideplugin/topling-sst ${TOPLING_SST_GIT_VER_SRC} +endif +ifneq (,$(wildcard sideplugin/topling-zip_table_reader/src/table)) +sideplugin/topling-zip_table_reader/${TOPLING_ZIP_TABLE_READER_GIT_VER_SRC}: \ + $(wildcard sideplugin/topling-zip_table_reader/src/table/*.h) \ + $(wildcard sideplugin/topling-zip_table_reader/src/table/*.cc) \ + sideplugin/topling-zip_table_reader/Makefile + +make -C sideplugin/topling-zip_table_reader ${TOPLING_ZIP_TABLE_READER_GIT_VER_SRC} +endif +ifneq (,$(wildcard sideplugin/topling-dcompact/src/dcompact)) +sideplugin/topling-dcompact/${TOPLING_DCOMPACT_GIT_VER_SRC}: \ + $(wildcard sideplugin/topling-dcompact/src/dcompact/*.h) \ + $(wildcard sideplugin/topling-dcompact/src/dcompact/*.cc) \ + $(wildcard sideplugin/topling-dcompact/tools/dcompact/*.cpp) \ + sideplugin/topling-dcompact/Makefile + +make -C sideplugin/topling-dcompact ${TOPLING_DCOMPACT_GIT_VER_SRC} +.PHONY: dcompact_worker +dcompact_worker: ${SHARED1} +ifeq (${MAKE_UNIT_TEST},1) + @echo rocksdb unit test, skip dcompact_worker +else + +make -C sideplugin/topling-dcompact/tools/dcompact ${OBJ_DIR}/dcompact_worker.exe CHECK_TERARK_FSA_LIB_UPDATE=0 + cp -a sideplugin/topling-dcompact/tools/dcompact/${OBJ_DIR}/dcompact_worker.exe ${OBJ_DIR} +endif +endif + +${OBJ_DIR}/sideplugin/rockside/src/topling/web/civetweb.o: CFLAGS += -DUSE_ZLIB + # Remove the rules for which dependencies should not be generated and see if any are left. #If so, include the dependencies; if not, do not include the dependency files ROCKS_DEP_RULES=$(filter-out clean format check-format check-buck-targets check-headers check-sources jclean jtest package analyze tags rocksdbjavastatic% unity.% unity_test checkout_folly, $(MAKECMDGOALS)) diff --git a/README-zh_cn.md b/README-zh_cn.md new file mode 100644 index 0000000000..c6d77a0f85 --- /dev/null +++ b/README-zh_cn.md @@ -0,0 +1,136 @@ +## ToplingDB: 一个外存上的持久化 Key-Value 存储引擎 +ToplingDB 由[北京拓扑岭科技有限公司](https://topling.cn)开发与维护，从 [RocksDB](https://github.com/facebook/rocksdb) 分叉而来，详情参考 [ToplingDB 分支名称约定](https://github.com/topling/toplingdb/wiki/ToplingDB-Branch-Name-Convention)。 + +ToplingDB 的子模块 **[rockside](https://github.com/topling/rockside)** 是 ToplingDB 的入口，详情参考 **[SidePlugin wiki](https://github.com/topling/rockside/wiki)**。 + +ToplingDB 兼容 RocksDB API 的同时，增加了很多非常重要的功能与改进： +1. [SidePlugin](https://github.com/topling/rockside/wiki) 让用户可以通过 json/yaml 文件来定义 DB 配置 +1. [内嵌 Http](https://github.com/topling/rockside/wiki/WebView) 让用户可以通过 Web 查看几乎所有 DB 信息，这是 [SidePlugin](https://github.com/topling/rockside/wiki) 的一个子功能 +1. [内嵌 Http](https://github.com/topling/rockside/wiki/WebView) 让用户可以无需重启进程，[在线修改](https://github.com/topling/rockside/wiki/Online-Change-Options) 各种 db/cf 配置，包括修改 DB 元对象（例如 MemTabFactory, TableFactory, WriteBufferManager ...） +1. 为提升性能和可扩展性而实施的很多重构与改进，例如 MemTable 的重构 +1. 对事务处理的改进，特别是 TransactionDB 中 Lock 的管理，热点代码有 5x 以上的性能提升 +1. MultiGet 中使用 fiber/coroutine + io_uring 实现了并发 IO，比 RocksDB 自身的异步 MultiGet 又快又简洁，相应的代码量要少 100 倍不止 +1. [去虚拟化](https://github.com/topling/rockside/wiki/Devirtualization-And-Key-Prefix-Cache-Principle)，消除热点代码中的虚函数调用（主要是 Comparator），并且增加了 Key 前缀缓存，参考相应 [bechmarks](https://github.com/topling/rockside/wiki/Devirtualization-And-Key-Prefix-Cache-Benchmark) +1. 点查和迭代器扫描中的 Zero Copy，对大 Value 效果尤其显著 +1. 将现存的 RocksDB 组件作为**内置插件**纳入 SidePlugin 体系，例如 Cache, Comparator, TableFactory, MemTableFactory... +1. 内置 Prometheus 指标的支持，这是在[内嵌 Http](https://github.com/topling/rockside/wiki/WebView) 中实现的 +1. 修复了很多 RocksDB 的 bug，我们已将其中易于合并到 RocksDB 的很多修复与改进给上游 RocksDB 发了 [Pull Request](https://github.com/facebook/rocksdb/pulls?q=is%3Apr+author%3Arockeet) + +## ToplingDB 云原生数据库服务 +1. [MyTopling](https://github.com/topling/mytopling)(MySQL on ToplingDB), [阿里云上的托管 MyTopling](https://topling.cn/products/mytopling/) +1. [Todis](https://github.com/topling/todis)(Redis on ToplingDB), [阿里云上的托管 Todis](https://topling.cn/products/todis-enterprise/) + +## ToplingDB 组件 +通过 SidePlugin 的实现机制，插件（组件）可以与 ToplingDB 的核心代码实现物理隔离 +1. 可以编译为一个单独的动态库，实现运行时动态加载 +1. 应用代码不需要为插件做任何改变，只需要修改 json/yaml 配置 + +### git 仓库的目录结构 +```bash +toplingdb + \__ sideplugin + \__ rockside (submodule , sideplugin core and framework) + \__ topling-zip (auto clone, zip and core lib) + \__ cspp-memtab (auto clone, sideplugin component) + \__ cspp-wbwi (auto clone, sideplugin component) + \__ topling-sst (auto clone, sideplugin component) + \__ topling-rocks (auto clone, sideplugin component) + \__ topling-zip_table_reader (auto clone, sideplugin component) + \__ topling-dcompact (auto clone, sideplugin component) + \_ tools/dcompact (dcompact-worker binary app) +``` + 仓库 | 权限 | 说明 +-------------- | ---------- | ----------- +[ToplingDB](https://github.com/topling/toplingdb) | public | 顶级仓库，分叉自 [RocksDB](https://github.com/facebook/rocksdb)，增加了我们的改进与修复 +[rockside](https://github.com/topling/rockside) | public | ToplingDB 子模块，包含： +[cspp-wbwi
(**W**rite**B**atch**W**ith**I**ndex)](https://github.com/topling/cspp-wbwi) | public | 使用 Topling CSPP Trie 实现的 **CSPP_WBWI** 相比 rocksdb SkipList WBWI 最多有 20 倍以上的性能提升 +[cspp-memtable](https://github.com/topling/cspp-memtable) | public | (**CSPP** is **C**rash **S**afe **P**arallel **P**atricia trie) MemTab, 相比 SkipList：内存用量更低，单线程性能提升 7 倍，多线程线性提升，可[直接转化为 SST](https://github.com/topling/cspp-memtable#%E4%BA%8Cmemtable-%E7%9B%B4%E6%8E%A5%E8%BD%AC%E5%8C%96%E6%88%90-sst) +[topling-sst](https://github.com/topling/topling-sst) | public | 1. [SingleFastTable](https://github.com/topling/rockside/wiki/SingleFastTable)(主要用于 L0 和 L1)
2. VecAutoSortTable(主要用于 MyTopling bulk_load).
3. 已弃用：[ToplingFastTable](https://github.com/topling/rockside/wiki/ToplingFastTable), CSPPAutoSortTable +[topling-dcompact](https://github.com/topling/topling-dcompact) | public | 分布式 Compact 与通用的 dcompact_worker 程序, 将 Compact 转移到弹性计算集群。
相比 RocksDB 自身的 Remote Compaction，ToplingDB 的分布式 Compact 功能完备，使用便捷，对上层应用非常友好 +[topling-rocks](https://github.com/topling/topling-rocks) | **private** | 创建 [Topling**Zip**Table](https://github.com/topling/rockside/wiki/ToplingZipTable)，基于 Topling 可检索内存压缩算法的 SST，压缩率更高，且内存占用更低，一般用于 L2 及更深层 SST +[topling-zip_table_reader](https://github.com/topling/topling-zip_table_reader) | public | 让社区版用户可以读取 Topling**Zip**Table，但创建需要私有仓库 [topling-rocks](https://github.com/topling/topling-rocks) + +为了简化编译流程，ToplingDB 在 Makefile 中会自动 clone 各个组件的 github 仓库，社区版用户可以成功 clone 公开的仓库，但克隆私有仓库（例如 topling-rocks）会失败，所以社区版用户编译出来的 ToplingDB 无法创建 Topling**Zip**Table，但可以读取 Topling**Zip**Table。 + +## 运行 db_bench +ToplingDB 需要 C++17，推荐 gcc 8.3 以上，或者 clang 也行。 + +即便没有 Topling**Zip**Table，ToplingDB 也比 RocksDB 要快得多，您可以通过运行 db_bench 来验证性能： +```bash +sudo yum -y install git libaio-devel gcc-c++ gflags-devel zlib-devel bzip2-devel libcurl-devel liburing-devel +#sudo apt-get update -y && sudo apt-get install -y libjemalloc-dev libaio-dev libgflags-dev zlib1g-dev libbz2-dev libcurl4-gnutls-dev liburing-dev libsnappy-dev libbz2-dev liblz4-dev libzstd-dev +git clone https://github.com/topling/toplingdb +cd toplingdb +make -j`nproc` db_bench DEBUG_LEVEL=0 +cp sideplugin/rockside/src/topling/web/{style.css,index.html} ${/path/to/dbdir} +cp sideplugin/rockside/sample-conf/db_bench_*.yaml . +export LD_LIBRARY_PATH=`find sideplugin -name lib_shared` +# change db_bench_community.yaml as your needs +# 1. use default path(/dev/shm) if you have no fast disk(such as a cloud server) +# 2. change max_background_compactions to your cpu core num +# 3. if you have github repo topling-rocks permissions, you can use db_bench_enterprise.yaml +# 4. use db_bench_community.yaml is faster than upstream RocksDB +# 5. use db_bench_enterprise.yaml is much faster than db_bench_community.yaml +# command option -json can accept json and yaml files, here use yaml file for more human readable +./db_bench -json=db_bench_community.yaml -num=10000000 -disable_wal=true -value_size=20 -benchmarks=fillrandom,readrandom -batch_size=10 +# you can access http://127.0.0.1:2011 to see webview +# you can see this db_bench is much faster than RocksDB +``` +## 可配置的功能 +为了性能和简化，ToplingDB 默认禁用了一些 RocksDB 的功能： + +功能|控制参数（预编译宏） +-------|------------- +动态创建 ColumnFamily | ROCKSDB_DYNAMIC_CREATE_CF +用户层 timestamp | TOPLINGDB_WITH_TIMESTAMP +宽列 | TOPLINGDB_WITH_WIDE_COLUMNS + +**注意**: SidePlugin 暂不支持动态创建 ColumnFamily，混用 SidePlugin 和动态创建 ColumnFamily时，动态创建的 ColumnFamily 不能在 Web 中展示 + +为了启用这些功能，需要为 make 命令显式添加 `EXTRA_CXXFLAGS="-D${MACRO_1} -D${MACRO_2} ..."`，例如编译带动态创建 ColumnFamily 的 rocksdbjava: +``` +make -j`nproc` EXTRA_CXXFLAGS='-DROCKSDB_DYNAMIC_CREATE_CF' rocksdbjava +``` +## License +为了兼容开源协议，下列原先禁止字节跳动使用本软件的条款从 2023-04-24 起已被删除，也就是说，字节跳动使用 ToplingDB 的行为不再是非法的，也不是无耻的。 + +~~我们禁止字节跳动使用本软件，其它条款与上游 RocksDB 完全相同，~~ 详情参考 [LICENSE.Apache](LICENSE.Apache), [COPYING](COPYING), [LICENSE.leveldb](LICENSE.leveldb). + +相应 LICENSE 文件中禁止字节跳动使用本软件的条款也已经删除：[LICENSE.Apache](LICENSE.Apache), [COPYING](COPYING), [LICENSE.leveldb](LICENSE.leveldb). + +
+以下是上游 RocksDB 的原版 README +
+
+ +## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage + +[![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb) +[![Appveyor Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/main?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/main) +[![PPC64le Build Status](http://140-211-168-68-openstack.osuosl.org:8080/buildStatus/icon?job=rocksdb&style=plastic)](http://140-211-168-68-openstack.osuosl.org:8080/job/rocksdb) + +RocksDB is developed and maintained by Facebook Database Engineering Team. +It is built on earlier work on [LevelDB](https://github.com/google/leveldb) by Sanjay Ghemawat (sanjay@google.com) +and Jeff Dean (jeff@google.com) + +This code is a library that forms the core building block for a fast +key-value server, especially suited for storing data on flash drives. +It has a Log-Structured-Merge-Database (LSM) design with flexible tradeoffs +between Write-Amplification-Factor (WAF), Read-Amplification-Factor (RAF) +and Space-Amplification-Factor (SAF). It has multi-threaded compactions, +making it especially suitable for storing multiple terabytes of data in a +single database. + +Start with example usage here: https://github.com/facebook/rocksdb/tree/main/examples + +See the [github wiki](https://github.com/facebook/rocksdb/wiki) for more explanation. + +The public interface is in `include/`. Callers should not include or +rely on the details of any other header files in this package. Those +internal APIs may be changed without warning. + +Questions and discussions are welcome on the [RocksDB Developers Public](https://www.facebook.com/groups/rocksdb.dev/) Facebook group and [email list](https://groups.google.com/g/rocksdb) on Google Groups. + +## License + +RocksDB is dual-licensed under both the GPLv2 (found in the COPYING file in the root directory) and Apache 2.0 License (found in the LICENSE.Apache file in the root directory). You may select, at your option, one of the above-listed licenses. diff --git a/README.md b/README.md index 8fcc4abc2c..d68c65285e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,114 @@ +## [中文版](README-zh_cn.md) +## ToplingDB: A Persistent Key-Value Store for External Storage +ToplingDB is developed and maintained by [Topling Inc](https://topling.cn). It is built with [RocksDB](https://github.com/facebook/rocksdb). See [ToplingDB Branch Name Convention](https://github.com/topling/toplingdb/wiki/ToplingDB-Branch-Name-Convention). + +ToplingDB's submodule **[rockside](https://github.com/topling/rockside)** is the entry point of ToplingDB, see **[SidePlugin wiki](https://github.com/topling/rockside/wiki)**. + +ToplingDB has much more key features than RocksDB: +1. [SidePlugin](https://github.com/topling/rockside/wiki) enables users to write a json(or yaml) to define DB configs +1. [Embedded Http Server](https://github.com/topling/rockside/wiki/WebView) enables users to view almost all DB info on web, this is a component of [SidePlugin](https://github.com/topling/rockside/wiki) +1. [Embedded Http Server](https://github.com/topling/rockside/wiki/WebView) enables users to [online change](https://github.com/topling/rockside/wiki/Online-Change-Options) db/cf options and all db meta objects(such as MemTabFactory, TableFactory, WriteBufferManager ...) without restart the running process +1. Many improvements and refactories on RocksDB, aimed for performance and extendibility +1. Topling transaction lock management, 5x faster than rocksdb +1. MultiGet with concurrent IO by fiber/coroutine + io_uring, much faster than RocksDB's async MultiGet +1. Topling [de-virtualization](https://github.com/topling/rockside/wiki/Devirtualization-And-Key-Prefix-Cache-Principle), de-virtualize hotspot (virtual) functions, and key prefix caches, [bechmarks](https://github.com/topling/rockside/wiki/Devirtualization-And-Key-Prefix-Cache-Benchmark) +1. Topling zero copy for point search(Get/MultiGet) and Iterator +1. Builtin SidePlugin**s** for existing RocksDB components(Cache, Comparator, TableFactory, MemTableFactory...) +1. Builtin Prometheus metrics support, this is based on [Embedded Http Server](https://github.com/topling/rockside/wiki/WebView) +1. Many bugfixes for RocksDB, a small part of such fixes was [Pull Requested](https://github.com/facebook/rocksdb/pulls?q=is%3Apr+author%3Arockeet) to [upstream RocksDB](https://github.com/facebook/rocksdb) + +## ToplingDB cloud native DB services +1. [MyTopling](https://github.com/topling/mytopling)(MySQL on ToplingDB), [Managed MyTopling on aliyun](https://topling.cn/products/mytopling/) +1. [Todis](https://github.com/topling/todis)(Redis on ToplingDB), [Managed Todis on aliyun](https://topling.cn/products/todis-enterprise/) + +## ToplingDB Components +With SidePlugin mechanics, plugins/components can be physically separated from core toplingdb +1. Can be compiled to a separated dynamic lib and loaded at runtime +2. User code need not any changes, just change json/yaml files +3. Topling's non-open-source enterprise plugins/components are delivered in this way + +### Repository dir structure +```bash +toplingdb + \__ sideplugin + \__ rockside (submodule , sideplugin core and framework) + \__ topling-zip (auto clone, zip and core lib) + \__ cspp-memtab (auto clone, sideplugin component) + \__ cspp-wbwi (auto clone, sideplugin component) + \__ topling-sst (auto clone, sideplugin component) + \__ topling-rocks (auto clone, sideplugin component) + \__ topling-zip_table_reader (auto clone, sideplugin component) + \__ topling-dcompact (auto clone, sideplugin component) + \_ tools/dcompact (dcompact-worker binary app) +``` + + Repository | Permission | Description (and components) +-------------- | ---------- | ----------- +[ToplingDB](https://github.com/topling/toplingdb) | public | Top repository, forked from [RocksDB](https://github.com/facebook/rocksdb) with our fixes, refactories and enhancements +[rockside](https://github.com/topling/rockside) | public | This is a submodule, contains: +[cspp-wbwi
(**W**rite**B**atch**W**ith**I**ndex)](https://github.com/topling/cspp-wbwi) | public | With CSPP and carefully coding, **CSPP_WBWI** is 20x faster than rocksdb SkipList based WBWI +[cspp-memtable](https://github.com/topling/cspp-memtable) | public | (**CSPP** is **C**rash **S**afe **P**arallel **P**atricia trie) MemTab, which outperforms SkipList on all aspects: 3x lower memory usage, 7x single thread performance, perfect multi-thread scaling) +[topling-sst](https://github.com/topling/topling-sst) | public | 1. [SingleFastTable](https://github.com/topling/rockside/wiki/SingleFastTable)(designed for L0 and L1)
2. VecAutoSortTable(designed for MyTopling bulk_load).
3. Deprecated [ToplingFastTable](https://github.com/topling/rockside/wiki/ToplingFastTable), CSPPAutoSortTable +[topling-dcompact](https://github.com/topling/topling-dcompact) | public | Distributed Compaction with general dcompact_worker application, offload compactions to elastic computing clusters, much more powerful than RocksDB's Remote Compaction +[topling-rocks](https://github.com/topling/topling-rocks) | **private** | [Topling**Zip**Table](https://github.com/topling/rockside/wiki/ToplingZipTable), an SST implementation optimized for RAM and SSD space, aimed for L2+ level compaction, which uses topling dedicated searchable in-memory data compression algorithms +[topling-zip_table_reader](https://github.com/topling/topling-zip_table_reader) | public | For read Topling**Zip**Table by community users, builder of Topling**Zip**Table is in [topling-rocks](https://github.com/topling/topling-rocks) + +To simplify the compiling, repo**s** are auto cloned in ToplingDB's Makefile, community users will auto clone public repo successfully but fail to auto clone **private** repo, thus ToplingDB is built without **private** components, this is so called **community** version. + +## Run db_bench +ToplingDB requires C++17, gcc 8.3 or newer is recommended, clang also works. + +Even without ToplingZipTable, ToplingDB is much faster than upstream RocksDB: +```bash +sudo yum -y install git libaio-devel gcc-c++ gflags-devel zlib-devel bzip2-devel libcurl-devel liburing-devel +#sudo apt-get update -y && sudo apt-get install -y libjemalloc-dev libaio-dev libgflags-dev zlib1g-dev libbz2-dev libcurl4-gnutls-dev liburing-dev libsnappy-dev libbz2-dev liblz4-dev libzstd-dev +git clone https://github.com/topling/toplingdb +cd toplingdb +make -j`nproc` db_bench DEBUG_LEVEL=0 +cp sideplugin/rockside/src/topling/web/{style.css,index.html} ${/path/to/dbdir} +cp sideplugin/rockside/sample-conf/db_bench_*.yaml . +export LD_LIBRARY_PATH=`find sideplugin -name lib_shared` +# change db_bench_community.yaml as your needs +# 1. use default path(/dev/shm) if you have no fast disk(such as a cloud server) +# 2. change max_background_compactions to your cpu core num +# 3. if you have github repo topling-rocks permissions, you can use db_bench_enterprise.yaml +# 4. use db_bench_community.yaml is faster than upstream RocksDB +# 5. use db_bench_enterprise.yaml is much faster than db_bench_community.yaml +# command option -json can accept json and yaml files, here use yaml file for more human readable +./db_bench -json=db_bench_community.yaml -num=10000000 -disable_wal=true -value_size=20 -benchmarks=fillrandom,readrandom -batch_size=10 +# you can access http://127.0.0.1:2011 to see webview +# you can see this db_bench is much faster than RocksDB +``` +## Configurable features +For performance and simplicity, ToplingDB disabled some RocksDB features by default: + +Feature|Control MACRO +-------|------------- +Dynamic creation of ColumnFamily | ROCKSDB_DYNAMIC_CREATE_CF +User level timestamp on key | TOPLINGDB_WITH_TIMESTAMP +Wide Columns | TOPLINGDB_WITH_WIDE_COLUMNS + +**Note**: Dynamic creation of ColumnFamily is not supported by SidePlugin + +To enable these features, add `-D${MACRO_NAME}` to var `EXTRA_CXXFLAGS`, such as build ToplingDB for java with dynamic ColumnFamily: +``` +make -j`nproc` EXTRA_CXXFLAGS='-DROCKSDB_DYNAMIC_CREATE_CF' rocksdbjava +``` +## License +To conform open source license, the following term of disallowing bytedance is deleted since 2023-04-24, +that is say: bytedance using ToplingDB is no longer illeagal and is not a shame. + +~~We disallow bytedance using this software, other terms are identidal with +upstream rocksdb license,~~ see [LICENSE.Apache](LICENSE.Apache), [COPYING](COPYING) and +[LICENSE.leveldb](LICENSE.leveldb). + +The terms of disallowing bytedance are also deleted in [LICENSE.Apache](LICENSE.Apache), [COPYING](COPYING) and +[LICENSE.leveldb](LICENSE.leveldb). + +
+
+
+ ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage [![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb) diff --git a/TARGETS b/TARGETS index e8aaf325d4..8a851d7502 100644 --- a/TARGETS +++ b/TARGETS @@ -39,6 +39,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[ "db/c.cc", "db/column_family.cc", "db/compaction/compaction.cc", + "db/compaction/compaction_executor.cc", "db/compaction/compaction_iterator.cc", "db/compaction/compaction_job.cc", "db/compaction/compaction_outputs.cc", @@ -176,6 +177,15 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[ "port/win/port_win.cc", "port/win/win_logger.cc", "port/win/win_thread.cc", + "sideplugin/rockside/src/topling/block_based_table_side_plugin.cc", + "sideplugin/rockside/src/topling/builtin_db_open.cc", + "sideplugin/rockside/src/topling/builtin_plugin_basic.cc", + "sideplugin/rockside/src/topling/builtin_plugin_misc.cc", + "sideplugin/rockside/src/topling/builtin_table_factory.cc", + "sideplugin/rockside/src/topling/side_plugin_repo.cc", + "sideplugin/rockside/src/topling/side_plugin_tpl_inst.cc", + "sideplugin/rockside/src/topling/web/CivetServer.cc", + "sideplugin/rockside/src/topling/web/json_civetweb.cc", "table/adaptive/adaptive_table_factory.cc", "table/block_based/binary_search_index_reader.cc", "table/block_based/block.cc", diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index a5e2b5aa2f..2a46b0209b 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -49,7 +49,7 @@ fi if [ "$ROCKSDB_CXX_STANDARD" ]; then PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD" else - PLATFORM_CXXFLAGS="-std=c++17" + PLATFORM_CXXFLAGS="-std=gnu++17" fi # we currently depend on POSIX platform @@ -238,7 +238,7 @@ EOF Cygwin) PLATFORM=CYGWIN PLATFORM_SHARED_CFLAGS="" - PLATFORM_CXXFLAGS="-std=gnu++11" + PLATFORM_CXXFLAGS="-std=gnu++17" COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN" if [ -z "$USE_CLANG" ]; then COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" @@ -334,6 +334,9 @@ EOF then COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" + else + echo Not found: GFLAGS 1>&2 + exit 1 fi fi @@ -347,6 +350,9 @@ EOF COMMON_FLAGS="$COMMON_FLAGS -DZLIB" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz" JAVA_LDFLAGS="$JAVA_LDFLAGS -lz" + else + echo Not found: zlib "(for gzip)" 1>&2 + exit 1 fi fi diff --git a/cache/lru_cache.cc b/cache/lru_cache.cc index 9d16952243..3080acfd29 100644 --- a/cache/lru_cache.cc +++ b/cache/lru_cache.cc @@ -528,14 +528,14 @@ LRUHandle* LRUCacheShard::CreateHandle(const Slice& key, uint32_t hash, // Allocate the memory here outside of the mutex. // If the cache is full, we'll have to release it. // It shouldn't happen very often though. - LRUHandle* e = - static_cast(malloc(sizeof(LRUHandle) - 1 + key.size())); - + static_assert(sizeof(LRUHandle) == 64); + auto e = static_cast(malloc(sizeof(LRUHandle) + key.size())); + e->padding = 0; // padding makes key_data aligned better e->value = value; e->m_flags = 0; e->im_flags = 0; e->helper = helper; - e->key_length = key.size(); + e->key_length = (uint32_t)key.size(); e->hash = hash; e->refs = 0; e->next = e->prev = nullptr; diff --git a/cache/lru_cache.h b/cache/lru_cache.h index 1a9ba04425..33fdc79a73 100644 --- a/cache/lru_cache.h +++ b/cache/lru_cache.h @@ -54,7 +54,7 @@ struct LRUHandle { LRUHandle* next; LRUHandle* prev; size_t total_charge; // TODO(opt): Only allow uint32_t? - size_t key_length; + uint32_t key_length; // The hash of key(). Used for fast sharding and comparisons. uint32_t hash; // The number of external refs to this entry. The cache itself is not counted. @@ -87,8 +87,10 @@ struct LRUHandle { IM_IS_STANDALONE = (1 << 2), }; + uint16_t padding; + // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!) - char key_data[1]; + char key_data[0]; Slice key() const { return Slice(key_data, key_length); } diff --git a/db/arena_wrapped_db_iter.cc b/db/arena_wrapped_db_iter.cc index e6dcb66962..3a8b37d466 100644 --- a/db/arena_wrapped_db_iter.cc +++ b/db/arena_wrapped_db_iter.cc @@ -19,14 +19,28 @@ namespace ROCKSDB_NAMESPACE { -inline static SequenceNumber GetSeqNum(const DBImpl* db, const Snapshot* s) { - if (s) { +static constexpr size_t KEEP_SNAPSHOT = 16; + +inline static +SequenceNumber GetSeqNum(const DBImpl* db, const Snapshot* s, const DBIter* i) { + if (size_t(s) == KEEP_SNAPSHOT) + return i->get_sequence(); + else if (s) + //return static_cast_with_check(s)->number_; return s->GetSequenceNumber(); - } else { + else return db->GetLatestSequenceNumber(); - } } +Status Iterator::RefreshKeepSnapshot(bool keep_iter_pos) { + return Refresh(reinterpret_cast(KEEP_SNAPSHOT), keep_iter_pos); +} + +ArenaWrappedDBIter::ArenaWrappedDBIter() { + // do nothing +} +#define db_iter_ (&db_iter_obj_) + Status ArenaWrappedDBIter::GetProperty(std::string prop_name, std::string* prop) { if (prop_name == "rocksdb.iterator.super-version-number") { @@ -45,14 +59,15 @@ void ArenaWrappedDBIter::Init( const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration, uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) { - auto mem = arena_.AllocateAligned(sizeof(DBIter)); - db_iter_ = + auto mem = db_iter_; new (mem) DBIter(env, read_options, ioptions, mutable_cf_options, ioptions.user_comparator, /* iter */ nullptr, version, sequence, true, max_sequential_skip_in_iteration, read_callback, db_impl, cfd, expose_blob_index); + db_iter_inited_ = true; sv_number_ = version_number; read_options_ = read_options; + read_options_.pinning_tls = nullptr; // must set null allow_refresh_ = allow_refresh; memtable_range_tombstone_iter_ = nullptr; @@ -62,12 +77,17 @@ void ArenaWrappedDBIter::Init( } } -Status ArenaWrappedDBIter::Refresh() { return Refresh(nullptr); } +Status ArenaWrappedDBIter::Refresh() { + return Refresh(nullptr, false); // do not keep iter pos +} -Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) { +// when keep_iter_pos is true, user code should ensure ReadOptions's +// lower_bound and upper_bound are not changed +Status ArenaWrappedDBIter::Refresh(const Snapshot* snap, bool keep_iter_pos) { if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) { return Status::NotSupported("Creating renew iterator is not allowed."); } + assert(db_iter_inited_); assert(db_iter_ != nullptr); // TODO(yiwu): For last_seq_same_as_publish_seq_==false, this is not the // correct behavior. Will be corrected automatically when we take a snapshot @@ -80,7 +100,25 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) { TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1"); TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2"); auto reinit_internal_iter = [&]() { + std::string curr_key, curr_val; + bool is_valid = this->Valid(); + SequenceNumber old_iter_seq = db_iter_->get_sequence(); + SequenceNumber latest_seq = GetSeqNum(db_impl_, snap, db_iter_); + if (is_valid && keep_iter_pos) { + curr_key = this->key().ToString(); + curr_val = this->value().ToString(); + } + Snapshot* pin_snap = nullptr; + if (size_t(snap) == KEEP_SNAPSHOT) { + // pin the snapshot latest_seq to avoid race condition caused by + // the the snapshot latest_seq being garbage collected by a + // compaction, which may cause many errors, for example an external + // behavior is Seek on belowing new iterator failed(with same + // read_opt.lower_bound/upper_bound...) + pin_snap = db_impl_->GetSnapshotImpl(latest_seq, false); + } Env* env = db_iter_->env(); + db_iter_inited_ = false; db_iter_->~DBIter(); arena_.~Arena(); new (&arena_) Arena(); @@ -101,13 +139,36 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) { read_options_, cfd_, sv, &arena_, read_seq, /* allow_unprepared_value */ true, /* db_iter */ this); SetIterUnderDBIter(internal_iter); + if (is_valid && keep_iter_pos) { + this->Seek(curr_key); + if (old_iter_seq == latest_seq) { + ROCKSDB_VERIFY_F(this->Valid(), + "curr_key = %s, seq = %lld, snap = %p, pin_snap = %p", + Slice(curr_key).hex().c_str(), + (long long)latest_seq, snap, pin_snap); + ROCKSDB_VERIFY_F(key() == curr_key, "%s %s", + key().hex().c_str(), Slice(curr_key).hex().c_str()); + ROCKSDB_VERIFY_F(value() == curr_val, "%s %s", + value().hex().c_str(), Slice(curr_val).hex().c_str()); + } + } + if (pin_snap) { + db_impl_->ReleaseSnapshot(pin_snap); + } }; while (true) { if (sv_number_ != cur_sv_number) { reinit_internal_iter(); break; + } else if (size_t(snap) == KEEP_SNAPSHOT) { + break; } else { SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot); + SequenceNumber latest_seq = snap ? snap->GetSequenceNumber() + : db_impl_->GetLatestSequenceNumber(); + if (latest_seq == db_iter_->get_sequence()) { + break; + } // Refresh range-tombstones in MemTable if (!read_options_.ignore_range_deletions) { SuperVersion* sv = cfd_->GetThreadLocalSuperVersion(db_impl_); @@ -143,6 +204,9 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) { } db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv); } + // Refresh latest sequence number + db_iter_->set_sequence(latest_seq); + // db_iter_->set_valid(false); // comment out for ToplingDB // Check again if the latest super version number is changed uint64_t latest_sv_number = cfd_->GetSuperVersionNumber(); if (latest_sv_number != cur_sv_number) { @@ -156,15 +220,24 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) { break; } } + if (size_t(snap) > KEEP_SNAPSHOT) { + this->read_options_.snapshot = snap; + } return Status::OK(); } ArenaWrappedDBIter* NewArenaWrappedDbIterator( - Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, - const MutableCFOptions& mutable_cf_options, const Version* version, - const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations, - uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl, - ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) { + const ReadOptions& read_options, const SuperVersion* sv, + SequenceNumber sequence, ReadCallback* read_callback, DBImpl* db_impl, + bool expose_blob_index, bool allow_refresh) { + auto version = sv->current; + auto version_number = sv->version_number; + auto env = version->env(); + auto cfd = sv->cfd; + const auto& ioptions = *cfd->ioptions(); + const auto& mutable_cf_options = sv->mutable_cf_options; + auto max_sequential_skip_in_iterations = + mutable_cf_options.max_sequential_skip_in_iterations; ArenaWrappedDBIter* iter = new ArenaWrappedDBIter(); iter->Init(env, read_options, ioptions, mutable_cf_options, version, sequence, max_sequential_skip_in_iterations, version_number, read_callback, diff --git a/db/arena_wrapped_db_iter.h b/db/arena_wrapped_db_iter.h index 678ea3e78d..2088ed4390 100644 --- a/db/arena_wrapped_db_iter.h +++ b/db/arena_wrapped_db_iter.h @@ -33,10 +33,12 @@ class Version; // to allocate. // When using the class's Iterator interface, the behavior is exactly // the same as the inner DBIter. +#define db_iter_ (&db_iter_obj_) class ArenaWrappedDBIter : public Iterator { public: + ArenaWrappedDBIter(); ~ArenaWrappedDBIter() override { - if (db_iter_ != nullptr) { + if (db_iter_inited_) { db_iter_->~DBIter(); } else { assert(false); @@ -70,17 +72,21 @@ class ArenaWrappedDBIter : public Iterator { } void Next() override { db_iter_->Next(); } void Prev() override { db_iter_->Prev(); } + ROCKSDB_FLATTEN Slice key() const override { return db_iter_->key(); } + ROCKSDB_FLATTEN Slice value() const override { return db_iter_->value(); } const WideColumns& columns() const override { return db_iter_->columns(); } Status status() const override { return db_iter_->status(); } Slice timestamp() const override { return db_iter_->timestamp(); } + ROCKSDB_FLATTEN + bool PrepareValue() override { return db_iter_->PrepareValue(); } bool IsBlob() const { return db_iter_->IsBlob(); } Status GetProperty(std::string prop_name, std::string* prop) override; Status Refresh() override; - Status Refresh(const Snapshot*) override; + Status Refresh(const Snapshot*, bool keep_iter_pos) override; void Init(Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, @@ -101,7 +107,7 @@ class ArenaWrappedDBIter : public Iterator { } private: - DBIter* db_iter_ = nullptr; + union { DBIter db_iter_obj_; }; Arena arena_; uint64_t sv_number_; ColumnFamilyData* cfd_ = nullptr; @@ -110,19 +116,18 @@ class ArenaWrappedDBIter : public Iterator { ReadCallback* read_callback_; bool expose_blob_index_ = false; bool allow_refresh_ = true; + bool db_iter_inited_ = false; // If this is nullptr, it means the mutable memtable does not contain range // tombstone when added under this DBIter. TruncatedRangeDelIterator** memtable_range_tombstone_iter_ = nullptr; }; +#undef db_iter_ // Generate the arena wrapped iterator class. // `db_impl` and `cfd` are used for reneweal. If left null, renewal will not // be supported. extern ArenaWrappedDBIter* NewArenaWrappedDbIterator( - Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions, - const MutableCFOptions& mutable_cf_options, const Version* version, - const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations, - uint64_t version_number, ReadCallback* read_callback, - DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr, + const ReadOptions&, const SuperVersion*, SequenceNumber sequence, + ReadCallback*, DBImpl* db_impl = nullptr, bool expose_blob_index = false, bool allow_refresh = true); } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_counting_iterator_test.cc b/db/blob/blob_counting_iterator_test.cc index c7bbc8f587..eced3f2167 100644 --- a/db/blob/blob_counting_iterator_test.cc +++ b/db/blob/blob_counting_iterator_test.cc @@ -136,7 +136,7 @@ TEST(BlobCountingIteratorTest, CountBlobs) { { IterateResult result; ASSERT_TRUE(blob_counter.NextAndGetResult(&result)); - ASSERT_EQ(result.key, keys[1]); + ASSERT_EQ(result.key(), keys[1]); ASSERT_EQ(blob_counter.user_key(), user_key1); ASSERT_TRUE(blob_counter.Valid()); ASSERT_OK(blob_counter.status()); @@ -151,7 +151,7 @@ TEST(BlobCountingIteratorTest, CountBlobs) { { IterateResult result; ASSERT_TRUE(blob_counter.NextAndGetResult(&result)); - ASSERT_EQ(result.key, keys[2]); + ASSERT_EQ(result.key(), keys[2]); ASSERT_EQ(blob_counter.user_key(), user_key2); ASSERT_TRUE(blob_counter.Valid()); ASSERT_OK(blob_counter.status()); diff --git a/db/blob/blob_fetcher.h b/db/blob/blob_fetcher.h index 8aeaf965d2..ad6dda64b3 100644 --- a/db/blob/blob_fetcher.h +++ b/db/blob/blob_fetcher.h @@ -19,6 +19,7 @@ class BlobIndex; // A thin wrapper around the blob retrieval functionality of Version. class BlobFetcher { public: + virtual ~BlobFetcher() = default; BlobFetcher(const Version* version, const ReadOptions& read_options) : version_(version), read_options_(read_options) {} @@ -32,6 +33,14 @@ class BlobFetcher { private: const Version* version_; - ReadOptions read_options_; + const ReadOptions& read_options_; }; + +class BlobFetcherCopyReadOptions : public BlobFetcher { + const ReadOptions read_options_copy_; +public: + BlobFetcherCopyReadOptions(const Version* v, const ReadOptions& ro) + : BlobFetcher(v, read_options_copy_), read_options_copy_(ro) {} +}; + } // namespace ROCKSDB_NAMESPACE diff --git a/db/builder.cc b/db/builder.cc index d3040ee9e2..a5070297ee 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -413,13 +413,20 @@ Status BuildTable( OutputValidator file_validator(tboptions.internal_comparator, /*enable_order_check=*/true, /*enable_hash=*/true); + file_validator.m_file_number = meta->fd.GetNumber(); for (it->SeekToFirst(); it->Valid(); it->Next()) { // Generate a rolling 64-bit hash of the key and values file_validator.Add(it->key(), it->value()).PermitUncheckedError(); } s = it->status(); if (s.ok() && !output_validator.CompareValidator(file_validator)) { - s = Status::Corruption("Paranoid checksums do not match"); + #if !defined(ROCKSDB_UNIT_TEST) + auto& fd = meta->fd; + ROCKSDB_DIE("BuildTable: Paranoid checksums do not match(%d:%lld.sst)", + fd.GetPathId(), (long long)fd.GetNumber()); + #else + s = Status::Corruption("BuildTable: Paranoid checksums do not match"); + #endif } } } diff --git a/db/column_family.cc b/db/column_family.cc index 9782cd31a7..964b5861a1 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -79,10 +79,10 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() { } } -uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd()->GetID(); } +uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd_->GetID(); } const std::string& ColumnFamilyHandleImpl::GetName() const { - return cfd()->GetName(); + return cfd_->GetName(); } Status ColumnFamilyHandleImpl::GetDescriptor(ColumnFamilyDescriptor* desc) { @@ -93,7 +93,25 @@ Status ColumnFamilyHandleImpl::GetDescriptor(ColumnFamilyDescriptor* desc) { } const Comparator* ColumnFamilyHandleImpl::GetComparator() const { - return cfd()->user_comparator(); + return cfd_->user_comparator(); +} +ColumnFamilyHandle* ColumnFamilyHandleImpl::CloneHandle() const { + return new ColumnFamilyHandleImpl(cfd_, db_, mutex_); +} + +uint32_t ColumnFamilyHandleInternal::GetID() const { + return internal_cfd_->GetID(); +} +const std::string& ColumnFamilyHandleInternal::GetName() const { + return internal_cfd_->GetName(); +} +const Comparator* ColumnFamilyHandleInternal::GetComparator() const { + return internal_cfd_->user_comparator(); +} +ColumnFamilyHandle* ColumnFamilyHandleInternal::CloneHandle() const { + auto p = new ColumnFamilyHandleInternal(); + p->SetCFD(internal_cfd_); + return p; } void GetIntTblPropCollectorFactory( @@ -541,7 +559,7 @@ ColumnFamilyData::ColumnFamilyData( ioptions_.max_write_buffer_size_to_maintain), super_version_(nullptr), super_version_number_(0), - local_sv_(new ThreadLocalPtr(&SuperVersionUnrefHandle)), + local_sv_(&SuperVersionUnrefHandle), next_(nullptr), prev_(nullptr), log_number_(0), @@ -717,7 +735,12 @@ bool ColumnFamilyData::UnrefAndTryDelete() { super_version_ = nullptr; // Release SuperVersion references kept in ThreadLocalPtr. - local_sv_.reset(); + #if 0 + local_sv_.~ThreadLocalPtr(); + new(&local_sv_)ThreadLocalPtr(&SuperVersionUnrefHandle); + #else + local_sv_.Destroy(); + #endif if (sv->Unref()) { // Note: sv will delete this ColumnFamilyData during Cleanup() @@ -763,7 +786,11 @@ uint64_t ColumnFamilyData::OldestLogToKeep() { return current_log; } +#if defined(ROCKSDB_UNIT_TEST) const double kIncSlowdownRatio = 0.8; +#else +const double kIncSlowdownRatio = 0.97; // topling specific +#endif const double kDecSlowdownRatio = 1 / kIncSlowdownRatio; const double kNearStopSlowdownRatio = 0.6; const double kDelayRecoverSlowdownRatio = 1.4; @@ -1122,10 +1149,65 @@ uint64_t ColumnFamilyData::GetLiveSstFilesSize() const { return current_->GetSstFilesSize(); } +void ColumnFamilyData::PrepareNewMemtableInBackground( + const MutableCFOptions& mutable_cf_options) { + #if !defined(ROCKSDB_UNIT_TEST) + { + std::lock_guard lk(precreated_memtable_mutex_); + if (precreated_memtable_list_.full()) { + // do nothing + return; + } + } + auto beg = ioptions_.clock->NowNanos(); + auto tab = new MemTable(internal_comparator_, ioptions_, mutable_cf_options, + write_buffer_manager_, 0/*earliest_seq*/, id_); + auto end = ioptions_.clock->NowNanos(); + RecordInHistogram(ioptions_.stats, MEMTAB_CONSTRUCT_NANOS, end - beg); + { + std::lock_guard lk(precreated_memtable_mutex_); + if (LIKELY(!precreated_memtable_list_.full())) { + precreated_memtable_list_.emplace_back(tab); + tab = nullptr; + } + } + if (UNLIKELY(nullptr != tab)) { // precreated_memtable_list_ is full + // this is very rare, we have not put `tab` to precreated_memtable_list_, + // but this thread must keep going on, just delete `tab` + ROCKS_LOG_WARN(ioptions_.info_log, + "precreated_memtable_list_ is full, discard the newly created memtab"); + delete tab; + } + #endif +} + MemTable* ColumnFamilyData::ConstructNewMemtable( const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) { - return new MemTable(internal_comparator_, ioptions_, mutable_cf_options, + MemTable* tab = nullptr; + #if !defined(ROCKSDB_UNIT_TEST) + { + std::lock_guard lk(precreated_memtable_mutex_); + if (!precreated_memtable_list_.empty()) { + tab = precreated_memtable_list_.front().release(); + precreated_memtable_list_.pop_front(); + } + } + #endif + if (tab) { + tab->SetCreationSeq(earliest_seq); + tab->SetEarliestSequenceNumber(earliest_seq); + } else { + #if !defined(ROCKSDB_UNIT_TEST) + auto beg = ioptions_.clock->NowNanos(); + #endif + tab = new MemTable(internal_comparator_, ioptions_, mutable_cf_options, write_buffer_manager_, earliest_seq, id_); + #if !defined(ROCKSDB_UNIT_TEST) + auto end = ioptions_.clock->NowNanos(); + RecordInHistogram(ioptions_.stats, MEMTAB_CONSTRUCT_NANOS, end - beg); + #endif + } + return tab; } void ColumnFamilyData::CreateNewMemtable( @@ -1252,6 +1334,12 @@ SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(DBImpl* db) { return sv; } +template +inline T NoAtomicLoad(const std::atomic& x) { + static_assert(sizeof(x) == sizeof(T)); + return reinterpret_cast(x); +} + SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) { // The SuperVersion is cached in thread local storage to avoid acquiring // mutex when SuperVersion does not change since the last use. When a new @@ -1264,7 +1352,7 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) { // have swapped in kSVObsolete. We re-check the value at when returning // SuperVersion back to thread local, with an atomic compare and swap. // The superversion will need to be released if detected to be stale. - void* ptr = local_sv_->Swap(SuperVersion::kSVInUse); + void* ptr = local_sv_.Swap(SuperVersion::kSVInUse); // Invariant: // (1) Scrape (always) installs kSVObsolete in ThreadLocal storage // (2) the Swap above (always) installs kSVInUse, ThreadLocal storage @@ -1286,7 +1374,7 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) { assert(sv != nullptr); // Put the SuperVersion back void* expected = SuperVersion::kSVInUse; - if (local_sv_->CompareAndSwap(static_cast(sv), expected)) { + if (local_sv_.CompareAndSwap(static_cast(sv), expected)) { // When we see kSVInUse in the ThreadLocal, we are sure ThreadLocal // storage has not been altered and no Scrape has happened. The // SuperVersion is still current. @@ -1354,7 +1442,7 @@ void ColumnFamilyData::InstallSuperVersion( void ColumnFamilyData::ResetThreadLocalSuperVersions() { autovector sv_ptrs; - local_sv_->Scrape(&sv_ptrs, SuperVersion::kSVObsolete); + local_sv_.Scrape(&sv_ptrs, SuperVersion::kSVObsolete); for (auto ptr : sv_ptrs) { assert(ptr); if (ptr == SuperVersion::kSVInUse) { @@ -1608,6 +1696,10 @@ void ColumnFamilyData::RecoverEpochNumbers() { vstorage->RecoverEpochNumbers(this); } +const std::string& ColumnFamilyData::GetDBName() const { + return column_family_set_->db_name_; +} + ColumnFamilySet::ColumnFamilySet(const std::string& dbname, const ImmutableDBOptions* db_options, const FileOptions& file_options, diff --git a/db/column_family.h b/db/column_family.h index 2a38feb731..696e2705be 100644 --- a/db/column_family.h +++ b/db/column_family.h @@ -29,6 +29,8 @@ #include "util/hash_containers.h" #include "util/thread_local.h" +#include + namespace ROCKSDB_NAMESPACE { class Version; @@ -167,12 +169,13 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle { InstrumentedMutex* mutex); // destroy without mutex virtual ~ColumnFamilyHandleImpl(); - virtual ColumnFamilyData* cfd() const { return cfd_; } + virtual ColumnFamilyData* cfd() const override { return cfd_; } virtual uint32_t GetID() const override; virtual const std::string& GetName() const override; virtual Status GetDescriptor(ColumnFamilyDescriptor* desc) override; virtual const Comparator* GetComparator() const override; + virtual ColumnFamilyHandle* CloneHandle() const override; private: ColumnFamilyData* cfd_; @@ -194,6 +197,10 @@ class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl { void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; } virtual ColumnFamilyData* cfd() const override { return internal_cfd_; } + uint32_t GetID() const final; + const std::string& GetName() const final; + const Comparator* GetComparator() const override; + ColumnFamilyHandle* CloneHandle() const override; private: ColumnFamilyData* internal_cfd_; @@ -371,6 +378,8 @@ class ColumnFamilyData { // calculate the oldest log needed for the durability of this column family uint64_t OldestLogToKeep(); + void PrepareNewMemtableInBackground(const MutableCFOptions&); + // See Memtable constructor for explanation of earliest_seq param. MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq); @@ -450,6 +459,9 @@ class ColumnFamilyData { uint64_t GetSuperVersionNumber() const { return super_version_number_.load(); } + uint64_t GetSuperVersionNumberNoAtomic() const { + return reinterpret_cast(super_version_number_); + } // will return a pointer to SuperVersion* if previous SuperVersion // if its reference count is zero and needs deletion or nullptr if not // As argument takes a pointer to allocated SuperVersion to enable @@ -518,7 +530,7 @@ class ColumnFamilyData { // user's setting. Called by background flush job. bool ShouldPostponeFlushToRetainUDT(uint64_t max_memtable_id); - ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); } + ThreadLocalPtr* TEST_GetLocalSV() { return &local_sv_; } WriteBufferManager* write_buffer_mgr() { return write_buffer_manager_; } std::shared_ptr GetFileMetadataCacheReservationManager() { @@ -549,6 +561,8 @@ class ColumnFamilyData { // of its files (if missing) void RecoverEpochNumbers(); + const std::string& GetDBName() const; + private: friend class ColumnFamilySet; ColumnFamilyData(uint32_t id, const std::string& name, @@ -590,6 +604,12 @@ class ColumnFamilyData { WriteBufferManager* write_buffer_manager_; + #if !defined(ROCKSDB_UNIT_TEST) + // precreated_memtable_list_.size() is normally 1 + terark::fixed_circular_queue, 4> precreated_memtable_list_; + std::mutex precreated_memtable_mutex_; + #endif + MemTable* mem_; MemTableList imm_; SuperVersion* super_version_; @@ -601,7 +621,7 @@ class ColumnFamilyData { // Thread's local copy of SuperVersion pointer // This needs to be destructed before mutex_ - std::unique_ptr local_sv_; + ThreadLocalPtr local_sv_; // pointers for a circular linked list. we use it to support iterations over // all column families that are alive (note: dropped column families can also diff --git a/db/compaction/clipping_iterator.h b/db/compaction/clipping_iterator.h index 3f50cdd9dd..db7131db74 100644 --- a/db/compaction/clipping_iterator.h +++ b/db/compaction/clipping_iterator.h @@ -17,14 +17,54 @@ namespace ROCKSDB_NAMESPACE { // iterator has already performed the bounds checking, it relies on that result; // otherwise, it performs the necessary key comparisons itself. Both bounds // are optional. -class ClippingIterator : public InternalIterator { +template +struct ClippingIterBounds; + +template<> struct ClippingIterBounds { + Slice m_start, m_end; + ClippingIterBounds(const Slice* start, const Slice* end) + : m_start(*start), m_end(*end) { + assert(nullptr != start); + assert(nullptr != end); + } + const Slice* start_() const { return &m_start; } + const Slice* end_() const { return &m_end; } +}; +template<> struct ClippingIterBounds { + Slice m_start; + ClippingIterBounds(const Slice* start, const Slice* end) + : m_start(*start) { + assert(nullptr != start); + assert(nullptr == end); + } + const Slice* start_() const { return &m_start; } + const Slice* end_() const { return nullptr; } +}; +template<> struct ClippingIterBounds { + Slice m_end; + ClippingIterBounds(const Slice* start, const Slice* end) + : m_end(*end) { + assert(nullptr == start); + assert(nullptr != end); + } + const Slice* start_() const { return nullptr; } + const Slice* end_() const { return &m_end; } +}; + +template +class ClippingIterator final : public InternalIterator, ClippingIterBounds, LessCMP { + using bounds = ClippingIterBounds; + using bounds::start_; + using bounds::end_; + bool less(const Slice& x, const Slice& y) const { + return static_cast(*this)(x, y); + } public: ClippingIterator(InternalIterator* iter, const Slice* start, const Slice* end, - const CompareInterface* cmp) - : iter_(iter), start_(start), end_(end), cmp_(cmp), valid_(false) { + const LessCMP& cmp) + : bounds(start, end), LessCMP(cmp), iter_(iter), valid_(false) { assert(iter_); - assert(cmp_); - assert(!start_ || !end_ || cmp_->Compare(*start_, *end_) <= 0); + assert(!start || !end || !less(*end, *start)); UpdateAndEnforceBounds(); } @@ -32,71 +72,77 @@ class ClippingIterator : public InternalIterator { bool Valid() const override { return valid_; } void SeekToFirst() override { - if (start_) { - iter_->Seek(*start_); + if (start_()) { + iter_->Seek(*start_()); } else { iter_->SeekToFirst(); } + UpdateValid(); UpdateAndEnforceUpperBound(); } void SeekToLast() override { - if (end_) { - iter_->SeekForPrev(*end_); + if (end_()) { + iter_->SeekForPrev(*end_()); // Upper bound is exclusive, so we need a key which is strictly smaller - if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) { + if (iter_->Valid() && !less(iter_->key(), *end_())) { iter_->Prev(); } } else { iter_->SeekToLast(); } + UpdateValid(); UpdateAndEnforceLowerBound(); } void Seek(const Slice& target) override { - if (start_ && cmp_->Compare(target, *start_) < 0) { - iter_->Seek(*start_); + if (start_() && less(target, *start_())) { + iter_->Seek(*start_()); + UpdateValid(); UpdateAndEnforceUpperBound(); return; } - if (end_ && cmp_->Compare(target, *end_) >= 0) { + if (end_() && !less(target, *end_())) { valid_ = false; return; } iter_->Seek(target); + UpdateValid(); UpdateAndEnforceUpperBound(); } void SeekForPrev(const Slice& target) override { - if (start_ && cmp_->Compare(target, *start_) < 0) { + if (start_() && less(target, *start_())) { valid_ = false; return; } - if (end_ && cmp_->Compare(target, *end_) >= 0) { - iter_->SeekForPrev(*end_); + if (end_() && !less(target, *end_())) { + iter_->SeekForPrev(*end_()); // Upper bound is exclusive, so we need a key which is strictly smaller - if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) { + if (iter_->Valid() && !less(iter_->key(), *end_())) { iter_->Prev(); } + UpdateValid(); UpdateAndEnforceLowerBound(); return; } iter_->SeekForPrev(target); + UpdateValid(); UpdateAndEnforceLowerBound(); } void Next() override { assert(valid_); - iter_->Next(); + valid_ = iter_->NextAndCheckValid(); UpdateAndEnforceUpperBound(); } @@ -104,30 +150,28 @@ class ClippingIterator : public InternalIterator { assert(valid_); assert(result); - IterateResult res; - valid_ = iter_->NextAndGetResult(&res); + valid_ = iter_->NextAndGetResult(result); - if (!valid_) { + if (UNLIKELY(!valid_)) { return false; } - if (end_) { - EnforceUpperBoundImpl(res.bound_check_result); - + if (end_()) { + EnforceUpperBoundImpl(result->bound_check_result); + result->is_valid = valid_; if (!valid_) { return false; } } - res.bound_check_result = IterBoundCheck::kInbound; - *result = res; + result->bound_check_result = IterBoundCheck::kInbound; return true; } void Prev() override { assert(valid_); - iter_->Prev(); + valid_ = iter_->PrevAndCheckValid(); UpdateAndEnforceLowerBound(); } @@ -201,18 +245,18 @@ class ClippingIterator : public InternalIterator { } void EnforceUpperBoundImpl(IterBoundCheck bound_check_result) { - if (bound_check_result == IterBoundCheck::kInbound) { + if (UNLIKELY(bound_check_result == IterBoundCheck::kInbound)) { return; } - if (bound_check_result == IterBoundCheck::kOutOfBound) { + if (UNLIKELY(bound_check_result == IterBoundCheck::kOutOfBound)) { valid_ = false; return; } assert(bound_check_result == IterBoundCheck::kUnknown); - if (cmp_->Compare(key(), *end_) >= 0) { + if (!less(key(), *end_())) { valid_ = false; } } @@ -222,7 +266,7 @@ class ClippingIterator : public InternalIterator { return; } - if (!end_) { + if (!end_()) { return; } @@ -234,7 +278,7 @@ class ClippingIterator : public InternalIterator { return; } - if (!start_) { + if (!start_()) { return; } @@ -242,14 +286,14 @@ class ClippingIterator : public InternalIterator { return; } - if (cmp_->Compare(key(), *start_) < 0) { + if (less(key(), *start_())) { valid_ = false; } } void AssertBounds() { - assert(!valid_ || !start_ || cmp_->Compare(key(), *start_) >= 0); - assert(!valid_ || !end_ || cmp_->Compare(key(), *end_) < 0); + assert(!valid_ || !start_() || !less(key(), *start_())); + assert(!valid_ || !end_() || less(key(), *end_())); } void UpdateAndEnforceBounds() { @@ -260,22 +304,55 @@ class ClippingIterator : public InternalIterator { } void UpdateAndEnforceUpperBound() { - UpdateValid(); EnforceUpperBound(); AssertBounds(); } void UpdateAndEnforceLowerBound() { - UpdateValid(); EnforceLowerBound(); AssertBounds(); } InternalIterator* iter_; - const Slice* start_; - const Slice* end_; - const CompareInterface* cmp_; bool valid_; }; +template +std::unique_ptr +MakeClippingIteratorAux(InternalIterator* iter, + const Slice* start, const Slice* end, LessCMP cmp) { + if (nullptr == start) + return std::make_unique >(iter, start, end, cmp); + else if (nullptr == end) + return std::make_unique >(iter, start, end, cmp); + else + return std::make_unique >(iter, start, end, cmp); +} + +inline +std::unique_ptr +MakeClippingIterator(InternalIterator* iter, + const Slice* start, const Slice* end, + const InternalKeyComparator* cmp) { + if (cmp->IsForwardBytewise()) + return MakeClippingIteratorAux(iter, start, end, {}); + else if (cmp->IsReverseBytewise()) + return MakeClippingIteratorAux(iter, start, end, {}); + else + return MakeClippingIteratorAux(iter, start, end, {cmp}); +} + +inline +std::unique_ptr +MakeClippingIterator(InternalIterator* iter, + const Slice* start, const Slice* end, + const Comparator* cmp) { + if (cmp->IsForwardBytewise()) + return MakeClippingIteratorAux(iter, start, end, {}); + else if (cmp->IsReverseBytewise()) + return MakeClippingIteratorAux(iter, start, end, {}); + else + return MakeClippingIteratorAux(iter, start, end, {cmp}); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/compaction/clipping_iterator_test.cc b/db/compaction/clipping_iterator_test.cc index b2b1670489..31a0a4e00b 100644 --- a/db/compaction/clipping_iterator_test.cc +++ b/db/compaction/clipping_iterator_test.cc @@ -38,12 +38,14 @@ class BoundsCheckingVectorIterator : public VectorIterator { Next(); if (!Valid()) { + result->is_valid = false; return false; } - result->key = key(); + result->SetKey(this->key()); result->bound_check_result = UpperBoundCheckResult(); result->value_prepared = true; + result->is_valid = true; return true; } @@ -109,7 +111,8 @@ TEST_P(ClippingIteratorTest, Clip) { &end, BytewiseComparator()) : new VectorIterator(input_keys, input_values, BytewiseComparator())); - ClippingIterator clip(input.get(), &start, &end, BytewiseComparator()); + auto p_clip = MakeClippingIterator(input.get(), &start, &end, BytewiseComparator()); + auto& clip = *p_clip; // The range the clipping iterator should return values from. This is // essentially the intersection of the input range [1, 4) and the clipping @@ -168,7 +171,7 @@ TEST_P(ClippingIteratorTest, Clip) { for (size_t i = data_start_idx + 1; i < data_end_idx; ++i) { IterateResult result; ASSERT_TRUE(clip.NextAndGetResult(&result)); - ASSERT_EQ(result.key, keys[i]); + ASSERT_EQ(result.key(), keys[i]); ASSERT_EQ(result.bound_check_result, IterBoundCheck::kInbound); ASSERT_TRUE(clip.Valid()); ASSERT_EQ(clip.key(), keys[i]); diff --git a/db/compaction/compaction.cc b/db/compaction/compaction.cc index bbab8f79fb..516940f2b7 100644 --- a/db/compaction/compaction.cc +++ b/db/compaction/compaction.cc @@ -24,8 +24,10 @@ namespace ROCKSDB_NAMESPACE { const uint64_t kRangeTombstoneSentinel = PackSequenceAndType(kMaxSequenceNumber, kTypeRangeDeletion); -int sstableKeyCompare(const Comparator* uc, const Slice& a, const Slice& b) { - auto c = uc->CompareWithoutTimestamp(ExtractUserKey(a), ExtractUserKey(b)); +template +ROCKSDB_FLATTEN +int sstableKeyCompare(CmpNoTS ucmp, const Slice& a, const Slice& b) { + auto c = ucmp(ExtractUserKey(a), ExtractUserKey(b)); if (c != 0) { return c; } @@ -40,27 +42,26 @@ int sstableKeyCompare(const Comparator* uc, const Slice& a, const Slice& b) { } return 0; } +#define sstableKeyCompareInstantiate(CmpNoTS) \ + template int sstableKeyCompare(CmpNoTS, const Slice&, const Slice&) -int sstableKeyCompare(const Comparator* user_cmp, const InternalKey* a, - const InternalKey& b) { - if (a == nullptr) { - return -1; - } - return sstableKeyCompare(user_cmp, *a, b); -} +sstableKeyCompareInstantiate(ForwardBytewiseCompareUserKeyNoTS); +sstableKeyCompareInstantiate(ReverseBytewiseCompareUserKeyNoTS); +sstableKeyCompareInstantiate(VirtualFunctionCompareUserKeyNoTS); -int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, - const InternalKey* b) { - if (b == nullptr) { - return -1; +uint64_t TotalFileSize(const std::vector& files) { + uint64_t sum = 0; + for (size_t i = 0; i < files.size() && files[i]; i++) { + sum += files[i]->fd.GetFileSize(); } - return sstableKeyCompare(user_cmp, a, *b); + return sum; } -uint64_t TotalFileSize(const std::vector& files) { +uint64_t TotalFileRawKV(const std::vector& files) { uint64_t sum = 0; for (size_t i = 0; i < files.size() && files[i]; i++) { - sum += files[i]->fd.GetFileSize(); + if (auto reader = files[i]->fd.table_reader) + sum += reader->GetTableProperties()->raw_size(); } return sum; } @@ -342,12 +343,16 @@ Compaction::Compaction( ? Compaction::kInvalidLevel : EvaluatePenultimateLevel(vstorage, immutable_options_, start_level_, output_level_)) { + is_compaction_woker_ = IsCompactionWorker(); // preload to speed up MarkFilesBeingCompacted(true); if (is_manual_compaction_) { compaction_reason_ = CompactionReason::kManualCompaction; } if (max_subcompactions_ == 0) { - max_subcompactions_ = _mutable_db_options.max_subcompactions; + if (output_level_ > 0 && 0 == start_level_ && _mutable_db_options.max_level1_subcompactions) + max_subcompactions_ = _mutable_db_options.max_level1_subcompactions; + else + max_subcompactions_ = _mutable_db_options.max_subcompactions; } // for the non-bottommost levels, it tries to build files match the target @@ -378,6 +383,7 @@ Compaction::Compaction( // Every compaction regardless of any compaction reason may respect the // existing compact cursor in the output level to split output files output_split_key_ = nullptr; +#if defined(ROCKSDB_UNIT_TEST) if (immutable_options_.compaction_style == kCompactionStyleLevel && immutable_options_.compaction_pri == kRoundRobin) { const InternalKey* cursor = @@ -395,6 +401,7 @@ Compaction::Compaction( } } } +#endif PopulatePenultimateLevelOutputRange(); } @@ -516,6 +523,10 @@ bool Compaction::InputCompressionMatchesOutput() const { return matches; } +bool TableFactory::InputCompressionMatchesOutput(const Compaction* c) const { + return c->InputCompressionMatchesOutput(); +} + bool Compaction::IsTrivialMove() const { // Avoid a move if there is lots of overlapping grandparent data. // Otherwise, the move could create a parent file that will require @@ -550,6 +561,17 @@ bool Compaction::IsTrivialMove() const { return false; } +#if !defined(ROCKSDB_UNIT_TEST) // ToplingDB specific + if (kCompactionStyleLevel == immutable_options_.compaction_style) { + auto& cfo = mutable_cf_options_; + if (1 == output_level_ && + immutable_options_.compaction_executor_factory && + cfo.write_buffer_size > cfo.target_file_size_base * 3/2) { + return false; + } + } +#endif + // Used in universal compaction, where trivial move can be done if the // input files are non overlapping if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) && @@ -560,7 +582,7 @@ bool Compaction::IsTrivialMove() const { if (!(start_level_ != output_level_ && num_input_levels() == 1 && input(0, 0)->fd.GetPathId() == output_path_id() && - InputCompressionMatchesOutput())) { + immutable_options_.table_factory->InputCompressionMatchesOutput(this))) { return false; } @@ -611,6 +633,8 @@ bool Compaction::KeyNotExistsBeyondOutputLevel( assert(level_ptrs->size() == static_cast(number_levels_)); if (bottommost_level_) { return true; + } else if (is_compaction_woker_) { + return false; } else if (output_level_ != 0 && cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { // Maybe use binary search to find right entry instead of linear search? @@ -829,6 +853,7 @@ std::unique_ptr Compaction::CreateCompactionFilter() const { context.input_start_level = start_level_; context.column_family_id = cfd_->GetID(); context.reason = TableFileCreationReason::kCompaction; +<<<<<<< HEAD context.input_table_properties = GetInputTableProperties(); if (context.input_table_properties.empty()) { ROCKS_LOG_WARN( @@ -837,6 +862,9 @@ std::unique_ptr Compaction::CreateCompactionFilter() const { "for compaction."); } +======= + context.smallest_seqno = GetSmallestSeqno(); +>>>>>>> sideplugin-8.04.0-2023-06-20-2926e071 return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter( context); } @@ -852,6 +880,7 @@ std::unique_ptr Compaction::CreateSstPartitioner() const { context.output_level = output_level_; context.smallest_user_key = smallest_user_key_; context.largest_user_key = largest_user_key_; + context.target_output_file_size = target_output_file_size_; return immutable_options_.sst_partitioner_factory->CreatePartitioner(context); } @@ -864,12 +893,14 @@ bool Compaction::ShouldFormSubcompactions() const { return false; } +#if defined(ROCKSDB_UNIT_TEST) // Round-Robin pri under leveled compaction allows subcompactions by default // and the number of subcompactions can be larger than max_subcompactions_ if (cfd_->ioptions()->compaction_pri == kRoundRobin && cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { return output_level_ > 0; } +#endif if (max_subcompactions_ <= 1) { return false; @@ -985,4 +1016,14 @@ int Compaction::EvaluatePenultimateLevel( return penultimate_level; } +uint64_t Compaction::GetSmallestSeqno() const { + uint64_t smallest_seqno = UINT64_MAX; + for (auto& eachlevel : inputs_) { + for (auto& eachfile : eachlevel.files) + if (smallest_seqno > eachfile->fd.smallest_seqno) + smallest_seqno = eachfile->fd.smallest_seqno; + } + return smallest_seqno; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/compaction/compaction.h b/db/compaction/compaction.h index 50c75f70b2..2ba7e70053 100644 --- a/db/compaction/compaction.h +++ b/db/compaction/compaction.h @@ -31,23 +31,39 @@ namespace ROCKSDB_NAMESPACE { // that key never appears in the database. We don't want adjacent sstables to // be considered overlapping if they are separated by the range tombstone // sentinel. -int sstableKeyCompare(const Comparator* user_cmp, const Slice&, const Slice&); -inline int sstableKeyCompare(const Comparator* user_cmp, const Slice& a, - const InternalKey& b) { - return sstableKeyCompare(user_cmp, a, b.Encode()); + +template +extern int sstableKeyCompare(CmpNoTS, const Slice& a, const Slice& b); +inline int +sstableKeyCompare(const Comparator* uc, const Slice& a, const Slice& b) { + return sstableKeyCompare(VirtualFunctionCompareUserKeyNoTS{uc}, a, b); +} +template inline int +sstableKeyCompare(CmpNoTS cmp, const Slice& a, const InternalKey& b) { + return sstableKeyCompare(cmp, a, b.Encode()); +} +template inline int +sstableKeyCompare(CmpNoTS cmp, const InternalKey& a, const Slice& b) { + return sstableKeyCompare(cmp, a.Encode(), b); +} +template inline int +sstableKeyCompare(CmpNoTS cmp, const InternalKey& a, const InternalKey& b) { + return sstableKeyCompare(cmp, a.Encode(), b.Encode()); } -inline int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, - const Slice& b) { - return sstableKeyCompare(user_cmp, a.Encode(), b); +template inline int +sstableKeyCompare(CmpNoTS cmp, const InternalKey* a, const InternalKey& b) { + if (a == nullptr) + return -1; + else + return sstableKeyCompare(cmp, *a, b); } -inline int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, - const InternalKey& b) { - return sstableKeyCompare(user_cmp, a.Encode(), b.Encode()); +template inline int +sstableKeyCompare(CmpNoTS cmp, const InternalKey& a, const InternalKey* b) { + if (b == nullptr) + return -1; + else + return sstableKeyCompare(cmp, a, *b); } -int sstableKeyCompare(const Comparator* user_cmp, const InternalKey* a, - const InternalKey& b); -int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a, - const InternalKey* b); // An AtomicCompactionUnitBoundary represents a range of keys [smallest, // largest] that exactly spans one ore more neighbouring SSTs on the same @@ -173,7 +189,7 @@ class Compaction { return &inputs_[compaction_input_level].files; } - const std::vector* inputs() { return &inputs_; } + const std::vector* inputs() const { return &inputs_; } // Returns the LevelFilesBrief of the specified compaction input level. const LevelFilesBrief* input_levels(size_t compaction_input_level) const { @@ -197,6 +213,11 @@ class Compaction { // Whether need to write output file to second DB path. uint32_t output_path_id() const { return output_path_id_; } + const DbPath& output_path() const { + ROCKSDB_VERIFY_LT(output_path_id_, immutable_options_.cf_paths.size()); + return immutable_options_.cf_paths[output_path_id_]; + } + // Is this a trivial compaction that can be implemented by just // moving a single input file to the next level (no merging or splitting) bool IsTrivialMove() const; @@ -243,6 +264,8 @@ class Compaction { // Is this compaction creating a file in the bottom most level? bool bottommost_level() const { return bottommost_level_; } + void set_bottommost_level(bool v) { bottommost_level_ = v; } + // Is the compaction compact to the last level bool is_last_level() const { return output_level_ == immutable_options_.num_levels - 1; @@ -415,6 +438,7 @@ class Compaction { bool ShouldNotifyOnCompactionCompleted() const { return notify_on_compaction_completion_; } + uint64_t GetSmallestSeqno() const; static constexpr int kInvalidLevel = -1; @@ -504,6 +528,7 @@ class Compaction { // logic might pick a subset of the files that aren't overlapping. if // that is the case, set the value to false. Otherwise, set it true. bool l0_files_might_overlap_; + bool is_compaction_woker_; // Compaction input files organized by level. Constant after construction const std::vector inputs_; @@ -517,7 +542,7 @@ class Compaction { const double score_; // score that was used to pick this compaction. // Is this compaction creating a file in the bottom most level? - const bool bottommost_level_; + bool bottommost_level_; // Does this compaction include all sst files? const bool is_full_compaction_; @@ -533,6 +558,7 @@ class Compaction { // Does input compression match the output compression? bool InputCompressionMatchesOutput() const; + friend class TableFactory; // use InputCompressionMatchesOutput TablePropertiesCollection input_table_properties_; TablePropertiesCollection output_table_properties_; @@ -597,4 +623,7 @@ struct PerKeyPlacementContext { // Return sum of sizes of all files in `files`. extern uint64_t TotalFileSize(const std::vector& files); +// Return sum of raw kv sizes of all files in `files`. +extern uint64_t TotalFileRawKV(const std::vector& files); + } // namespace ROCKSDB_NAMESPACE diff --git a/db/compaction/compaction_executor.cc b/db/compaction/compaction_executor.cc new file mode 100644 index 0000000000..727acc3beb --- /dev/null +++ b/db/compaction/compaction_executor.cc @@ -0,0 +1,332 @@ +// +// Created by leipeng on 2021/1/11. +// + +#include "compaction_executor.h" + +namespace ROCKSDB_NAMESPACE { + +CompactionParams::CompactionParams() { + is_deserialized = false; +} +CompactionParams::~CompactionParams() { + if (is_deserialized) { + ROCKSDB_VERIFY(IsCompactionWorker()); + /* + for (auto& x : *inputs) { + for (auto& e : x.atomic_compaction_unit_boundaries) { + delete e.smallest; + delete e.largest; + } + } + */ + if (grandparents) { + for (auto meta : *grandparents) { + delete meta; + } + delete grandparents; + } + if (inputs) { + for (auto& level_files : *inputs) { + for (auto meta : level_files.files) + delete meta; + } + delete inputs; + } + delete existing_snapshots; + //delete compaction_job_stats; + } + else { + //ROCKSDB_VERIFY(!IsCompactionWorker()); + } +} + +#if defined(_MSC_VER) +static std::string html_user_key_decode(const CompactionParams&, Slice uk) { + return uk.ToString(true); +} +#else +std::string __attribute__((weak)) +CompactionParams_html_user_key_decode(const CompactionParams&, Slice); +static std::string html_user_key_decode(const CompactionParams& cp, Slice uk) { + if (CompactionParams_html_user_key_decode) + return CompactionParams_html_user_key_decode(cp, uk); + else + return uk.ToString(true); +} +#endif + +static void PrintVersionSetSerDe(FILE* fp, const VersionSetSerDe& v) { + fprintf(fp, "VersionSetSerDe\n"); + fprintf(fp, " last_sequence = %zd, " + "last_allocated_sequence = %zd, " + "last_published_sequence = %zd\n", + size_t(v.last_sequence), + size_t(v.last_allocated_sequence), + size_t(v.last_published_sequence)); + fprintf(fp, " next_file_number = %zd, " + "min_log_number_to_keep_2pc = %zd, " + "manifest_file_number = %zd, " + "options_file_number = %zd, " + "prev_log_number = %zd, " + "current_version_number = %zd\n", + size_t(v.next_file_number), + #if ROCKSDB_MAJOR < 7 + size_t(v.min_log_number_to_keep_2pc), + #else + size_t(v.min_log_number_to_keep), + #endif + size_t(v.manifest_file_number), + size_t(v.options_file_number), + size_t(v.prev_log_number), + size_t(v.current_version_number)); +} +static void PrintFileMetaData(const CompactionParams& cp, + FILE* fp, const FileMetaData* f) { + Slice temperature = enum_name(f->temperature); + std::string lo = html_user_key_decode(cp, f->smallest.user_key()); + std::string hi = html_user_key_decode(cp, f->largest.user_key()); + fprintf(fp, + " %06zd.sst : entries = %zd, del = %zd, rks = %zd, rvs = %zd, " + "fsize = %zd : %zd, temp = %.*s, seq = %zd : %zd, rng = %.*s : %.*s\n", + size_t(f->fd.GetNumber()), + size_t(f->num_entries), size_t(f->num_deletions), + size_t(f->raw_key_size), size_t(f->raw_value_size), + size_t(f->fd.file_size), size_t(f->compensated_file_size), + int(temperature.size_), temperature.data_, + size_t(f->fd.smallest_seqno), size_t(f->fd.largest_seqno), + int(lo.size()), lo.data(), int(hi.size()), hi.data()); +} + +std::string CompactionParams::DebugString() const { + size_t mem_len = 0; + char* mem_buf = nullptr; + FILE* fp = open_memstream(&mem_buf, &mem_len); + fprintf(fp, "job_id = %d, output_level = %d, dbname = %s, cfname = %s\n", + job_id, output_level, dbname.c_str(), cf_name.c_str()); + fprintf(fp, "bottommost_level = %d, compaction_reason = %s\n", + bottommost_level, enum_cstr(compaction_reason)); + fprintf(fp, "smallest_user_key = %s\n", html_user_key_decode(*this, smallest_user_key).c_str()); + fprintf(fp, "llargest_user_key = %s\n", html_user_key_decode(*this, largest_user_key).c_str()); + for (size_t i = 0; i < inputs->size(); ++i) { + auto& l = inputs->at(i); + fprintf(fp, "inputs.size = %zd : %zd : level = %d, size = %3zd\n", + inputs->size(), i, l.level, l.size()); + for (auto fmd : l.files) { + PrintFileMetaData(*this, fp, fmd); + } + } + if (grandparents) { + fprintf(fp, "grandparents.size = %zd\n", grandparents->size()); + for (size_t i = 0; i < grandparents->size(); ++i) { + FileMetaData* fmd = grandparents->at(i); + PrintFileMetaData(*this, fp, fmd); + } + } + else { + fprintf(fp, "grandparents = nullptr\n"); + } + if (existing_snapshots) { + fprintf(fp, "existing_snapshots.size = %zd\n", existing_snapshots->size()); + } + else { + fprintf(fp, "existing_snapshots = nullptr\n"); + } + fprintf(fp, "level_compaction_dynamic_file_size = %s", + level_compaction_dynamic_file_size ? "true" : "false"); + PrintVersionSetSerDe(fp, version_set); + fclose(fp); + std::string result(mem_buf, mem_len); + free(mem_buf); + return result; +} + +// res[0] : raw +// res[1] : zip +void CompactionParams::InputBytes(size_t* res) const { + size_t raw = 0, zip = 0; + for (auto& eachlevel : *inputs) { + for (auto& eachfile : eachlevel.files) { + zip += eachfile->fd.file_size; + raw += eachfile->raw_key_size + eachfile->raw_value_size; + } + } + res[0] = raw; + res[1] = zip; +} + +CompactionResults::CompactionResults() { + curl_time_usec = 0; + work_time_usec = 0; + mount_time_usec = 0; + prepare_time_usec = 0; + waiting_time_usec = 0; + output_index_size = 0; + output_data_size = 0; +} +CompactionResults::~CompactionResults() {} + +struct MyVersionSet : VersionSet { + void From(const VersionSetSerDe& version_set) { + next_file_number_ = version_set.next_file_number; + last_sequence_ = version_set.last_sequence; + // below are not necessary fields, but we serialize it for + // for completeness debugging + last_allocated_sequence_ = version_set.last_allocated_sequence; + last_published_sequence_ = version_set.last_published_sequence; + #if ROCKSDB_MAJOR < 7 + min_log_number_to_keep_2pc_ = version_set.min_log_number_to_keep_2pc; + #else + min_log_number_to_keep_ = version_set.min_log_number_to_keep; + #endif + manifest_file_number_ = version_set.manifest_file_number; + options_file_number_ = version_set.options_file_number; + //pending_manifest_file_number_ is temporal on running, do NOT serilize! + //pending_manifest_file_number_ = version_set.pending_manifest_file_number; + prev_log_number_ = version_set.prev_log_number; + current_version_number_ = version_set.current_version_number; + } + void To(VersionSetSerDe& version_set) const { + version_set.next_file_number = next_file_number_; + version_set.last_sequence = last_sequence_; + // below are not necessary fields, but we serialize it for + // for completeness debugging + version_set.last_allocated_sequence = last_allocated_sequence_; + version_set.last_published_sequence = last_published_sequence_; + #if ROCKSDB_MAJOR < 7 + version_set.min_log_number_to_keep_2pc = min_log_number_to_keep_2pc_; + #else + version_set.min_log_number_to_keep = min_log_number_to_keep_; + #endif + version_set.manifest_file_number = manifest_file_number_; + version_set.options_file_number = options_file_number_; + //pending_manifest_file_number_ is temporal on running, do NOT serilize! + //version_set.pending_manifest_file_number = pending_manifest_file_number_; + version_set.prev_log_number = prev_log_number_; + version_set.current_version_number = current_version_number_; + } +}; +void VersionSetSerDe::From(const VersionSet* vs) { + static_cast(vs)->To(*this); // NOLINT +} +void VersionSetSerDe::To(VersionSet* vs) const { + static_cast(vs)->From(*this); // NOLINT +} + +CompactionExecutor::~CompactionExecutor() = default; +CompactionExecutorFactory::~CompactionExecutorFactory() = default; + +std::string CompactionExecutorFactory::JobUrl(const std::string&, int, int) const { + return std::string(); // empty string +} + +static bool g_is_compaction_worker = false; +bool IsCompactionWorker() { + return g_is_compaction_worker; +} +void SetAsCompactionWorker() { + g_is_compaction_worker = true; +} + +///////////////////////////////////////////////////////////////////////////// +std::string GetDirFromEnv(const char* name, const char* Default) { + const char* dir = getenv(name); + if (nullptr == dir) { + ROCKSDB_VERIFY(nullptr != Default); + dir = Default; + } + size_t dir_name_len = strlen(dir); + ROCKSDB_VERIFY(dir_name_len > 0); + while (dir_name_len && '/' == dir[dir_name_len-1]) { + dir_name_len--; + } + ROCKSDB_VERIFY(dir_name_len > 0); + return std::string(dir, dir_name_len); +} + +bool ReplacePrefix(Slice Old, Slice New, Slice str, std::string* res) { + ROCKSDB_VERIFY(Old.size_ > 0); + ROCKSDB_VERIFY(New.size_ > 0); + while (Old.size_ && Old.data_[Old.size_-1] == '/') { + --Old.size_; + } + while (New.size_ && New.data_[New.size_-1] == '/') { + --New.size_; + } + ROCKSDB_VERIFY(Old.size_ > 0); + ROCKSDB_VERIFY(New.size_ > 0); + if (str.starts_with(Old)) { + size_t suffixLen = str.size_ - Old.size_; + res->reserve(New.size_ + suffixLen); + res->assign(New.data_, New.size_); + res->append(str.data_ + Old.size_, suffixLen); + return true; + } + return false; +} + +std::string ReplacePrefix(Slice Old, Slice New, Slice str) { + std::string res; + if (ReplacePrefix(Old, New, str, &res)) { + return res; + } + ROCKSDB_DIE("str = '%.*s' does not start with Old='%.*s'", + int(str.size()), str.data(), int(Old.size()), Old.data()); +} + +void ReplaceAll(std::string& str, Slice from, Slice to) { + if (from.empty()) return; + size_t start_pos = 0; + while ((start_pos = str.find(from.data(), start_pos)) != std::string::npos) { + str.replace(start_pos, from.size(), to.data(), to.size()); + start_pos += to.size(); + } +} +std::string ReplaceAll(Slice str, Slice from, Slice to) { + std::string tmp(str.data(), str.size()); + ReplaceAll(tmp, from, to); + return tmp; +} +std::string MakePath(std::string dir, Slice sub) { + while (!dir.empty() && '/' == dir.back()) { + dir.pop_back(); + } + dir.reserve(dir.size() + 1 + sub.size()); + ROCKSDB_VERIFY(!sub.empty()); + while (!sub.empty() && '/' == sub[0]) { + sub.remove_prefix(1); + } + ROCKSDB_VERIFY(!sub.empty()); + dir.push_back('/'); + dir.append(sub.data(), sub.size()); + return dir; +} + +std::string& AppendJobID(std::string& dir, int job_id) { + while (!dir.empty() && '/' == dir.back()) { + dir.pop_back(); + } + char buf[32]; + dir.append(buf, snprintf(buf, sizeof(buf), "/job-%05d", job_id)); + return dir; +} +std::string CatJobID(const std::string& dir, int job_id) { + std::string output_path = dir; + AppendJobID(output_path, job_id); + return output_path; +} +std::string& AppendAttempt(std::string& dir, int attempt) { + while (!dir.empty() && '/' == dir.back()) { + dir.pop_back(); + } + char buf[32]; + dir.append(buf, snprintf(buf, sizeof(buf), "/att-%02d", attempt)); + return dir; +} +std::string CatAttempt(const std::string& dir, int attempt) { + std::string output_path = dir; + AppendAttempt(output_path, attempt); + return output_path; +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/compaction/compaction_executor.h b/db/compaction/compaction_executor.h new file mode 100644 index 0000000000..47c32969c5 --- /dev/null +++ b/db/compaction/compaction_executor.h @@ -0,0 +1,192 @@ +// +// Created by leipeng on 2021/1/11. +// +#pragma once +#include "compaction_job.h" + +namespace ROCKSDB_NAMESPACE { + +struct ObjectRpcParam { + std::string clazz; + std::string params; // construction json params + typedef std::function serde_fn_t; + serde_fn_t serde; +}; +struct VersionSetSerDe { + uint64_t last_sequence; + uint64_t last_allocated_sequence; + uint64_t last_published_sequence; + uint64_t next_file_number; + #if ROCKSDB_MAJOR < 7 + uint64_t min_log_number_to_keep_2pc; + #else + uint64_t min_log_number_to_keep; + #endif + uint64_t manifest_file_number; + uint64_t options_file_number; + //uint64_t pending_manifest_file_number; + uint64_t prev_log_number; + uint64_t current_version_number; + void From(const VersionSet*); + void To(VersionSet*) const; +}; +struct CompactionParams { + CompactionParams(const CompactionParams&) = delete; + CompactionParams& operator=(const CompactionParams&) = delete; + CompactionParams(); + ~CompactionParams(); + int job_id; + int num_levels; + int output_level; + uint32_t cf_id; + std::string cf_name; + const std::vector* inputs = nullptr; + VersionSetSerDe version_set; + uint64_t target_file_size; + uint64_t max_compaction_bytes; + + // we add a dedicated path to compaction worker's cf_path as + // output path, thus reduce changes to the existing rocksdb code. + // the output_path_id should be the last elem of cf_paths, so it + // needs not the field output_path_id. + //uint32_t output_path_id; // point to the extra cf_path + //std::string output_path; // will append to cfopt.cf_paths on remote node? + std::vector cf_paths; + + uint32_t max_subcompactions; // num_threads + CompressionType compression; + CompressionOptions compression_opts; + const std::vector* grandparents = nullptr; + double score; + bool manual_compaction; + bool deletion_compaction; + InfoLogLevel compaction_log_level; + CompactionReason compaction_reason; + + //VersionSet* version_set; + SequenceNumber preserve_deletes_seqnum; + const std::vector* existing_snapshots = nullptr; + SequenceNumber smallest_seqno; + SequenceNumber earliest_write_conflict_snapshot; + bool paranoid_file_checks; + uint32_t code_version; + std::string code_githash; + std::string hoster_root; + std::string instance_name; + std::string dbname; + std::string db_id; + std::string db_session_id; + std::string full_history_ts_low; + //CompactionJobStats* compaction_job_stats = nullptr; // this is out param + //SnapshotChecker* snapshot_checker; // not used + //FSDirectory* db_directory; + //FSDirectory* output_directory; + //FSDirectory* blob_output_directory; + + std::string smallest_user_key; // serialization must before + std::string largest_user_key; // ObjectRpcParam fields + //ObjectRpcParam compaction_filter; // don't use compaction_filter + ObjectRpcParam compaction_filter_factory; // always use + ObjectRpcParam merge_operator; + ObjectRpcParam user_comparator; + ObjectRpcParam table_factory; + ObjectRpcParam prefix_extractor; + ObjectRpcParam sst_partitioner_factory; + ObjectRpcParam html_user_key_coder; + + //bool skip_filters; + bool allow_ingest_behind; + bool preserve_deletes; + bool bottommost_level; + bool is_deserialized; + bool level_compaction_dynamic_file_size; + CompactionStyle compaction_style; + CompactionPri compaction_pri; + std::vector listeners; + std::vector table_properties_collector_factories; + std::string extensible_js_data; + + // CompactionFilterFactory ... can have individual serde files + mutable std::vector extra_serde_files; + Logger* info_log = nullptr; // do not serialize, just for running process + mutable class UserKeyCoder* p_html_user_key_coder = nullptr; + const std::atomic* shutting_down = nullptr; // do not serialize + + std::string DebugString() const; + void InputBytes(size_t* res) const; +}; + +struct CompactionResults { + CompactionResults(const CompactionResults&) = delete; + CompactionResults& operator=(const CompactionResults&) = delete; + CompactionResults(); + ~CompactionResults(); + struct FileMinMeta { + uint64_t file_number; + uint64_t file_size; + uint64_t smallest_seqno; + uint64_t largest_seqno; + InternalKey smallest_ikey; + InternalKey largest_ikey; + bool marked_for_compaction; + }; + // collect remote statistics + struct RawStatistics { + uint64_t tickers[INTERNAL_TICKER_ENUM_MAX] = {0}; + HistogramStat histograms[INTERNAL_HISTOGRAM_ENUM_MAX]; + }; + + std::string output_dir; + std::vector > output_files; + InternalStats::CompactionStats compaction_stats; + CompactionJobStats job_stats; + RawStatistics statistics; + Status status; + size_t curl_time_usec; // set by CompactionExecutor, not worker + size_t work_time_usec; + size_t mount_time_usec; // mount nfs + size_t prepare_time_usec; // open nfs params/results + size_t waiting_time_usec; // wait in work queue + + uint64_t output_index_size; // not serialized, just for DB side convenient + uint64_t output_data_size; // not serialized, just for DB side convenient + + size_t all_time_usec() const { + return curl_time_usec + mount_time_usec + prepare_time_usec + work_time_usec; + } +}; + +class CompactionExecutor { + public: + virtual ~CompactionExecutor(); + virtual void SetParams(CompactionParams*, const Compaction*) = 0; + virtual Status CopyOneFile(const std::string& src, const std::string& dst, off_t fsize) = 0; + virtual Status RenameFile(const std::string& src, const std::string& dst, off_t fsize) = 0; + virtual Status Execute(const CompactionParams&, CompactionResults*) = 0; + virtual void CleanFiles(const CompactionParams&, const CompactionResults&) = 0; +}; + +class CompactionExecutorFactory { + public: + virtual ~CompactionExecutorFactory(); + virtual bool ShouldRunLocal(const Compaction*) const = 0; + virtual bool AllowFallbackToLocal() const = 0; + virtual CompactionExecutor* NewExecutor(const Compaction*) const = 0; + virtual const char* Name() const = 0; + virtual std::string JobUrl(const std::string& dbname, int job_id, int attempt) const; +}; + +///////////////////////////////////////////////////////////////////////////// + +std::string GetDirFromEnv(const char* name, const char* Default = nullptr); +bool ReplacePrefix(Slice Old, Slice New, Slice str, std::string* res); +std::string ReplacePrefix(Slice Old, Slice New, Slice str); +void ReplaceAll(std::string& str, Slice from, Slice to); +std::string ReplaceAll(Slice str, Slice from, Slice to); +std::string MakePath(std::string dir, Slice sub); +std::string& AppendJobID(std::string& path, int job_id); +std::string CatJobID(const std::string& path, int job_id); +std::string& AppendAttempt(std::string& path, int attempt); +std::string CatAttempt(const std::string& path, int attempt); + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/compaction/compaction_iterator.cc b/db/compaction/compaction_iterator.cc index 85d1c039bd..4fc614de4d 100644 --- a/db/compaction/compaction_iterator.cc +++ b/db/compaction/compaction_iterator.cc @@ -79,6 +79,8 @@ CompactionIterator::CompactionIterator( clock_(env_->GetSystemClock().get()), report_detailed_time_(report_detailed_time), expect_valid_internal_key_(expect_valid_internal_key), + allow_ingest_behind_(compaction && compaction->allow_ingest_behind()), + supports_per_key_placement_(compaction && compaction->SupportsPerKeyPlacement()), range_del_agg_(range_del_agg), blob_file_builder_(blob_file_builder), compaction_(std::move(compaction)), @@ -117,11 +119,17 @@ CompactionIterator::CompactionIterator( if (compaction_ != nullptr) { level_ptrs_ = std::vector(compaction_->number_levels(), 0); + if (auto c = compaction_->real_compaction()) { + if (level_ >= 0 && level_ < c->mutable_cf_options()->min_filter_level) { + compaction_filter_ = nullptr; // ignore compaction_filter_ + } + } } #ifndef NDEBUG // findEarliestVisibleSnapshot assumes this ordering. for (size_t i = 1; i < snapshots_->size(); ++i) { - assert(snapshots_->at(i - 1) < snapshots_->at(i)); + ROCKSDB_VERIFY_F(snapshots_->at(i - 1) < snapshots_->at(i), + "[%zd]: %zd %zd", i, snapshots_->at(i - 1), snapshots_->at(i)); } assert(timestamp_size_ == 0 || !full_history_ts_low_ || timestamp_size_ == full_history_ts_low_->size()); @@ -347,24 +355,34 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip, env_ != nullptr && report_detailed_time_ ? timer.ElapsedNanos() : 0; } - if (decision == CompactionFilter::Decision::kUndetermined) { + switch (decision) { + default: + ROCKSDB_DIE("Bad decision = %d", int(decision)); + break; + case CompactionFilter::Decision::kUndetermined: // Should not reach here, since FilterV2/FilterV3 should never return // kUndetermined. status_ = Status::NotSupported( "FilterV2/FilterV3 should never return kUndetermined"); validity_info_.Invalidate(); return false; - } - - if (decision == CompactionFilter::Decision::kRemoveAndSkipUntil && - cmp_->Compare(*compaction_filter_skip_until_.rep(), ikey_.user_key) <= + case CompactionFilter::Decision::kRemoveAndSkipUntil: + if (cmp_->Compare(compaction_filter_skip_until_.Encode(), ikey_.user_key) <= 0) { - // Can't skip to a key smaller than the current one. - // Keep the key as per FilterV2/FilterV3 documentation. - decision = CompactionFilter::Decision::kKeep; - } - - if (decision == CompactionFilter::Decision::kRemove) { + // Can't skip to a key smaller than the current one. + // Keep the key as per FilterV2/FilterV3 documentation. + // decision = CompactionFilter::Decision::kKeep; + } else { + *need_skip = true; + compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber, + kValueTypeForSeek); + *skip_until = compaction_filter_skip_until_.Encode(); + } + break; + case CompactionFilter::Decision::kKeep: + // do nothing + break; + case CompactionFilter::Decision::kRemove: // convert the current key to a delete; key_ is pointing into // current_key_ at this point, so updating current_key_ updates key() ikey_.type = kTypeDeletion; @@ -372,7 +390,8 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip, // no value associated with delete value_.clear(); iter_stats_.num_record_drop_user++; - } else if (decision == CompactionFilter::Decision::kPurge) { + break; + case CompactionFilter::Decision::kPurge: // convert the current key to a single delete; key_ is pointing into // current_key_ at this point, so updating current_key_ updates key() ikey_.type = kTypeSingleDeletion; @@ -380,19 +399,16 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip, // no value associated with single delete value_.clear(); iter_stats_.num_record_drop_user++; - } else if (decision == CompactionFilter::Decision::kChangeValue) { + break; + case CompactionFilter::Decision::kChangeValue: if (ikey_.type != kTypeValue) { ikey_.type = kTypeValue; current_key_.UpdateInternalKey(ikey_.sequence, kTypeValue); } value_ = compaction_filter_value_; - } else if (decision == CompactionFilter::Decision::kRemoveAndSkipUntil) { - *need_skip = true; - compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber, - kValueTypeForSeek); - *skip_until = compaction_filter_skip_until_.Encode(); - } else if (decision == CompactionFilter::Decision::kChangeBlobIndex) { + break; + case CompactionFilter::Decision::kChangeBlobIndex: // Only the StackableDB-based BlobDB impl's compaction filter should return // kChangeBlobIndex. Decision about rewriting blob and changing blob index // in the integrated BlobDB impl is made in subsequent call to @@ -411,18 +427,18 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip, } value_ = compaction_filter_value_; - } else if (decision == CompactionFilter::Decision::kIOError) { + break; + case CompactionFilter::Decision::kIOError: if (!compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) { status_ = Status::NotSupported( "CompactionFilter for integrated BlobDB should not return kIOError"); - validity_info_.Invalidate(); - return false; + } else { + status_ = Status::IOError("Failed to access blob during compaction filter"); } - - status_ = Status::IOError("Failed to access blob during compaction filter"); validity_info_.Invalidate(); return false; - } else if (decision == CompactionFilter::Decision::kChangeWideColumnEntity) { + case CompactionFilter::Decision::kChangeWideColumnEntity: + { WideColumns sorted_columns; sorted_columns.reserve(new_columns.size()); @@ -448,7 +464,9 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip, } value_ = compaction_filter_value_; - } + } + break; + } // switch return true; } @@ -466,7 +484,7 @@ void CompactionIterator::NextFromInput() { is_range_del_ = input_.IsDeleteRangeSentinelKey(); Status pik_status = ParseInternalKey(key_, &ikey_, allow_data_in_errors_); - if (!pik_status.ok()) { + if (UNLIKELY(!pik_status.ok())) { iter_stats_.num_input_corrupt_records++; // If `expect_valid_internal_key_` is false, return the corrupted key @@ -483,7 +501,7 @@ void CompactionIterator::NextFromInput() { break; } TEST_SYNC_POINT_CALLBACK("CompactionIterator:ProcessKV", &ikey_); - if (is_range_del_) { + if (UNLIKELY(is_range_del_)) { validity_info_.SetValid(kRangeDeletion); break; } @@ -796,6 +814,7 @@ void CompactionIterator::NextFromInput() { // is an unexpected Merge or Delete. We will compact it out // either way. We will maintain counts of how many mismatches // happened + ROCKSDB_ASSUME(next_ikey.type < kTypeMaxValid); if (next_ikey.type != kTypeValue && next_ikey.type != kTypeBlobIndex && next_ikey.type != kTypeWideColumnEntity) { @@ -1033,8 +1052,10 @@ void CompactionIterator::NextFromInput() { // trim_ts. bool should_delete = false; if (!timestamp_size_ || cmp_with_history_ts_low_ < 0) { + if (!range_del_agg_->IsEmpty()) { should_delete = range_del_agg_->ShouldDelete( key_, RangeDelPositioningMode::kForwardTraversal); + } } if (should_delete) { ++iter_stats_.num_record_drop_hidden; @@ -1248,6 +1269,7 @@ void CompactionIterator::DecideOutputLevel() { } } +ROCKSDB_FLATTEN void CompactionIterator::PrepareOutput() { if (Valid()) { if (LIKELY(!is_range_del_)) { @@ -1260,7 +1282,7 @@ void CompactionIterator::PrepareOutput() { // For range del sentinel, we don't use it to cut files for bottommost // compaction. So it should not make a difference which output level we // decide. - if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) { + if (compaction_ != nullptr && supports_per_key_placement_) { DecideOutputLevel(); } } @@ -1277,7 +1299,7 @@ void CompactionIterator::PrepareOutput() { // Can we do the same for levels above bottom level as long as // KeyNotExistsBeyondOutputLevel() return true? if (Valid() && compaction_ != nullptr && - !compaction_->allow_ingest_behind() && bottommost_level_ && + !allow_ingest_behind_ && bottommost_level_ && DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) && ikey_.type != kTypeMerge && current_key_committed_ && !output_to_penultimate_level_ && @@ -1322,15 +1344,19 @@ void CompactionIterator::PrepareOutput() { inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot( SequenceNumber in, SequenceNumber* prev_snapshot) { + auto const snapshots_beg = snapshots_->begin(); + auto const snapshots_end = snapshots_->end(); + auto const snapshots_num = snapshots_end - snapshots_beg; assert(snapshots_->size()); - if (snapshots_->size() == 0) { + if (snapshots_num == 0) { ROCKS_LOG_FATAL(info_log_, "No snapshot left in findEarliestVisibleSnapshot"); } auto snapshots_iter = - std::lower_bound(snapshots_->begin(), snapshots_->end(), in); + //std::lower_bound(snapshots_->begin(), snapshots_->end(), in); + snapshots_beg + terark::lower_bound_0(snapshots_beg, snapshots_num, in); assert(prev_snapshot != nullptr); - if (snapshots_iter == snapshots_->begin()) { + if (snapshots_iter == snapshots_beg) { *prev_snapshot = 0; } else { *prev_snapshot = *std::prev(snapshots_iter); @@ -1343,11 +1369,11 @@ inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot( } } if (snapshot_checker_ == nullptr) { - return snapshots_iter != snapshots_->end() ? *snapshots_iter + return snapshots_iter != snapshots_end ? *snapshots_iter : kMaxSequenceNumber; } bool has_released_snapshot = !released_snapshots_.empty(); - for (; snapshots_iter != snapshots_->end(); ++snapshots_iter) { + for (; snapshots_iter != snapshots_end; ++snapshots_iter) { auto cur = *snapshots_iter; if (in > cur) { ROCKS_LOG_FATAL(info_log_, @@ -1417,7 +1443,7 @@ std::unique_ptr CompactionIterator::CreateBlobFetcherIfNeeded( read_options.io_activity = Env::IOActivity::kCompaction; read_options.fill_cache = false; - return std::unique_ptr(new BlobFetcher(version, read_options)); + return std::make_unique(version, read_options); } std::unique_ptr diff --git a/db/compaction/compaction_iterator.h b/db/compaction/compaction_iterator.h index 1ff9c88692..dbf315ba1b 100644 --- a/db/compaction/compaction_iterator.h +++ b/db/compaction/compaction_iterator.h @@ -369,6 +369,8 @@ class CompactionIterator { SystemClock* clock_; const bool report_detailed_time_; const bool expect_valid_internal_key_; + const bool allow_ingest_behind_; + const bool supports_per_key_placement_; CompactionRangeDelAggregator* range_del_agg_; BlobFileBuilder* blob_file_builder_; std::unique_ptr compaction_; diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 99b099759d..ed3ecc8b48 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -8,6 +8,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/compaction/compaction_job.h" +#include "compaction_executor.h" #include #include @@ -47,6 +48,8 @@ #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/options.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/sst_partitioner.h" #include "rocksdb/statistics.h" #include "rocksdb/status.h" #include "rocksdb/table.h" @@ -195,6 +198,11 @@ CompactionJob::CompactionJob( ThreadStatusUtil::SetEnableTracking(db_options_.enable_thread_tracking); ThreadStatusUtil::SetColumnFamily(cfd); ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION); + for (auto& level : *compaction->inputs()) { + for (auto& file : level.files) { + file->job_id = job_id; + } + } ReportStartedCompaction(compaction); } @@ -356,6 +364,7 @@ uint64_t CompactionJob::GetSubcompactionsLimit() { void CompactionJob::AcquireSubcompactionResources( int num_extra_required_subcompactions) { +#if defined(ROCKSDB_UNIT_TEST) TEST_SYNC_POINT("CompactionJob::AcquireSubcompactionResources:0"); TEST_SYNC_POINT("CompactionJob::AcquireSubcompactionResources:1"); int max_db_compactions = @@ -392,9 +401,11 @@ void CompactionJob::AcquireSubcompactionResources( } else { *bg_compaction_scheduled_ += extra_num_subcompaction_threads_reserved_; } +#endif } void CompactionJob::ShrinkSubcompactionResources(uint64_t num_extra_resources) { +#if defined(ROCKSDB_UNIT_TEST) // Do nothing when we have zero resources to shrink if (num_extra_resources == 0) return; db_mutex_->Lock(); @@ -419,9 +430,11 @@ void CompactionJob::ShrinkSubcompactionResources(uint64_t num_extra_resources) { } db_mutex_->Unlock(); TEST_SYNC_POINT("CompactionJob::ShrinkSubcompactionResources:0"); +#endif } void CompactionJob::ReleaseSubcompactionResources() { +#if defined(ROCKSDB_UNIT_TEST) if (extra_num_subcompaction_threads_reserved_ == 0) { return; } @@ -440,6 +453,7 @@ void CompactionJob::ReleaseSubcompactionResources() { 1 + extra_num_subcompaction_threads_reserved_); } ShrinkSubcompactionResources(extra_num_subcompaction_threads_reserved_); +#endif } struct RangeWithSize { @@ -478,11 +492,15 @@ void CompactionJob::GenSubcompactionBoundaries() { // cause relatively small inaccuracy. const ReadOptions read_options(Env::IOActivity::kCompaction); auto* c = compact_->compaction; +#if defined(ROCKSDB_UNIT_TEST) if (c->max_subcompactions() <= 1 && !(c->immutable_options()->compaction_pri == kRoundRobin && c->immutable_options()->compaction_style == kCompactionStyleLevel)) { return; } +#else + if (c->max_subcompactions() <= 1) return; +#endif auto* cfd = c->column_family_data(); const Comparator* cfd_comparator = cfd->user_comparator(); const InternalKeyComparator& icomp = cfd->internal_comparator(); @@ -548,6 +566,7 @@ void CompactionJob::GenSubcompactionBoundaries() { }), all_anchors.end()); +#if defined(ROCKSDB_UNIT_TEST) // Get the number of planned subcompactions, may update reserve threads // and update extra_num_subcompaction_threads_reserved_ for round-robin uint64_t num_planned_subcompactions; @@ -580,6 +599,9 @@ void CompactionJob::GenSubcompactionBoundaries() { } else { num_planned_subcompactions = GetSubcompactionsLimit(); } +#else + uint64_t num_planned_subcompactions = std::max(1u, c->max_subcompactions()); +#endif TEST_SYNC_POINT_CALLBACK("CompactionJob::GenSubcompactionBoundaries:0", &num_planned_subcompactions); @@ -620,6 +642,23 @@ void CompactionJob::GenSubcompactionBoundaries() { } Status CompactionJob::Run() { + auto icf_opt = compact_->compaction->immutable_options(); + auto exec = icf_opt->compaction_executor_factory.get(); + if (!exec || exec->ShouldRunLocal(compact_->compaction)) { + return RunLocal(); + } + Status s = RunRemote(); + if (!s.ok()) { + if (exec->AllowFallbackToLocal()) { + s = RunLocal(); + } else { + // fatal, rocksdb does not handle compact errors properly + } + } + return s; +} + +Status CompactionJob::RunLocal() { AutoThreadOperationStageUpdater stage_updater( ThreadStatus::STAGE_COMPACTION_RUN); TEST_SYNC_POINT("CompactionJob::Run():Start"); @@ -646,6 +685,26 @@ Status CompactionJob::Run() { for (auto& thread : thread_pool) { thread.join(); } + auto GetPath = [this]() { + size_t pathId = compact_->compaction->output_path_id(); + auto& paths = compact_->compaction->immutable_options()->cf_paths; + return paths[std::min(paths.size()-1, pathId)].path.c_str(); + }; + for (const auto& state : compact_->sub_compact_states) { + std::string filelist; + long long size = 0; + for (const auto& output : state.GetOutputs()) { + auto& fd = output.meta.fd; + char buf[32]; + auto len = sprintf(buf, "%06lld,", (long long)fd.GetNumber()); + filelist.append(buf, len); + size += fd.file_size; + } + if (!filelist.empty()) filelist.pop_back(); + ROCKS_LOG_INFO(db_options_.info_log, + "job-%05d: subcompact[%d], size: %.6f G, files: %s [%s]", + job_id_, state.sub_job_id, size/1e9, GetPath(), filelist.c_str()); + } compaction_stats_.SetMicros(db_options_.clock->NowMicros() - start_micros); @@ -654,8 +713,28 @@ Status CompactionJob::Run() { state.RemoveLastEmptyOutput(); } - RecordTimeToHistogram(stats_, COMPACTION_TIME, - compaction_stats_.stats.micros); + for (size_t i = 0; i < compact_->sub_compact_states.size(); i++) { + auto& sub = compact_->sub_compact_states[i]; + for (size_t j = 0; j < sub.outputs.size(); ++j) { + auto& meta = sub.outputs[j].meta; + auto raw = meta.raw_key_size + meta.raw_value_size; + auto zip = meta.fd.file_size; + RecordTick(stats_, LCOMPACT_WRITE_BYTES_RAW, raw); + RecordTimeToHistogram(stats_, LCOMPACTION_OUTPUT_FILE_RAW_SIZE, raw); + RecordTimeToHistogram(stats_, LCOMPACTION_OUTPUT_FILE_ZIP_SIZE, zip); + } + } + uint64_t sum_raw = 0, sum_zip = 0; + for (auto& each_level : *compact_->compaction->inputs()) { + for (FileMetaData* fmd : each_level.files) { + sum_raw += fmd->raw_key_size + fmd->raw_value_size; + sum_zip += fmd->fd.file_size; + } + } + RecordTimeToHistogram(stats_, LCOMPACTION_INPUT_RAW_BYTES, sum_raw); + RecordTimeToHistogram(stats_, LCOMPACTION_INPUT_ZIP_BYTES, sum_zip); + + RecordTimeToHistogram(stats_, COMPACTION_TIME, compaction_stats_.stats.micros); RecordTimeToHistogram(stats_, COMPACTION_CPU_TIME, compaction_stats_.stats.cpu_micros); @@ -752,6 +831,8 @@ Status CompactionJob::Run() { OutputValidator validator(cfd->internal_comparator(), /*_enable_order_check=*/true, /*_enable_hash=*/true); + auto& fd = files_output[file_idx]->meta.fd; + validator.m_file_number = fd.GetNumber(); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { s = validator.Add(iter->key(), iter->value()); if (!s.ok()) { @@ -763,7 +844,13 @@ Status CompactionJob::Run() { } if (s.ok() && !validator.CompareValidator(files_output[file_idx]->validator)) { - s = Status::Corruption("Paranoid checksums do not match"); + #if !defined(ROCKSDB_UNIT_TEST) + ROCKSDB_DIE("Compact: Paranoid checksums do not match(%s/%lld.sst)", + compact_->compaction->output_path().path.c_str(), + (long long)fd.GetNumber()); + #else + s = Status::Corruption("Compact: Paranoid checksums do not match"); + #endif } } @@ -850,9 +937,269 @@ Status CompactionJob::Run() { return status; } +void CompactionJob::GetSubCompactOutputs( + std::vector >* outputs) const { + outputs->clear(); + outputs->reserve(compact_->sub_compact_states.size()); + for (const auto& state : compact_->sub_compact_states) { + outputs->emplace_back(); + auto& cur_sub = outputs->back(); + for (const auto& output : state.outputs) { + cur_sub.push_back(&output.meta); + } + } +} + +Status CompactionJob::RunRemote() +try { + ROCKSDB_VERIFY_F(nullptr == snapshot_checker_, + "dcompact does not support snapshot_checker, ex: WritePreparedTxnDB " + "and WriteUnpreparedTxnDB are not supported because they use " + "WritePreparedSnapshotChecker" + ); + + AutoThreadOperationStageUpdater stage_updater( + ThreadStatus::STAGE_COMPACTION_RUN); + TEST_SYNC_POINT("CompactionJob::RunRemote():Start"); + log_buffer_->FlushBufferToLog(); + LogCompaction(); + + size_t num_threads = compact_->sub_compact_states.size(); + assert(num_threads > 0); + const Compaction* c = compact_->compaction; + ColumnFamilyData* cfd = c->column_family_data(); + auto imm_cfo = c->immutable_options(); + auto mut_cfo = c->mutable_cf_options(); + + // if with compaction filter, always use compaction filter factory + assert(nullptr == imm_cfo->compaction_filter); + CompactionParams rpc_params; + CompactionResults rpc_results; + + rpc_results.status = Status::Incomplete("Just Created"); + rpc_params.job_id = job_id_; + rpc_params.version_set.From(versions_); + #if (ROCKSDB_MAJOR * 10000 + ROCKSDB_MINOR * 10 + ROCKSDB_PATCH) < 70030 + rpc_params.preserve_deletes_seqnum = preserve_deletes_seqnum_; + #endif + rpc_params.existing_snapshots = &existing_snapshots_; + rpc_params.earliest_write_conflict_snapshot = earliest_write_conflict_snapshot_; + rpc_params.paranoid_file_checks = paranoid_file_checks_; + rpc_params.dbname = this->dbname_; + rpc_params.db_id = this->db_id_; + rpc_params.db_session_id = this->db_session_id_; + rpc_params.full_history_ts_low = this->full_history_ts_low_; +//rpc_params.compaction_job_stats = this->compaction_job_stats_; +//rpc_params.max_subcompactions = uint32_t(num_threads); + rpc_params.max_subcompactions = c->max_subcompactions(); + rpc_params.shutting_down = this->shutting_down_; + + const uint64_t start_micros = env_->NowMicros(); + auto exec_factory = imm_cfo->compaction_executor_factory.get(); + assert(nullptr != exec_factory); + auto exec = exec_factory->NewExecutor(c); + std::unique_ptr exec_auto_del(exec); + exec->SetParams(&rpc_params, c); + Status s = exec->Execute(rpc_params, &rpc_results); + if (!s.ok()) { + compact_->status = s; + return s; + } + if (!rpc_results.status.ok()) { + compact_->status = rpc_results.status; + return rpc_results.status; + } + //exec->NotifyResults(&rpc_results, c); + + // remote compact fabricates a version_set, which may cause + // GenSubcompactionBoundaries yield different num of sub_compact_states, + // thus makes the following assert fail: + //assert(rpc_results.output_files.size() == num_threads); // can be diff + + const uint64_t elapsed_us = env_->NowMicros() - start_micros; + compaction_stats_.stats = rpc_results.compaction_stats; + *compaction_job_stats_ = rpc_results.job_stats; + + // remote statistics will be merged to stat_ later: stats_->Merge(..) + //RecordTimeToHistogram(stats_, COMPACTION_TIME, compaction_stats_.micros); + //RecordTimeToHistogram(stats_, COMPACTION_CPU_TIME, compaction_stats_.cpu_micros); + + TablePropertiesCollection tp_map; + auto& cf_paths = imm_cfo->cf_paths; + compact_->num_output_files = 0; + + if (rpc_results.output_files.size() != num_threads) { + size_t result_sub_num = rpc_results.output_files.size(); + // this will happen, but is rare, log it + ROCKS_LOG_INFO(db_options_.info_log, + "job-%05d: subcompact num diff: rpc = %zd, local = %zd", + job_id_, result_sub_num, num_threads); + num_threads = result_sub_num; + auto& sub_vec = compact_->sub_compact_states; + while (sub_vec.size() < result_sub_num) { + int sub_job_id = (int)sub_vec.size(); + sub_vec.emplace_back(compact_->compaction, nullptr, nullptr, sub_job_id); + } + while (sub_vec.size() > result_sub_num) { + sub_vec.pop_back(); + } + } + + long long rename_t0 = env_->NowMicros(); + size_t out_raw_bytes = 0; + uint64_t epoch_number = c->MinInputFileEpochNumber(); + for (size_t i = 0; i < num_threads; ++i) { + auto& sub_state = compact_->sub_compact_states[i]; + for (const auto& min_meta : rpc_results.output_files[i]) { + auto old_fnum = min_meta.file_number; + auto old_fname = MakeTableFileName(rpc_results.output_dir, old_fnum); + auto path_id = c->output_path_id(); + uint64_t file_number = versions_->NewFileNumber(); + std::string new_fname = TableFileName(cf_paths, file_number, path_id); + Status st = exec->RenameFile(old_fname, new_fname, min_meta.file_size); + if (!st.ok()) { + ROCKS_LOG_ERROR(db_options_.info_log, "rename(%s, %s) = %s", + old_fname.c_str(), new_fname.c_str(), st.ToString().c_str()); + compact_->status = st; + return st; + } + FileDescriptor fd(file_number, path_id, min_meta.file_size, + min_meta.smallest_seqno, min_meta.largest_seqno); + FileMetaData meta; + meta.fd = fd; + TableCache* tc = cfd->table_cache(); + TableCache::TypedHandle* ch = nullptr; + auto& icmp = cfd->internal_comparator(); + auto& fopt = *cfd->soptions(); // file_options + auto& pref_ext = mut_cfo->prefix_extractor; + st = tc->FindTable(ReadOptions(), fopt, icmp, meta, &ch, + mut_cfo->block_protection_bytes_per_key, pref_ext); + if (!st.ok()) { + compact_->status = st; + return st; + } + assert(nullptr != ch); + TableReader* tr = tc->GetTableReaderFromHandle(ch); + auto tp = tr->GetTableProperties(); + tp_map[new_fname] = tp; + out_raw_bytes += tp->raw_key_size + tp->raw_value_size; + tc->ReleaseHandle(ch); // end use of TableReader in handle + meta.smallest = min_meta.smallest_ikey; + meta.largest = min_meta.largest_ikey; + meta.num_deletions = tp->num_deletions; + meta.num_entries = tp->num_entries; + meta.raw_key_size = tp->raw_key_size; + meta.raw_value_size = tp->raw_value_size; + meta.marked_for_compaction = min_meta.marked_for_compaction; + meta.epoch_number = epoch_number; + bool enable_order_check = mut_cfo->check_flush_compaction_key_order; + bool enable_hash = paranoid_file_checks_; + uint64_t precalculated_hash = 0; + sub_state.outputs.emplace_back(std::move(meta), icmp, + enable_order_check, enable_hash, true, precalculated_hash); + sub_state.total_bytes += min_meta.file_size; + sub_state.num_output_records += tp->num_entries; + rpc_results.output_index_size += tp->index_size; + rpc_results.output_data_size += tp->data_size; + } + // instead AggregateStatistics: + compact_->num_output_files += sub_state.outputs.size(); + compact_->total_bytes += sub_state.total_bytes; + compact_->num_output_records += sub_state.num_output_records; + } + compact_->compaction->SetOutputTableProperties(std::move(tp_map)); + long long rename_t1 = env_->NowMicros(); + + { + Compaction::InputLevelSummaryBuffer inputs_summary; // NOLINT + double work_time_us = rpc_results.work_time_usec; + if (work_time_us <= 1) work_time_us = 1; + ROCKS_LOG_INFO(db_options_.info_log, + "[%s] [JOB %d] Dcompacted %s [%zd] => time sec: " + "curl = %6.3f, mount = %6.3f, prepare = %6.3f, " + "wait = %6.3f, work = %6.3f, e2e = %6.3f, rename = %6.3f, " + "out zip = %9.6f GB %8.3f MB/sec, " + "out raw = %9.6f GB %8.3f MB/sec", + c->column_family_data()->GetName().c_str(), job_id_, + c->InputLevelSummary(&inputs_summary), compact_->num_output_files, + rpc_results.curl_time_usec/1e6, + rpc_results.mount_time_usec/1e6, + rpc_results.prepare_time_usec/1e6, + (elapsed_us - work_time_us)/1e6, // wait is non-work + work_time_us/1e6, elapsed_us/1e6, (rename_t1 - rename_t0)/1e9, + compact_->total_bytes/1e9, compact_->total_bytes/work_time_us, + out_raw_bytes/1e9, out_raw_bytes/work_time_us); + } + // Finish up all book-keeping to unify the subcompaction results + // these were run on remote compaction worker node + //AggregateStatistics(); + //UpdateCompactionStats(); + //compaction_job_stats_->Add(rpc_results.job_stats); // instead AggregateStatistics + + //RecordCompactionIOStats(); // update remote statistics to local -->> +#if defined(__GNUC__) && !defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif +#define MoveHG(dst,src) \ + memcpy(&rpc_results.statistics.histograms[dst], \ + &rpc_results.statistics.histograms[src], \ + sizeof rpc_results.statistics.histograms[src]), \ + rpc_results.statistics.histograms[src].Clear() + MoveHG(DCOMPACTION_INPUT_RAW_BYTES, LCOMPACTION_INPUT_RAW_BYTES); + MoveHG(DCOMPACTION_INPUT_ZIP_BYTES, LCOMPACTION_INPUT_ZIP_BYTES); + MoveHG(DCOMPACTION_OUTPUT_FILE_RAW_SIZE, LCOMPACTION_OUTPUT_FILE_RAW_SIZE); + MoveHG(DCOMPACTION_OUTPUT_FILE_ZIP_SIZE, LCOMPACTION_OUTPUT_FILE_ZIP_SIZE); +#if defined(__GNUC__) && !defined(__clang__) + #pragma GCC diagnostic pop +#endif + +#define MoveTK(dst, src) \ + rpc_results.statistics.tickers[dst] = rpc_results.statistics.tickers[src]; \ + rpc_results.statistics.tickers[src] = 0 + + MoveTK(DCOMPACT_WRITE_BYTES_RAW, LCOMPACT_WRITE_BYTES_RAW); + MoveTK(REMOTE_COMPACT_READ_BYTES, COMPACT_READ_BYTES); + MoveTK(REMOTE_COMPACT_WRITE_BYTES, COMPACT_WRITE_BYTES); + + stats_->Merge(rpc_results.statistics.tickers, + rpc_results.statistics.histograms); + + LogFlush(db_options_.info_log); + TEST_SYNC_POINT("CompactionJob::RunRemote():End"); + + exec->CleanFiles(rpc_params, rpc_results); + + compact_->status = Status::OK(); + return Status::OK(); +} +catch (const std::exception& ex) { + compact_->status = Status::Corruption(ROCKSDB_FUNC, ex.what()); + return compact_->status; +} +catch (const Status& s) { + compact_->status = s; + return s; +} + Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options, bool* compaction_released) { assert(compact_); +#if 0 + // this fails unit test: + // DBCompactionTestBlobError/DBCompactionTestBlobError.CompactionError/1 + // and does not help for error checking + if (!compact_->status.ok()) { // caller does not check retval of Run() + ColumnFamilyData* cfd = compact_->compaction->column_family_data(); + assert(cfd); + ROCKS_LOG_BUFFER(log_buffer_, "[%s] compaction failed, job_id = %d : %s", + cfd->GetName().c_str(), job_id_, + compact_->status.ToString().c_str()); + Status s = compact_->status; + CleanupCompaction(); + return s; + } +#endif AutoThreadOperationStageUpdater stage_updater( ThreadStatus::STAGE_COMPACTION_INSTALL); @@ -952,6 +1299,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options, UpdateCompactionJobStats(stats); auto stream = event_logger_->LogToBuffer(log_buffer_, 8192); + stream << "cf" << cfd->GetName(); stream << "job" << job_id_ << "event" << "compaction_finished" << "compaction_time_micros" << stats.micros @@ -1196,7 +1544,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) { std::unique_ptr clip; if (start.has_value() || end.has_value()) { - clip = std::make_unique( + clip = MakeClippingIterator( raw_input.get(), start.has_value() ? &start_slice : nullptr, end.has_value() ? &end_slice : nullptr, &cfd->internal_comparator()); input = clip.get(); @@ -1621,6 +1969,10 @@ Status CompactionJob::FinishCompactionOutputFile( TableProperties tp; if (s.ok()) { tp = outputs.GetTableProperties(); + meta->num_entries = tp.num_entries; + meta->num_deletions = tp.num_deletions; + meta->raw_key_size = tp.raw_key_size; + meta->raw_value_size = tp.raw_value_size; } if (s.ok() && current_entries == 0 && tp.num_range_deletions == 0) { @@ -1773,6 +2125,7 @@ Status CompactionJob::InstallCompactionResults( stats.GetBytes()); } +#if defined(ROCKSDB_UNIT_TEST) if ((compaction->compaction_reason() == CompactionReason::kLevelMaxLevelSize || compaction->compaction_reason() == CompactionReason::kRoundRobinTtl) && @@ -1785,6 +2138,7 @@ Status CompactionJob::InstallCompactionResults( start_level, compaction->num_input_files(0))); } } +#endif auto manifest_wcb = [&compaction, &compaction_released](const Status& s) { compaction->ReleaseCompactionFiles(s); @@ -2077,15 +2431,17 @@ void CompactionJob::LogCompaction() { if (db_options_.info_log_level <= InfoLogLevel::INFO_LEVEL) { Compaction::InputLevelSummaryBuffer inputs_summary; ROCKS_LOG_INFO( - db_options_.info_log, "[%s] [JOB %d] Compacting %s, score %.2f", + db_options_.info_log, + "[%s] [JOB %d] Compacting %s, score %.2f, subcompactions %d : %zd", cfd->GetName().c_str(), job_id_, - compaction->InputLevelSummary(&inputs_summary), compaction->score()); + compaction->InputLevelSummary(&inputs_summary), compaction->score(), + compaction->max_subcompactions(), compact_->sub_compact_states.size()); char scratch[2345]; compaction->Summary(scratch, sizeof(scratch)); ROCKS_LOG_INFO(db_options_.info_log, "[%s]: Compaction start summary: %s\n", cfd->GetName().c_str(), scratch); // build event logger report - auto stream = event_logger_->Log(); + auto stream = event_logger_->LogToBuffer(log_buffer_, 64*1024); stream << "job" << job_id_ << "event" << "compaction_started" << "compaction_reason" diff --git a/db/compaction/compaction_job.h b/db/compaction/compaction_job.h index e812cfc72a..acaf657525 100644 --- a/db/compaction/compaction_job.h +++ b/db/compaction/compaction_job.h @@ -194,6 +194,10 @@ class CompactionJob { // Return the IO status IOStatus io_status() const { return io_status_; } + void GetSubCompactOutputs(std::vector >*) const; + CompactionJobStats* GetCompactionJobStats() const { return compaction_job_stats_; } + const InternalStats::CompactionStatsFull& GetCompactionStats() const { return compaction_stats_; } + protected: // Update the following stats in compaction_stats_.stats // - num_input_files_in_non_output_levels @@ -289,6 +293,9 @@ class CompactionJob { void NotifyOnSubcompactionCompleted(SubcompactionState* sub_compact); + Status RunLocal(); + Status RunRemote(); + uint32_t job_id_; // DBImpl state @@ -362,6 +369,8 @@ class CompactionJob { // the last level (output to penultimate level). SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber; + std::vector > rand_key_store_; + // Get table file name in where it's outputting to, which should also be in // `output_directory_`. virtual std::string GetTableFileName(uint64_t file_number); diff --git a/db/compaction/compaction_outputs.cc b/db/compaction/compaction_outputs.cc index eb76cd849a..62949191a4 100644 --- a/db/compaction/compaction_outputs.cc +++ b/db/compaction/compaction_outputs.cc @@ -121,20 +121,29 @@ bool CompactionOutputs::UpdateFilesToCutForTTLStates( return false; } -size_t CompactionOutputs::UpdateGrandparentBoundaryInfo( - const Slice& internal_key) { - size_t curr_key_boundary_switched_num = 0; - const std::vector& grandparents = compaction_->grandparents(); - - if (grandparents.empty()) { - return curr_key_boundary_switched_num; +ROCKSDB_FLATTEN +size_t CompactionOutputs::UpdateGrandparentBoundaryInfo(const Slice& ikey) { + if (0 == grandparents_size_) { + return 0; } - const Comparator* ucmp = compaction_->column_family_data()->user_comparator(); + if (cmp_meta_.IsForwardBytewise()) + return UpdateGrandparentBoundaryInfoTmpl(ForwardBytewiseCompareUserKeyNoTS(), ikey); + if (cmp_meta_.IsReverseBytewise()) + return UpdateGrandparentBoundaryInfoTmpl(ReverseBytewiseCompareUserKeyNoTS(), ikey); + else + return UpdateGrandparentBoundaryInfoTmpl(VirtualFunctionCompareUserKeyNoTS + {compaction_->immutable_options()->user_comparator}, ikey); +} +template +size_t CompactionOutputs::UpdateGrandparentBoundaryInfoTmpl(UKCmpNoTS ucmp, const Slice& ikey) { + size_t curr_key_boundary_switched_num = 0; + const auto grandparents = grandparents_data_; + const auto grandparents_size = grandparents_size_; // Move the grandparent_index_ to the file containing the current user_key. // If there are multiple files containing the same user_key, make sure the // index points to the last file containing the key. - while (grandparent_index_ < grandparents.size()) { + while (grandparent_index_ < grandparents_size) { if (being_grandparent_gap_) { if (sstableKeyCompare(ucmp, internal_key, grandparents[grandparent_index_]->smallest) < 0) { @@ -154,8 +163,8 @@ size_t CompactionOutputs::UpdateGrandparentBoundaryInfo( // one. if (cmp_result < 0 || (cmp_result == 0 && - (grandparent_index_ == grandparents.size() - 1 || - sstableKeyCompare(ucmp, internal_key, + (grandparent_index_ == grandparents_size - 1 || + sstableKeyCompare(ucmp, ikey, grandparents[grandparent_index_ + 1]->smallest) < 0))) { break; @@ -174,7 +183,7 @@ size_t CompactionOutputs::UpdateGrandparentBoundaryInfo( if (!seen_key_ && !being_grandparent_gap_) { assert(grandparent_overlapped_bytes_ == 0); grandparent_overlapped_bytes_ = - GetCurrentKeyGrandparentOverlappedBytes(internal_key); + GetCurrentKeyGrandparentOverlappedBytes(ikey); } seen_key_ = true; @@ -189,7 +198,7 @@ uint64_t CompactionOutputs::GetCurrentKeyGrandparentOverlappedBytes( } uint64_t overlapped_bytes = 0; - const std::vector& grandparents = compaction_->grandparents(); + const auto grandparents = grandparents_data_; const Comparator* ucmp = compaction_->column_family_data()->user_comparator(); InternalKey ikey; ikey.DecodeFrom(internal_key); @@ -201,7 +210,7 @@ uint64_t CompactionOutputs::GetCurrentKeyGrandparentOverlappedBytes( assert( cmp_result < 0 || (cmp_result == 0 && - (grandparent_index_ == grandparents.size() - 1 || + (grandparent_index_ == grandparents_size_ - 1 || sstableKeyCompare( ucmp, ikey, grandparents[grandparent_index_ + 1]->smallest) < 0))); assert(sstableKeyCompare(ucmp, ikey, @@ -236,15 +245,13 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) { } #endif // NDEBUG const uint64_t previous_overlapped_bytes = grandparent_overlapped_bytes_; - const InternalKeyComparator* icmp = - &compaction_->column_family_data()->internal_comparator(); size_t num_grandparent_boundaries_crossed = 0; bool should_stop_for_ttl = false; // Always update grandparent information like overlapped file number, size // etc., and TTL states. // If compaction_->output_level() == 0, there is no need to update grandparent // info, and that `grandparent` should be empty. - if (compaction_->output_level() > 0) { + if (output_level_ > 0) { num_grandparent_boundaries_crossed = UpdateGrandparentBoundaryInfo(internal_key); should_stop_for_ttl = UpdateFilesToCutForTTLStates(internal_key); @@ -260,24 +267,25 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) { // If there's user defined partitioner, check that first if (partitioner_ && partitioner_->ShouldPartition(PartitionerRequest( - last_key_for_partitioner_, c_iter.user_key(), + SliceOf(last_key_for_partitioner_), c_iter.user_key(), current_output_file_size_)) == kRequired) { return true; } // files output to Level 0 won't be split - if (compaction_->output_level() == 0) { + if (output_level_ == 0) { return false; } // reach the max file size - if (current_output_file_size_ >= compaction_->max_output_file_size()) { + if (current_output_file_size_ >= max_output_file_size_) { return true; } // Check if it needs to split for RoundRobin // Invalid local_output_split_key indicates that we do not need to split if (local_output_split_key_ != nullptr && !is_split_) { + auto icmp = &compaction_->immutable_options()->internal_comparator; // Split occurs when the next key is larger than/equal to the cursor if (icmp->Compare(internal_key, local_output_split_key_->Encode()) >= 0) { is_split_ = true; @@ -293,7 +301,7 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) { // max_compaction_bytes. Which is to prevent future bigger than // max_compaction_bytes compaction from the current output level. if (grandparent_overlapped_bytes_ + current_output_file_size_ > - compaction_->max_compaction_bytes()) { + max_compaction_bytes_) { return true; } @@ -315,13 +323,12 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) { // More details, check PR #1963 const size_t num_skippable_boundaries_crossed = being_grandparent_gap_ ? 2 : 3; - if (compaction_->immutable_options()->compaction_style == - kCompactionStyleLevel && - compaction_->immutable_options()->level_compaction_dynamic_file_size && + if (compaction_style_ == kCompactionStyleLevel && + level_compaction_dynamic_file_size_ && num_grandparent_boundaries_crossed >= num_skippable_boundaries_crossed && grandparent_overlapped_bytes_ - previous_overlapped_bytes > - compaction_->target_output_file_size() / 8) { + target_output_file_size_ / 8) { return true; } @@ -337,11 +344,10 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) { // target file size. The test shows it can generate larger files than a // static threshold like 75% and has a similar write amplification // improvement. - if (compaction_->immutable_options()->compaction_style == - kCompactionStyleLevel && - compaction_->immutable_options()->level_compaction_dynamic_file_size && + if (compaction_style_ == kCompactionStyleLevel && + level_compaction_dynamic_file_size_ && current_output_file_size_ >= - ((compaction_->target_output_file_size() + 99) / 100) * + ((target_output_file_size_ + 99) / 100) * (50 + std::min(grandparent_boundary_switched_num_ * 5, size_t{40}))) { return true; @@ -355,17 +361,16 @@ Status CompactionOutputs::AddToOutput( const CompactionIterator& c_iter, const CompactionFileOpenFunc& open_file_func, const CompactionFileCloseFunc& close_file_func) { - Status s; bool is_range_del = c_iter.IsDeleteRangeSentinelKey(); if (is_range_del && compaction_->bottommost_level()) { // We don't consider range tombstone for bottommost level since: // 1. there is no grandparent and hence no overlap to consider // 2. range tombstone may be dropped at bottommost level. - return s; + return Status::OK(); } const Slice& key = c_iter.key(); if (ShouldStopBefore(c_iter) && HasBuilder()) { - s = close_file_func(*this, c_iter.InputStatus(), key); + Status s = close_file_func(*this, c_iter.InputStatus(), key); if (!s.ok()) { return s; } @@ -384,7 +389,7 @@ Status CompactionOutputs::AddToOutput( // Open output file if necessary if (!HasBuilder()) { - s = open_file_func(*this); + Status s = open_file_func(*this); if (!s.ok()) { return s; } @@ -398,13 +403,12 @@ Status CompactionOutputs::AddToOutput( } if (UNLIKELY(is_range_del)) { - return s; + return Status::OK(); } assert(builder_ != nullptr); const Slice& value = c_iter.value(); - s = current_output().validator.Add(key, value); - if (!s.ok()) { + if (Status s = current_output().validator.Add(key, value); !s.ok()) { return s; } builder_->Add(key, value); @@ -413,15 +417,14 @@ Status CompactionOutputs::AddToOutput( current_output_file_size_ = builder_->EstimatedFileSize(); if (blob_garbage_meter_) { - s = blob_garbage_meter_->ProcessOutFlow(key, value); - } - - if (!s.ok()) { - return s; + Status s = blob_garbage_meter_->ProcessOutFlow(key, value); + if (!s.ok()) { + return s; + } } const ParsedInternalKey& ikey = c_iter.ikey(); - s = current_output().meta.UpdateBoundaries(key, value, ikey.sequence, + Status s = current_output().meta.UpdateBoundaries(key, value, ikey.sequence, ikey.type); return s; @@ -780,6 +783,17 @@ void CompactionOutputs::FillFilesToCutForTtl() { CompactionOutputs::CompactionOutputs(const Compaction* compaction, const bool is_penultimate_level) : compaction_(compaction), is_penultimate_level_(is_penultimate_level) { + auto& io = *compaction->immutable_options(); + cmp_meta_ = *io.user_comparator; + compaction_style_ = io.compaction_style; + level_compaction_dynamic_file_size_ = io.level_compaction_dynamic_file_size; + output_level_ = compaction->output_level(); + max_compaction_bytes_ = compaction->max_compaction_bytes(); + max_output_file_size_ = compaction->max_output_file_size(); + target_output_file_size_ = compaction->target_output_file_size(); + grandparents_data_ = compaction->grandparents().data(); + grandparents_size_ = compaction->grandparents().size(); + partitioner_ = compaction->output_level() == 0 ? nullptr : compaction->CreateSstPartitioner(); diff --git a/db/compaction/compaction_outputs.h b/db/compaction/compaction_outputs.h index 18246cf2fa..560ef95cdd 100644 --- a/db/compaction/compaction_outputs.h +++ b/db/compaction/compaction_outputs.h @@ -15,6 +15,7 @@ #include "db/compaction/compaction_iterator.h" #include "db/internal_stats.h" #include "db/output_validator.h" +#include namespace ROCKSDB_NAMESPACE { @@ -190,6 +191,8 @@ class CompactionOutputs { return range_del_agg_ && !range_del_agg_->IsEmpty(); } + std::vector& GetOutputs() { return outputs_; } + private: friend class SubcompactionState; @@ -240,6 +243,9 @@ class CompactionOutputs { // It returns how many boundaries it crosses by including current key. size_t UpdateGrandparentBoundaryInfo(const Slice& internal_key); + template + size_t UpdateGrandparentBoundaryInfoTmpl(UKCmpNoTS ucmp, const Slice& internal_key); + // helper function to get the overlapped grandparent files size, it's only // used for calculating the first key's overlap. uint64_t GetCurrentKeyGrandparentOverlappedBytes( @@ -310,19 +316,39 @@ class CompactionOutputs { // Basic compaction output stats for this level's outputs InternalStats::CompactionOutputsStats stats_; + // indicate if this CompactionOutputs obj for penultimate_level, should always // be false if per_key_placement feature is not enabled. const bool is_penultimate_level_; - std::unique_ptr range_del_agg_ = nullptr; - - // partitioner information - std::string last_key_for_partitioner_; - std::unique_ptr partitioner_; // A flag determines if this subcompaction has been split by the cursor // for RoundRobin compaction bool is_split_ = false; + // if the output key is being grandparent files gap, so: + // key > grandparents[grandparent_index_ - 1].largest && + // key < grandparents[grandparent_index_].smallest + bool being_grandparent_gap_ = true; + + // A flag determines whether the key has been seen in ShouldStopBefore() + bool seen_key_ = false; + + ComparatorMetaData cmp_meta_; + CompactionStyle compaction_style_; + bool level_compaction_dynamic_file_size_; + int output_level_; + uint64_t max_compaction_bytes_; + uint64_t max_output_file_size_; + uint64_t target_output_file_size_; + FileMetaData* const * grandparents_data_; + size_t grandparents_size_; + + std::unique_ptr range_del_agg_ = nullptr; + + // partitioner information + terark::valvec32 last_key_for_partitioner_; + std::unique_ptr partitioner_; + // We also maintain the output split key for each subcompaction to avoid // repetitive comparison in ShouldStopBefore() const InternalKey* local_output_split_key_ = nullptr; @@ -337,18 +363,10 @@ class CompactionOutputs { // An index that used to speed up ShouldStopBefore(). size_t grandparent_index_ = 0; - // if the output key is being grandparent files gap, so: - // key > grandparents[grandparent_index_ - 1].largest && - // key < grandparents[grandparent_index_].smallest - bool being_grandparent_gap_ = true; - // The number of bytes overlapping between the current output and // grandparent files used in ShouldStopBefore(). uint64_t grandparent_overlapped_bytes_ = 0; - // A flag determines whether the key has been seen in ShouldStopBefore() - bool seen_key_ = false; - // for the current output file, how many file boundaries has it crossed, // basically number of files overlapped * 2 size_t grandparent_boundary_switched_num_ = 0; diff --git a/db/compaction/compaction_picker.cc b/db/compaction/compaction_picker.cc index 4d40ab5034..f3525f194f 100644 --- a/db/compaction/compaction_picker.cc +++ b/db/compaction/compaction_picker.cc @@ -46,10 +46,9 @@ bool FindIntraL0Compaction(const std::vector& level_files, size_t limit; // Pull in files until the amount of compaction work per deleted file begins // increasing or maximum total compaction size is reached. - size_t new_compact_bytes_per_del_file = 0; for (limit = start + 1; limit < level_files.size(); ++limit) { compact_bytes += static_cast(level_files[limit]->fd.file_size); - new_compact_bytes_per_del_file = compact_bytes / (limit - start); + size_t new_compact_bytes_per_del_file = compact_bytes / (limit - start); if (level_files[limit]->being_compacted || new_compact_bytes_per_del_file > compact_bytes_per_del_file || compact_bytes > max_compaction_bytes) { diff --git a/db/compaction/compaction_picker_level.cc b/db/compaction/compaction_picker_level.cc index c436689bb6..79d36afb77 100644 --- a/db/compaction/compaction_picker_level.cc +++ b/db/compaction/compaction_picker_level.cc @@ -444,6 +444,15 @@ bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() { return false; } + if (CompactionReason::kFilesMarkedForCompaction == compaction_reason_) { + const CompactionInputFiles* inputs[] = { + &start_level_inputs_, &output_level_inputs_, + }; + if (!ioptions_.table_factory->ShouldCompactMarkForCompaction(inputs, 2)) { + return false; + } + } + compaction_inputs_.push_back(start_level_inputs_); if (!output_level_inputs_.empty()) { compaction_inputs_.push_back(output_level_inputs_); @@ -858,6 +867,9 @@ bool LevelCompactionBuilder::PickFileToCompact() { } bool LevelCompactionBuilder::PickIntraL0Compaction() { + if (mutable_db_options_.max_level1_subcompactions > 1) { + return false; + } start_level_inputs_.clear(); const std::vector& level_files = vstorage_->LevelFiles(0 /* level */); diff --git a/db/compaction/compaction_state.h b/db/compaction/compaction_state.h index cc5b66c682..e2d3d16fe4 100644 --- a/db/compaction/compaction_state.h +++ b/db/compaction/compaction_state.h @@ -27,6 +27,11 @@ class CompactionState { // REQUIRED: subcompaction states are stored in order of increasing key-range std::vector sub_compact_states; Status status; + size_t num_output_files = 0; + uint64_t total_bytes = 0; + size_t num_blob_output_files = 0; + uint64_t total_blob_bytes = 0; + uint64_t num_output_records = 0; void AggregateCompactionStats( InternalStats::CompactionStatsFull& compaction_stats, diff --git a/db/compaction/sst_partitioner.cc b/db/compaction/sst_partitioner.cc index 2f4d879357..325dfdb2ca 100644 --- a/db/compaction/sst_partitioner.cc +++ b/db/compaction/sst_partitioner.cc @@ -27,11 +27,11 @@ SstPartitionerFixedPrefixFactory::SstPartitionerFixedPrefixFactory(size_t len) PartitionerResult SstPartitionerFixedPrefix::ShouldPartition( const PartitionerRequest& request) { - Slice last_key_fixed(*request.prev_user_key); + Slice last_key_fixed(request.prev_user_key); if (last_key_fixed.size() > len_) { last_key_fixed.size_ = len_; } - Slice current_key_fixed(*request.current_user_key); + Slice current_key_fixed(request.current_user_key); if (current_key_fixed.size() > len_) { current_key_fixed.size_ = len_; } diff --git a/db/compaction/subcompaction_state.h b/db/compaction/subcompaction_state.h index b933a62a51..bef663a9ec 100644 --- a/db/compaction/subcompaction_state.h +++ b/db/compaction/subcompaction_state.h @@ -215,6 +215,10 @@ class SubcompactionState { CompactionOutputs* current_outputs_ = &compaction_outputs_; bool is_current_penultimate_level_ = false; bool has_penultimate_level_outputs_ = false; +public: + std::vector& outputs = compaction_outputs_.GetOutputs(); + size_t total_bytes = 0; + size_t num_output_records = 0; }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/compaction/tiered_compaction_test.cc b/db/compaction/tiered_compaction_test.cc index 779b980d82..360b981f69 100644 --- a/db/compaction/tiered_compaction_test.cc +++ b/db/compaction/tiered_compaction_test.cc @@ -1943,13 +1943,13 @@ class ThreeRangesPartitioner : public SstPartitioner { PartitionerResult ShouldPartition( const PartitionerRequest& request) override { - if ((cmp->CompareWithoutTimestamp(*request.current_user_key, + if ((cmp->CompareWithoutTimestamp(request.current_user_key, DBTestBase::Key(20)) >= 0 && - cmp->CompareWithoutTimestamp(*request.prev_user_key, + cmp->CompareWithoutTimestamp(request.prev_user_key, DBTestBase::Key(20)) < 0) || - (cmp->CompareWithoutTimestamp(*request.current_user_key, + (cmp->CompareWithoutTimestamp(request.current_user_key, DBTestBase::Key(40)) >= 0 && - cmp->CompareWithoutTimestamp(*request.prev_user_key, + cmp->CompareWithoutTimestamp(request.prev_user_key, DBTestBase::Key(40)) < 0)) { return kRequired; } else { diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index 9bd5c11b6d..2f8a4b2ef7 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -51,7 +51,7 @@ T Pop(T& var) { return rv; } PerfContextByLevel& GetLevelPerfContext(uint32_t level) { - return (*(get_perf_context()->level_to_perf_context))[level]; + return ((get_perf_context()->level_to_perf_context))[level]; } } // anonymous namespace @@ -508,9 +508,9 @@ TEST_F(DBBloomFilterTest, WholeKeyFilterProp) { EXPECT_EQ(PopTicker(options, BLOOM_FILTER_PREFIX_USEFUL), 0); EXPECT_EQ(PopTicker(options, BLOOM_FILTER_USEFUL), 0); uint64_t bloom_filter_useful_all_levels = 0; - for (auto& kv : (*(get_perf_context()->level_to_perf_context))) { - if (kv.second.bloom_filter_useful > 0) { - bloom_filter_useful_all_levels += kv.second.bloom_filter_useful; + for (auto& perf : get_perf_context()->level_to_perf_context) { + if (perf.bloom_filter_useful > 0) { + bloom_filter_useful_all_levels += perf.bloom_filter_useful; } } ASSERT_EQ(12, bloom_filter_useful_all_levels); @@ -2582,9 +2582,9 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) { ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 65000 * 2); ASSERT_LT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 120000 * 2); uint64_t bloom_filter_useful_all_levels = 0; - for (auto& kv : (*(get_perf_context()->level_to_perf_context))) { - if (kv.second.bloom_filter_useful > 0) { - bloom_filter_useful_all_levels += kv.second.bloom_filter_useful; + for (auto& perf : get_perf_context()->level_to_perf_context) { + if (perf.bloom_filter_useful > 0) { + bloom_filter_useful_all_levels += perf.bloom_filter_useful; } } ASSERT_GT(bloom_filter_useful_all_levels, 65000 * 2); diff --git a/db/db_impl/compacted_db_impl.cc b/db/db_impl/compacted_db_impl.cc index 3b665ea26b..0a8157c869 100644 --- a/db/db_impl/compacted_db_impl.cc +++ b/db/db_impl/compacted_db_impl.cc @@ -189,8 +189,8 @@ std::vector CompactedDBImpl::MultiGet( int idx = 0; for (auto* r : reader_list) { if (r != nullptr) { - PinnableSlice pinnable_val; - std::string& value = (*values)[idx]; + PinnableSlice pinnable_val(&(*values)[idx]); + pinnable_val.GetSelf()->clear(); LookupKey lkey(keys[idx], kMaxSequenceNumber, read_options.timestamp); std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr; GetContext get_context( @@ -204,7 +204,7 @@ std::vector CompactedDBImpl::MultiGet( if (!s.ok() && !s.IsNotFound()) { statuses[idx] = s; } else { - value.assign(pinnable_val.data(), pinnable_val.size()); + pinnable_val.SyncToString(); if (get_context.State() == GetContext::kFound) { statuses[idx] = Status::OK(); } diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 297c6aceb7..cd10b57af1 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -108,6 +108,15 @@ #include "util/udt_util.h" #include "utilities/trace/replayer_impl.h" +#if defined(__clang__) + #pragma clang diagnostic ignored "-Wshorten-64-to-32" + #pragma clang diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include +#include +#include + namespace ROCKSDB_NAMESPACE { const std::string kDefaultColumnFamilyName("default"); @@ -115,6 +124,36 @@ const std::string kPersistentStatsColumnFamilyName( "___rocksdb_stats_history___"); void DumpRocksDBBuildVersion(Logger* log); +// ensure fiber thread locals are constructed first +// because FiberPool.m_channel must be destructed first +static ROCKSDB_STATIC_TLS thread_local terark::FiberPool gt_fiber_pool( + boost::fibers::context::active_pp()); +struct ToplingMGetCtx : protected MergeContext { + MergeContext& merge_context() { return *this; } + SequenceNumber max_covering_tombstone_seq = 0; + static constexpr uint32_t FLAG_done = 1; + static constexpr uint32_t FLAG_lkey_initialized = 2; + +#if defined(TOPLINGDB_WITH_TIMESTAMP) + std::string* timestamp = nullptr; +#endif + union { + LookupKey lkey; + }; + void InitLookupKey(const Slice& user_key, SequenceNumber seq, + const Slice* ts) { + new(&lkey)LookupKey(user_key, seq, ts); + this->ext_flags_ |= FLAG_lkey_initialized; + } + ToplingMGetCtx() {} + ~ToplingMGetCtx() { + if (this->ext_flags_ & FLAG_lkey_initialized) + lkey.~LookupKey(); + } + void set_done() { this->ext_flags_ |= FLAG_done; } + bool is_done() const { return (this->ext_flags_ & FLAG_done) != 0; } +}; + CompressionType GetCompressionFlush( const ImmutableCFOptions& ioptions, const MutableCFOptions& mutable_cf_options) { @@ -149,8 +188,72 @@ void DumpSupportInfo(Logger* logger) { ROCKS_LOG_HEADER(logger, "DMutex implementation: %s", DMutex::kName()); } + +// A structure to hold the information required to process MultiGet of keys +// belonging to one column family. For a multi column family MultiGet, there +// will be a container of these objects. +struct MultiGetColumnFamilyData { + ColumnFamilyHandle* cf; + ColumnFamilyData* cfd; + + // For the batched MultiGet which relies on sorted keys, start specifies + // the index of first key belonging to this column family in the sorted + // list. + size_t start; + + // For the batched MultiGet case, num_keys specifies the number of keys + // belonging to this column family in the sorted list + size_t num_keys; + + // SuperVersion for the column family obtained in a manner that ensures a + // consistent view across all column families in the DB + SuperVersion* super_version; + MultiGetColumnFamilyData(ColumnFamilyHandle* column_family, SuperVersion* sv) + : cf(column_family), + cfd(static_cast(cf)->cfd()), + start(0), + num_keys(0), + super_version(sv) {} + + MultiGetColumnFamilyData(ColumnFamilyHandle* column_family, size_t first, + size_t count, SuperVersion* sv) + : cf(column_family), + cfd(static_cast(cf)->cfd()), + start(first), + num_keys(count), + super_version(sv) {} + + MultiGetColumnFamilyData() = default; +}; + +template +static inline auto iter_deref_func(const Iter& i) + -> std::common_type_tsecond)> { + return &i->second; +} + +template +static inline auto iter_deref_func(const Iter& i) + -> std::common_type_t { + return &*i; +} + } // namespace +InstrumentedMutex* Get_DB_mutex(const DB* db) { + db = const_cast(db)->GetRootDB(); + auto dbi = dynamic_cast(db); + ROCKSDB_VERIFY(nullptr != dbi); + return dbi->mutex(); +} + +int Get_DB_next_job_id(const DB* db) { + db = const_cast(db)->GetRootDB(); + auto dbi = dynamic_cast(db); + ROCKSDB_VERIFY(nullptr != dbi); + return dbi->next_job_id(); +} + DBImpl::DBImpl(const DBOptions& options, const std::string& dbname, const bool seq_per_batch, const bool batch_per_txn, bool read_only) @@ -169,7 +272,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname, mutex_(stats_, immutable_db_options_.clock, DB_MUTEX_WAIT_MICROS, &bg_cv_, immutable_db_options_.use_adaptive_mutex), #else // COERCE_CONTEXT_SWITCH - mutex_(stats_, immutable_db_options_.clock, DB_MUTEX_WAIT_MICROS, + mutex_(stats_, immutable_db_options_.clock, immutable_db_options_.use_adaptive_mutex), #endif // COERCE_CONTEXT_SWITCH default_cf_handle_(nullptr), @@ -1283,9 +1386,11 @@ Status DBImpl::SetDBOptions( s = GetMutableDBOptionsFromStrings(mutable_db_options_, options_map, &new_options); +#ifdef ROCKSDB_UNIT_TEST // the document says bytes_per_sync == 0 means turn off if (new_options.bytes_per_sync == 0) { new_options.bytes_per_sync = 1024 * 1024; } +#endif if (MutableDBOptionsAreEqual(mutable_db_options_, new_options)) { ROCKS_LOG_INFO(immutable_db_options_.info_log, @@ -2159,6 +2264,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, assert(get_impl_options.column_family); +#if defined(TOPLINGDB_WITH_TIMESTAMP) if (read_options.timestamp) { const Status s = FailIfTsMismatchCf(get_impl_options.column_family, *(read_options.timestamp)); @@ -2179,6 +2285,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, } GetWithTimestampReadCallback read_cb(0); // Will call Refresh +#endif PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); StopWatch sw(immutable_db_options_.clock, stats_, DB_GET); @@ -2188,7 +2295,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, get_impl_options.column_family); auto cfd = cfh->cfd(); - if (tracer_) { + if (UNLIKELY(tracer_ != nullptr)) { // TODO: This mutex should be removed later, to improve performance when // tracing is enabled. InstrumentedMutexLock lock(&trace_mutex_); @@ -2198,7 +2305,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, } } - if (get_impl_options.get_merge_operands_options != nullptr) { + if (UNLIKELY(get_impl_options.get_merge_operands_options != nullptr)) { for (int i = 0; i < get_impl_options.get_merge_operands_options ->expected_max_number_of_operands; ++i) { @@ -2207,7 +2314,8 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, } // Acquire SuperVersion - SuperVersion* sv = GetAndRefSuperVersion(cfd); + SuperVersion* sv = GetAndRefSuperVersion(cfd, &read_options); +#if defined(TOPLINGDB_WITH_TIMESTAMP) if (read_options.timestamp && read_options.timestamp->size() > 0) { const Status s = FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp)); @@ -2216,6 +2324,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, return s; } } +#endif TEST_SYNC_POINT_CALLBACK("DBImpl::GetImpl:AfterAcquireSv", nullptr); TEST_SYNC_POINT("DBImpl::GetImpl:1"); @@ -2256,6 +2365,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, snapshot = get_impl_options.callback->max_visible_seq(); } } +#if defined(TOPLINGDB_WITH_TIMESTAMP) // If timestamp is used, we use read callback to ensure is returned // only if t <= read_opts.timestamp and s <= snapshot. // HACK: temporarily overwrite input struct field but restore @@ -2268,6 +2378,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, read_cb.Refresh(snapshot); get_impl_options.callback = &read_cb; } +#endif TEST_SYNC_POINT("DBImpl::GetImpl:3"); TEST_SYNC_POINT("DBImpl::GetImpl:4"); @@ -2286,46 +2397,41 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, has_unpersisted_data_.load(std::memory_order_relaxed)); bool done = false; std::string* timestamp = +#if defined(TOPLINGDB_WITH_TIMESTAMP) ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr; +#else + nullptr; +#endif if (!skip_memtable) { // Get value associated with key if (get_impl_options.get_value) { - if (sv->mem->Get( + if (!sv->mem->IsEmpty() && sv->mem->Get( lkey, - get_impl_options.value ? get_impl_options.value->GetSelf() - : nullptr, + get_impl_options.value, get_impl_options.columns, timestamp, &s, &merge_context, &max_covering_tombstone_seq, read_options, false /* immutable_memtable */, get_impl_options.callback, get_impl_options.is_blob_index)) { done = true; - if (get_impl_options.value) { - get_impl_options.value->PinSelf(); - } - RecordTick(stats_, MEMTABLE_HIT); } else if ((s.ok() || s.IsMergeInProgress()) && + !sv->imm->IsEmpty() && sv->imm->Get(lkey, - get_impl_options.value - ? get_impl_options.value->GetSelf() - : nullptr, + get_impl_options.value, get_impl_options.columns, timestamp, &s, &merge_context, &max_covering_tombstone_seq, read_options, get_impl_options.callback, get_impl_options.is_blob_index)) { done = true; - if (get_impl_options.value) { - get_impl_options.value->PinSelf(); - } - RecordTick(stats_, MEMTABLE_HIT); } } else { // Get Merge Operands associated with key, Merge Operands should not be // merged and raw values should be returned to the user. - if (sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, + if (!sv->mem->IsEmpty() && + sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, /*timestamp=*/nullptr, &s, &merge_context, &max_covering_tombstone_seq, read_options, false /* immutable_memtable */, nullptr, nullptr, @@ -2333,6 +2439,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, done = true; RecordTick(stats_, MEMTABLE_HIT); } else if ((s.ok() || s.IsMergeInProgress()) && + !sv->imm->IsEmpty() && sv->imm->GetMergeOperands(lkey, &s, &merge_context, &max_covering_tombstone_seq, read_options)) { @@ -2341,7 +2448,8 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, } } if (!done && !s.ok() && !s.IsMergeInProgress()) { - ReturnAndCleanupSuperVersion(cfd, sv); + if (!read_options.pinning_tls) + ReturnAndCleanupSuperVersion(cfd, sv); return s; } } @@ -2449,7 +2557,8 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key, PERF_COUNTER_ADD(get_read_bytes, size); } - ReturnAndCleanupSuperVersion(cfd, sv); + if (!read_options.pinning_tls) + ReturnAndCleanupSuperVersion(cfd, sv); RecordInHistogram(stats_, BYTES_PER_READ, size); } @@ -2494,6 +2603,7 @@ std::vector DBImpl::MultiGet( read_options.io_activity = Env::IOActivity::kMultiGet; } +#if defined(TOPLINGDB_WITH_TIMESTAMP) bool should_fail = false; for (size_t i = 0; i < num_keys; ++i) { assert(column_family[i]); @@ -2503,11 +2613,6 @@ std::vector DBImpl::MultiGet( if (!stat_list[i].ok()) { should_fail = true; } - } else { - stat_list[i] = FailIfCfHasTs(column_family[i]); - if (!stat_list[i].ok()) { - should_fail = true; - } } } @@ -2520,8 +2625,9 @@ std::vector DBImpl::MultiGet( } return stat_list; } +#endif - if (tracer_) { + if (UNLIKELY(tracer_ != nullptr)) { // TODO: This mutex should be removed later, to improve performance when // tracing is enabled. InstrumentedMutexLock lock(&trace_mutex_); @@ -2536,33 +2642,13 @@ std::vector DBImpl::MultiGet( for (auto cf : column_family) { auto cfh = static_cast_with_check(cf); auto cfd = cfh->cfd(); - if (multiget_cf_data.find(cfd->GetID()) == multiget_cf_data.end()) { - multiget_cf_data.emplace(cfd->GetID(), - MultiGetColumnFamilyData(cfh, nullptr)); - } + multiget_cf_data.try_emplace(cfd->GetID(), cfh, nullptr); } - std::function::iterator&)> - iter_deref_lambda = - [](UnorderedMap::iterator& - cf_iter) { return &cf_iter->second; }; - SequenceNumber consistent_seqnum; bool sv_from_thread_local; - Status status = - MultiCFSnapshot>( - read_options, nullptr, iter_deref_lambda, &multiget_cf_data, - &consistent_seqnum, &sv_from_thread_local); - - if (!status.ok()) { - for (auto& s : stat_list) { - if (s.ok()) { - s = status; - } - } - return stat_list; - } + bool unref_only = MultiCFSnapshot(read_options, nullptr, &multiget_cf_data, + &consistent_seqnum, &sv_from_thread_local); TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum1"); TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum2"); @@ -2572,9 +2658,11 @@ std::vector DBImpl::MultiGet( // Note: this always resizes the values array values->resize(num_keys); +#if defined(TOPLINGDB_WITH_TIMESTAMP) if (timestamps) { timestamps->resize(num_keys); } +#endif // Keep track of bytes that we read for statistics-recording later uint64_t bytes_read = 0; @@ -2588,18 +2676,27 @@ std::vector DBImpl::MultiGet( size_t keys_read; uint64_t curr_value_size = 0; +#if defined(TOPLINGDB_WITH_TIMESTAMP) GetWithTimestampReadCallback timestamp_read_callback(0); ReadCallback* read_callback = nullptr; if (read_options.timestamp && read_options.timestamp->size() > 0) { timestamp_read_callback.Refresh(consistent_seqnum); read_callback = ×tamp_read_callback; } +#else + ReadCallback* read_callback = nullptr; +#endif for (keys_read = 0; keys_read < num_keys; ++keys_read) { merge_context.Clear(); Status& s = stat_list[keys_read]; std::string* value = &(*values)[keys_read]; + value->clear(); +#if defined(TOPLINGDB_WITH_TIMESTAMP) std::string* timestamp = timestamps ? &(*timestamps)[keys_read] : nullptr; +#else + std::string* timestamp = nullptr; +#endif LookupKey lkey(keys[keys_read], consistent_seqnum, read_options.timestamp); auto cfh = static_cast_with_check( @@ -2614,22 +2711,25 @@ std::vector DBImpl::MultiGet( has_unpersisted_data_.load(std::memory_order_relaxed)); bool done = false; if (!skip_memtable) { + PinnableSlice pin(value); if (super_version->mem->Get( - lkey, value, /*columns=*/nullptr, timestamp, &s, &merge_context, + lkey, &pin, /*columns=*/nullptr, timestamp, &s, &merge_context, &max_covering_tombstone_seq, read_options, false /* immutable_memtable */, read_callback)) { done = true; + pin.SyncToString(value); RecordTick(stats_, MEMTABLE_HIT); - } else if (super_version->imm->Get(lkey, value, /*columns=*/nullptr, + } else if (super_version->imm->Get(lkey, &pin, /*columns=*/nullptr, timestamp, &s, &merge_context, &max_covering_tombstone_seq, read_options, read_callback)) { done = true; + pin.SyncToString(value); RecordTick(stats_, MEMTABLE_HIT); } } if (!done) { - PinnableSlice pinnable_val; + PinnableSlice pinnable_val(value); PERF_TIMER_GUARD(get_from_output_files_time); PinnedIteratorsManager pinned_iters_mgr; super_version->current->Get(read_options, lkey, &pinnable_val, @@ -2638,7 +2738,7 @@ std::vector DBImpl::MultiGet( &pinned_iters_mgr, /*value_found=*/nullptr, /*key_exists=*/nullptr, /*seq=*/nullptr, read_callback); - value->assign(pinnable_val.data(), pinnable_val.size()); + pinnable_val.SyncToString(value); RecordTick(stats_, MEMTABLE_MISS); } @@ -2694,6 +2794,7 @@ std::vector DBImpl::MultiGet( RecordTick(stats_, NUMBER_MULTIGET_KEYS_FOUND, num_found); RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read); RecordInHistogram(stats_, BYTES_PER_MULTIGET, bytes_read); + RecordInHistogram(stats_, NUMBER_PER_MULTIGET, num_keys); PERF_COUNTER_ADD(multiget_read_bytes, bytes_read); PERF_TIMER_STOP(get_post_process_time); @@ -2703,8 +2804,6 @@ std::vector DBImpl::MultiGet( template Status DBImpl::MultiCFSnapshot( const ReadOptions& read_options, ReadCallback* callback, - std::function& - iter_deref_func, T* cf_list, SequenceNumber* snapshot, bool* sv_from_thread_local) { PERF_TIMER_GUARD(get_snapshot_time); @@ -2865,6 +2964,16 @@ void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys, /* timestamps */ nullptr, statuses, sorted_input); } +template +bool all_same(const T* a, size_t n) { + assert(n > 0); + T p = a[0]; + for (size_t i = 1; i < n; ++i) + if (a[i] != p) + return false; + return true; +} + void DBImpl::MultiGet(const ReadOptions& _read_options, const size_t num_keys, ColumnFamilyHandle** column_families, const Slice* keys, PinnableSlice* values, std::string* timestamps, @@ -2896,9 +3005,11 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, PinnableWideColumns* columns, std::string* timestamps, Status* statuses, const bool sorted_input) { - if (num_keys == 0) { + if (UNLIKELY(num_keys == 0)) { return; } + +#if defined(TOPLINGDB_WITH_TIMESTAMP) bool should_fail = false; for (size_t i = 0; i < num_keys; ++i) { ColumnFamilyHandle* cfh = column_families[i]; @@ -2923,8 +3034,9 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, } return; } +#endif - if (tracer_) { + if (UNLIKELY(tracer_ != nullptr)) { // TODO: This mutex should be removed later, to improve performance when // tracing is enabled. InstrumentedMutexLock lock(&trace_mutex_); @@ -2936,6 +3048,7 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, autovector key_context; autovector sorted_keys; + key_context.reserve(num_keys); sorted_keys.resize(num_keys); for (size_t i = 0; i < num_keys; ++i) { PinnableSlice* val = nullptr; @@ -2958,7 +3071,8 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, for (size_t i = 0; i < num_keys; ++i) { sorted_keys[i] = &key_context[i]; } - PrepareMultiGetKeys(num_keys, sorted_input, &sorted_keys); + bool same_cf = all_same(column_families, num_keys); + PrepareMultiGetKeys(num_keys, sorted_input, same_cf, &sorted_keys); autovector multiget_cf_data; @@ -2976,22 +3090,10 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, multiget_cf_data.emplace_back(cf, cf_start, num_keys - cf_start, nullptr); - std::function::iterator&)> - iter_deref_lambda = - [](autovector::iterator& cf_iter) { - return &(*cf_iter); - }; - SequenceNumber consistent_seqnum; bool sv_from_thread_local; - Status s = MultiCFSnapshot< - autovector>( - read_options, nullptr, iter_deref_lambda, &multiget_cf_data, - &consistent_seqnum, &sv_from_thread_local); - + bool unref_only = MultiCFSnapshot(read_options, nullptr, &multiget_cf_data, + &consistent_seqnum, &sv_from_thread_local); if (!s.ok()) { for (size_t i = 0; i < num_keys; ++i) { if (statuses[i].ok()) { @@ -3001,12 +3103,16 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, return; } +#if defined(TOPLINGDB_WITH_TIMESTAMP) GetWithTimestampReadCallback timestamp_read_callback(0); ReadCallback* read_callback = nullptr; if (read_options.timestamp && read_options.timestamp->size() > 0) { timestamp_read_callback.Refresh(consistent_seqnum); read_callback = ×tamp_read_callback; } +#else + ReadCallback* read_callback = nullptr; +#endif auto cf_iter = multiget_cf_data.begin(); for (; cf_iter != multiget_cf_data.end(); ++cf_iter) { @@ -3064,10 +3170,19 @@ struct CompareKeyContext { } }; +struct CompareKeyContextSameCF { + const Comparator* comparator; + inline bool operator()(const KeyContext* lhs, const KeyContext* rhs) { + int cmp = comparator->CompareWithoutTimestamp( + *(lhs->key), /*a_has_ts=*/false, *(rhs->key), /*b_has_ts=*/false); + return cmp < 0; + } +}; + } // anonymous namespace void DBImpl::PrepareMultiGetKeys( - size_t num_keys, bool sorted_input, + size_t num_keys, bool sorted_input, bool same_cf, autovector* sorted_keys) { if (sorted_input) { #ifndef NDEBUG @@ -3077,8 +3192,16 @@ void DBImpl::PrepareMultiGetKeys( return; } - std::sort(sorted_keys->begin(), sorted_keys->begin() + num_keys, - CompareKeyContext()); + ROCKSDB_VERIFY_LE(sorted_keys->size(), num_keys); + if (same_cf) { + auto uc = sorted_keys->front()->column_family->GetComparator(); + std::sort(sorted_keys->begin(), sorted_keys->end(), + CompareKeyContextSameCF{uc}); + } + else { + std::sort(sorted_keys->begin(), sorted_keys->end(), + CompareKeyContext()); + } } void DBImpl::MultiGet(const ReadOptions& read_options, @@ -3089,6 +3212,12 @@ void DBImpl::MultiGet(const ReadOptions& read_options, /* timestamps */ nullptr, statuses, sorted_input); } +#if defined(ROCKSDB_UNIT_TEST) +static bool const g_MultiGetUseFiber = terark::getEnvBool("MultiGetUseFiber", false); +#else +static bool const g_MultiGetUseFiber = terark::getEnvBool("MultiGetUseFiber", true); +#endif + void DBImpl::MultiGet(const ReadOptions& _read_options, ColumnFamilyHandle* column_family, const size_t num_keys, const Slice* keys, PinnableSlice* values, @@ -3121,7 +3250,7 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, PinnableSlice* values, PinnableWideColumns* columns, std::string* timestamps, Status* statuses, bool sorted_input) { - if (tracer_) { + if (UNLIKELY(tracer_ != nullptr)) { // TODO: This mutex should be removed later, to improve performance when // tracing is enabled. InstrumentedMutexLock lock(&trace_mutex_); @@ -3130,8 +3259,10 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, tracer_->MultiGet(num_keys, column_family, keys).PermitUncheckedError(); } } +if (UNLIKELY(!g_MultiGetUseFiber)) { autovector key_context; autovector sorted_keys; + key_context.reserve(num_keys); sorted_keys.resize(num_keys); for (size_t i = 0; i < num_keys; ++i) { PinnableSlice* val = nullptr; @@ -3154,8 +3285,212 @@ void DBImpl::MultiGetCommon(const ReadOptions& read_options, for (size_t i = 0; i < num_keys; ++i) { sorted_keys[i] = &key_context[i]; } - PrepareMultiGetKeys(num_keys, sorted_input, &sorted_keys); + bool same_cf = true; + PrepareMultiGetKeys(num_keys, sorted_input, same_cf, &sorted_keys); MultiGetWithCallbackImpl(read_options, column_family, nullptr, &sorted_keys); + +} else { // topling MultiGet with fiber + + // copy from GetImpl with modify + +#if defined(TOPLINGDB_WITH_TIMESTAMP) + if (read_options.timestamp) { + const Status s = FailIfTsMismatchCf(column_family, + *(read_options.timestamp), + /*ts_for_read=*/true); + if (!s.ok()) { + for (size_t i = 0; i < num_keys; ++i) statuses[i] = s; + return; + } + } else { + const Status s = FailIfCfHasTs(column_family); + if (!s.ok()) { + for (size_t i = 0; i < num_keys; ++i) statuses[i] = s; + return; + } + } + + // Clear the timestamps for returning results so that we can distinguish + // between tombstone or key that has never been written + if (timestamps) { + for (size_t i = 0; i < num_keys; i++) + timestamps[i].clear(); + } + + GetWithTimestampReadCallback read_cb(0); // Will call Refresh +#endif + + PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock); + StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET); + PERF_TIMER_GUARD(get_snapshot_time); + + auto cfh = static_cast_with_check(column_family); + auto cfd = cfh->cfd(); + + // Acquire SuperVersion + SuperVersion* sv = GetAndRefSuperVersion(cfd, &read_options); + +// TEST_SYNC_POINT("DBImpl::MultiGet:1"); +// TEST_SYNC_POINT("DBImpl::MultiGet:2"); + + SequenceNumber snapshot; + ReadCallback* callback = read_options.read_callback; +// begin copied from GetImpl + if (read_options.snapshot != nullptr) { + if (callback) { + // Already calculated based on read_options.snapshot + snapshot = callback->max_visible_seq(); + } else { + snapshot = + reinterpret_cast(read_options.snapshot)->number_; + } + } else { + // Note that the snapshot is assigned AFTER referencing the super + // version because otherwise a flush happening in between may compact away + // data for the snapshot, so the reader would see neither data that was be + // visible to the snapshot before compaction nor the newer data inserted + // afterwards. + snapshot = GetLastPublishedSequence(); + if (callback) { + // The unprep_seqs are not published for write unprepared, so it could be + // that max_visible_seq is larger. Seek to the std::max of the two. + // However, we still want our callback to contain the actual snapshot so + // that it can do the correct visibility filtering. + callback->Refresh(snapshot); + + // Internally, WriteUnpreparedTxnReadCallback::Refresh would set + // max_visible_seq = max(max_visible_seq, snapshot) + // + // Currently, the commented out assert is broken by + // InvalidSnapshotReadCallback, but if write unprepared recovery followed + // the regular transaction flow, then this special read callback would not + // be needed. + // + // assert(callback->max_visible_seq() >= snapshot); + snapshot = callback->max_visible_seq(); + } + } +#if defined(TOPLINGDB_WITH_TIMESTAMP) + // If timestamp is used, we use read callback to ensure is returned + // only if t <= read_opts.timestamp and s <= snapshot. + // HACK: temporarily overwrite input struct field but restore + SaveAndRestore restore_callback(&callback); + const Comparator* ucmp = cfh->GetComparator(); + assert(ucmp); + if (ucmp->timestamp_size() > 0) { + assert(!callback); // timestamp with callback is not supported + read_cb.Refresh(snapshot); + callback = &read_cb; + } +#endif +// end copied from GetImpl + + //TEST_SYNC_POINT("DBImpl::GetImpl:3"); + //TEST_SYNC_POINT("DBImpl::GetImpl:4"); + + // First look in the memtable, then in the immutable memtable (if any). + // s is both in/out. When in, s could either be OK or MergeInProgress. + // merge_operands will contain the sequence of merges in the latter case. + PERF_TIMER_STOP(get_snapshot_time); + std::vector ctx_vec(num_keys); + for (size_t i = 0; i < num_keys; i++) { + ctx_vec[i].InitLookupKey(keys[i], snapshot, read_options.timestamp); + } + for (size_t i = 0; i < num_keys; i++) values[i].Reset(); + for (size_t i = 0; i < num_keys; i++) statuses[i].SetAsOK(); + + bool skip_memtable = (read_options.read_tier == kPersistedTier && + has_unpersisted_data_.load(std::memory_order_relaxed)); + + std::string* timestamp = nullptr; + bool* is_blob_index = nullptr; + PinnableWideColumns* columns = nullptr; + if (!skip_memtable) { + size_t hits = 0; + for (size_t i = 0; i < num_keys; i++) { + auto& max_covering_tombstone_seq = ctx_vec[i].max_covering_tombstone_seq; + MergeContext& merge_context = ctx_vec[i].merge_context(); + Status& s = statuses[i]; + if (sv->mem->Get(ctx_vec[i].lkey, &values[i], columns, + timestamp, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false, // immutable_memtable + callback, is_blob_index)) { + ctx_vec[i].set_done(); + hits++; + } else if ((s.ok() || s.IsMergeInProgress()) && + sv->imm->Get(ctx_vec[i].lkey, &values[i], columns, + timestamp, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + callback, is_blob_index)) { + ctx_vec[i].set_done(); + hits++; + } + } + RecordTick(stats_, MEMTABLE_HIT, hits); + } + //TEST_SYNC_POINT("DBImpl::GetImpl:PostMemTableGet:0"); + //TEST_SYNC_POINT("DBImpl::GetImpl:PostMemTableGet:1"); + size_t counting = 0; + auto get_in_sst = [&](size_t i, size_t/*unused*/ = 0) { + MergeContext& merge_context = ctx_vec[i].merge_context(); + PinnedIteratorsManager pinned_iters_mgr; + auto& max_covering_tombstone_seq = ctx_vec[i].max_covering_tombstone_seq; + //PERF_TIMER_GUARD(get_from_output_files_time); + bool* value_found = nullptr; + bool get_value = true; + sv->current->Get( + read_options, ctx_vec[i].lkey, &values[i], columns, + timestamp, &statuses[i], + &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr, + value_found, + nullptr, nullptr, + callback, + is_blob_index, + get_value); + counting++; + }; + if (read_options.async_io) { + gt_fiber_pool.update_fiber_count(read_options.async_queue_depth); + } + size_t memtab_miss = 0; + for (size_t i = 0; i < num_keys; i++) { + if (!ctx_vec[i].is_done()) { + if (read_options.async_io) { + gt_fiber_pool.push({TERARK_C_CALLBACK(get_in_sst), i}); + } else { + get_in_sst(i); + } + memtab_miss++; + } + } + while (counting < memtab_miss) { + gt_fiber_pool.unchecked_yield(); + } + + // Post processing (decrement reference counts and record statistics) + RecordTick(stats_, MEMTABLE_MISS, memtab_miss); + PERF_TIMER_GUARD(get_post_process_time); + size_t num_found = 0; + uint64_t bytes_read = 0; + for (size_t i = 0; i < num_keys; ++i) { + if (statuses[i].ok()) { + bytes_read += values[i].size(); + num_found++; + } + } + RecordTick(stats_, NUMBER_MULTIGET_CALLS); + RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys); + RecordTick(stats_, NUMBER_MULTIGET_KEYS_FOUND, num_found); + RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read); + RecordInHistogram(stats_, BYTES_PER_MULTIGET, bytes_read); + RecordInHistogram(stats_, NUMBER_PER_MULTIGET, num_keys); + PERF_COUNTER_ADD(multiget_read_bytes, bytes_read); + PERF_TIMER_STOP(get_post_process_time); + + if (!read_options.pinning_tls) + ReturnAndCleanupSuperVersion(cfd, sv); +} // g_MultiGetUseFiber } void DBImpl::MultiGetWithCallback( @@ -3181,19 +3516,12 @@ void DBImpl::MultiGetWithCallbackImpl( autovector* sorted_keys) { std::array multiget_cf_data; multiget_cf_data[0] = MultiGetColumnFamilyData(column_family, nullptr); - std::function::iterator&)> - iter_deref_lambda = - [](std::array::iterator& cf_iter) { - return &(*cf_iter); - }; size_t num_keys = sorted_keys->size(); SequenceNumber consistent_seqnum; bool sv_from_thread_local; - Status s = MultiCFSnapshot>( - read_options, callback, iter_deref_lambda, &multiget_cf_data, - &consistent_seqnum, &sv_from_thread_local); + Status s = MultiCFSnapshot(read_options, callback, &multiget_cf_data, + &consistent_seqnum, &sv_from_thread_local); if (!s.ok()) { return; } @@ -3223,6 +3551,7 @@ void DBImpl::MultiGetWithCallbackImpl( consistent_seqnum = callback->max_visible_seq(); } +#if defined(TOPLINGDB_WITH_TIMESTAMP) GetWithTimestampReadCallback timestamp_read_callback(0); ReadCallback* read_callback = callback; if (read_options.timestamp && read_options.timestamp->size() > 0) { @@ -3230,6 +3559,9 @@ void DBImpl::MultiGetWithCallbackImpl( timestamp_read_callback.Refresh(consistent_seqnum); read_callback = ×tamp_read_callback; } +#else + ReadCallback* read_callback = callback; +#endif s = MultiGetImpl(read_options, 0, num_keys, sorted_keys, multiget_cf_data[0].super_version, consistent_seqnum, @@ -3257,6 +3589,7 @@ Status DBImpl::MultiGetImpl( StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET); assert(sorted_keys); +#if defined(TOPLINGDB_WITH_TIMESTAMP) // Clear the timestamps for returning results so that we can distinguish // between tombstone or key that has never been written for (auto* kctx : *sorted_keys) { @@ -3265,6 +3598,7 @@ Status DBImpl::MultiGetImpl( kctx->timestamp->clear(); } } +#endif // For each of the given keys, apply the entire "get" process as follows: // First look in the memtable, then in the immutable memtable (if any). @@ -3365,6 +3699,7 @@ Status DBImpl::MultiGetImpl( RecordTick(stats_, NUMBER_MULTIGET_KEYS_FOUND, num_found); RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read); RecordInHistogram(stats_, BYTES_PER_MULTIGET, bytes_read); + RecordInHistogram(stats_, NUMBER_PER_MULTIGET, num_keys); PERF_COUNTER_ADD(multiget_read_bytes, bytes_read); PERF_TIMER_STOP(get_post_process_time); @@ -3770,14 +4105,15 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options, // TODO: plumb Env::IOActivity ReadOptions roptions = read_options; roptions.read_tier = kBlockCacheTier; // read from block cache only - PinnableSlice pinnable_val; + value->clear(); + PinnableSlice pinnable_val(value); GetImplOptions get_impl_options; get_impl_options.column_family = column_family; get_impl_options.value = &pinnable_val; get_impl_options.value_found = value_found; get_impl_options.timestamp = timestamp; auto s = GetImpl(roptions, key, get_impl_options); - value->assign(pinnable_val.data(), pinnable_val.size()); + pinnable_val.SyncToString(value); // If block_cache is enabled and the index block of the table didn't // not present in block_cache, the return value will be Status::Incomplete. @@ -3921,9 +4257,7 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl( // likely that any iterator pointer is close to the iterator it points to so // that they are likely to be in the same cache line and/or page. ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator( - env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, sv->current, - snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations, - sv->version_number, read_callback, this, cfd, expose_blob_index, + read_options, sv, snapshot, read_callback, this, expose_blob_index, allow_refresh); InternalIterator* internal_iter = NewInternalIterator( @@ -4072,6 +4406,12 @@ Status DBImpl::GetTimestampedSnapshots( SnapshotImpl* DBImpl::GetSnapshotImpl(bool is_write_conflict_boundary, bool lock) { + return GetSnapshotImpl(kMaxSequenceNumber, is_write_conflict_boundary, lock); +} + +SnapshotImpl* DBImpl::GetSnapshotImpl(SequenceNumber snapshot_seq, + bool is_write_conflict_boundary, + bool lock) { int64_t unix_time = 0; immutable_db_options_.clock->GetCurrentTime(&unix_time) .PermitUncheckedError(); // Ignore error @@ -4090,7 +4430,9 @@ SnapshotImpl* DBImpl::GetSnapshotImpl(bool is_write_conflict_boundary, delete s; return nullptr; } - auto snapshot_seq = GetLastPublishedSequence(); + if (kMaxSequenceNumber == snapshot_seq) { + snapshot_seq = GetLastPublishedSequence(); + } SnapshotImpl* snapshot = snapshots_.New(s, snapshot_seq, unix_time, is_write_conflict_boundary); if (lock) { @@ -4520,6 +4862,115 @@ bool DBImpl::GetAggregatedIntProperty(const Slice& property, return ret; } +template struct ToplingDB_size_to_uint; +template<> struct ToplingDB_size_to_uint<4> { typedef unsigned int type; }; +template<> struct ToplingDB_size_to_uint<8> { typedef unsigned long long type; }; + +terark_pure_func inline static size_t ThisThreadID() { +#if defined(_MSC_VER) + auto id = std::this_thread::get_id(); + return (size_t)(ToplingDB_size_to_uint::type&)(id); +#else + // gnu pthread_self impl + size_t __self; + asm("movq %%fs:%c1,%q0" : "=r" (__self) : "i" (16)); + return __self; +#endif +} + +struct ReadOptionsTLS { + size_t thread_id = size_t(-1); + SuperVersion* sv = nullptr; + DBImpl* db_impl = nullptr; + std::vector cfsv; + SuperVersion*& GetSuperVersionRef(size_t cfid); + void FinishPin(); + ReadOptionsTLS(); + ~ReadOptionsTLS(); +}; + +ReadOptionsTLS::ReadOptionsTLS() { + // do nothing +} +ReadOptionsTLS::~ReadOptionsTLS() { + FinishPin(); +} +inline SuperVersion*& ReadOptionsTLS::GetSuperVersionRef(size_t cfid) { + if (0 == cfid) { + return sv; + } else { + if (UNLIKELY(cfsv.size() < cfid)) { + cfsv.resize(cfid, nullptr); + } + return cfsv[cfid - 1]; + } +} + +void ReadOptionsTLS::FinishPin() { + if (sv) { + db_impl->ReturnAndCleanupSuperVersion(sv->cfd, sv); + sv = nullptr; + } + for (auto& x : cfsv) { + if (x) { + db_impl->ReturnAndCleanupSuperVersion(x->cfd, x); + x = nullptr; + } + } + cfsv.resize(0); + db_impl = nullptr; +} + +void ReadOptions::StartPin() { + if (!pinning_tls) { + pinning_tls = std::make_shared(); + } else { + ROCKSDB_VERIFY_EQ(nullptr, pinning_tls->db_impl); + ROCKSDB_VERIFY_EQ(nullptr, pinning_tls->sv); + ROCKSDB_VERIFY_EQ(pinning_tls->cfsv.size(), 0); + } + pinning_tls->thread_id = ThisThreadID(); +} +void ReadOptions::FinishPin() { + // some applications(such as myrocks/mytopling) clean the working area which + // needs to call FinishPin before StartPin, so we need to allow such usage + if (pinning_tls) { + ROCKSDB_VERIFY_EQ(pinning_tls->thread_id, ThisThreadID()); + pinning_tls->FinishPin(); + } +} +ReadOptions::~ReadOptions() { + if (pinning_tls) + this->FinishPin(); +} + +SuperVersion* +DBImpl::GetAndRefSuperVersion(ColumnFamilyData* cfd, const ReadOptions* ro) { + auto tls = ro->pinning_tls.get(); + if (!tls) { // do not use zero copy, same as old behavior + return GetAndRefSuperVersion(cfd); + } + ROCKSDB_ASSERT_EQ(tls->thread_id, ThisThreadID()); + size_t cfid = cfd->GetID(); + SuperVersion*& sv = tls->GetSuperVersionRef(cfid); + if (sv) { + if (LIKELY(sv->version_number == cfd->GetSuperVersionNumberNoAtomic())) { + ROCKSDB_ASSERT_EQ(sv->cfd, cfd); + return sv; + } + ReturnAndCleanupSuperVersion(cfd, sv); + } + // slow path + ROCKSDB_VERIFY_EQ(tls->thread_id, ThisThreadID()); + if (!tls->db_impl) { + tls->db_impl = this; + } else { + ROCKSDB_VERIFY_EQ(this, tls->db_impl); + } + sv = GetAndRefSuperVersion(cfd); + return sv; +} + SuperVersion* DBImpl::GetAndRefSuperVersion(ColumnFamilyData* cfd) { // TODO(ljin): consider using GetReferencedSuperVersion() directly return cfd->GetThreadLocalSuperVersion(this); @@ -4641,19 +5092,40 @@ Status DBImpl::GetApproximateSizes(const SizeApproximationOptions& options, if (!options.include_memtables && !options.include_files) { return Status::InvalidArgument("Invalid options"); } + if (UNLIKELY(n <= 0)) { + return Status::OK(); + } +#if defined(TOPLINGDB_WITH_TIMESTAMP) const Comparator* const ucmp = column_family->GetComparator(); assert(ucmp); size_t ts_sz = ucmp->timestamp_size(); +#endif Version* v; auto cfh = static_cast_with_check(column_family); auto cfd = cfh->cfd(); - SuperVersion* sv = GetAndRefSuperVersion(cfd); + bool zero_copy = options.read_options && options.read_options->pinning_tls; + SuperVersion* sv = zero_copy ? GetAndRefSuperVersion(cfd, options.read_options) + : GetAndRefSuperVersion(cfd); v = sv->current; // TODO: plumb Env::IOActivity - const ReadOptions read_options; + const auto& read_options = options.read_options ? *options.read_options : ReadOptions(); + + size_t len1 = range[0].start.size_; + size_t len2 = range[0].limit.size_; + for (int i = 1; i < n; i++) { + len1 = std::max(len1, range[i].start.size_); + len2 = std::max(len2, range[i].limit.size_); + } +#if defined(TOPLINGDB_WITH_TIMESTAMP) + len1 += ts_sz; + len2 += ts_sz; +#endif + char* k1 = (char*)alloca(len1 + 8); + char* k2 = (char*)alloca(len2 + 8); + for (int i = 0; i < n; i++) { // Add timestamp if needed std::string start_with_ts, limit_with_ts; @@ -4663,21 +5135,24 @@ Status DBImpl::GetApproximateSizes(const SizeApproximationOptions& options, assert(start.has_value()); assert(limit.has_value()); // Convert user_key into a corresponding internal key. - InternalKey k1(start.value(), kMaxSequenceNumber, kValueTypeForSeek); - InternalKey k2(limit.value(), kMaxSequenceNumber, kValueTypeForSeek); + SetInternalKey(k1, start, kMaxSequenceNumber, kValueTypeForSeek); + SetInternalKey(k2, limit, kMaxSequenceNumber, kValueTypeForSeek); sizes[i] = 0; + Slice ik1(k1, start.size_ + 8); + Slice ik2(k2, limit.size_ + 8); if (options.include_files) { sizes[i] += versions_->ApproximateSize( - options, read_options, v, k1.Encode(), k2.Encode(), /*start_level=*/0, + options, read_options, v, ik1, ik2, /*start_level=*/0, /*end_level=*/-1, TableReaderCaller::kUserApproximateSize); } if (options.include_memtables) { - sizes[i] += sv->mem->ApproximateStats(k1.Encode(), k2.Encode()).size; - sizes[i] += sv->imm->ApproximateStats(k1.Encode(), k2.Encode()).size; + sizes[i] += sv->mem->ApproximateStats(ik1, ik2).size; + sizes[i] += sv->imm->ApproximateStats(ik1, ik2).size; } } - - ReturnAndCleanupSuperVersion(cfd, sv); + if (!zero_copy) { + ReturnAndCleanupSuperVersion(cfd, sv); + } return Status::OK(); } @@ -5012,10 +5487,12 @@ Status DBImpl::CheckConsistency() { uint64_t fsize = 0; TEST_SYNC_POINT("DBImpl::CheckConsistency:BeforeGetFileSize"); Status s = env_->GetFileSize(file_path, &fsize); +#ifdef ROCKSDB_SUPPORT_LEVELDB_FILE_LDB if (!s.ok() && env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) { s = Status::OK(); } +#endif // ROCKSDB_SUPPORT_LEVELDB_FILE_LDB if (!s.ok()) { corruption_messages += "Can't access " + md.name + ": " + s.ToString() + "\n"; @@ -5303,7 +5780,20 @@ Status DestroyDB(const std::string& dbname, const Options& options, return result; } +static bool g_KICK_OUT_OPTIONS_FILE() { + static bool val = []() { + if (auto env = getenv("ROCKSDB_KICK_OUT_OPTIONS_FILE")) { + return atoi(env) != 0; + } + return false; + }(); + return val; +} + Status DBImpl::WriteOptionsFile(bool db_mutex_already_held) { + if (g_KICK_OUT_OPTIONS_FILE()) { + return Status::OK(); + } options_mutex_.AssertHeld(); if (db_mutex_already_held) { @@ -5535,6 +6025,7 @@ Status DBImpl::GetLatestSequenceForKey( ReadOptions read_options; SequenceNumber current_seq = versions_->LastSequence(); +#if defined(TOPLINGDB_WITH_TIMESTAMP) ColumnFamilyData* cfd = sv->cfd; assert(cfd); const Comparator* const ucmp = cfd->user_comparator(); @@ -5550,6 +6041,12 @@ Status DBImpl::GetLatestSequenceForKey( Slice ts(ts_buf); LookupKey lkey(key, current_seq, ts_sz == 0 ? nullptr : &ts); +#else + #if !defined(NDEBUG) + constexpr size_t ts_sz = 0; + #endif + LookupKey lkey(key, current_seq, nullptr); +#endif *seq = kMaxSequenceNumber; *found_record_for_key = false; @@ -5752,8 +6249,7 @@ Status DBImpl::IngestExternalFiles( uint64_t start_file_number = next_file_number; for (size_t i = 1; i != num_cfs; ++i) { start_file_number += args[i - 1].external_files.size(); - auto* cfd = - static_cast(args[i].column_family)->cfd(); + auto* cfd = args[i].column_family->cfd(); SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); Status es = ingestion_jobs[i].Prepare( args[i].external_files, args[i].files_checksums, @@ -5768,8 +6264,7 @@ Status DBImpl::IngestExternalFiles( TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeLastJobPrepare:0"); TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeLastJobPrepare:1"); { - auto* cfd = - static_cast(args[0].column_family)->cfd(); + auto* cfd = args[0].column_family->cfd(); SuperVersion* super_version = cfd->GetReferencedSuperVersion(this); Status es = ingestion_jobs[0].Prepare( args[0].external_files, args[0].files_checksums, @@ -5821,8 +6316,7 @@ Status DBImpl::IngestExternalFiles( bool at_least_one_cf_need_flush = false; std::vector need_flush(num_cfs, false); for (size_t i = 0; i != num_cfs; ++i) { - auto* cfd = - static_cast(args[i].column_family)->cfd(); + auto* cfd = args[i].column_family->cfd(); if (cfd->IsDropped()) { // TODO (yanqin) investigate whether we should abort ingestion or // proceed with other non-dropped column families. @@ -5854,9 +6348,7 @@ Status DBImpl::IngestExternalFiles( for (size_t i = 0; i != num_cfs; ++i) { if (need_flush[i]) { mutex_.Unlock(); - auto* cfd = - static_cast(args[i].column_family) - ->cfd(); + auto* cfd = args[i].column_family->cfd(); status = FlushMemTable(cfd, flush_opts, FlushReason::kExternalFileIngestion, true /* entered_write_thread */); @@ -5885,8 +6377,7 @@ Status DBImpl::IngestExternalFiles( autovector> edit_lists; uint32_t num_entries = 0; for (size_t i = 0; i != num_cfs; ++i) { - auto* cfd = - static_cast(args[i].column_family)->cfd(); + auto* cfd = args[i].column_family->cfd(); if (cfd->IsDropped()) { continue; } @@ -5938,8 +6429,7 @@ Status DBImpl::IngestExternalFiles( if (status.ok()) { for (size_t i = 0; i != num_cfs; ++i) { - auto* cfd = - static_cast(args[i].column_family)->cfd(); + auto* cfd = args[i].column_family->cfd(); if (!cfd->IsDropped()) { InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i], *cfd->GetLatestMutableCFOptions()); @@ -5993,8 +6483,7 @@ Status DBImpl::IngestExternalFiles( } if (status.ok()) { for (size_t i = 0; i != num_cfs; ++i) { - auto* cfd = - static_cast(args[i].column_family)->cfd(); + auto* cfd = args[i].column_family->cfd(); if (!cfd->IsDropped()) { NotifyOnExternalFileIngested(cfd, ingestion_jobs[i]); } @@ -6423,6 +6912,9 @@ void DBImpl::NotifyOnExternalFileIngested( Status DBImpl::StartTrace(const TraceOptions& trace_options, std::unique_ptr&& trace_writer) { InstrumentedMutexLock lock(&trace_mutex_); + if (tracer_) { + return Status::Busy("Working tracer existed"); + } tracer_.reset(new Tracer(immutable_db_options_.clock, trace_options, std::move(trace_writer))); return Status::OK(); diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index 34a5f33989..0b0b929177 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -182,6 +182,8 @@ class DBImpl : public DB { virtual ~DBImpl(); + bool opened_successfully() const { return this->opened_successfully_; } + // ---- Implementations of the DB interface ---- using DB::Resume; @@ -872,6 +874,8 @@ class DBImpl : public DB { // sends the signals. void CancelAllBackgroundWork(bool wait); + SuperVersion* GetAndRefSuperVersion(ColumnFamilyData*, const ReadOptions*); + // Find Super version and reference it. Based on options, it might return // the thread local cached one. // Call ReturnAndCleanupSuperVersion() when it is no longer needed. @@ -1283,6 +1287,10 @@ class DBImpl : public DB { bool seq_per_batch() const { return seq_per_batch_; } + int next_job_id() const noexcept { + return next_job_id_.load(std::memory_order_relaxed); + } + protected: const std::string dbname_; // TODO(peterd): unify with VersionSet::db_id_ @@ -1620,7 +1628,6 @@ class DBImpl : public DB { friend class WriteUnpreparedTransactionTest_RecoveryTest_Test; #endif - struct CompactionState; struct PrepickedCompaction; struct PurgeFileInfo; @@ -2209,6 +2216,11 @@ class DBImpl : public DB { SnapshotImpl* GetSnapshotImpl(bool is_write_conflict_boundary, bool lock = true); +public: + SnapshotImpl* GetSnapshotImpl(SequenceNumber snapshot_seq, + bool is_write_conflict_boundary, + bool lock = true); +private: // If snapshot_seq != kMaxSequenceNumber, then this function can only be // called from the write thread that publishes sequence numbers to readers. @@ -2287,8 +2299,9 @@ class DBImpl : public DB { // Utility function to do some debug validation and sort the given vector // of MultiGet keys + static void PrepareMultiGetKeys( - const size_t num_keys, bool sorted, + const size_t num_keys, bool sorted, bool same_cf, autovector* key_ptrs); void MultiGetCommon(const ReadOptions& options, @@ -2303,44 +2316,6 @@ class DBImpl : public DB { std::string* timestamps, Status* statuses, bool sorted_input); - // A structure to hold the information required to process MultiGet of keys - // belonging to one column family. For a multi column family MultiGet, there - // will be a container of these objects. - struct MultiGetColumnFamilyData { - ColumnFamilyHandle* cf; - ColumnFamilyData* cfd; - - // For the batched MultiGet which relies on sorted keys, start specifies - // the index of first key belonging to this column family in the sorted - // list. - size_t start; - - // For the batched MultiGet case, num_keys specifies the number of keys - // belonging to this column family in the sorted list - size_t num_keys; - - // SuperVersion for the column family obtained in a manner that ensures a - // consistent view across all column families in the DB - SuperVersion* super_version; - MultiGetColumnFamilyData(ColumnFamilyHandle* column_family, - SuperVersion* sv) - : cf(column_family), - cfd(static_cast(cf)->cfd()), - start(0), - num_keys(0), - super_version(sv) {} - - MultiGetColumnFamilyData(ColumnFamilyHandle* column_family, size_t first, - size_t count, SuperVersion* sv) - : cf(column_family), - cfd(static_cast(cf)->cfd()), - start(first), - num_keys(count), - super_version(sv) {} - - MultiGetColumnFamilyData() = default; - }; - // A common function to obtain a consistent snapshot, which can be implicit // if the user doesn't specify a snapshot in read_options, across // multiple column families for MultiGet. It will attempt to get an implicit @@ -2361,8 +2336,6 @@ class DBImpl : public DB { template Status MultiCFSnapshot( const ReadOptions& read_options, ReadCallback* callback, - std::function& - iter_deref_func, T* cf_list, SequenceNumber* snapshot, bool* sv_from_thread_local); // The actual implementation of the batching MultiGet. The caller is expected diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 71c23de95a..1c92166499 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -1170,6 +1170,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, // Small lies about compaction range context.smallest_user_key = *begin; context.largest_user_key = *end; + context.target_output_file_size = 0; partitioner = partitioner_factory->CreatePartitioner(context); } @@ -2872,6 +2873,9 @@ void DBImpl::MaybeScheduleFlushOrCompaction() { env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this, &DBImpl::UnscheduleCompactionCallback); } + ROCKS_LOG_DEBUG(immutable_db_options_.info_log.get(), + "bg_compaction_scheduled = %d, unscheduled_compactions = %d", + bg_compaction_scheduled_, unscheduled_compactions_); } DBImpl::BGJobLimits DBImpl::GetBGJobLimits() const { @@ -2900,7 +2904,11 @@ DBImpl::BGJobLimits DBImpl::GetBGJobLimits(int max_background_flushes, } if (!parallelize_compactions) { // throttle background compactions until we deem necessary + #if defined(ROCKSDB_UNIT_TEST) + // this line cause compact jiggling, we should delete this line, + // but we keep it for making rocksdb unit test happy res.max_compactions = 1; + #endif } return res; } diff --git a/db/db_impl/db_impl_files.cc b/db/db_impl/db_impl_files.cc index bd48796474..0c4f457494 100644 --- a/db/db_impl/db_impl_files.cc +++ b/db/db_impl/db_impl_files.cc @@ -192,10 +192,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force, for (size_t path_id = 0; path_id < cfd->ioptions()->cf_paths.size(); path_id++) { auto& path = cfd->ioptions()->cf_paths[path_id].path; - - if (paths.find(path) == paths.end()) { - paths.insert(path); - } + paths.insert(path); } } @@ -999,10 +996,8 @@ Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) { // path ends with '/' or '\\' const std::string normalized_fpath = path + fname; largest_file_number = std::max(largest_file_number, number); - if (type == kTableFile && number >= next_file_number && - recovery_ctx->files_to_delete_.find(normalized_fpath) == - recovery_ctx->files_to_delete_.end()) { - recovery_ctx->files_to_delete_.emplace(normalized_fpath, path); + if (type == kTableFile && number >= next_file_number) { + recovery_ctx->files_to_delete_.emplace(normalized_fpath); } } } diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 074fa86214..923e825765 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -81,11 +81,13 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src, result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_flushes, Env::Priority::HIGH); +#ifdef ROCKSDB_UNIT_TEST // the document says bytes_per_sync == 0 means turn off if (result.rate_limiter.get() != nullptr) { if (result.bytes_per_sync == 0) { result.bytes_per_sync = 1024 * 1024; } } +#endif if (result.delayed_write_rate == 0) { if (result.rate_limiter.get() != nullptr) { diff --git a/db/db_impl/db_impl_readonly.cc b/db/db_impl/db_impl_readonly.cc index 997a4e2edf..a258141e11 100644 --- a/db/db_impl/db_impl_readonly.cc +++ b/db/db_impl/db_impl_readonly.cc @@ -91,14 +91,11 @@ Status DBImplReadOnly::GetImpl(const ReadOptions& read_options, SequenceNumber max_covering_tombstone_seq = 0; LookupKey lkey(key, snapshot, read_options.timestamp); PERF_TIMER_STOP(get_snapshot_time); - // Look up starts here - if (super_version->mem->Get( - lkey, - get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr, - get_impl_options.columns, ts, &s, &merge_context, - &max_covering_tombstone_seq, read_options, - false /* immutable_memtable */, &read_cb)) { + if (super_version->mem->Get(lkey, pinnable_val, + get_impl_options.columns, ts, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, &read_cb)) { if (get_impl_options.value) { get_impl_options.value->PinSelf(); } @@ -174,10 +171,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options, : latest_snapshot; ReadCallback* read_callback = nullptr; // No read callback provided. auto db_iter = NewArenaWrappedDbIterator( - env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options, - super_version->current, read_seq, - super_version->mutable_cf_options.max_sequential_skip_in_iterations, - super_version->version_number, read_callback); + read_options, super_version, read_seq, read_callback); auto internal_iter = NewInternalIterator( db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(), read_seq, /* allow_unprepared_value */ true, db_iter); @@ -242,10 +236,7 @@ Status DBImplReadOnly::NewIterators( assert(cfd_to_sv.size() == column_families.size()); for (auto [cfd, sv] : cfd_to_sv) { auto* db_iter = NewArenaWrappedDbIterator( - env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, - sv->current, read_seq, - sv->mutable_cf_options.max_sequential_skip_in_iterations, - sv->version_number, read_callback); + read_options, sv, read_seq, read_callback); auto* internal_iter = NewInternalIterator( db_iter->GetReadOptions(), cfd, sv, db_iter->GetArena(), read_seq, /* allow_unprepared_value */ true, db_iter); diff --git a/db/db_impl/db_impl_secondary.cc b/db/db_impl/db_impl_secondary.cc index 235a528ba0..df2a4be8dd 100644 --- a/db/db_impl/db_impl_secondary.cc +++ b/db/db_impl/db_impl_secondary.cc @@ -302,11 +302,8 @@ Status DBImplSecondary::RecoverLogFiles( if (cfd == nullptr) { continue; } - std::unordered_map::iterator iter = - cfd_to_current_log_.find(cfd); - if (iter == cfd_to_current_log_.end()) { - cfd_to_current_log_.insert({cfd, log_number}); - } else if (log_number > iter->second) { + auto [iter, success] = cfd_to_current_log_.emplace(cfd, log_number); + if (!success && log_number > iter->second) { iter->second = log_number; } } @@ -339,6 +336,21 @@ Status DBImplSecondary::RecoverLogFiles( return status; } +#if defined(ROCKSDB_UNIT_TEST) +// Implementation of the DB interface +Status DBImplSecondary::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value) { + return GetImpl(read_options, column_family, key, value, + /*timestamp*/ nullptr); +} + +Status DBImplSecondary::Get(const ReadOptions& read_options, + ColumnFamilyHandle* column_family, const Slice& key, + PinnableSlice* value, std::string* timestamp) { + return GetImpl(read_options, column_family, key, value, timestamp); +} + Status DBImplSecondary::GetImpl(const ReadOptions& read_options, const Slice& key, GetImplOptions& get_impl_options) { @@ -388,7 +400,9 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, } // Acquire SuperVersion - SuperVersion* super_version = GetAndRefSuperVersion(cfd); + SuperVersion* super_version = GetAndRefSuperVersion(cfd, &read_options); + SequenceNumber snapshot = versions_->LastSequence(); + GetWithTimestampReadCallback read_cb(snapshot); if (read_options.timestamp && read_options.timestamp->size() > 0) { s = FailIfReadCollapsedHistory(cfd, super_version, *(read_options.timestamp)); @@ -402,14 +416,13 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, LookupKey lkey(key, snapshot, read_options.timestamp); PERF_TIMER_STOP(get_snapshot_time); bool done = false; - - // Look up starts here - if (super_version->mem->Get( - lkey, - get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr, - get_impl_options.columns, ts, &s, &merge_context, - &max_covering_tombstone_seq, read_options, - false /* immutable_memtable */, &read_cb)) { + const Comparator* ucmp = column_family->GetComparator(); + assert(ucmp); + std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr; + if (super_version->mem->Get(lkey, pinnable_val, + get_impl_options.columns, ts, &s, &merge_context, + &max_covering_tombstone_seq, read_options, + false /* immutable_memtable */, &read_cb)) { done = true; if (get_impl_options.value) { get_impl_options.value->PinSelf(); @@ -417,11 +430,9 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, RecordTick(stats_, MEMTABLE_HIT); } else if ((s.ok() || s.IsMergeInProgress()) && super_version->imm->Get( - lkey, - get_impl_options.value ? get_impl_options.value->GetSelf() - : nullptr, - get_impl_options.columns, ts, &s, &merge_context, - &max_covering_tombstone_seq, read_options, &read_cb)) { + lkey, pinnable_val, get_impl_options.columns, ts, &s, + &merge_context, &max_covering_tombstone_seq, read_options, + &read_cb)) { done = true; if (get_impl_options.value) { get_impl_options.value->PinSelf(); @@ -429,7 +440,8 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, RecordTick(stats_, MEMTABLE_HIT); } if (!done && !s.ok() && !s.IsMergeInProgress()) { - ReturnAndCleanupSuperVersion(cfd, super_version); + if (!read_options.pinning_tls) + ReturnAndCleanupSuperVersion(cfd, super_version); return s; } if (!done) { @@ -445,7 +457,8 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options, } { PERF_TIMER_GUARD(get_post_process_time); - ReturnAndCleanupSuperVersion(cfd, super_version); + if (!read_options.pinning_tls) + ReturnAndCleanupSuperVersion(cfd, super_version); RecordTick(stats_, NUMBER_KEYS_READ); size_t size = 0; if (get_impl_options.value) { @@ -503,9 +516,16 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& _read_options, return NewErrorIterator(Status::NotSupported( "tailing iterator not supported in secondary mode")); } else if (read_options.snapshot != nullptr) { + #if defined(ROCKSDB_UNIT_TEST) // TODO (yanqin) support snapshot. return NewErrorIterator( Status::NotSupported("snapshot not supported in secondary mode")); + #else + // I dont know why does not support iterator, I just add snapshot + // read stupidly + SequenceNumber snapshot(read_options.snapshot->GetSequenceNumber()); + result = NewIteratorImpl(read_options, cfd, snapshot, read_callback); + #endif } else { SequenceNumber snapshot(kMaxSequenceNumber); SuperVersion* sv = cfd->GetReferencedSuperVersion(this); @@ -531,10 +551,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl( snapshot = versions_->LastSequence(); assert(snapshot != kMaxSequenceNumber); auto db_iter = NewArenaWrappedDbIterator( - env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options, - super_version->current, snapshot, - super_version->mutable_cf_options.max_sequential_skip_in_iterations, - super_version->version_number, read_callback, this, cfd, + read_options, super_version, snapshot, read_callback, this, expose_blob_index, allow_refresh); auto internal_iter = NewInternalIterator( db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(), @@ -622,6 +639,7 @@ Status DBImplSecondary::NewIterators( } return Status::OK(); } +#endif // ROCKSDB_UNIT_TEST Status DBImplSecondary::CheckConsistency() { mutex_.AssertHeld(); @@ -652,11 +670,17 @@ Status DBImplSecondary::CheckConsistency() { uint64_t fsize = 0; s = env_->GetFileSize(file_path, &fsize); +#ifdef ROCKSDB_SUPPORT_LEVELDB_FILE_LDB if (!s.ok() && (env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok() || s.IsPathNotFound())) { s = Status::OK(); } +#else + if (s.IsPathNotFound()) { + s = Status::OK(); + } +#endif // ROCKSDB_SUPPORT_LEVELDB_FILE_LDB if (!s.ok()) { corruption_messages += "Can't access " + md.name + ": " + s.ToString() + "\n"; @@ -679,7 +703,7 @@ Status DBImplSecondary::TryCatchUpWithPrimary() { ->ReadAndApply(&mutex_, &manifest_reader_, manifest_reader_status_.get(), &cfds_changed); - ROCKS_LOG_INFO(immutable_db_options_.info_log, "Last sequence is %" PRIu64, + ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "Last sequence is %" PRIu64, static_cast(versions_->LastSequence())); for (ColumnFamilyData* cfd : cfds_changed) { if (cfd->IsDropped()) { diff --git a/db/db_impl/db_impl_secondary.h b/db/db_impl/db_impl_secondary.h index 12a8bbdd70..5f5d766a27 100644 --- a/db/db_impl/db_impl_secondary.h +++ b/db/db_impl/db_impl_secondary.h @@ -85,6 +85,9 @@ class DBImplSecondary : public DBImpl { bool error_if_data_exists_in_wals, uint64_t* = nullptr, RecoveryContext* recovery_ctx = nullptr) override; +#if defined(ROCKSDB_UNIT_TEST) + // Implementations of the DB interface. + using DB::Get; // Can return IOError due to files being deleted by the primary. To avoid // IOError in this case, application can coordinate between primary and // secondaries so that primary will not delete files that are currently being @@ -121,6 +124,7 @@ class DBImplSecondary : public DBImpl { Status NewIterators(const ReadOptions& _read_options, const std::vector& column_families, std::vector* iterators) override; +#endif // ROCKSDB_UNIT_TEST using DBImpl::Put; Status Put(const WriteOptions& /*options*/, diff --git a/db/db_impl/db_impl_write.cc b/db/db_impl/db_impl_write.cc index df67ba8c8f..2cc5cd26b9 100644 --- a/db/db_impl/db_impl_write.cc +++ b/db/db_impl/db_impl_write.cc @@ -325,7 +325,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options, if (w.ShouldWriteToMemtable()) { PERF_TIMER_STOP(write_pre_and_post_process_time); - PERF_TIMER_GUARD(write_memtable_time); + PERF_TIMER_WITH_HISTOGRAM(write_memtable_time, MEMTAB_WRITE_KV_NANOS, stats_); ColumnFamilyMemTablesImpl column_family_memtables( versions_->GetColumnFamilySet()); @@ -502,7 +502,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options, assert(log_context.log_file_number_size); LogFileNumberSize& log_file_number_size = *(log_context.log_file_number_size); - PERF_TIMER_GUARD(write_wal_time); + PERF_TIMER_WITH_HISTOGRAM(write_wal_time, WRITE_WAL_NANOS, stats_); io_s = WriteToWAL(write_group, log_context.writer, log_used, log_context.need_log_sync, log_context.need_log_dir_sync, @@ -510,7 +510,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options, } } else { if (status.ok() && !write_options.disableWAL) { - PERF_TIMER_GUARD(write_wal_time); + PERF_TIMER_WITH_HISTOGRAM(write_wal_time, WRITE_WAL_NANOS, stats_); // LastAllocatedSequence is increased inside WriteToWAL under // wal_write_mutex_ to ensure ordered events in WAL io_s = ConcurrentWriteToWAL(write_group, log_used, &last_sequence, @@ -557,7 +557,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options, } if (status.ok()) { - PERF_TIMER_GUARD(write_memtable_time); + PERF_TIMER_WITH_HISTOGRAM(write_memtable_time, MEMTAB_WRITE_KV_NANOS, stats_); if (!parallel) { // w.sequence will be set inside InsertInto @@ -746,7 +746,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options, io_s.PermitUncheckedError(); // Allow io_s to be uninitialized if (w.status.ok() && !write_options.disableWAL) { - PERF_TIMER_GUARD(write_wal_time); + PERF_TIMER_WITH_HISTOGRAM(write_wal_time, WRITE_WAL_NANOS, stats_); stats->AddDBStats(InternalStats::kIntStatsWriteDoneBySelf, 1); RecordTick(stats_, WRITE_DONE_BY_SELF, 1); if (wal_write_group.size > 1) { @@ -797,7 +797,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options, WriteThread::WriteGroup memtable_write_group; if (w.state == WriteThread::STATE_MEMTABLE_WRITER_LEADER) { - PERF_TIMER_GUARD(write_memtable_time); + PERF_TIMER_WITH_HISTOGRAM(write_memtable_time, MEMTAB_WRITE_KV_NANOS, stats_); assert(w.ShouldWriteToMemtable()); write_thread_.EnterAsMemTableWriter(&w, &memtable_write_group); if (memtable_write_group.size > 1 && @@ -1004,7 +1004,7 @@ Status DBImpl::WriteImplWALOnly( PERF_TIMER_STOP(write_pre_and_post_process_time); - PERF_TIMER_GUARD(write_wal_time); + PERF_TIMER_WITH_HISTOGRAM(write_wal_time, WRITE_WAL_NANOS, stats_); // LastAllocatedSequence is increased inside WriteToWAL under // wal_write_mutex_ to ensure ordered events in WAL size_t seq_inc = 0 /* total_count */; @@ -1180,7 +1180,10 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options, assert(num_cfs >= 1); if (num_cfs > 1) { WaitForPendingWrites(); + auto beg = immutable_db_options_.clock->NowNanos(); status = SwitchWAL(write_context); + auto end = immutable_db_options_.clock->NowNanos(); + RecordInHistogram(stats_, SWITCH_WAL_NANOS, end - beg); } } @@ -1192,18 +1195,28 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options, // suboptimal but still correct. InstrumentedMutexLock l(&mutex_); WaitForPendingWrites(); + auto beg = immutable_db_options_.clock->NowNanos(); status = HandleWriteBufferManagerFlush(write_context); + auto end = immutable_db_options_.clock->NowNanos(); + RecordInHistogram(stats_, SWITCH_WAL_NANOS, end - beg); } if (UNLIKELY(status.ok() && !trim_history_scheduler_.Empty())) { - InstrumentedMutexLock l(&mutex_); + auto beg = immutable_db_options_.clock->NowNanos(); + mutex_.Lock(); status = TrimMemtableHistory(write_context); + mutex_.Unlock(); + auto end = immutable_db_options_.clock->NowNanos(); + RecordInHistogram(stats_, SWITCH_WAL_NANOS, end - beg); } if (UNLIKELY(status.ok() && !flush_scheduler_.Empty())) { InstrumentedMutexLock l(&mutex_); WaitForPendingWrites(); + auto beg = immutable_db_options_.clock->NowNanos(); status = ScheduleFlushes(write_context); + auto end = immutable_db_options_.clock->NowNanos(); + RecordInHistogram(stats_, SWITCH_WAL_NANOS, end - beg); } PERF_TIMER_STOP(write_scheduling_flushes_compactions_time); @@ -1839,8 +1852,7 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, delay = 0; } TEST_SYNC_POINT("DBImpl::DelayWrite:Start"); - start_time = immutable_db_options_.clock->NowMicros(); - + start_time = StopWatch::s_now_micros(immutable_db_options_.clock); if (delay > 0) { if (write_options.no_slowdown) { return Status::Incomplete("Write stall"); @@ -1859,7 +1871,7 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, const uint64_t kDelayInterval = 1001; uint64_t stall_end = start_time + delay; while (write_controller_.NeedsDelay()) { - if (immutable_db_options_.clock->NowMicros() >= stall_end) { + if (StopWatch::s_now_micros(immutable_db_options_.clock) >= stall_end) { // We already delayed this write `delay` microseconds break; } @@ -1898,7 +1910,8 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread, } assert(!delayed || !write_options.no_slowdown); if (delayed) { - auto time_delayed = immutable_db_options_.clock->NowMicros() - start_time; + uint64_t now = StopWatch::s_now_micros(immutable_db_options_.clock); + uint64_t time_delayed = now - start_time; default_cf_internal_stats_->AddDBStats( InternalStats::kIntStatsWriteStallMicros, time_delayed); RecordTick(stats_, STALL_MICROS, time_delayed); diff --git a/db/db_iter.cc b/db/db_iter.cc index 507bb2577b..a66b9e66d7 100644 --- a/db/db_iter.cc +++ b/db/db_iter.cc @@ -37,6 +37,10 @@ namespace ROCKSDB_NAMESPACE { +#if !defined(TOPLINGDB_WITH_TIMESTAMP) +std::string DBIter::saved_timestamp_; +#endif + DBIter::DBIter(Env* _env, const ReadOptions& read_options, const ImmutableOptions& ioptions, const MutableCFOptions& mutable_cf_options, @@ -47,7 +51,9 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options, ColumnFamilyData* cfd, bool expose_blob_index) : prefix_extractor_(mutable_cf_options.prefix_extractor.get()), env_(_env), +#if !defined(CLOCK_MONOTONIC) || defined(ROCKSDB_UNIT_TEST) clock_(ioptions.clock), +#endif logger_(ioptions.logger), user_comparator_(cmp), merge_operator_(ioptions.merge_operator.get()), @@ -57,7 +63,7 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options, sequence_(s), statistics_(ioptions.stats), max_skip_(max_sequential_skip_in_iterations), - max_skippable_internal_keys_(read_options.max_skippable_internal_keys), + max_skippable_internal_keys_(read_options.max_skippable_internal_keys?:UINT64_MAX), num_internal_keys_skipped_(0), iterate_lower_bound_(read_options.iterate_lower_bound), iterate_upper_bound_(read_options.iterate_upper_bound), @@ -68,7 +74,9 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options, prefix_same_as_start_(mutable_cf_options.prefix_extractor ? read_options.prefix_same_as_start : false), +#if defined(ROCKSDB_UNIT_TEST) pin_thru_lifetime_(read_options.pin_data), +#endif expect_total_order_inner_iter_(prefix_extractor_ == nullptr || read_options.total_order_seek || read_options.auto_prefix_mode), @@ -81,9 +89,14 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options, io_activity_(read_options.io_activity), db_impl_(db_impl), cfd_(cfd), + #if defined(TOPLINGDB_WITH_TIMESTAMP) timestamp_ub_(read_options.timestamp), timestamp_lb_(read_options.iter_start_ts), - timestamp_size_(timestamp_ub_ ? timestamp_ub_->size() : 0) { + timestamp_size_(timestamp_ub_ ? timestamp_ub_->size() : 0), + saved_timestamp_(), + #endif + db_impl_(db_impl), + cfd_(cfd) { RecordTick(statistics_, NO_ITERATOR_CREATED); if (pin_thru_lifetime_) { pinned_iters_mgr_.StartPinning(); @@ -94,6 +107,7 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options, status_.PermitUncheckedError(); assert(timestamp_size_ == user_comparator_.user_comparator()->timestamp_size()); + enable_perf_timer_ = perf_level >= PerfLevel::kEnableTimeExceptForMutex; } Status DBIter::GetProperty(std::string prop_name, std::string* prop) { @@ -120,7 +134,9 @@ Status DBIter::GetProperty(std::string prop_name, std::string* prop) { return Status::InvalidArgument("Unidentified property."); } +__always_inline bool DBIter::ParseKey(ParsedInternalKey* ikey) { +#if 0 Status s = ParseInternalKey(iter_.key(), ikey, false /* log_err_key */); if (!s.ok()) { status_ = Status::Corruption("In DBIter: ", s.getState()); @@ -130,8 +146,13 @@ bool DBIter::ParseKey(ParsedInternalKey* ikey) { } else { return true; } +#else + ikey->FastParseInternalKey(iter_.key()); + return true; +#endif } +ROCKSDB_FLATTEN void DBIter::Next() { assert(valid_); assert(status_.ok()); @@ -146,10 +167,12 @@ void DBIter::Next() { local_stats_.skip_count_--; num_internal_keys_skipped_ = 0; bool ok = true; - if (direction_ == kReverse) { + if (UNLIKELY(direction_ == kReverse)) { is_key_seqnum_zero_ = false; if (!ReverseToForward()) { ok = false; + } else { + ok = iter_.Valid(); } } else if (!current_entry_is_merged_) { // If the current value is not a merge, the iter position is the @@ -158,12 +181,14 @@ void DBIter::Next() { // If the current key is a merge, very likely iter already points // to the next internal position. assert(iter_.Valid()); - iter_.Next(); + ok = iter_.Next(); PERF_COUNTER_ADD(internal_key_skipped_count, 1); + } else { + ok = iter_.Valid(); } local_stats_.next_count_++; - if (ok && iter_.Valid()) { + if (ok) { ClearSavedValue(); if (prefix_same_as_start_) { @@ -173,14 +198,16 @@ void DBIter::Next() { } else { FindNextUserEntry(true /* skipping the current user key */, nullptr); } + if (LIKELY(valid_)) { + local_stats_.next_found_count_++; + local_stats_.bytes_read_ += saved_key_.Size(); + if (is_value_prepared_) + local_stats_.bytes_read_ += value_.size_; + } } else { is_key_seqnum_zero_ = false; valid_ = false; } - if (statistics_ != nullptr && valid_) { - local_stats_.next_found_count_++; - local_stats_.bytes_read_ += (key().size() + value().size()); - } } bool DBIter::SetBlobValueIfNeeded(const Slice& user_key, @@ -223,6 +250,7 @@ bool DBIter::SetBlobValueIfNeeded(const Slice& user_key, } bool DBIter::SetValueAndColumnsFromEntity(Slice slice) { +#if defined(TOPLINGDB_WITH_WIDE_COLUMNS) assert(value_.empty()); assert(wide_columns_.empty()); @@ -237,6 +265,7 @@ bool DBIter::SetValueAndColumnsFromEntity(Slice slice) { if (WideColumnsHelper::HasDefaultColumn(wide_columns_)) { value_ = WideColumnsHelper::GetDefaultColumn(wide_columns_); } +#endif return true; } @@ -280,13 +309,60 @@ bool DBIter::SetValueAndColumnsFromMergeResult(const Status& merge_status, // within the prefix, and the iterator needs to be made invalid, if no // more entry for the prefix can be found. bool DBIter::FindNextUserEntry(bool skipping_saved_key, const Slice* prefix) { - PERF_TIMER_GUARD(find_next_user_entry_time); - return FindNextUserEntryInternal(skipping_saved_key, prefix); + if (enable_perf_timer_) { + PERF_TIMER_GUARD(find_next_user_entry_time); + return FindNextUserEntryInternal(skipping_saved_key, prefix); + } else { + return FindNextUserEntryInternal(skipping_saved_key, prefix); + } } +struct BytewiseCmpNoTS { + bool equal(const Slice& x, const Slice& y) const { return x == y; } + bool operator()(const Slice& x, const Slice& y) const { return x < y; } + int compare(const Slice& x, const Slice& y) const { return x.compare(y); } +}; + +struct RevBytewiseCmpNoTS { + bool equal(const Slice& x, const Slice& y) const { return x == y; } + bool operator()(const Slice& x, const Slice& y) const { return y < x; } + int compare(const Slice& x, const Slice& y) const { return y.compare(x); } +}; + +struct VirtualCmpNoTS { + bool equal(const Slice& x, const Slice& y) const { + return cmp->CompareWithoutTimestamp(x, y) == 0; + } + bool operator()(const Slice& x, const Slice& y) const { + return cmp->CompareWithoutTimestamp(x, false, y, false) < 0; + } + int compare(const Slice& x, const Slice& y) const { + return cmp->CompareWithoutTimestamp(x, y); + } + const Comparator* cmp; +}; + // Actual implementation of DBIter::FindNextUserEntry() bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, const Slice* prefix) { + if (user_comparator_.IsForwardBytewise()) { + ROCKSDB_ASSERT_EZ(user_comparator_.timestamp_size()); + BytewiseCmpNoTS cmp; + return FindNextUserEntryInternalTmpl(skipping_saved_key, prefix, cmp); + } else if (user_comparator_.IsReverseBytewise()) { + ROCKSDB_ASSERT_EZ(user_comparator_.timestamp_size()); + RevBytewiseCmpNoTS cmp; + return FindNextUserEntryInternalTmpl(skipping_saved_key, prefix, cmp); + } else { + VirtualCmpNoTS cmp{user_comparator_.user_comparator()}; + return FindNextUserEntryInternalTmpl(skipping_saved_key, prefix, cmp); + } +} + +template +bool DBIter::FindNextUserEntryInternalTmpl(bool skipping_saved_key, + const Slice* prefix, + CmpNoTS cmpNoTS) { // Loop until we hit an acceptable entry to yield assert(iter_.Valid()); assert(status_.ok()); @@ -311,17 +387,19 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, // an infinite loop of reseeks. To avoid that, we limit the number of reseeks // to one. bool reseek_done = false; + is_value_prepared_ = true; do { // Will update is_key_seqnum_zero_ as soon as we parsed the current key // but we need to save the previous value to be used in the loop. bool is_prev_key_seqnum_zero = is_key_seqnum_zero_; - if (!ParseKey(&ikey_)) { - is_key_seqnum_zero_ = false; - return false; - } + ParsedInternalKey ikey_(iter_.key()); // ToplingDB, move field as local var +#if defined(TOPLINGDB_WITH_TIMESTAMP) Slice user_key_without_ts = StripTimestampFromUserKey(ikey_.user_key, timestamp_size_); +#else + Slice& user_key_without_ts = ikey_.user_key; +#endif is_key_seqnum_zero_ = (ikey_.sequence == 0); @@ -331,10 +409,10 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, user_key_without_ts, /*a_has_ts=*/false, *iterate_upper_bound_, /*b_has_ts=*/false) < 0); if (iterate_upper_bound_ != nullptr && - iter_.UpperBoundCheckResult() != IterBoundCheck::kInbound && - user_comparator_.CompareWithoutTimestamp( - user_key_without_ts, /*a_has_ts=*/false, *iterate_upper_bound_, - /*b_has_ts=*/false) >= 0) { + // ToplingDB: for speed up, do not call UpperBoundCheckResult() + // The following cmpNoTS has same semantic as UpperBoundCheckResult() + // iter_.UpperBoundCheckResult() != IterBoundCheck::kInbound && + !cmpNoTS(user_key_without_ts, *iterate_upper_bound_)) { break; } @@ -365,7 +443,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, // level. This may change in the future. if ((!is_prev_key_seqnum_zero || timestamp_size_ > 0) && skipping_saved_key && - CompareKeyForSkip(ikey_.user_key, saved_key_.GetUserKey()) <= 0) { + EqKeyForSkip(saved_key_.GetUserKey(), ikey_.user_key, cmpNoTS)) { num_skipped++; // skip this entry PERF_COUNTER_ADD(internal_key_skipped_count, 1); } else { @@ -392,6 +470,18 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, } break; case kTypeValue: + #if !defined(TOPLINGDB_WITH_WIDE_COLUMNS) + if (timestamp_lb_) { + saved_key_.SetInternalKey(ikey_); + } else { + saved_key_.SetUserKey( + ikey_.user_key, !pin_thru_lifetime_ || + !iter_.iter()->IsKeyPinned() /* copy */); + } + is_value_prepared_ = false; + valid_ = true; + return true; + #endif case kTypeBlobIndex: case kTypeWideColumnEntity: if (!iter_.PrepareValue()) { @@ -427,7 +517,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, return true; break; case kTypeMerge: - if (!iter_.PrepareValue()) { + if (UNLIKELY(!iter_.PrepareValue())) { assert(!iter_.status().ok()); valid_ = false; return false; @@ -456,14 +546,14 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, // This key was inserted after our snapshot was taken or skipped by // timestamp range. If this happens too many times in a row for the same // user key, we want to seek to the target sequence number. - int cmp = user_comparator_.CompareWithoutTimestamp( + int cmp = cmpNoTS.compare( ikey_.user_key, saved_key_.GetUserKey()); if (cmp == 0 || (skipping_saved_key && cmp < 0)) { num_skipped++; } else { saved_key_.SetUserKey( ikey_.user_key, - !iter_.iter()->IsKeyPinned() || !pin_thru_lifetime_ /* copy */); + !pin_thru_lifetime_ || !iter_.iter()->IsKeyPinned() /* copy */); skipping_saved_key = false; num_skipped = 0; reseek_done = false; @@ -522,7 +612,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key, iter_.Seek(last_key); RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION); } else { - iter_.Next(); + if (iter_.Next()) continue; else break; // omit iter_.Valid() } } while (iter_.Valid()); @@ -567,6 +657,7 @@ bool DBIter::MergeValuesNewToOld() { // hit the next user key, stop right here break; } + ROCKSDB_ASSUME(ikey.type < kTypeMaxValid); if (kTypeDeletion == ikey.type || kTypeSingleDeletion == ikey.type || kTypeDeletionWithTimestamp == ikey.type) { // hit a delete with the same user key, stop right here @@ -673,7 +764,7 @@ void DBIter::Prev() { ResetValueAndColumns(); ResetInternalKeysSkippedCounter(); bool ok = true; - if (direction_ == kForward) { + if (UNLIKELY(direction_ == kForward)) { if (!ReverseToBackward()) { ok = false; } @@ -774,10 +865,11 @@ bool DBIter::ReverseToBackward() { } void DBIter::PrevInternal(const Slice* prefix) { + is_value_prepared_ = true; while (iter_.Valid()) { saved_key_.SetUserKey( ExtractUserKey(iter_.key()), - !iter_.iter()->IsKeyPinned() || !pin_thru_lifetime_ /* copy */); + !pin_thru_lifetime_ || !iter_.iter()->IsKeyPinned() /* copy */); assert(prefix == nullptr || prefix_extractor_ != nullptr); if (prefix != nullptr && @@ -853,9 +945,12 @@ bool DBIter::FindValueForCurrentKey() { // last_key_entry_type is initialized to kTypeDeletion. bool valid_entry_seen = false; + ParsedInternalKey ikey_; // ToplingDB, move field as local var + // Temporarily pin blocks that hold (merge operands / the value) ReleaseTempPinnedData(); TempPinData(); + Slice pinned_value_; size_t num_skipped = 0; while (iter_.Valid()) { ParsedInternalKey ikey; @@ -886,9 +981,11 @@ bool DBIter::FindValueForCurrentKey() { break; } +#if defined(TOPLINGDB_WITH_TIMESTAMP) // ts may need runtime check if (!ts.empty()) { saved_timestamp_.assign(ts.data(), ts.size()); } +#endif if (TooManyInternalKeysSkipped()) { return false; @@ -1003,6 +1100,7 @@ bool DBIter::FindValueForCurrentKey() { return true; case kTypeMerge: current_entry_is_merged_ = true; + ROCKSDB_ASSUME(last_not_merge_type < kTypeMaxValid); if (last_not_merge_type == kTypeDeletion || last_not_merge_type == kTypeSingleDeletion || last_not_merge_type == kTypeDeletionWithTimestamp) { @@ -1149,10 +1247,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { Slice ts = ExtractTimestampFromUserKey(ikey.user_key, timestamp_size_); saved_timestamp_.assign(ts.data(), ts.size()); } + ROCKSDB_ASSUME(ikey.type < kTypeMaxValid); if (ikey.type == kTypeValue || ikey.type == kTypeBlobIndex || ikey.type == kTypeWideColumnEntity) { assert(iter_.iter()->IsValuePinned()); - pinned_value_ = iter_.value(); + Slice pinned_value_ = iter_.value(); if (ikey.type == kTypeBlobIndex) { if (!SetBlobValueIfNeeded(ikey.user_key, pinned_value_)) { return false; @@ -1160,10 +1259,12 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { SetValueAndColumnsFromPlain(expose_blob_index_ ? pinned_value_ : blob_value_); +#if defined(TOPLINGDB_WITH_WIDE_COLUMNS) } else if (ikey.type == kTypeWideColumnEntity) { if (!SetValueAndColumnsFromEntity(pinned_value_)) { return false; } +#endif } else { assert(ikey.type == kTypeValue); SetValueAndColumnsFromPlain(pinned_value_); @@ -1203,6 +1304,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() { saved_key_.GetUserKey())) { break; } + ROCKSDB_ASSUME(ikey.type < kTypeMaxValid); if (ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion || ikey.type == kTypeDeletionWithTimestamp) { break; @@ -1282,6 +1384,7 @@ bool DBIter::MergeWithNoBaseValue(const Slice& user_key) { // `op_failure_scope` (an output parameter) is not provided (set to nullptr) // since a failure must be propagated regardless of its value. ValueType result_type; + Slice pinned_value_; const Status s = MergeHelper::TimedFullMerge( merge_operator_, user_key, MergeHelper::kNoBaseValue, merge_context_.GetOperands(), logger_, statistics_, clock_, @@ -1295,6 +1398,7 @@ bool DBIter::MergeWithPlainBaseValue(const Slice& value, // `op_failure_scope` (an output parameter) is not provided (set to nullptr) // since a failure must be propagated regardless of its value. ValueType result_type; + Slice pinned_value_; const Status s = MergeHelper::TimedFullMerge( merge_operator_, user_key, MergeHelper::kPlainBaseValue, value, merge_context_.GetOperands(), logger_, statistics_, clock_, @@ -1308,6 +1412,7 @@ bool DBIter::MergeWithWideColumnBaseValue(const Slice& entity, // `op_failure_scope` (an output parameter) is not provided (set to nullptr) // since a failure must be propagated regardless of its value. ValueType result_type; + Slice pinned_value_; const Status s = MergeHelper::TimedFullMerge( merge_operator_, user_key, MergeHelper::kWideBaseValue, entity, merge_context_.GetOperands(), logger_, statistics_, clock_, @@ -1381,9 +1486,9 @@ bool DBIter::FindUserKeyBeforeSavedKey() { return true; } +__always_inline bool DBIter::TooManyInternalKeysSkipped(bool increment) { - if ((max_skippable_internal_keys_ > 0) && - (num_internal_keys_skipped_ > max_skippable_internal_keys_)) { + if (UNLIKELY(num_internal_keys_skipped_ > max_skippable_internal_keys_)) { valid_ = false; status_ = Status::Incomplete("Too many internal keys skipped."); return true; @@ -1393,6 +1498,7 @@ bool DBIter::TooManyInternalKeysSkipped(bool increment) { return false; } +__always_inline bool DBIter::IsVisible(SequenceNumber sequence, const Slice& ts, bool* more_recent) { // Remember that comparator orders preceding timestamp as larger. @@ -1461,6 +1567,7 @@ void DBIter::SetSavedKeyToSeekForPrevTarget(const Slice& target) { } } +ROCKSDB_FLATTEN void DBIter::Seek(const Slice& target) { PERF_COUNTER_ADD(iter_seek_count, 1); PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_); @@ -1637,7 +1744,7 @@ void DBIter::SeekToFirst() { if (iter_.Valid()) { saved_key_.SetUserKey( ExtractUserKey(iter_.key()), - !iter_.iter()->IsKeyPinned() || !pin_thru_lifetime_ /* copy */); + !pin_thru_lifetime_ || !iter_.iter()->IsKeyPinned() /* copy */); FindNextUserEntry(false /* not skipping saved_key */, nullptr /* no prefix check */); if (statistics_ != nullptr) { diff --git a/db/db_iter.h b/db/db_iter.h index 5022405c32..0a4d1141d0 100644 --- a/db/db_iter.h +++ b/db/db_iter.h @@ -158,17 +158,50 @@ class DBIter final : public Iterator { return Slice(ukey_and_ts.data(), ukey_and_ts.size() - timestamp_size_); } } + Slice value() const override { assert(valid_); - + #if defined(TOPLINGDB_WITH_WIDE_COLUMNS) + assert(is_value_prepared_); + #endif + if (!is_value_prepared_) { + auto mut = const_cast(this); + if (LIKELY(mut->iter_.PrepareAndGetValue(&mut->value_))) { + mut->is_value_prepared_ = true; + mut->local_stats_.bytes_read_ += value_.size_; + } else { // Can not go on, die with message + ROCKSDB_DIE("PrepareAndGetValue() failed, status = %s", + iter_.status().ToString().c_str()); + } + } return value_; } + // without PrepareValue, user can not check iter_.PrepareAndGetValue(), + // thus must die in DBIter::value() if iter_.PrepareAndGetValue() fails. + bool PrepareValue() override { // enable error check for lazy load + assert(valid_); + if (!is_value_prepared_) { + if (LIKELY(iter_.PrepareAndGetValue(&value_))) { + is_value_prepared_ = true; + local_stats_.bytes_read_ += value_.size_; + } else { + valid_ = false; + status_ = iter_.status(); + ROCKSDB_VERIFY(!status_.ok()); + return false; + } + } + return true; + } + +#if defined(TOPLINGDB_WITH_WIDE_COLUMNS) const WideColumns& columns() const override { assert(valid_); return wide_columns_; } +#endif Status status() const override { if (status_.ok()) { @@ -204,6 +237,7 @@ class DBIter final : public Iterator { void SeekToFirst() final override; void SeekToLast() final override; Env* env() const { return env_; } + uint64_t get_sequence() const { return sequence_; } void set_sequence(uint64_t s) { sequence_ = s; if (read_callback_) { @@ -238,6 +272,8 @@ class DBIter final : public Iterator { bool FindNextUserEntry(bool skipping_saved_key, const Slice* prefix); // Internal implementation of FindNextUserEntry(). bool FindNextUserEntryInternal(bool skipping_saved_key, const Slice* prefix); + template + bool FindNextUserEntryInternalTmpl(bool, const Slice* prefix, CmpNoTS); bool ParseKey(ParsedInternalKey* key); bool MergeValuesNewToOld(); @@ -294,6 +330,20 @@ class DBIter final : public Iterator { : user_comparator_.CompareWithoutTimestamp(a, b); } + template + inline bool CmpKeyForSkip(const Slice& a, const Slice& b, const CmpNoTS& c) { + return timestamp_lb_ != nullptr + ? user_comparator_.Compare(a, b) < 0 + : c(a, b); + } + + template + inline bool EqKeyForSkip(const Slice& a, const Slice& b, const CmpNoTS& c) { + return timestamp_lb_ != nullptr // semantic exactly same with origin code + ? user_comparator_.Compare(a, b) >= 0 // ^^^^^^^^^^^^^^^^^^^^^ + : c.equal(a, b); + } + // Retrieves the blob value for the specified user key using the given blob // index when using the integrated BlobDB implementation. bool SetBlobValueIfNeeded(const Slice& user_key, const Slice& blob_index); @@ -305,10 +355,12 @@ class DBIter final : public Iterator { void SetValueAndColumnsFromPlain(const Slice& slice) { assert(value_.empty()); - assert(wide_columns_.empty()); - value_ = slice; + +#if defined(TOPLINGDB_WITH_WIDE_COLUMNS) + assert(wide_columns_.empty()); wide_columns_.emplace_back(kDefaultWideColumnName, slice); +#endif } bool SetValueAndColumnsFromEntity(Slice slice); @@ -318,7 +370,9 @@ class DBIter final : public Iterator { void ResetValueAndColumns() { value_.clear(); +#if defined(TOPLINGDB_WITH_WIDE_COLUMNS) wide_columns_.clear(); +#endif } // The following methods perform the actual merge operation for the @@ -330,7 +384,11 @@ class DBIter final : public Iterator { const SliceTransform* prefix_extractor_; Env* const env_; +#if !defined(CLOCK_MONOTONIC) || defined(ROCKSDB_UNIT_TEST) SystemClock* clock_; +#else + static constexpr SystemClock* clock_ = nullptr; +#endif Logger* logger_; UserComparatorWrapper user_comparator_; const MergeOperator* const merge_operator_; @@ -345,15 +403,17 @@ class DBIter final : public Iterator { // Reusable internal key data structure. This is only used inside one function // and should not be used across functions. Reusing this object can reduce // overhead of calling construction of the function if creating it each time. - ParsedInternalKey ikey_; + //ParsedInternalKey ikey_; std::string saved_value_; - Slice pinned_value_; + //Slice pinned_value_; // for prefix seek mode to support prev() PinnableSlice blob_value_; // Value of the default column Slice value_; +#if defined(TOPLINGDB_WITH_WIDE_COLUMNS) // All columns (i.e. name-value pairs) WideColumns wide_columns_; +#endif Statistics* statistics_; uint64_t max_skip_; uint64_t max_skippable_internal_keys_; @@ -371,6 +431,7 @@ class DBIter final : public Iterator { Status status_; Direction direction_; bool valid_; + bool is_value_prepared_; bool current_entry_is_merged_; // True if we know that the current entry's seqnum is 0. // This information is used as that the next entry will be for another @@ -379,7 +440,11 @@ class DBIter final : public Iterator { const bool prefix_same_as_start_; // Means that we will pin all data blocks we read as long the Iterator // is not deleted, will be true if ReadOptions::pin_data is true +#if defined(ROCKSDB_UNIT_TEST) const bool pin_thru_lifetime_; +#else + static constexpr bool pin_thru_lifetime_ = false; +#endif // Expect the inner iterator to maintain a total order. // prefix_extractor_ must be non-NULL if the value is false. const bool expect_total_order_inner_iter_; @@ -391,17 +456,25 @@ class DBIter final : public Iterator { bool expose_blob_index_; bool is_blob_; bool arena_mode_; + bool enable_perf_timer_; const Env::IOActivity io_activity_; // List of operands for merge operator. MergeContext merge_context_; LocalStatistics local_stats_; PinnedIteratorsManager pinned_iters_mgr_; - DBImpl* db_impl_; - ColumnFamilyData* cfd_; +#if defined(TOPLINGDB_WITH_TIMESTAMP) const Slice* const timestamp_ub_; const Slice* const timestamp_lb_; const size_t timestamp_size_; std::string saved_timestamp_; +#else + static constexpr const Slice* const timestamp_ub_ = nullptr; + static constexpr const Slice* const timestamp_lb_ = nullptr; + static constexpr size_t timestamp_size_ = 0; + static std::string saved_timestamp_; +#endif + DBImpl* db_impl_; + ColumnFamilyData* cfd_; }; // Return a new iterator that converts internal keys (yielded by diff --git a/db/db_iterator_test.cc b/db/db_iterator_test.cc index 29c39f6ad1..e157b5f46d 100644 --- a/db/db_iterator_test.cc +++ b/db/db_iterator_test.cc @@ -2516,6 +2516,11 @@ TEST_P(DBIteratorTest, RefreshWithSnapshot) { ASSERT_OK(iter->Refresh(snapshot)); } + ASSERT_OK(iter->status()); + Status s = iter->Refresh(); + ASSERT_TRUE(s.ok()); + s = iter->Refresh(snapshot, false); + ASSERT_TRUE(s.ok()); delete iter; db_->ReleaseSnapshot(snapshot); db_->ReleaseSnapshot(snapshot2); @@ -2587,7 +2592,7 @@ TEST_P(DBIteratorTest, TableFilter) { { std::set unseen{1, 2, 3}; ReadOptions opts; - opts.table_filter = [&](const TableProperties& props) { + opts.table_filter = [&](const TableProperties& props, const FileMetaData&) { auto it = unseen.find(props.num_entries); if (it == unseen.end()) { ADD_FAILURE() << "saw table properties with an unexpected " @@ -2621,7 +2626,7 @@ TEST_P(DBIteratorTest, TableFilter) { // during iteration. { ReadOptions opts; - opts.table_filter = [](const TableProperties& props) { + opts.table_filter = [](const TableProperties& props, const FileMetaData&) { return props.num_entries != 2; }; auto iter = NewIterator(opts); diff --git a/db/db_kv_checksum_test.cc b/db/db_kv_checksum_test.cc index 614399243e..838f7937e7 100644 --- a/db/db_kv_checksum_test.cc +++ b/db/db_kv_checksum_test.cc @@ -681,200 +681,6 @@ TEST_F(DbKVChecksumWALToWriteBatchTest, WriteBatchChecksumHandoff) { }; // TODO (cbi): add DeleteRange coverage once it is implemented -class DbMemtableKVChecksumTest : public DbKvChecksumTest { - public: - DbMemtableKVChecksumTest() : DbKvChecksumTest() {} - - protected: - // Indices in the memtable entry that we will not corrupt. - // For memtable entry format, see comments in MemTable::Add(). - // We do not corrupt key length and value length fields in this test - // case since it causes segfault and ASAN will complain. - // For this test case, key and value are all of length 3, so - // key length field is at index 0 and value length field is at index 12. - const std::set index_not_to_corrupt{0, 12}; - - void SkipNotToCorruptEntry() { - if (index_not_to_corrupt.find(corrupt_byte_offset_) != - index_not_to_corrupt.end()) { - corrupt_byte_offset_++; - } - } -}; - -INSTANTIATE_TEST_CASE_P( - DbMemtableKVChecksumTest, DbMemtableKVChecksumTest, - ::testing::Combine(::testing::Range(static_cast(0), - WriteBatchOpType::kDeleteRange), - ::testing::Values(2, 103, 251), - ::testing::Range(static_cast(0), - WriteMode::kWriteOptionProtectedBatch), - // skip 1 byte checksum as it makes test flaky - ::testing::Values(2, 4, 8)), - [](const testing::TestParamInfo< - std::tuple>& args) { - std::ostringstream oss; - oss << GetOpTypeString(std::get<0>(args.param)) << "Add" - << static_cast( - static_cast(std::get<1>(args.param))) - << GetWriteModeString(std::get<2>(args.param)) - << static_cast(std::get<3>(args.param)); - return oss.str(); - }); - -TEST_P(DbMemtableKVChecksumTest, GetWithCorruptAfterMemtableInsert) { - // Record memtable entry size. - // Not corrupting memtable entry here since it will segfault - // or fail some asserts inside memtablerep implementation - // e.g., when key_len is corrupted. - SyncPoint::GetInstance()->SetCallBack( - "MemTable::Add:BeforeReturn:Encoded", [&](void* arg) { - Slice encoded = *static_cast(arg); - entry_len_ = encoded.size(); - }); - - SyncPoint::GetInstance()->SetCallBack( - "Memtable::SaveValue:Begin:entry", [&](void* entry) { - char* buf = *static_cast(entry); - buf[corrupt_byte_offset_] += corrupt_byte_addend_; - ++corrupt_byte_offset_; - }); - SyncPoint::GetInstance()->EnableProcessing(); - Options options = CurrentOptions(); - options.memtable_protection_bytes_per_key = - memtable_protection_bytes_per_key_; - if (op_type_ == WriteBatchOpType::kMerge) { - options.merge_operator = MergeOperators::CreateStringAppendOperator(); - } - - SkipNotToCorruptEntry(); - while (MoreBytesToCorrupt()) { - Reopen(options); - ASSERT_OK(ExecuteWrite(nullptr)); - std::string val; - ASSERT_TRUE(db_->Get(ReadOptions(), "key", &val).IsCorruption()); - Destroy(options); - SkipNotToCorruptEntry(); - } -} - -TEST_P(DbMemtableKVChecksumTest, - GetWithColumnFamilyCorruptAfterMemtableInsert) { - // Record memtable entry size. - // Not corrupting memtable entry here since it will segfault - // or fail some asserts inside memtablerep implementation - // e.g., when key_len is corrupted. - SyncPoint::GetInstance()->SetCallBack( - "MemTable::Add:BeforeReturn:Encoded", [&](void* arg) { - Slice encoded = *static_cast(arg); - entry_len_ = encoded.size(); - }); - - SyncPoint::GetInstance()->SetCallBack( - "Memtable::SaveValue:Begin:entry", [&](void* entry) { - char* buf = *static_cast(entry); - buf[corrupt_byte_offset_] += corrupt_byte_addend_; - ++corrupt_byte_offset_; - }); - SyncPoint::GetInstance()->EnableProcessing(); - Options options = CurrentOptions(); - options.memtable_protection_bytes_per_key = - memtable_protection_bytes_per_key_; - if (op_type_ == WriteBatchOpType::kMerge) { - options.merge_operator = MergeOperators::CreateStringAppendOperator(); - } - - SkipNotToCorruptEntry(); - while (MoreBytesToCorrupt()) { - Reopen(options); - CreateAndReopenWithCF({"pikachu"}, options); - ASSERT_OK(ExecuteWrite(handles_[1])); - std::string val; - ASSERT_TRUE( - db_->Get(ReadOptions(), handles_[1], "key", &val).IsCorruption()); - Destroy(options); - SkipNotToCorruptEntry(); - } -} - -TEST_P(DbMemtableKVChecksumTest, IteratorWithCorruptAfterMemtableInsert) { - SyncPoint::GetInstance()->SetCallBack( - "MemTable::Add:BeforeReturn:Encoded", - std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, - std::placeholders::_1)); - SyncPoint::GetInstance()->EnableProcessing(); - Options options = CurrentOptions(); - options.memtable_protection_bytes_per_key = - memtable_protection_bytes_per_key_; - if (op_type_ == WriteBatchOpType::kMerge) { - options.merge_operator = MergeOperators::CreateStringAppendOperator(); - } - - SkipNotToCorruptEntry(); - while (MoreBytesToCorrupt()) { - Reopen(options); - ASSERT_OK(ExecuteWrite(nullptr)); - Iterator* it = db_->NewIterator(ReadOptions()); - it->SeekToFirst(); - ASSERT_FALSE(it->Valid()); - ASSERT_TRUE(it->status().IsCorruption()); - delete it; - Destroy(options); - SkipNotToCorruptEntry(); - } -} - -TEST_P(DbMemtableKVChecksumTest, - IteratorWithColumnFamilyCorruptAfterMemtableInsert) { - SyncPoint::GetInstance()->SetCallBack( - "MemTable::Add:BeforeReturn:Encoded", - std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, - std::placeholders::_1)); - SyncPoint::GetInstance()->EnableProcessing(); - Options options = CurrentOptions(); - options.memtable_protection_bytes_per_key = - memtable_protection_bytes_per_key_; - if (op_type_ == WriteBatchOpType::kMerge) { - options.merge_operator = MergeOperators::CreateStringAppendOperator(); - } - - SkipNotToCorruptEntry(); - while (MoreBytesToCorrupt()) { - Reopen(options); - CreateAndReopenWithCF({"pikachu"}, options); - ASSERT_OK(ExecuteWrite(handles_[1])); - Iterator* it = db_->NewIterator(ReadOptions(), handles_[1]); - it->SeekToFirst(); - ASSERT_FALSE(it->Valid()); - ASSERT_TRUE(it->status().IsCorruption()); - delete it; - Destroy(options); - SkipNotToCorruptEntry(); - } -} - -TEST_P(DbMemtableKVChecksumTest, FlushWithCorruptAfterMemtableInsert) { - SyncPoint::GetInstance()->SetCallBack( - "MemTable::Add:BeforeReturn:Encoded", - std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this, - std::placeholders::_1)); - SyncPoint::GetInstance()->EnableProcessing(); - Options options = CurrentOptions(); - options.memtable_protection_bytes_per_key = - memtable_protection_bytes_per_key_; - if (op_type_ == WriteBatchOpType::kMerge) { - options.merge_operator = MergeOperators::CreateStringAppendOperator(); - } - - SkipNotToCorruptEntry(); - // Not corruping each byte like other tests since Flush() is relatively slow. - Reopen(options); - ASSERT_OK(ExecuteWrite(nullptr)); - ASSERT_TRUE(Flush().IsCorruption()); - // DB enters read-only state when flush reads corrupted data - ASSERT_TRUE(dbfull()->TEST_GetBGError().IsCorruption()); - Destroy(options); -} } // namespace ROCKSDB_NAMESPACE diff --git a/db/db_memtable_test.cc b/db/db_memtable_test.cc index cae592db36..049ec2d572 100644 --- a/db/db_memtable_test.cc +++ b/db/db_memtable_test.cc @@ -39,11 +39,25 @@ class MockMemTableRep : public MemTableRep { last_hint_out_ = *hint; } - bool Contains(const char* key) const override { return rep_->Contains(key); } + bool InsertKeyValue(const Slice& ikey, const Slice& value) override { + return rep_->InsertKeyValue(ikey, value); + } + + bool InsertKeyValueWithHint(const Slice& ikey, + const Slice& value, void** hint) override { + num_insert_with_hint_++; + EXPECT_NE(nullptr, hint); + last_hint_in_ = *hint; + bool ret = rep_->InsertKeyValueWithHint(ikey, value, hint); + last_hint_out_ = *hint; + return ret; + } + + bool Contains(const Slice& key) const override { return rep_->Contains(key); } - void Get(const LookupKey& k, void* callback_args, - bool (*callback_func)(void* arg, const char* entry)) override { - rep_->Get(k, callback_args, callback_func); + void Get(const ReadOptions& ro, const LookupKey& k, void* callback_args, + bool (*callback_func)(void* arg, const KeyValuePair&)) override { + rep_->Get(ro, k, callback_args, callback_func); } size_t ApproximateMemoryUsage() override { @@ -65,12 +79,45 @@ class MockMemTableRep : public MemTableRep { int num_insert_with_hint_; }; +static auto g_cspp_fac = []()-> std::shared_ptr { + const char* memtab_opt = getenv("MemTableRepFactory"); + if (memtab_opt && strncmp(memtab_opt, "cspp:", 5) == 0) { + #ifdef HAS_TOPLING_CSPP_MEMTABLE + extern MemTableRepFactory* NewCSPPMemTabForPlain(const std::string&); + return std::shared_ptr(NewCSPPMemTabForPlain(memtab_opt + 5)); + #else + fprintf(stderr, "env MemTableRepFactory is cspp but HAS_TOPLING_CSPP_MEMTABLE is not defined\n"); + #endif + } + return nullptr; +}(); + class MockMemTableRepFactory : public MemTableRepFactory { public: MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& cmp, Allocator* allocator, const SliceTransform* transform, Logger* logger) override { + return CreateMemTableRep(cmp, allocator, transform, logger, 0); + } + + virtual MemTableRep* CreateMemTableRep( + const std::string& level0_dir, + const MutableCFOptions& mcfo, + const MemTableRep::KeyComparator& cmp, Allocator* allocator, + const SliceTransform* transform, Logger* logger, + uint32_t column_family_id) { + last_column_family_id_ = column_family_id; + if (g_cspp_fac) { + auto ucmp = cmp.icomparator()->user_comparator(); + if (IsBytewiseComparator(ucmp)) { + auto rep = g_cspp_fac->CreateMemTableRep + (level0_dir, mcfo, cmp, allocator, transform, logger, column_family_id); + mock_rep_ = new MockMemTableRep(allocator, rep); + return mock_rep_; + } + fprintf(stderr, "MemTableTest skip %s\n", ucmp->Name()); + } SkipListFactory factory; MemTableRep* skiplist_rep = factory.CreateMemTableRep(cmp, allocator, transform, logger); @@ -83,8 +130,9 @@ class MockMemTableRepFactory : public MemTableRepFactory { const SliceTransform* transform, Logger* logger, uint32_t column_family_id) override { - last_column_family_id_ = column_family_id; - return CreateMemTableRep(cmp, allocator, transform, logger); + MutableCFOptions mcfo; + return CreateMemTableRep("/tmp", mcfo, cmp, allocator, transform, logger, + column_family_id); } const char* Name() const override { return "MockMemTableRepFactory"; } @@ -262,12 +310,13 @@ TEST_F(DBMemTableTest, ConcurrentMergeWrite) { ReadOptions roptions; SequenceNumber max_covering_tombstone_seq = 0; LookupKey lkey("key", kMaxSequenceNumber); - bool res = mem->Get(lkey, &value, /*columns=*/nullptr, /*timestamp=*/nullptr, + PinnableSlice pin; + bool res = mem->Get(lkey, &pin, /*columns=*/nullptr, /*timestamp=*/nullptr, &status, &merge_context, &max_covering_tombstone_seq, roptions, false /* immutable_memtable */); ASSERT_OK(status); ASSERT_TRUE(res); - uint64_t ivalue = DecodeFixed64(Slice(value).data()); + uint64_t ivalue = DecodeFixed64(pin.data()); uint64_t sum = 0; for (int seq = 0; seq < num_ops; seq++) { sum += seq; @@ -278,6 +327,9 @@ TEST_F(DBMemTableTest, ConcurrentMergeWrite) { } TEST_F(DBMemTableTest, InsertWithHint) { + if (g_cspp_fac) { + return; // skip this test for cspp + } Options options; options.allow_concurrent_memtable_write = false; options.create_if_missing = true; diff --git a/db/db_range_del_test.cc b/db/db_range_del_test.cc index 003117eec9..a84cad4c56 100644 --- a/db/db_range_del_test.cc +++ b/db/db_range_del_test.cc @@ -2214,7 +2214,7 @@ class TombstoneTestSstPartitioner : public SstPartitioner { PartitionerResult ShouldPartition( const PartitionerRequest& request) override { - if (cmp->Compare(*request.current_user_key, DBTestBase::Key(5)) == 0) { + if (cmp->Compare(request.current_user_key, DBTestBase::Key(5)) == 0) { return kRequired; } else { return kNotRequired; diff --git a/db/db_rate_limiter_test.cc b/db/db_rate_limiter_test.cc index 05419db44a..4f09a2f17b 100644 --- a/db/db_rate_limiter_test.cc +++ b/db/db_rate_limiter_test.cc @@ -221,6 +221,9 @@ TEST_P(DBRateLimiterOnReadTest, Iterator) { } } ASSERT_OK(iter->status()); + if (GetReadOptions().cache_sst_file_iter) { + return; + } // Reverse scan does not read evenly (one block per iteration) due to // descending seqno ordering, so wait until after the loop to check total. ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER)); diff --git a/db/db_secondary_test.cc b/db/db_secondary_test.cc index 987756906e..102a9ea482 100644 --- a/db/db_secondary_test.cc +++ b/db/db_secondary_test.cc @@ -493,6 +493,7 @@ class TraceFileEnv : public EnvWrapper { char* scratch) const override { return target_->Read(offset, n, result, scratch); } + intptr_t FileDescriptor() const final { return target_->FileDescriptor(); } private: std::unique_ptr target_; diff --git a/db/db_sst_test.cc b/db/db_sst_test.cc index 7590aa2f11..e5a9a7b0ff 100644 --- a/db/db_sst_test.cc +++ b/db/db_sst_test.cc @@ -85,6 +85,7 @@ TEST_F(DBSSTTest, DontDeletePendingOutputs) { Compact("a", "b"); } +#ifdef ROCKSDB_SUPPORT_LEVELDB_FILE_LDB // 1 Create some SST files by inserting K-V pairs into DB // 2 Close DB and change suffix from ".sst" to ".ldb" for every other SST file // 3 Open DB and check if all key can be read @@ -133,6 +134,7 @@ TEST_F(DBSSTTest, SSTsWithLdbSuffixHandling) { } Destroy(options); } +#endif // ROCKSDB_SUPPORT_LEVELDB_FILE_LDB // Check that we don't crash when opening DB with // DBOptions::skip_checking_sst_file_sizes_on_db_open = true. diff --git a/db/db_statistics_test.cc b/db/db_statistics_test.cc index f430811d3e..07afb80f99 100644 --- a/db/db_statistics_test.cc +++ b/db/db_statistics_test.cc @@ -142,7 +142,7 @@ TEST_F(DBStatisticsTest, MutexWaitStatsDisabledByDefault) { ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, kMutexWaitDelay); ASSERT_OK(Put("hello", "rocksdb")); - ASSERT_EQ(TestGetTickerCount(options, DB_MUTEX_WAIT_MICROS), 0); + ASSERT_EQ(TestGetTickerCount(options, DB_MUTEX_WAIT_NANOS), 0); ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0); } @@ -156,7 +156,7 @@ TEST_F(DBStatisticsTest, MutexWaitStats) { ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, kMutexWaitDelay); ASSERT_OK(Put("hello", "rocksdb")); - ASSERT_GE(TestGetTickerCount(options, DB_MUTEX_WAIT_MICROS), kMutexWaitDelay); + ASSERT_GE(TestGetTickerCount(options, DB_MUTEX_WAIT_NANOS), kMutexWaitDelay); ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0); } diff --git a/db/db_test2.cc b/db/db_test2.cc index e471685b21..e164287c79 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -5491,6 +5491,8 @@ class DummyOldStats : public Statistics { } bool HistEnabledForType(uint32_t /*type*/) const override { return false; } std::string ToString() const override { return ""; } + void GetAggregated(uint64_t* tickers, struct HistogramStat*) const override {} + void Merge(const uint64_t* tickers, const struct HistogramStat*) override {} std::atomic num_rt{0}; std::atomic num_mt{0}; }; diff --git a/db/db_test_util.h b/db/db_test_util.h index 023784f615..2dea33965e 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -234,6 +234,8 @@ class SpecialEnv : public EnvWrapper { return base_->GetUniqueId(id, max_size); } uint64_t GetFileSize() final { return base_->GetFileSize(); } + intptr_t FileDescriptor() const final { return base_->FileDescriptor(); } + void SetFileSize(uint64_t fsize) final { base_->SetFileSize(fsize); } }; class ManifestFile : public WritableFile { public: @@ -272,6 +274,9 @@ class SpecialEnv : public EnvWrapper { return base_->Allocate(offset, len); } + intptr_t FileDescriptor() const final { return base_->FileDescriptor(); } + void SetFileSize(uint64_t fsize) { base_->SetFileSize(fsize); } + private: SpecialEnv* env_; std::unique_ptr base_; @@ -347,6 +352,8 @@ class SpecialEnv : public EnvWrapper { return base_->Allocate(offset, len); } uint64_t GetFileSize() final { return base_->GetFileSize(); } + intptr_t FileDescriptor() const final { return base_->FileDescriptor(); } + void SetFileSize(uint64_t fsize) { base_->SetFileSize(fsize); } private: SpecialEnv* env_; @@ -376,6 +383,8 @@ class SpecialEnv : public EnvWrapper { Status Allocate(uint64_t offset, uint64_t len) override { return base_->Allocate(offset, len); } + intptr_t FileDescriptor() const final { return base_->FileDescriptor(); } + void SetFileSize(uint64_t fsize) { base_->SetFileSize(fsize); } private: SpecialEnv* env_; @@ -452,6 +461,8 @@ class SpecialEnv : public EnvWrapper { return s; } + intptr_t FileDescriptor() const final { return target_->FileDescriptor(); } + private: std::unique_ptr target_; anon::AtomicCounter* counter_; @@ -478,6 +489,8 @@ class SpecialEnv : public EnvWrapper { return target_->Prefetch(offset, n); } + intptr_t FileDescriptor() const final { return target_->FileDescriptor(); } + private: std::unique_ptr target_; std::atomic* fail_cnt_; diff --git a/db/dbformat.cc b/db/dbformat.cc index 63bb354de8..f459235890 100644 --- a/db/dbformat.cc +++ b/db/dbformat.cc @@ -54,6 +54,7 @@ EntryType GetEntryType(ValueType value_type) { } void AppendInternalKey(std::string* result, const ParsedInternalKey& key) { + result->reserve(key.user_key.size() + 8); result->append(key.user_key.data(), key.user_key.size()); PutFixed64(result, PackSequenceAndType(key.sequence, key.type)); } @@ -62,6 +63,7 @@ void AppendInternalKeyWithDifferentTimestamp(std::string* result, const ParsedInternalKey& key, const Slice& ts) { assert(key.user_key.size() >= ts.size()); + result->reserve(key.user_key.size() + 8); result->append(key.user_key.data(), key.user_key.size() - ts.size()); result->append(ts.data(), ts.size()); PutFixed64(result, PackSequenceAndType(key.sequence, key.type)); @@ -213,28 +215,30 @@ int InternalKeyComparator::Compare(const ParsedInternalKey& a, LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s, const Slice* ts) { + static_assert(offsetof(LookupKey, longstart_) == 8); size_t usize = _user_key.size(); size_t ts_sz = (nullptr == ts) ? 0 : ts->size(); - size_t needed = usize + ts_sz + 13; // A conservative estimate + klength_ = uint32_t(usize + ts_sz + 8); + char buf[8]; + auto klen_len = EncodeVarint32(buf, klength_) - buf; + klen_len_ = char(klen_len); char* dst; - if (needed <= sizeof(space_)) { - dst = space_; + if (LIKELY(klength_ <= sizeof(space_) - 4)) { + dst = space_ + 4 - klen_len; } else { - dst = new char[needed]; + char* ptr = new char[usize + ts_sz + 16]; // precise space + dst = ptr + 8 - klen_len; + longstart_ = ptr + 8; } - start_ = dst; - // NOTE: We don't support users keys of more than 2GB :) - dst = EncodeVarint32(dst, static_cast(usize + ts_sz + 8)); - kstart_ = dst; + ROCKSDB_ASSUME(klen_len >= 1 && klen_len <= 5); + memcpy(dst, buf, klen_len); dst += klen_len; memcpy(dst, _user_key.data(), usize); dst += usize; - if (nullptr != ts) { + if (UNLIKELY(nullptr != ts)) { memcpy(dst, ts->data(), ts_sz); dst += ts_sz; } EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek)); - dst += 8; - end_ = dst; } void IterKey::EnlargeBuffer(size_t key_size) { @@ -246,4 +250,32 @@ void IterKey::EnlargeBuffer(size_t key_size) { buf_ = new char[key_size]; buf_size_ = key_size; } + +void IterKey::TrimAppend(const size_t shared_len, const char* non_shared_data, + const size_t non_shared_len) { + assert(shared_len <= key_size_); + size_t total_size = shared_len + non_shared_len; + + if (IsKeyPinned() /* key is not in buf_ */) { + // Copy the key from external memory to buf_ (copy shared_len bytes) + EnlargeBufferIfNeeded(total_size); + memcpy(buf(), key_, shared_len); + } else if (total_size > buf_size_) { + // Need to allocate space, delete previous space + char* p = new char[total_size]; + memcpy(p, key_, shared_len); + + if (buf_size_ != sizeof(space_)) { + delete[] buf_; + } + + buf_ = p; + buf_size_ = total_size; + } + + memcpy(buf() + shared_len, non_shared_data, non_shared_len); + key_ = buf(); + key_size_ = total_size; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/dbformat.h b/db/dbformat.h index 981866c09d..0052160b2f 100644 --- a/db/dbformat.h +++ b/db/dbformat.h @@ -15,6 +15,8 @@ #include #include "rocksdb/comparator.h" +#include "rocksdb/enum_reflection.h" +#include "rocksdb/filter_policy.h" #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" #include "rocksdb/types.h" @@ -36,7 +38,7 @@ class InternalKey; // data structures. // The highest bit of the value type needs to be reserved to SST tables // for them to do more flexible encoding. -enum ValueType : unsigned char { +ROCKSDB_ENUM_PLAIN(ValueType, unsigned char, kTypeDeletion = 0x0, kTypeValue = 0x1, kTypeMerge = 0x2, @@ -71,7 +73,7 @@ enum ValueType : unsigned char { kTypeMaxValid, // Should be after the last valid type, only used for // validation kMaxValue = 0x7F // Not used for storing records. -}; +); // Defined in dbformat.cc extern const ValueType kValueTypeForSeek; @@ -110,6 +112,9 @@ struct ParsedInternalKey { Slice user_key; SequenceNumber sequence; ValueType type; + unsigned char ext_ui08 = 0; + uint16_t ext_ui16 = 0; + uint32_t ext_ui32 = 0; ParsedInternalKey() : sequence(kMaxSequenceNumber), @@ -118,6 +123,25 @@ struct ParsedInternalKey { // u contains timestamp if user timestamp feature is enabled. ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t) : user_key(u), sequence(seq), type(t) {} + ParsedInternalKey(const Slice& u, uint64_t seqvt) + : user_key(u), sequence(seqvt >> 8), type(ValueType(seqvt)) {} + explicit ParsedInternalKey(const Slice& ik) + : user_key(ik.data_, ik.size_ - 8) { + ROCKSDB_ASSERT_GE(ik.size_, 8); + auto seqvt = GetUnaligned(ik.data_ + ik.size_ - 8); + sequence = seqvt >> 8; + type = ValueType(seqvt); + } + // same as cons ParsedInternalKey(const Slice& ik) + inline void FastParseInternalKey(const Slice& ik) { + user_key.data_ = ik.data_; + user_key.size_ = ik.size_ - 8; + ROCKSDB_ASSERT_GE(ik.size_, 8); + auto seqvt = GetUnaligned(ik.data_ + ik.size_ - 8); + sequence = seqvt >> 8; + type = ValueType(seqvt); + } + inline uint64_t GetTag() const { return sequence << 8 | uint64_t(type); } std::string DebugString(bool log_err_key, bool hex) const; void clear() { @@ -138,6 +162,7 @@ struct ParsedInternalKey { return Slice(const_cast(addr), ts_sz); } }; +static_assert(sizeof(ParsedInternalKey) == 32); // Return the length of the encoding of "key". inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) { @@ -165,8 +190,40 @@ inline void UnPackSequenceAndType(uint64_t packed, uint64_t* seq, // assert(IsExtendedValueType(*t)); } +inline void UnPackSequenceAndType(uint64_t packed, ParsedInternalKey* pikey) { + pikey->sequence = packed >> 8; + pikey->type = static_cast(packed & 0xff); +} + +inline std::pair +UnPackSequenceAndType(uint64_t packed) { + return {packed >> 8, ValueType(packed & 0xff)}; +} + EntryType GetEntryType(ValueType value_type); +inline void SetInternalKey(std::string* result, Slice ukey, uint64_t seqvt) { + result->assign(ukey.data(), ukey.size()); + PutFixed64(result, seqvt); +} +inline void SetInternalKey(std::string* result, Slice ukey, + SequenceNumber seq, ValueType vt) { + result->assign(ukey.data(), ukey.size()); + PutFixed64(result, PackSequenceAndType(seq, vt)); +} + +// user code should ensure buf size is at least ukey.size() + 8 +inline void SetInternalKey(char* buf, Slice ukey, + SequenceNumber seq, ValueType vt) { + memcpy(buf, ukey.data_, ukey.size_); + auto value = PackSequenceAndType(seq, vt); + if (port::kLittleEndian) { + memcpy(buf + ukey.size_, &value, sizeof(value)); + } else { + EncodeFixed64(buf + ukey.size_, value); + } +} + // Append the serialization of "key" to *result. // // input [internal key]: @@ -277,17 +334,14 @@ inline Slice ExtractUserKey(const Slice& internal_key) { // output : inline Slice ExtractUserKeyAndStripTimestamp(const Slice& internal_key, size_t ts_sz) { - Slice ret = internal_key; - ret.remove_suffix(kNumInternalBytes + ts_sz); - return ret; + return Slice(internal_key.data(), + internal_key.size() - (kNumInternalBytes + ts_sz)); } // input [user key]: // output: inline Slice StripTimestampFromUserKey(const Slice& user_key, size_t ts_sz) { - Slice ret = user_key; - ret.remove_suffix(ts_sz); - return ret; + return Slice(user_key.data(), user_key.size() - ts_sz); } // input [user key]: @@ -371,6 +425,12 @@ class InternalKeyComparator // value `kDisableGlobalSequenceNumber`. int Compare(const Slice& a, SequenceNumber a_global_seqno, const Slice& b, SequenceNumber b_global_seqno) const; + + size_t timestamp_size() const noexcept { return user_comparator_.timestamp_size(); } + uint8_t opt_cmp_type() const noexcept { return user_comparator_.opt_cmp_type(); } + bool IsForwardBytewise() const noexcept { return user_comparator_.IsForwardBytewise(); } + bool IsReverseBytewise() const noexcept { return user_comparator_.IsReverseBytewise(); } + bool IsBytewise() const noexcept { return user_comparator_.IsBytewise(); } }; // The class represent the internal key in encoded form. @@ -416,6 +476,7 @@ class InternalKey { Slice user_key() const { return ExtractUserKey(rep_); } size_t size() const { return rep_.size(); } + bool empty() const { return rep_.empty(); } void Set(const Slice& _user_key, SequenceNumber s, ValueType t) { SetFrom(ParsedInternalKey(_user_key, s, t)); @@ -463,7 +524,7 @@ inline Status ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result, bool log_err_key) { const size_t n = internal_key.size(); - if (n < kNumInternalBytes) { + if (UNLIKELY(n < kNumInternalBytes)) { return Status::Corruption("Corrupted Key: Internal Key too small. Size=" + std::to_string(n) + ". "); } @@ -475,7 +536,7 @@ inline Status ParseInternalKey(const Slice& internal_key, assert(result->type <= ValueType::kMaxValue); result->user_key = Slice(internal_key.data(), n - kNumInternalBytes); - if (IsExtendedValueType(result->type)) { + if (LIKELY(IsExtendedValueType(static_cast(c)))) { return Status::OK(); } else { return Status::Corruption("Corrupted Key", @@ -503,6 +564,18 @@ inline uint64_t GetInternalKeySeqno(const Slice& internal_key) { return num >> 8; } +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#pragma GCC diagnostic ignored "-Wstringop-overflow" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-warning-option" +#pragma clang diagnostic ignored "-Warray-bounds" +#pragma clang diagnostic ignored "-Wstringop-overflow" +#pragma clang diagnostic ignored "-Wmaybe-uninitialized" +#endif // The class to store keys in an efficient way. It allows: // 1. Users can either copy the key into it, or have it point to an unowned // address. @@ -512,8 +585,7 @@ inline uint64_t GetInternalKeySeqno(const Slice& internal_key) { class IterKey { public: IterKey() - : buf_(space_), - key_(buf_), + : key_(space_), key_size_(0), buf_size_(sizeof(space_)), is_user_key_(true) {} @@ -556,31 +628,7 @@ class IterKey { // shared_len: bytes in [0, shard_len-1] would be remained // non_shared_data: data to be append, its length must be >= non_shared_len void TrimAppend(const size_t shared_len, const char* non_shared_data, - const size_t non_shared_len) { - assert(shared_len <= key_size_); - size_t total_size = shared_len + non_shared_len; - - if (IsKeyPinned() /* key is not in buf_ */) { - // Copy the key from external memory to buf_ (copy shared_len bytes) - EnlargeBufferIfNeeded(total_size); - memcpy(buf_, key_, shared_len); - } else if (total_size > buf_size_) { - // Need to allocate space, delete previous space - char* p = new char[total_size]; - memcpy(p, key_, shared_len); - - if (buf_ != space_) { - delete[] buf_; - } - - buf_ = p; - buf_size_ = total_size; - } - - memcpy(buf_ + shared_len, non_shared_data, non_shared_len); - key_ = buf_; - key_size_ = total_size; - } + const size_t non_shared_len); // A version of `TrimAppend` assuming the last bytes of length `ts_sz` in the // user key part of `key_` is not counted towards shared bytes. And the @@ -671,8 +719,9 @@ class IterKey { assert(IsKeyPinned() == true); Reserve(key_size_); - memcpy(buf_, key_, key_size_); - key_ = buf_; + char* bufp = buf(); + memcpy(bufp, key_, key_size_); + key_ = bufp; } // Update the sequence number in the internal key. Guarantees not to @@ -680,16 +729,17 @@ class IterKey { void UpdateInternalKey(uint64_t seq, ValueType t, const Slice* ts = nullptr) { assert(!IsKeyPinned()); assert(key_size_ >= kNumInternalBytes); + char* bufp = buf(); if (ts) { assert(key_size_ >= kNumInternalBytes + ts->size()); - memcpy(&buf_[key_size_ - kNumInternalBytes - ts->size()], ts->data(), + memcpy(&bufp[key_size_ - kNumInternalBytes - ts->size()], ts->data(), ts->size()); } uint64_t newval = (seq << 8) | t; - EncodeFixed64(&buf_[key_size_ - kNumInternalBytes], newval); + EncodeFixed64(&bufp[key_size_ - kNumInternalBytes], newval); } - bool IsKeyPinned() const { return (key_ != buf_); } + bool IsKeyPinned() const { return (key_ != buf()); } // If `ts` is provided, user_key should not contain timestamp, // and `ts` is appended after user_key. @@ -702,17 +752,18 @@ class IterKey { size_t usize = user_key.size(); size_t ts_sz = (ts != nullptr ? ts->size() : 0); EnlargeBufferIfNeeded(psize + usize + sizeof(uint64_t) + ts_sz); + char* bufp = buf(); if (psize > 0) { - memcpy(buf_, key_prefix.data(), psize); + memcpy(bufp, key_prefix.data(), psize); } - memcpy(buf_ + psize, user_key.data(), usize); + memcpy(bufp + psize, user_key.data(), usize); if (ts) { - memcpy(buf_ + psize + usize, ts->data(), ts_sz); + memcpy(bufp + psize + usize, ts->data(), ts_sz); } - EncodeFixed64(buf_ + usize + psize + ts_sz, + EncodeFixed64(bufp + usize + psize + ts_sz, PackSequenceAndType(s, value_type)); - key_ = buf_; + key_ = bufp; key_size_ = psize + usize + sizeof(uint64_t) + ts_sz; is_user_key_ = false; } @@ -741,41 +792,48 @@ class IterKey { void EncodeLengthPrefixedKey(const Slice& key) { auto size = key.size(); EnlargeBufferIfNeeded(size + static_cast(VarintLength(size))); - char* ptr = EncodeVarint32(buf_, static_cast(size)); + char* bufp = buf(); + char* ptr = EncodeVarint32(bufp, static_cast(size)); memcpy(ptr, key.data(), size); - key_ = buf_; + key_ = bufp; is_user_key_ = true; } bool IsUserKey() const { return is_user_key_; } private: - char* buf_; const char* key_; - size_t key_size_; - size_t buf_size_; - char space_[39]; // Avoid allocation for short keys - bool is_user_key_; - + size_t key_size_ : 32; + size_t buf_size_ : 31; + size_t is_user_key_ : 1; + union { + char* buf_; + char space_[48]; // Avoid allocation for short keys + }; + + char* buf() { return buf_size_ <= sizeof(space_) ? space_ : buf_ ; } + const char* buf() const { return buf_size_ <= sizeof(space_) ? space_ : buf_ ; } + + __always_inline Slice SetKeyImpl(const Slice& key, bool copy) { size_t size = key.size(); if (copy) { // Copy key to buf_ EnlargeBufferIfNeeded(size); - memcpy(buf_, key.data(), size); - key_ = buf_; + char* bufp = buf(); + key_ = bufp; + memcpy(bufp, key.data(), size); } else { // Update key_ to point to external memory key_ = key.data(); } key_size_ = size; - return Slice(key_, key_size_); + return Slice(key_, size); } void ResetBuffer() { - if (buf_ != space_) { + if (sizeof(space_) != buf_size_) { delete[] buf_; - buf_ = space_; } buf_size_ = sizeof(space_); key_size_ = 0; @@ -786,10 +844,11 @@ class IterKey { // larger than the static allocated buffer, another buffer is dynamically // allocated, until a larger key buffer is requested. In that case, we // reallocate buffer and delete the old one. + __always_inline void EnlargeBufferIfNeeded(size_t key_size) { // If size is smaller than buffer size, continue using current buffer, // or the static allocated one, as default - if (key_size > buf_size_) { + if (UNLIKELY(key_size > buf_size_)) { EnlargeBuffer(key_size); } } @@ -813,6 +872,11 @@ class IterKey { } } }; +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif // Convert from a SliceTransform of user keys, to a SliceTransform of // internal keys. @@ -1036,4 +1100,89 @@ struct ParsedInternalKeyComparator { const InternalKeyComparator* cmp; }; +/////////////////////////////////////////////////////////////////////////// + +__always_inline uint64_t GetUnalignedU64(const void* ptr) noexcept { + uint64_t x; + memcpy(&x, ptr, sizeof(uint64_t)); + return x; +} +struct BytewiseCompareInternalKey { + __always_inline bool operator()(Slice x, Slice y) const noexcept { + size_t n = std::min(x.size_, y.size_) - 8; + int cmp = memcmp(x.data_, y.data_, n); + if (0 != cmp) return cmp < 0; + if (x.size_ != y.size_) return x.size_ < y.size_; + return GetUnalignedU64(x.data_ + n) > GetUnalignedU64(y.data_ + n); + } + __always_inline bool operator()(uint64_t x, uint64_t y) const noexcept { + return x < y; + } + BytewiseCompareInternalKey(...) {} +}; +struct RevBytewiseCompareInternalKey { + __always_inline bool operator()(Slice x, Slice y) const noexcept { + size_t n = std::min(x.size_, y.size_) - 8; + int cmp = memcmp(x.data_, y.data_, n); + if (0 != cmp) return cmp > 0; + if (x.size_ != y.size_) return x.size_ > y.size_; + return GetUnalignedU64(x.data_ + n) > GetUnalignedU64(y.data_ + n); + } + __always_inline bool operator()(uint64_t x, uint64_t y) const noexcept { + return x > y; + } + RevBytewiseCompareInternalKey(...) {} +}; +struct FallbackVirtCmp { + __always_inline bool operator()(Slice x, Slice y) const { + return icmp->Compare(x, y) < 0; + } + const InternalKeyComparator* icmp; + FallbackVirtCmp(const InternalKeyComparator* ic) : icmp(ic) {} +}; + +struct ForwardBytewiseLessUserKey { + __always_inline bool operator()(Slice x, Slice y) const noexcept { + return x < y; + } + ForwardBytewiseLessUserKey(...) {} +}; +struct ReverseBytewiseLessUserKey { + __always_inline bool operator()(Slice x, Slice y) const noexcept { + return y < x; + } + ReverseBytewiseLessUserKey(...) {} +}; +struct VirtualFunctionLessUserKey { + __always_inline bool operator()(Slice x, Slice y) const noexcept { + return cmp->Compare(x, y) < 0; + } + const Comparator* cmp; +}; + +__always_inline int BytewiseCompare(Slice x, Slice y) noexcept { + size_t n = std::min(x.size_, y.size_); + int cmp = memcmp(x.data_, y.data_, n); + if (cmp) + return cmp; + else + return int(x.size_ - y.size_); // ignore key len larger than 2G-1 +} +struct ForwardBytewiseCompareUserKeyNoTS { + __always_inline int operator()(Slice x, Slice y) const noexcept { + return BytewiseCompare(x, y); + } +}; +struct ReverseBytewiseCompareUserKeyNoTS { + __always_inline int operator()(Slice x, Slice y) const noexcept { + return BytewiseCompare(y, x); + } +}; +struct VirtualFunctionCompareUserKeyNoTS { + __always_inline int operator()(Slice x, Slice y) const noexcept { + return cmp->CompareWithoutTimestamp(x, y); + } + const Comparator* cmp; +}; + } // namespace ROCKSDB_NAMESPACE diff --git a/db/event_helpers.cc b/db/event_helpers.cc index 65f6a5a486..eb57e38618 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -118,6 +118,7 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( << "num_range_deletions" << table_properties.num_range_deletions << "format_version" << table_properties.format_version << "fixed_key_len" << table_properties.fixed_key_len + << "fixed_value_len" << table_properties.fixed_value_len << "filter_policy" << table_properties.filter_policy_name << "column_family_name" << table_properties.column_family_name << "column_family_id" << table_properties.column_family_id diff --git a/db/external_sst_file_ingestion_job.cc b/db/external_sst_file_ingestion_job.cc index a4a1947145..5ce7effabc 100644 --- a/db/external_sst_file_ingestion_job.cc +++ b/db/external_sst_file_ingestion_job.cc @@ -110,7 +110,19 @@ Status ExternalSstFileIngestionJob::Prepare( if (ingestion_options_.move_files) { status = fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr); - if (status.ok()) { + #if !defined(ROCKSDB_UNIT_TEST) + if (!status.ok()) { + status = fs_->RenameFile( + path_outside_db, path_inside_db, IOOptions(), nullptr); + } + #endif + if (!status.ok() && status.subcode() == Status::kCrossDevice) { + status = CopyFile(fs_.get(), path_outside_db, path_inside_db, + f.fd.file_size, true, nullptr, Temperature::kUnknown); + if (status.ok()) + status = fs_->DeleteFile(path_outside_db, IOOptions(), nullptr); + } + if (status.ok() && ingestion_options_.sync_file) { // It is unsafe to assume application had sync the file and file // directory before ingest the file. For integrity of RocksDB we need // to sync the file. @@ -137,6 +149,8 @@ Status ExternalSstFileIngestionJob::Prepare( } } } + } else if (status.ok()) { + // ToplingDB: ingestion_options_.sync_file is false, do nothing } else if (status.IsNotSupported() && ingestion_options_.failed_move_fall_back_to_copy) { // Original file is on a different FS, use copy instead of hard linking. @@ -421,11 +435,11 @@ Status ExternalSstFileIngestionJob::Run() { // exclusive endpoint. ParsedInternalKey smallest_parsed, largest_parsed; if (status.ok()) { - status = ParseInternalKey(*f.smallest_internal_key.rep(), + status = ParseInternalKey(f.smallest_internal_key.Encode(), &smallest_parsed, false /* log_err_key */); } if (status.ok()) { - status = ParseInternalKey(*f.largest_internal_key.rep(), &largest_parsed, + status = ParseInternalKey(f.largest_internal_key.Encode(), &largest_parsed, false /* log_err_key */); } if (!status.ok()) { @@ -722,6 +736,14 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo( // Get the external file properties auto props = table_reader->GetTableProperties(); + +#if defined(ROCKSDB_UNIT_TEST) + // ToplingDB: now rocksdb store global_seqno in manifest file, we does not + // need to read global_seqno from sst, so version and global_seqno are + // all not needed, so we skip it! + // if we does not skip it, the ingest will failed when ingest sst files + // from MergeTables! + // Now global_seqno are load from TableReaderOptions::largest_seqno const auto& uprops = props->user_collected_properties; // Get table version @@ -759,6 +781,8 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo( } else { return Status::InvalidArgument("External file version is not supported"); } +#endif + // Get number of entries in table file_to_ingest->num_entries = props->num_entries; file_to_ingest->num_range_deletions = props->num_range_deletions; @@ -871,9 +895,12 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo( props->orig_file_number, &(file_to_ingest->unique_id)); if (!s.ok()) { + if (db_options_.verify_sst_unique_id_in_manifest) { ROCKS_LOG_WARN(db_options_.info_log, - "Failed to get SST unique id for file %s", - file_to_ingest->internal_file_path.c_str()); + "Failed to get SST unique id for file %s, reason = %s", + external_file.c_str(), + s.ToString().c_str()); + } file_to_ingest->unique_id = kNullUniqueId64x2; } @@ -1027,7 +1054,8 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile( return Status::OK(); } else if (!ingestion_options_.allow_global_seqno) { return Status::InvalidArgument("Global seqno is required, but disabled"); - } else if (file_to_ingest->global_seqno_offset == 0) { + } else if (file_to_ingest->global_seqno_offset == 0 && + ingestion_options_.write_global_seqno) { return Status::InvalidArgument( "Trying to set global seqno for a file that don't have a global seqno " "field"); diff --git a/db/flush_job.cc b/db/flush_job.cc index a3e168823a..47740b3220 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -688,7 +688,7 @@ bool FlushJob::MemPurgeDecider(double threshold) { Slice key_slice, value_slice; ParsedInternalKey res; SnapshotImpl min_snapshot; - std::string vget; + PinnableSlice vget; Status mget_s, parse_s; MergeContext merge_context; SequenceNumber max_covering_tombstone_seq = 0, sqno = 0, @@ -920,9 +920,6 @@ Status FlushJob::WriteLevel0Table() { << GetFlushReasonString(flush_reason_); { - ScopedArenaIterator iter( - NewMergingIterator(&cfd_->internal_comparator(), memtables.data(), - static_cast(memtables.size()), &arena)); ROCKS_LOG_INFO(db_options_.info_log, "[%s] [JOB %d] Level-0 flush table #%" PRIu64 ": started", cfd_->GetName().c_str(), job_context_->job_id, @@ -954,13 +951,6 @@ Status FlushJob::WriteLevel0Table() { meta_.oldest_ancester_time = oldest_ancester_time; meta_.file_creation_time = current_time; - uint64_t num_input_entries = 0; - uint64_t memtable_payload_bytes = 0; - uint64_t memtable_garbage_bytes = 0; - IOStatus io_s; - - const std::string* const full_history_ts_low = - (full_history_ts_low_.empty()) ? nullptr : &full_history_ts_low_; TableBuilderOptions tboptions( *cfd_->ioptions(), mutable_cf_options_, cfd_->internal_comparator(), cfd_->int_tbl_prop_collector_factories(), output_compression_, @@ -969,9 +959,49 @@ Status FlushJob::WriteLevel0Table() { TableFileCreationReason::kFlush, oldest_key_time, current_time, db_id_, db_session_id_, 0 /* target_file_size */, meta_.fd.GetNumber()); + if (mems_.size() == 1 && mems_.front()->SupportConvertToSST()) { + // convert MemTable to sst + MemTable* memtable = mems_.front(); + // pass these fields to ConvertToSST, to fill TableProperties + meta_.num_entries = memtable->num_entries(); + meta_.num_deletions = memtable->num_deletes(); + meta_.num_merges = memtable->num_merges(); + meta_.num_range_deletions = 0; + meta_.raw_key_size = memtable->raw_key_size(); + meta_.raw_value_size = memtable->raw_value_size(); + s = memtable->ConvertToSST(&meta_, tboptions); + if (!s.ok()) { + ROCKS_LOG_BUFFER(log_buffer_, + "[%s] [JOB %d] Level-0 ConvertToSST #%" PRIu64 ": ApproximateMemoryUsage %" PRIu64 + " bytes %s", + cfd_->GetName().c_str(), job_context_->job_id, + meta_.fd.GetNumber(), memtable->ApproximateMemoryUsage(), + s.ToString().c_str()); + goto UseBuildTable; + } + meta_.fd.smallest_seqno = std::min(memtable->GetEarliestSequenceNumber(), + memtable->GetFirstSequenceNumber()); + meta_.fd.largest_seqno = memtable->largest_seqno(); + meta_.marked_for_compaction = true; + for (auto* p_iter : memtables) { // memtables is vec of memtab iters + std::destroy_at(p_iter); // Attention!!! must! + } + memtables.clear(); + } + else { // call BuildTable +UseBuildTable: + uint64_t num_input_entries = 0; + uint64_t memtable_payload_bytes = 0; + uint64_t memtable_garbage_bytes = 0; + IOStatus io_s; + const std::string* const full_history_ts_low = + (full_history_ts_low_.empty()) ? nullptr : &full_history_ts_low_; const SequenceNumber job_snapshot_seq = job_context_->GetJobSnapshotSequence(); const ReadOptions read_options(Env::IOActivity::kFlush); + ScopedArenaIterator iter( + NewMergingIterator(&cfd_->internal_comparator(), memtables.data(), + static_cast(memtables.size()), &arena)); s = BuildTable(dbname_, versions_, db_options_, tboptions, file_options_, read_options, cfd_->table_cache(), iter.get(), std::move(range_del_iters), &meta_, &blob_file_additions, @@ -1007,6 +1037,7 @@ Status FlushJob::WriteLevel0Table() { memtable_garbage_bytes); } LogFlush(db_options_.info_log); + } // end call BuildTable } ROCKS_LOG_BUFFER(log_buffer_, "[%s] [JOB %d] Level-0 flush table #%" PRIu64 ": %" PRIu64 @@ -1023,6 +1054,9 @@ Status FlushJob::WriteLevel0Table() { DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced)); } TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table", &mems_); + + cfd_->PrepareNewMemtableInBackground(mutable_cf_options_); + db_mutex_->Lock(); } base_->Unref(); diff --git a/db/forward_iterator.cc b/db/forward_iterator.cc index c7691560eb..c5c7131a94 100644 --- a/db/forward_iterator.cc +++ b/db/forward_iterator.cc @@ -493,7 +493,7 @@ void ForwardIterator::SeekInternal(const Slice& internal_key, } uint32_t f_idx = 0; if (!seek_to_first && !seek_after_async_io) { - f_idx = FindFileInRange(level_files, internal_key, 0, + f_idx = vstorage->FindFileInRange(level, internal_key, 0, static_cast(level_files.size())); } @@ -1038,18 +1038,6 @@ bool ForwardIterator::TEST_CheckDeletedIters(int* pdeleted_iters, return retval; } -uint32_t ForwardIterator::FindFileInRange( - const std::vector& files, const Slice& internal_key, - uint32_t left, uint32_t right) { - auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool { - return cfd_->internal_comparator().InternalKeyComparator::Compare( - f->largest.Encode(), k) < 0; - }; - const auto& b = files.begin(); - return static_cast( - std::lower_bound(b + left, b + right, internal_key, cmp) - b); -} - void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) { if (iter == nullptr) { return; diff --git a/db/forward_iterator.h b/db/forward_iterator.h index cb418aeeb0..049e5969ac 100644 --- a/db/forward_iterator.h +++ b/db/forward_iterator.h @@ -107,10 +107,6 @@ class ForwardIterator : public InternalIterator { void UpdateCurrent(); bool NeedToSeekImmutable(const Slice& internal_key); void DeleteCurrentIter(); - uint32_t FindFileInRange(const std::vector& files, - const Slice& internal_key, uint32_t left, - uint32_t right); - bool IsOverUpperBound(const Slice& internal_key) const; // Set PinnedIteratorsManager for all children Iterators, this function should diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 6ef4b43023..e64721159e 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -31,6 +31,11 @@ #include "util/hash_containers.h" #include "util/string_util.h" +#if defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wnonnull" // for boost::replace_all_copy +#endif +#include + namespace ROCKSDB_NAMESPACE { @@ -40,6 +45,7 @@ const std::map InternalStats::compaction_level_stats = {LevelStatType::COMPACTED_FILES, LevelStat{"CompactedFiles", "CompactedFiles"}}, {LevelStatType::SIZE_BYTES, LevelStat{"SizeBytes", "Size"}}, + {LevelStatType::SIZE_RAW_KV, LevelStat{"SizeRawKV", "RawKV"}}, {LevelStatType::SCORE, LevelStat{"Score", "Score"}}, {LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB)"}}, {LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB)"}}, @@ -87,10 +93,11 @@ const std::map namespace { const double kMB = 1048576.0; const double kGB = kMB * 1024; +const double kTB = kGB * 1024; const double kMicrosInSec = 1000000.0; void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name, - const std::string& group_by) { + const char* group_by) { int written_size = snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str()); written_size = std::min(written_size, static_cast(len)); @@ -99,11 +106,33 @@ void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name, }; int line_size = snprintf( buf + written_size, len - written_size, - "%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s " - "%s\n", + "%-8s " // group_by + "%s " // NUM_FILES + "%s " // SIZE_BYTES + "%8s " // SIZE_RAW_KV + " %s " // SCORE + " %s " // READ_GB + " %s " // RN_GB + " %s " // RNP1_GB + " %s " // WRITE_GB + " %s " // W_NEW_GB + "%s " // MOVED_GB + "%s " // WRITE_AMP + "%s " // READ_MBPS + "%s " // WRITE_MBPS + " %s " // COMP_SEC + "%s " // COMP_CPU_SEC + " %s " // COMP_COUNT + "%s " // AVG_SEC + " %s " // KEY_IN + "%s " // KEY_DROP + " %s " // R_BLOB_GB + " %s\n", // W_BLOB_GB // Note that we skip COMPACTED_FILES and merge it with Files column - group_by.c_str(), hdr(LevelStatType::NUM_FILES), - hdr(LevelStatType::SIZE_BYTES), hdr(LevelStatType::SCORE), + group_by, hdr(LevelStatType::NUM_FILES), + hdr(LevelStatType::SIZE_BYTES), + hdr(LevelStatType::SIZE_RAW_KV), + hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB), hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB), hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB), hdr(LevelStatType::MOVED_GB), @@ -122,7 +151,8 @@ void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name, void PrepareLevelStats(std::map* level_stats, int num_files, int being_compacted, - double total_file_size, double score, double w_amp, + double total_file_size, double total_raw_kv, + double score, double w_amp, const InternalStats::CompactionStats& stats) { const uint64_t bytes_read = stats.bytes_read_non_output_levels + stats.bytes_read_output_level + @@ -134,6 +164,7 @@ void PrepareLevelStats(std::map* level_stats, (*level_stats)[LevelStatType::NUM_FILES] = num_files; (*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted; (*level_stats)[LevelStatType::SIZE_BYTES] = total_file_size; + (*level_stats)[LevelStatType::SIZE_RAW_KV] = total_raw_kv; (*level_stats)[LevelStatType::SCORE] = score; (*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB; (*level_stats)[LevelStatType::RN_GB] = @@ -163,31 +194,35 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name, snprintf( buf, len, "%4s " /* Level */ - "%6d/%-3d " /* Files */ - "%8s " /* Size */ - "%5.1f " /* Score */ - "%8.1f " /* Read(GB) */ - "%7.1f " /* Rn(GB) */ - "%8.1f " /* Rnp1(GB) */ - "%9.1f " /* Write(GB) */ - "%8.1f " /* Wnew(GB) */ + "%6d/%-4d " /* Files */ + "%10s " /* Size */ + "%10s " /* SIZE_RAW_KV */ + "%6.1f " /* Score */ + "%9.1f " /* Read(GB) */ + "%8.1f " /* Rn(GB) */ + "%9.1f " /* Rnp1(GB) */ + "%10.1f " /* Write(GB) */ + "%9.1f " /* Wnew(GB) */ "%9.1f " /* Moved(GB) */ "%5.1f " /* W-Amp */ "%8.1f " /* Rd(MB/s) */ "%8.1f " /* Wr(MB/s) */ - "%9.2f " /* Comp(sec) */ + "%11.2f " /* Comp(sec) */ "%17.2f " /* CompMergeCPU(sec) */ - "%9d " /* Comp(cnt) */ + "%10d " /* Comp(cnt) */ "%8.3f " /* Avg(sec) */ "%7s " /* KeyIn */ "%6s " /* KeyDrop */ - "%9.1f " /* Rblob(GB) */ - "%9.1f\n", /* Wblob(GB) */ + "%11.1f " /* Rblob(GB) */ + "%11.1f\n", /* Wblob(GB) */ name.c_str(), static_cast(stat_value.at(LevelStatType::NUM_FILES)), static_cast(stat_value.at(LevelStatType::COMPACTED_FILES)), BytesToHumanString( static_cast(stat_value.at(LevelStatType::SIZE_BYTES))) .c_str(), + BytesToHumanString( + static_cast(stat_value.at(LevelStatType::SIZE_RAW_KV))) + .c_str(), stat_value.at(LevelStatType::SCORE), stat_value.at(LevelStatType::READ_GB), stat_value.at(LevelStatType::RN_GB), @@ -214,10 +249,12 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name, void PrintLevelStats(char* buf, size_t len, const std::string& name, int num_files, int being_compacted, double total_file_size, + double total_raw_kv, double score, double w_amp, const InternalStats::CompactionStats& stats) { std::map level_stats; PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size, + total_raw_kv, score, w_amp, stats); PrintLevelStats(buf, len, name, level_stats); } @@ -1497,7 +1534,13 @@ bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/, Cache* InternalStats::GetBlockCacheForStats() { auto* table_factory = cfd_->ioptions()->table_factory.get(); assert(table_factory != nullptr); +#if 0 return table_factory->GetOptions(TableFactory::kBlockCacheOpts()); +#else + // defined in rockside: builtin_table_factory.cc + Cache* GetBlockCacheFromAnyTableFactory(TableFactory*); + return GetBlockCacheFromAnyTableFactory(table_factory); +#endif } bool InternalStats::HandleBlockCacheCapacity(uint64_t* value, DBImpl* /*db*/, @@ -1691,6 +1734,34 @@ void InternalStats::DumpDBMapStatsWriteStall( } } +static void DumpWriteStalls(std::ostringstream& str, + std::map& stats_map) { + str << "Write Stall (count): "; + + for (auto iter = stats_map.begin(); iter != stats_map.end(); ) { + std::string name = boost::replace_all_copy(iter->first, "-delays", ""); + str << name << ": delays " << iter->second; + ++iter; + if (stats_map.end() == iter) { + break; // should not goes here, check for safe + } + std::string name2 = boost::replace_all_copy(iter->first, "-stops", ""); + if (name2 == name) { + str << ", stops " << iter->second; + } + else { // should not goes here + str << iter->first << ": " << iter->second; + } + auto next = std::next(iter); + if (stats_map.end() == next) { + str << "\n"; + } else { + str << " | "; + } + iter = next; + } +} + void InternalStats::DumpDBStatsWriteStall(std::string* value) { assert(value); @@ -1698,19 +1769,7 @@ void InternalStats::DumpDBStatsWriteStall(std::string* value) { DumpDBMapStatsWriteStall(&write_stall_stats_map); std::ostringstream str; - str << "Write Stall (count): "; - - for (auto write_stall_stats_map_iter = write_stall_stats_map.begin(); - write_stall_stats_map_iter != write_stall_stats_map.end(); - write_stall_stats_map_iter++) { - const auto& name_and_stat = *write_stall_stats_map_iter; - str << name_and_stat.first << ": " << name_and_stat.second; - if (std::next(write_stall_stats_map_iter) == write_stall_stats_map.end()) { - str << "\n"; - } else { - str << ", "; - } - } + DumpWriteStalls(str, write_stall_stats_map); *value = str.str(); } @@ -1774,6 +1833,7 @@ void InternalStats::DumpCFMapStats( int total_files = 0; int total_files_being_compacted = 0; double total_file_size = 0; + double total_file_raw_kv = 0; uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED]; uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE]; uint64_t curr_ingest = flush_ingest + add_file_ingest; @@ -1784,7 +1844,10 @@ void InternalStats::DumpCFMapStats( if (comp_stats_[level].micros > 0 || comp_stats_[level].cpu_micros > 0 || files > 0) { compaction_stats_sum->Add(comp_stats_[level]); - total_file_size += vstorage->NumLevelBytes(level); + auto level_bytes = vstorage->NumLevelBytes(level); + auto level_raw_kv = vstorage->NumLevelRawKV(level); + total_file_size += level_bytes; + total_file_raw_kv += level_raw_kv; uint64_t input_bytes; if (level == 0) { input_bytes = curr_ingest; @@ -1800,7 +1863,8 @@ void InternalStats::DumpCFMapStats( input_bytes; std::map level_stats; PrepareLevelStats(&level_stats, files, files_being_compacted[level], - static_cast(vstorage->NumLevelBytes(level)), + static_cast(level_bytes), + static_cast(level_raw_kv), compaction_score[level], w_amp, comp_stats_[level]); (*levels_stats)[level] = level_stats; } @@ -1814,7 +1878,9 @@ void InternalStats::DumpCFMapStats( // Stats summary across levels std::map sum_stats; PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted, - total_file_size, 0, w_amp, *compaction_stats_sum); + total_file_size, + total_file_raw_kv, + 0, w_amp, *compaction_stats_sum); (*levels_stats)[-1] = sum_stats; // -1 is for the Sum level } @@ -1825,6 +1891,7 @@ void InternalStats::DumpCFMapStatsByPriority( std::map priority_stats; PrepareLevelStats(&priority_stats, 0 /* num_files */, 0 /* being_compacted */, 0 /* total_file_size */, + 0 /* total_file_raw_kv */, 0 /* compaction_score */, 0 /* w_amp */, comp_stats_by_pri_[priority]); (*priorities_stats)[static_cast(priority)] = priority_stats; @@ -1889,19 +1956,7 @@ void InternalStats::DumpCFStatsWriteStall(std::string* value, DumpCFMapStatsWriteStall(&write_stall_stats_map); std::ostringstream str; - str << "Write Stall (count): "; - - for (auto write_stall_stats_map_iter = write_stall_stats_map.begin(); - write_stall_stats_map_iter != write_stall_stats_map.end(); - write_stall_stats_map_iter++) { - const auto& name_and_stat = *write_stall_stats_map_iter; - str << name_and_stat.first << ": " << name_and_stat.second; - if (std::next(write_stall_stats_map_iter) == write_stall_stats_map.end()) { - str << "\n"; - } else { - str << ", "; - } - } + DumpWriteStalls(str, write_stall_stats_map); if (total_stall_count) { *total_stall_count = @@ -1958,13 +2013,13 @@ void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic, uint64_t interval_add_file_inget = add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile; uint64_t interval_ingest = - interval_flush_ingest + interval_add_file_inget + 1; + interval_flush_ingest + interval_add_file_inget; CompactionStats interval_stats(compaction_stats_sum); interval_stats.Subtract(cf_stats_snapshot_.comp_stats); - double w_amp = + double w_amp = 0 == interval_ingest ? 0 : (interval_stats.bytes_written + interval_stats.bytes_written_blob) / static_cast(interval_ingest); - PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats); + PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, 0, w_amp, interval_stats); value->append(buf); PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName(), "Priority"); @@ -2040,11 +2095,15 @@ void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic, } snprintf(buf, sizeof(buf), - "Cumulative compaction: %.2f GB write, %.2f MB/s write, " - "%.2f GB read, %.2f MB/s read, %.1f seconds\n", - compact_bytes_write / kGB, + "Cumulative compaction: %11.6f %s write, %7.2f MB/s write, " + "%11.6f %s read, %7.2f MB/s read, %7.1f seconds\n", + compact_bytes_write / + (compact_bytes_write < (1LL<<40) ? kGB : kTB ), + (compact_bytes_write < (1LL<<40) ? "GB" : "TB"), compact_bytes_write / kMB / std::max(seconds_up, 0.001), - compact_bytes_read / kGB, + compact_bytes_read / + (compact_bytes_read < (1LL<<40) ? kGB : kTB ), + (compact_bytes_read < (1LL<<40) ? "GB" : "TB"), compact_bytes_read / kMB / std::max(seconds_up, 0.001), compact_micros / kMicrosInSec); value->append(buf); @@ -2059,8 +2118,8 @@ void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic, snprintf( buf, sizeof(buf), - "Interval compaction: %.2f GB write, %.2f MB/s write, " - "%.2f GB read, %.2f MB/s read, %.1f seconds\n", + "Interval compaction: %11.6f GB write, %7.2f MB/s write, " + "%11.6f GB read, %7.2f MB/s read, %7.1f seconds\n", interval_compact_bytes_write / kGB, interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001), interval_compact_bytes_read / kGB, diff --git a/db/internal_stats.h b/db/internal_stats.h index 85c1a6bb1e..b91c5ae0f7 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -66,6 +66,7 @@ enum class LevelStatType { NUM_FILES, COMPACTED_FILES, SIZE_BYTES, + SIZE_RAW_KV, SCORE, READ_GB, RN_GB, diff --git a/db/lookup_key.h b/db/lookup_key.h index 68851bddd1..aad533821a 100644 --- a/db/lookup_key.h +++ b/db/lookup_key.h @@ -13,10 +13,12 @@ #include "rocksdb/slice.h" #include "rocksdb/types.h" +#include "port/likely.h" namespace ROCKSDB_NAMESPACE { // A helper class useful for DBImpl::Get() +#pragma pack(push, 1) class LookupKey { public: // Initialize *this for looking up user_key at a snapshot with @@ -26,43 +28,83 @@ class LookupKey { ~LookupKey(); + const char* memtable_key_data() const { + if (LIKELY(klength_ <= sizeof(space_) - 4)) + return space_ + 4 - klen_len_; + else + return longstart_ - klen_len_; + } + +#if 0 // not used now // Return a key suitable for lookup in a MemTable. Slice memtable_key() const { - return Slice(start_, static_cast(end_ - start_)); + if (LIKELY(klength_ <= sizeof(space_) - 4)) + return Slice(space_ + 4 - klen_len_, klen_len_ + klength_); + else + return Slice(longstart_ - klen_len_, klen_len_ + klength_); } +#endif // Return an internal key (suitable for passing to an internal iterator) Slice internal_key() const { - return Slice(kstart_, static_cast(end_ - kstart_)); + if (LIKELY(klength_ <= sizeof(space_) - 4)) + return Slice(space_ + 4, klength_); + else + return Slice(longstart_, klength_); } // Return the user key. // If user-defined timestamp is enabled, then timestamp is included in the // result. Slice user_key() const { - return Slice(kstart_, static_cast(end_ - kstart_ - 8)); + if (LIKELY(klength_ <= sizeof(space_) - 4)) + return Slice(space_ + 4, klength_ - 8); + else + return Slice(longstart_, klength_ - 8); } private: // We construct a char array of the form: - // klength varint32 <-- start_ - // userkey char[klength] <-- kstart_ - // tag uint64 - // <-- end_ + // short keys: klength_ <= sizeof(space_) - 4 + // klen_len <-- space_[0], klen_offset = 4 - klen_len + // unused <-- space_[1 ~ klen_offset), + // klength varint32 <-- space_[klen_offset ~ 4) + // userkey char <-- space_[4 ~ 4 + ukey_len), aligned to 8 + // tag uint64 + // long keys: klength_ > sizeof(space_) - 4 + // klen_len_ <-- space_[0] + // unused <-- space_[1~4) + // longstart_ <-- ptr to key data, klen_offset = 8 - klen_len + // unused <-- longstart_[-8 ~ -8 + klen_offset) + // klength varint32 <-- longstart_[-klen_len, 0) + // userkey char <-- longstart_[0 ~ ukey_len), aligned to 8 + // tag uint64 + // // The array is a suitable MemTable key. // The suffix starting with "userkey" can be used as an InternalKey. - const char* start_; - const char* kstart_; - const char* end_; - char space_[200]; // Avoid allocation for short keys + uint32_t klength_; // internal key len + union { + char space_[124]; // Avoid allocation for short keys + struct { + char klen_len_; + char klen_data_[3]; // for short keys + const char* longstart_; // for long keys + }; + }; // No copying allowed LookupKey(const LookupKey&); void operator=(const LookupKey&); }; +#pragma pack(pop) + +static_assert(sizeof(LookupKey) == 128); inline LookupKey::~LookupKey() { - if (start_ != space_) delete[] start_; + if (UNLIKELY(klength_ > sizeof(space_) - 4)) { + assert(size_t(longstart_) % 8 == 0); // must be aligned to 8 + delete[] (longstart_ - 8); + } } } // namespace ROCKSDB_NAMESPACE diff --git a/db/memtable.cc b/db/memtable.cc index 0b8786bc2f..dc2f104e3f 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -55,6 +55,7 @@ ImmutableMemTableOptions::ImmutableMemTableOptions( mutable_cf_options.memtable_prefix_bloom_size_ratio) * 8u), memtable_huge_page_size(mutable_cf_options.memtable_huge_page_size), + allow_merge_memtables(mutable_cf_options.allow_merge_memtables), memtable_whole_key_filtering( mutable_cf_options.memtable_whole_key_filtering), inplace_update_support(ioptions.inplace_update_support), @@ -64,9 +65,7 @@ ImmutableMemTableOptions::ImmutableMemTableOptions( statistics(ioptions.stats), merge_operator(ioptions.merge_operator.get()), info_log(ioptions.logger), - allow_data_in_errors(ioptions.allow_data_in_errors), - protection_bytes_per_key( - mutable_cf_options.memtable_protection_bytes_per_key) {} + allow_data_in_errors(ioptions.allow_data_in_errors) {} MemTable::MemTable(const InternalKeyComparator& cmp, const ImmutableOptions& ioptions, @@ -86,6 +85,8 @@ MemTable::MemTable(const InternalKeyComparator& cmp, : nullptr, mutable_cf_options.memtable_huge_page_size), table_(ioptions.memtable_factory->CreateMemTableRep( + ioptions.cf_paths[0].path, // level0_dir + mutable_cf_options, comparator_, &arena_, mutable_cf_options.prefix_extractor.get(), ioptions.logger, column_family_id)), range_del_table_(SkipListFactory().CreateMemTableRep( @@ -96,6 +97,10 @@ MemTable::MemTable(const InternalKeyComparator& cmp, num_entries_(0), num_deletes_(0), num_range_deletes_(0), + num_merges_(0), + largest_seqno_(0), + raw_key_size_(0), + raw_value_size_(0), write_buffer_size_(mutable_cf_options.write_buffer_size), flush_in_progress_(false), flush_completed_(false), @@ -118,6 +123,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp, approximate_memory_usage_(0), memtable_max_range_deletions_( mutable_cf_options.memtable_max_range_deletions) { + needs_user_key_cmp_in_get_ = table_->NeedsUserKeyCompareInGet(); UpdateFlushState(); // something went wrong if we need to flush before inserting anything assert(!ShouldScheduleFlush()); @@ -158,7 +164,7 @@ MemTable::~MemTable() { } size_t MemTable::ApproximateMemoryUsage() { - autovector usages = { + size_t usages[] = { arena_.ApproximateMemoryUsage(), table_->ApproximateMemoryUsage(), range_del_table_->ApproximateMemoryUsage(), ROCKSDB_NAMESPACE::ApproximateMemoryUsage(insert_hints_)}; @@ -257,7 +263,7 @@ void MemTable::UpdateFlushState() { void MemTable::UpdateOldestKeyTime() { uint64_t oldest_key_time = oldest_key_time_.load(std::memory_order_relaxed); - if (oldest_key_time == std::numeric_limits::max()) { + if (UNLIKELY(oldest_key_time == std::numeric_limits::max())) { int64_t current_time = 0; auto s = clock_->GetCurrentTime(¤t_time); if (s.ok()) { @@ -270,56 +276,6 @@ void MemTable::UpdateOldestKeyTime() { } } -Status MemTable::VerifyEntryChecksum(const char* entry, - uint32_t protection_bytes_per_key, - bool allow_data_in_errors) { - if (protection_bytes_per_key == 0) { - return Status::OK(); - } - uint32_t key_length; - const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); - if (key_ptr == nullptr) { - return Status::Corruption("Unable to parse internal key length"); - } - if (key_length < 8) { - return Status::Corruption("Memtable entry internal key length too short."); - } - Slice user_key = Slice(key_ptr, key_length - 8); - - const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); - ValueType type; - SequenceNumber seq; - UnPackSequenceAndType(tag, &seq, &type); - - uint32_t value_length = 0; - const char* value_ptr = GetVarint32Ptr( - key_ptr + key_length, key_ptr + key_length + 5, &value_length); - if (value_ptr == nullptr) { - return Status::Corruption("Unable to parse internal key value"); - } - Slice value = Slice(value_ptr, value_length); - - const char* checksum_ptr = value_ptr + value_length; - bool match = - ProtectionInfo64() - .ProtectKVO(user_key, value, type) - .ProtectS(seq) - .Verify(static_cast(protection_bytes_per_key), checksum_ptr); - if (!match) { - std::string msg( - "Corrupted memtable entry, per key-value checksum verification " - "failed."); - if (allow_data_in_errors) { - msg.append("Unrecognized value type: " + - std::to_string(static_cast(type)) + ". "); - msg.append("User key: " + user_key.ToString(/*hex=*/true) + ". "); - msg.append("seq: " + std::to_string(seq) + "."); - } - return Status::Corruption(msg.c_str()); - } - return Status::OK(); -} - int MemTable::KeyComparator::operator()(const char* prefix_len_key1, const char* prefix_len_key2) const { // Internal keys are encoded as length-prefixed strings. @@ -339,16 +295,84 @@ void MemTableRep::InsertConcurrently(KeyHandle /*handle*/) { throw std::runtime_error("concurrent insert not supported"); } +const InternalKeyComparator* MemTable::KeyComparator::icomparator() const { + return &comparator; +} + Slice MemTableRep::UserKey(const char* key) const { Slice slice = GetLengthPrefixedSlice(key); return Slice(slice.data(), slice.size() - 8); } +size_t MemTableRep::EncodeKeyValueSize(const Slice& key, const Slice& value) { + size_t buf_size = 0; + buf_size += VarintLength(key.size()) + key.size(); + buf_size += VarintLength(value.size()) + value.size(); + return buf_size; +} + +KeyHandle MemTableRep::EncodeKeyValue(const Slice& key, const Slice& value) { + size_t buf_size = EncodeKeyValueSize(key, value); + char* buf = nullptr; + KeyHandle handle = Allocate(buf_size, &buf); + assert(nullptr != handle); + assert(nullptr != buf); + char* p = EncodeVarint32(buf, (uint32_t)key.size()); + memcpy(p, key.data(), key.size()); + p = EncodeVarint32(p + key.size(), (uint32_t)value.size()); + memcpy(p, value.data(), value.size()); + return handle; +} + +bool MemTableRep::InsertKeyValue(const Slice& internal_key, + const Slice& value) { + KeyHandle handle = EncodeKeyValue(internal_key, value); + return InsertKey(handle); +} + +bool MemTableRep::InsertKeyValueWithHint(const Slice& internal_key, + const Slice& value, void** hint) { + KeyHandle handle = EncodeKeyValue(internal_key, value); + return InsertKeyWithHint(handle, hint); +} + +bool MemTableRep::InsertKeyValueConcurrently(const Slice& internal_key, + const Slice& value) { + KeyHandle handle = EncodeKeyValue(internal_key, value); + return InsertKeyConcurrently(handle); +} + +bool MemTableRep::InsertKeyValueWithHintConcurrently(const Slice& internal_key, + const Slice& value, + void** hint) { + KeyHandle handle = EncodeKeyValue(internal_key, value); + return InsertKeyWithHintConcurrently(handle, hint); +} + KeyHandle MemTableRep::Allocate(const size_t len, char** buf) { *buf = allocator_->Allocate(len); return static_cast(*buf); } +void MemTableRep::Iterator::Seek(const Slice& ikey) { Seek(ikey, nullptr); } +void MemTableRep::Iterator::SeekForPrev(const Slice& ikey) { + return SeekForPrev(ikey, nullptr); +} +Status MemTableRep::Iterator::status() const { return Status::OK(); } +void MemTableRep::FinishHint(void* hint) { + delete[] reinterpret_cast(hint); +} +Status MemTableRep::ConvertToSST(struct FileMetaData*, + const struct TableBuilderOptions&) { + ROCKSDB_VERIFY(SupportConvertToSST()); + return Status::NotSupported("Not supported MemTableRep::ConvertToSST()"); +} +Status MemTable::ConvertToSST(struct FileMetaData* meta, + const struct TableBuilderOptions& tbo) { + ROCKSDB_VERIFY(table_->SupportConvertToSST()); + return table_->ConvertToSST(meta, tbo); +} + // Encode a suitable internal key target for "target" and return it. // Uses *scratch as scratch space, and the returned pointer will point // into this scratch space. @@ -384,7 +408,6 @@ class MemTableIterator : public InternalIterator { } else { iter_ = mem.table_->GetIterator(arena); } - status_.PermitUncheckedError(); } // No copying allowed MemTableIterator(const MemTableIterator&) = delete; @@ -410,7 +433,7 @@ class MemTableIterator : public InternalIterator { PinnedIteratorsManager* pinned_iters_mgr_ = nullptr; #endif - bool Valid() const override { return valid_ && status_.ok(); } + bool Valid() const override { return valid_; } void Seek(const Slice& k) override { PERF_TIMER_GUARD(seek_on_memtable_time); PERF_COUNTER_ADD(seek_on_memtable_count, 1); @@ -430,7 +453,6 @@ class MemTableIterator : public InternalIterator { } iter_->Seek(k, nullptr); valid_ = iter_->Valid(); - VerifyEntryChecksum(); } void SeekForPrev(const Slice& k) override { PERF_TIMER_GUARD(seek_on_memtable_time); @@ -450,8 +472,7 @@ class MemTableIterator : public InternalIterator { } iter_->Seek(k, nullptr); valid_ = iter_->Valid(); - VerifyEntryChecksum(); - if (!Valid() && status().ok()) { + if (!Valid()) { SeekToLast(); } while (Valid() && comparator_.comparator.Compare(k, key()) < 0) { @@ -461,53 +482,51 @@ class MemTableIterator : public InternalIterator { void SeekToFirst() override { iter_->SeekToFirst(); valid_ = iter_->Valid(); - VerifyEntryChecksum(); } void SeekToLast() override { iter_->SeekToLast(); valid_ = iter_->Valid(); - VerifyEntryChecksum(); } + ROCKSDB_FLATTEN void Next() override { + NextAndCheckValid(); // ignore return value + } + bool NextAndCheckValid() final { PERF_COUNTER_ADD(next_on_memtable_count, 1); assert(Valid()); - iter_->Next(); + bool is_valid = iter_->NextAndCheckValid(); TEST_SYNC_POINT_CALLBACK("MemTableIterator::Next:0", iter_); - valid_ = iter_->Valid(); - VerifyEntryChecksum(); + valid_ = is_valid; + return is_valid; } bool NextAndGetResult(IterateResult* result) override { - Next(); - bool is_valid = Valid(); - if (is_valid) { - result->key = key(); - result->bound_check_result = IterBoundCheck::kUnknown; - result->value_prepared = true; - } - return is_valid; + return iter_->NextAndGetResult(result); } + ROCKSDB_FLATTEN void Prev() override { + PrevAndCheckValid(); // ignore return value + } + bool PrevAndCheckValid() final { PERF_COUNTER_ADD(prev_on_memtable_count, 1); assert(Valid()); - iter_->Prev(); - valid_ = iter_->Valid(); - VerifyEntryChecksum(); + valid_ = iter_->PrevAndCheckValid(); + return valid_; } Slice key() const override { assert(Valid()); - return GetLengthPrefixedSlice(iter_->key()); + return iter_->key(); } Slice value() const override { assert(Valid()); - Slice key_slice = GetLengthPrefixedSlice(iter_->key()); - return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); + return iter_->value(); } - Status status() const override { return status_; } + Status status() const override { return Status::OK(); } bool IsKeyPinned() const override { - // memtable data is always pinned - return true; + // some memtable key may not pinned, such as a patricia trie + // which reconstruct key during search/iterate + return iter_->IsKeyPinned(); } bool IsValuePinned() const override { @@ -523,25 +542,21 @@ class MemTableIterator : public InternalIterator { bool valid_; bool arena_mode_; bool value_pinned_; - uint32_t protection_bytes_per_key_; Status status_; Logger* logger_; size_t ts_sz_; - - void VerifyEntryChecksum() { - if (protection_bytes_per_key_ > 0 && Valid()) { - status_ = MemTable::VerifyEntryChecksum(iter_->key(), - protection_bytes_per_key_); - if (!status_.ok()) { - ROCKS_LOG_ERROR(logger_, "In MemtableIterator: %s", status_.getState()); - } - } - } }; InternalIterator* MemTable::NewIterator(const ReadOptions& read_options, Arena* arena) { assert(arena != nullptr); +#if !defined(ROCKSDB_UNIT_TEST) + if (nullptr == bloom_filter_ && nullptr == prefix_extractor_ && + perf_level < PerfLevel::kEnableCount && + !moptions_.inplace_update_support) { + return table_->GetIterator(arena); + } +#endif auto mem = arena->AllocateAligned(sizeof(MemTableIterator)); return new (mem) MemTableIterator(*this, read_options, arena); } @@ -634,7 +649,8 @@ MemTable::MemTableStats MemTable::ApproximateStats(const Slice& start_ikey, return {entry_count * (data_size / n), entry_count}; } -Status MemTable::VerifyEncodedEntry(Slice encoded, +// encoded just contains key +Status MemTable::VerifyEncodedEntry(Slice ikey, Slice value, const ProtectionInfoKVOS64& kv_prot_info) { uint32_t ikey_len = 0; if (!GetVarint32(&encoded, &ikey_len)) { @@ -643,115 +659,55 @@ Status MemTable::VerifyEncodedEntry(Slice encoded, if (ikey_len < 8 + ts_sz_) { return Status::Corruption("Internal key length too short"); } - if (ikey_len > encoded.size()) { + if (ikey_len > ikey.size()) { return Status::Corruption("Internal key length too long"); } - uint32_t value_len = 0; const size_t user_key_len = ikey_len - 8; - Slice key(encoded.data(), user_key_len); - encoded.remove_prefix(user_key_len); + Slice key(ikey.data(), user_key_len); - uint64_t packed = DecodeFixed64(encoded.data()); + uint64_t packed = DecodeFixed64(key.end()); ValueType value_type = kMaxValue; SequenceNumber sequence_number = kMaxSequenceNumber; UnPackSequenceAndType(packed, &sequence_number, &value_type); - encoded.remove_prefix(8); - - if (!GetVarint32(&encoded, &value_len)) { - return Status::Corruption("Unable to parse value length"); - } - if (value_len < encoded.size()) { - return Status::Corruption("Value length too short"); - } - if (value_len > encoded.size()) { - return Status::Corruption("Value length too long"); - } - Slice value(encoded.data(), value_len); return kv_prot_info.StripS(sequence_number) .StripKVO(key, value, value_type) .GetStatus(); } -void MemTable::UpdateEntryChecksum(const ProtectionInfoKVOS64* kv_prot_info, - const Slice& key, const Slice& value, - ValueType type, SequenceNumber s, - char* checksum_ptr) { - if (moptions_.protection_bytes_per_key == 0) { - return; - } - - if (kv_prot_info == nullptr) { - ProtectionInfo64() - .ProtectKVO(key, value, type) - .ProtectS(s) - .Encode(static_cast(moptions_.protection_bytes_per_key), - checksum_ptr); - } else { - kv_prot_info->Encode( - static_cast(moptions_.protection_bytes_per_key), checksum_ptr); - } -} - +ROCKSDB_FLATTEN Status MemTable::Add(SequenceNumber s, ValueType type, const Slice& key, /* user key */ const Slice& value, const ProtectionInfoKVOS64* kv_prot_info, bool allow_concurrent, MemTablePostProcessInfo* post_process_info, void** hint) { - // Format of an entry is concatenation of: - // key_size : varint32 of internal_key.size() - // key bytes : char[internal_key.size()] - // value_size : varint32 of value.size() - // value bytes : char[value.size()] - // checksum : char[moptions_.protection_bytes_per_key] - uint32_t key_size = static_cast(key.size()); - uint32_t val_size = static_cast(value.size()); - uint32_t internal_key_size = key_size + 8; - const uint32_t encoded_len = VarintLength(internal_key_size) + - internal_key_size + VarintLength(val_size) + - val_size + moptions_.protection_bytes_per_key; - char* buf = nullptr; std::unique_ptr& table = type == kTypeRangeDeletion ? range_del_table_ : table_; - KeyHandle handle = table->Allocate(encoded_len, &buf); - - char* p = EncodeVarint32(buf, internal_key_size); - memcpy(p, key.data(), key_size); - Slice key_slice(p, key_size); - p += key_size; - uint64_t packed = PackSequenceAndType(s, type); - EncodeFixed64(p, packed); - p += 8; - p = EncodeVarint32(p, val_size); - memcpy(p, value.data(), val_size); - assert((unsigned)(p + val_size - buf + moptions_.protection_bytes_per_key) == - (unsigned)encoded_len); - - UpdateEntryChecksum(kv_prot_info, key, value, type, s, - buf + encoded_len - moptions_.protection_bytes_per_key); - Slice encoded(buf, encoded_len - moptions_.protection_bytes_per_key); + Slice key_slice((char*)memcpy(alloca(key.size_ + 8), key.data_, key.size_), + key.size_ + 8); + PutUnaligned((uint64_t*)(key_slice.data_ + key.size_), PackSequenceAndType(s, type)); if (kv_prot_info != nullptr) { - TEST_SYNC_POINT_CALLBACK("MemTable::Add:Encoded", &encoded); - Status status = VerifyEncodedEntry(encoded, *kv_prot_info); + TEST_SYNC_POINT_CALLBACK("MemTable::Add:Encoded", &key_slice); + Status status = VerifyEncodedEntry(key_slice, value, *kv_prot_info); if (!status.ok()) { return status; } } - Slice key_without_ts = StripTimestampFromUserKey(key, ts_sz_); - + size_t encoded_len = MemTableRep::EncodeKeyValueSize(key_slice, value); if (!allow_concurrent) { // Extract prefix for insert with hint. if (insert_with_hint_prefix_extractor_ != nullptr && insert_with_hint_prefix_extractor_->InDomain(key_slice)) { Slice prefix = insert_with_hint_prefix_extractor_->Transform(key_slice); - bool res = table->InsertKeyWithHint(handle, &insert_hints_[prefix]); + hint = &insert_hints_[prefix]; // overwrite hint? + bool res = table->InsertKeyValueWithHint(key_slice, value, hint); if (UNLIKELY(!res)) { return Status::TryAgain("key+seq exists"); } } else { - bool res = table->InsertKey(handle); + bool res = table->InsertKeyValue(key_slice, value); if (UNLIKELY(!res)) { return Status::TryAgain("key+seq exists"); } @@ -763,6 +719,10 @@ Status MemTable::Add(SequenceNumber s, ValueType type, std::memory_order_relaxed); data_size_.store(data_size_.load(std::memory_order_relaxed) + encoded_len, std::memory_order_relaxed); + raw_key_size_.store(raw_key_size_.load(std::memory_order_relaxed) + key_slice.size_, + std::memory_order_relaxed); + raw_value_size_.store(raw_value_size_.load(std::memory_order_relaxed) + value.size_, + std::memory_order_relaxed); if (type == kTypeDeletion || type == kTypeSingleDeletion || type == kTypeDeletionWithTimestamp) { num_deletes_.store(num_deletes_.load(std::memory_order_relaxed) + 1, @@ -771,13 +731,27 @@ Status MemTable::Add(SequenceNumber s, ValueType type, uint64_t val = num_range_deletes_.load(std::memory_order_relaxed) + 1; num_range_deletes_.store(val, std::memory_order_relaxed); } - - if (bloom_filter_ && prefix_extractor_ && - prefix_extractor_->InDomain(key_without_ts)) { - bloom_filter_->Add(prefix_extractor_->Transform(key_without_ts)); + else if (type == kTypeMerge) { + num_merges_.store(num_merges_.load(std::memory_order_relaxed) + 1, + std::memory_order_relaxed); } - if (bloom_filter_ && moptions_.memtable_whole_key_filtering) { - bloom_filter_->Add(key_without_ts); + if (largest_seqno_.load(std::memory_order_relaxed) < s) { + largest_seqno_.store(s, std::memory_order_relaxed); + } + + if (bloom_filter_) { + #if defined(TOPLINGDB_WITH_TIMESTAMP) + size_t ts_sz = GetInternalKeyComparator().user_comparator()->timestamp_size(); + Slice key_without_ts = StripTimestampFromUserKey(key, ts_sz); + #else + const Slice& key_without_ts = key; + #endif + if (prefix_extractor_ && prefix_extractor_->InDomain(key_without_ts)) { + bloom_filter_->Add(prefix_extractor_->Transform(key_without_ts)); + } + if (moptions_.memtable_whole_key_filtering) { + bloom_filter_->Add(key_without_ts); + } } // The first sequence number inserted into the memtable @@ -797,9 +771,10 @@ Status MemTable::Add(SequenceNumber s, ValueType type, MaybeUpdateNewestUDT(key_slice); UpdateFlushState(); } else { - bool res = (hint == nullptr) - ? table->InsertKeyConcurrently(handle) - : table->InsertKeyWithHintConcurrently(handle, hint); + bool res = + (hint == nullptr) + ? table->InsertKeyValueConcurrently(key_slice, value) + : table->InsertKeyValueWithHintConcurrently(key_slice, value, hint); if (UNLIKELY(!res)) { return Status::TryAgain("key+seq exists"); } @@ -807,17 +782,33 @@ Status MemTable::Add(SequenceNumber s, ValueType type, assert(post_process_info != nullptr); post_process_info->num_entries++; post_process_info->data_size += encoded_len; - if (type == kTypeDeletion) { + if (type == kTypeDeletion || type == kTypeSingleDeletion || + type == kTypeDeletionWithTimestamp) { post_process_info->num_deletes++; } - - if (bloom_filter_ && prefix_extractor_ && - prefix_extractor_->InDomain(key_without_ts)) { - bloom_filter_->AddConcurrently( - prefix_extractor_->Transform(key_without_ts)); + else if (type == kTypeMerge) { + post_process_info->num_merges++; } - if (bloom_filter_ && moptions_.memtable_whole_key_filtering) { - bloom_filter_->AddConcurrently(key_without_ts); + post_process_info->raw_key_size += key_slice.size_; + post_process_info->raw_value_size += value.size_; + if (post_process_info->largest_seqno < s) { + post_process_info->largest_seqno = s; + } + + if (bloom_filter_) { + #if defined(TOPLINGDB_WITH_TIMESTAMP) + size_t ts_sz = GetInternalKeyComparator().user_comparator()->timestamp_size(); + Slice key_without_ts = StripTimestampFromUserKey(key, ts_sz); + #else + const Slice& key_without_ts = key; + #endif + if (prefix_extractor_ && prefix_extractor_->InDomain(key_without_ts)) { + bloom_filter_->AddConcurrently( + prefix_extractor_->Transform(key_without_ts)); + } + if (moptions_.memtable_whole_key_filtering) { + bloom_filter_->AddConcurrently(key_without_ts); + } } // atomically update first_seqno_ and earliest_seqno_. @@ -832,7 +823,7 @@ Status MemTable::Add(SequenceNumber s, ValueType type, !earliest_seqno_.compare_exchange_weak(cur_earliest_seqno, s)) { } } - if (type == kTypeRangeDeletion) { + if (UNLIKELY(type == kTypeRangeDeletion)) { auto new_cache = std::make_shared(); size_t size = cached_range_tombstone_.Size(); if (allow_concurrent) { @@ -862,8 +853,6 @@ Status MemTable::Add(SequenceNumber s, ValueType type, is_range_del_table_empty_.store(false, std::memory_order_relaxed); } UpdateOldestKeyTime(); - - TEST_SYNC_POINT_CALLBACK("MemTable::Add:BeforeReturn:Encoded", &encoded); return Status::OK(); } @@ -873,9 +862,7 @@ namespace { struct Saver { Status* status; const LookupKey* key; - bool* found_final_value; // Is value set correctly? Used by KeyMayExist - bool* merge_in_progress; - std::string* value; + PinnableSlice* value; PinnableWideColumns* columns; SequenceNumber seq; std::string* timestamp; @@ -886,14 +873,17 @@ struct Saver { MemTable* mem; Logger* logger; Statistics* statistics; - bool inplace_update_support; - bool do_merge; SystemClock* clock; ReadCallback* callback_; bool* is_blob_index; + bool found_final_value; // Is value set correctly? Used by KeyMayExist + bool merge_in_progress; + bool inplace_update_support; + bool do_merge; bool allow_data_in_errors; - uint32_t protection_bytes_per_key; + bool is_zero_copy; + bool needs_user_key_cmp_in_get; bool CheckCallback(SequenceNumber _seq) { if (callback_) { return callback_->IsVisible(_seq); @@ -903,34 +893,23 @@ struct Saver { }; } // anonymous namespace -static bool SaveValue(void* arg, const char* entry) { - TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Begin:entry", &entry); +static bool SaveValue(void* arg, const MemTableRep::KeyValuePair& pair) { Saver* s = reinterpret_cast(arg); assert(s != nullptr); assert(!s->value || !s->columns); - if (s->protection_bytes_per_key > 0) { - *(s->status) = MemTable::VerifyEntryChecksum( - entry, s->protection_bytes_per_key, s->allow_data_in_errors); - if (!s->status->ok()) { - ROCKS_LOG_ERROR(s->logger, "In SaveValue: %s", s->status->getState()); - // Memtable entry corrupted - return false; - } - } - MergeContext* merge_context = s->merge_context; SequenceNumber max_covering_tombstone_seq = s->max_covering_tombstone_seq; const MergeOperator* merge_operator = s->merge_operator; assert(merge_context != nullptr); - // Refer to comments under MemTable::Add() for entry format. - // Check that it belongs to same user key. - uint32_t key_length = 0; - const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); - assert(key_length >= 8); - Slice user_key_slice = Slice(key_ptr, key_length - 8); + // Check that it belongs to same user key. We do not check the + // sequence number since the Seek() call above should have skipped + // all entries with overly large sequence numbers. + Slice v = pair.value; + const Slice& user_key_slice = pair.ukey; +#if defined(TOPLINGDB_WITH_TIMESTAMP) const Comparator* user_comparator = s->mem->GetInternalKeyComparator().user_comparator(); size_t ts_sz = user_comparator->timestamp_size(); @@ -938,10 +917,29 @@ static bool SaveValue(void* arg, const char* entry) { // timestamp should already be set to range tombstone timestamp assert(s->timestamp->size() == ts_sz); } - if (user_comparator->EqualWithoutTimestamp(user_key_slice, +#else + #if defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wparentheses" // fuck + #endif + const Comparator* user_comparator = nullptr; + constexpr size_t ts_sz = 0; // let compiler optimize it out +#endif + if (!s->needs_user_key_cmp_in_get || +#if !defined(TOPLINGDB_WITH_TIMESTAMP) + // user_comparator is not need if !needs_user_key_cmp_in_get without timestamp, + // omit load it from ptr to ptr + (user_comparator = s->mem->GetInternalKeyComparator().user_comparator(), true) && +#endif + user_comparator->EqualWithoutTimestamp(user_key_slice, s->key->user_key())) { +#if !defined(NDEBUG) + // In debug, user_comparator must be loaded + user_comparator = s->mem->GetInternalKeyComparator().user_comparator(); + assert(user_comparator->EqualWithoutTimestamp(user_key_slice, + s->key->user_key())); +#endif // Correct user key - const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); + const uint64_t tag = pair.tag; ValueType type; SequenceNumber seq; UnPackSequenceAndType(tag, &seq, &type); @@ -990,14 +988,14 @@ static bool SaveValue(void* arg, const char* entry) { if (!s->do_merge) { *(s->status) = Status::NotSupported( "GetMergeOperands not supported by stacked BlobDB"); - *(s->found_final_value) = true; + s->found_final_value = true; return false; } - if (*(s->merge_in_progress)) { + if (s->merge_in_progress) { *(s->status) = Status::NotSupported( "Merge operator not supported by stacked BlobDB"); - *(s->found_final_value) = true; + s->found_final_value = true; return false; } @@ -1006,7 +1004,7 @@ static bool SaveValue(void* arg, const char* entry) { *(s->status) = Status::NotSupported( "Encountered unexpected blob index. Please open DB with " "ROCKSDB_NAMESPACE::blob_db::BlobDB."); - *(s->found_final_value) = true; + s->found_final_value = true; return false; } @@ -1014,12 +1012,13 @@ static bool SaveValue(void* arg, const char* entry) { s->mem->GetLock(s->key->user_key())->ReadLock(); } - Slice v = GetLengthPrefixedSlice(key_ptr + key_length); - - *(s->status) = Status::OK(); + s->status->SetAsOK(); if (s->value) { - s->value->assign(v.data(), v.size()); + if (s->is_zero_copy) + s->value->PinSlice(v, nullptr); + else + s->value->PinSelf(v); } else if (s->columns) { s->columns->SetPlainValue(v); } @@ -1028,21 +1027,19 @@ static bool SaveValue(void* arg, const char* entry) { s->mem->GetLock(s->key->user_key())->ReadUnlock(); } - *(s->found_final_value) = true; + s->found_final_value = true; *(s->is_blob_index) = true; return false; } case kTypeValue: { - if (s->inplace_update_support) { + if (UNLIKELY(s->inplace_update_support)) { s->mem->GetLock(s->key->user_key())->ReadLock(); } - Slice v = GetLengthPrefixedSlice(key_ptr + key_length); + s->status->SetAsOK(); - *(s->status) = Status::OK(); - - if (!s->do_merge) { + if (UNLIKELY(!s->do_merge)) { // Preserve the value with the goal of returning it as part of // raw merge operands to the user // TODO(yanqin) update MergeContext so that timestamps information @@ -1050,7 +1047,7 @@ static bool SaveValue(void* arg, const char* entry) { merge_context->PushOperand( v, s->inplace_update_support == false /* operand_pinned */); - } else if (*(s->merge_in_progress)) { + } else if (UNLIKELY(s->merge_in_progress)) { assert(s->do_merge); if (s->value || s->columns) { @@ -1064,32 +1061,33 @@ static bool SaveValue(void* arg, const char* entry) { /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, s->value, s->columns); } - } else if (s->value) { - s->value->assign(v.data(), v.size()); + } else if (LIKELY(s->value != nullptr)) { + if (s->is_zero_copy) + s->value->PinSlice(v, nullptr); + else + s->value->PinSelf(v); } else if (s->columns) { s->columns->SetPlainValue(v); } - if (s->inplace_update_support) { + if (UNLIKELY(s->inplace_update_support)) { s->mem->GetLock(s->key->user_key())->ReadUnlock(); } - *(s->found_final_value) = true; + s->found_final_value = true; - if (s->is_blob_index != nullptr) { + if (UNLIKELY(s->is_blob_index != nullptr)) { *(s->is_blob_index) = false; } return false; } case kTypeWideColumnEntity: { - if (s->inplace_update_support) { + if (UNLIKELY(s->inplace_update_support)) { s->mem->GetLock(s->key->user_key())->ReadLock(); } - Slice v = GetLengthPrefixedSlice(key_ptr + key_length); - - *(s->status) = Status::OK(); + s->status->SetAsOK(); if (!s->do_merge) { // Preserve the value with the goal of returning it as part of @@ -1104,7 +1102,7 @@ static bool SaveValue(void* arg, const char* entry) { value_of_default, s->inplace_update_support == false /* operand_pinned */); } - } else if (*(s->merge_in_progress)) { + } else if (s->merge_in_progress) { assert(s->do_merge); if (s->value || s->columns) { @@ -1122,17 +1120,17 @@ static bool SaveValue(void* arg, const char* entry) { *(s->status) = WideColumnSerialization::GetValueOfDefaultColumn( v, value_of_default); if (s->status->ok()) { - s->value->assign(value_of_default.data(), value_of_default.size()); + s->value->PinSelf(value_of_default); } } else if (s->columns) { *(s->status) = s->columns->SetWideColumnValue(v); } - if (s->inplace_update_support) { + if (UNLIKELY(s->inplace_update_support)) { s->mem->GetLock(s->key->user_key())->ReadUnlock(); } - *(s->found_final_value) = true; + s->found_final_value = true; if (s->is_blob_index != nullptr) { *(s->is_blob_index) = false; @@ -1144,7 +1142,7 @@ static bool SaveValue(void* arg, const char* entry) { case kTypeDeletionWithTimestamp: case kTypeSingleDeletion: case kTypeRangeDeletion: { - if (*(s->merge_in_progress)) { + if (s->merge_in_progress) { if (s->value || s->columns) { // `op_failure_scope` (an output parameter) is not provided (set to // nullptr) since a failure must be propagated regardless of its @@ -1163,22 +1161,21 @@ static bool SaveValue(void* arg, const char* entry) { } else { *(s->status) = Status::NotFound(); } - *(s->found_final_value) = true; + s->found_final_value = true; return false; } case kTypeMerge: { - if (!merge_operator) { + if (UNLIKELY(!merge_operator)) { *(s->status) = Status::InvalidArgument( "merge_operator is not properly initialized."); // Normally we continue the loop (return true) when we see a merge // operand. But in case of an error, we should stop the loop // immediately and pretend we have found the value to stop further // seek. Otherwise, the later call will override this error status. - *(s->found_final_value) = true; + s->found_final_value = true; return false; } - Slice v = GetLengthPrefixedSlice(key_ptr + key_length); - *(s->merge_in_progress) = true; + s->merge_in_progress = true; merge_context->PushOperand( v, s->inplace_update_support == false /* operand_pinned */); PERF_COUNTER_ADD(internal_merge_point_lookup_count, 1); @@ -1196,7 +1193,7 @@ static bool SaveValue(void* arg, const char* entry) { /* op_failure_scope */ nullptr, s->value, s->columns); } - *(s->found_final_value) = true; + s->found_final_value = true; return false; } return true; @@ -1220,7 +1217,8 @@ static bool SaveValue(void* arg, const char* entry) { return false; } -bool MemTable::Get(const LookupKey& key, std::string* value, +ROCKSDB_FLATTEN +bool MemTable::Get(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, @@ -1228,7 +1226,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, bool immutable_memtable, ReadCallback* callback, bool* is_blob_index, bool do_merge) { // The sequence number is updated synchronously in version_set.h - if (IsEmpty()) { + if (UNLIKELY(IsEmpty())) { // Avoiding recording stats for speed. return false; } @@ -1253,12 +1251,15 @@ bool MemTable::Get(const LookupKey& key, std::string* value, } } - bool found_final_value = false; - bool merge_in_progress = s->IsMergeInProgress(); - bool may_contain = true; - Slice user_key_without_ts = StripTimestampFromUserKey(key.user_key(), ts_sz_); - bool bloom_checked = false; - if (bloom_filter_) { + if (UNLIKELY(bloom_filter_ != nullptr)) { + bool may_contain = true; + #if defined(TOPLINGDB_WITH_TIMESTAMP) + size_t ts_sz = GetInternalKeyComparator().user_comparator()->timestamp_size(); + Slice user_key_without_ts = StripTimestampFromUserKey(key.user_key(), ts_sz); + #else + Slice user_key_without_ts = key.user_key(); + #endif + bool bloom_checked = false; // when both memtable_whole_key_filtering and prefix_extractor_ are set, // only do whole key filtering for Get() to save CPU if (moptions_.memtable_whole_key_filtering) { @@ -1272,42 +1273,23 @@ bool MemTable::Get(const LookupKey& key, std::string* value, bloom_checked = true; } } - } - - if (bloom_filter_ && !may_contain) { - // iter is null if prefix bloom says the key does not exist - PERF_COUNTER_ADD(bloom_memtable_miss_count, 1); - *seq = kMaxSequenceNumber; - } else { - if (bloom_checked) { - PERF_COUNTER_ADD(bloom_memtable_hit_count, 1); + if (UNLIKELY(!may_contain)) { + // iter is null if prefix bloom says the key does not exist + PERF_COUNTER_ADD(bloom_memtable_miss_count, 1); + *seq = kMaxSequenceNumber; + PERF_COUNTER_ADD(get_from_memtable_count, 1); + return false; + } else { + if (UNLIKELY(bloom_checked)) { + PERF_COUNTER_ADD(bloom_memtable_hit_count, 1); + } } - GetFromTable(key, *max_covering_tombstone_seq, do_merge, callback, - is_blob_index, value, columns, timestamp, s, merge_context, - seq, &found_final_value, &merge_in_progress); } - // No change to value, since we have not yet found a Put/Delete - // Propagate corruption error - if (!found_final_value && merge_in_progress && !s->IsCorruption()) { - *s = Status::MergeInProgress(); - } - PERF_COUNTER_ADD(get_from_memtable_count, 1); - return found_final_value; -} - -void MemTable::GetFromTable(const LookupKey& key, - SequenceNumber max_covering_tombstone_seq, - bool do_merge, ReadCallback* callback, - bool* is_blob_index, std::string* value, - PinnableWideColumns* columns, - std::string* timestamp, Status* s, - MergeContext* merge_context, SequenceNumber* seq, - bool* found_final_value, bool* merge_in_progress) { Saver saver; saver.status = s; - saver.found_final_value = found_final_value; - saver.merge_in_progress = merge_in_progress; + saver.found_final_value = false; + saver.merge_in_progress = s->IsMergeInProgress(); saver.key = &key; saver.value = value; saver.columns = columns; @@ -1315,7 +1297,7 @@ void MemTable::GetFromTable(const LookupKey& key, saver.seq = kMaxSequenceNumber; saver.mem = this; saver.merge_context = merge_context; - saver.max_covering_tombstone_seq = max_covering_tombstone_seq; + saver.max_covering_tombstone_seq = *max_covering_tombstone_seq; saver.merge_operator = moptions_.merge_operator; saver.logger = moptions_.info_log; saver.inplace_update_support = moptions_.inplace_update_support; @@ -1325,9 +1307,21 @@ void MemTable::GetFromTable(const LookupKey& key, saver.is_blob_index = is_blob_index; saver.do_merge = do_merge; saver.allow_data_in_errors = moptions_.allow_data_in_errors; - saver.protection_bytes_per_key = moptions_.protection_bytes_per_key; - table_->Get(key, &saver, SaveValue); + saver.is_zero_copy = read_opts.pinning_tls != nullptr; + saver.needs_user_key_cmp_in_get = needs_user_key_cmp_in_get_; + if (LIKELY(value != nullptr)) { + value->Reset(); + } + table_->Get(read_opts, key, &saver, SaveValue); *seq = saver.seq; + + // No change to value, since we have not yet found a Put/Delete + // Propagate corruption error + if (!saver.found_final_value && saver.merge_in_progress && !s->IsCorruption()) { + *s = Status::MergeInProgress(); + } + PERF_COUNTER_ADD(get_from_memtable_count, 1); + return saver.found_final_value; } void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range, @@ -1373,8 +1367,6 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range, } } for (auto iter = temp_range.begin(); iter != temp_range.end(); ++iter) { - bool found_final_value{false}; - bool merge_in_progress = iter->s->IsMergeInProgress(); if (!no_range_del) { std::unique_ptr range_del_iter( NewRangeTombstoneIteratorInternal( @@ -1392,20 +1384,39 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range, } } } - SequenceNumber dummy_seq; - GetFromTable(*(iter->lkey), iter->max_covering_tombstone_seq, true, - callback, &iter->is_blob_index, - iter->value ? iter->value->GetSelf() : nullptr, iter->columns, - iter->timestamp, iter->s, &(iter->merge_context), &dummy_seq, - &found_final_value, &merge_in_progress); - - if (!found_final_value && merge_in_progress) { + Saver saver; + saver.status = iter->s; + saver.found_final_value = false; + saver.merge_in_progress = iter->s->IsMergeInProgress(); + saver.key = iter->lkey; + saver.value = iter->value; // not null + if (saver.value) + saver.value->Reset(); + saver.columns = iter->columns; + saver.timestamp = iter->timestamp; + saver.seq = kMaxSequenceNumber; // dummy_seq + saver.mem = this; + saver.merge_context = &(iter->merge_context); + saver.max_covering_tombstone_seq = iter->max_covering_tombstone_seq; + saver.merge_operator = moptions_.merge_operator; + saver.logger = moptions_.info_log; + saver.inplace_update_support = moptions_.inplace_update_support; + saver.statistics = moptions_.statistics; + saver.clock = clock_; + saver.callback_ = callback; + saver.is_blob_index = &iter->is_blob_index; + saver.do_merge = true; + saver.allow_data_in_errors = moptions_.allow_data_in_errors; + saver.is_zero_copy = read_options.pinning_tls != nullptr; + saver.needs_user_key_cmp_in_get = needs_user_key_cmp_in_get_; + table_->Get(read_options, *(iter->lkey), &saver, SaveValue); + + if (!saver.found_final_value && saver.merge_in_progress) { *(iter->s) = Status::MergeInProgress(); } - if (found_final_value) { + if (saver.found_final_value) { if (iter->value) { - iter->value->PinSelf(); range->AddValueSize(iter->value->size()); } else { assert(iter->columns); @@ -1431,21 +1442,20 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range, Status MemTable::Update(SequenceNumber seq, ValueType value_type, const Slice& key, const Slice& value, const ProtectionInfoKVOS64* kv_prot_info) { + assert(moptions_.inplace_update_support); LookupKey lkey(key, seq); - Slice mem_key = lkey.memtable_key(); std::unique_ptr iter( table_->GetDynamicPrefixIterator()); - iter->Seek(lkey.internal_key(), mem_key.data()); + iter->Seek(lkey.internal_key(), lkey.memtable_key_data()); if (iter->Valid()) { - // Refer to comments under MemTable::Add() for entry format. - // Check that it belongs to same user key. We do not check the // sequence number since the Seek() call above should have skipped // all entries with overly large sequence numbers. - const char* entry = iter->key(); - uint32_t key_length = 0; - const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + auto [internal_key, prev_value] = iter->GetKeyValue(); + size_t key_length = internal_key.size(); + const char* key_ptr = internal_key.data(); + assert(key_length >= 8); if (comparator_.comparator.user_comparator()->Equal( Slice(key_ptr, key_length - 8), lkey.user_key())) { // Correct user key @@ -1455,31 +1465,23 @@ Status MemTable::Update(SequenceNumber seq, ValueType value_type, UnPackSequenceAndType(tag, &existing_seq, &type); assert(existing_seq != seq); if (type == value_type) { - Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length); uint32_t prev_size = static_cast(prev_value.size()); uint32_t new_size = static_cast(value.size()); - // Update value, if new value size <= previous value size + // Update value, if new value size <= previous value size if (new_size <= prev_size) { char* p = - EncodeVarint32(const_cast(key_ptr) + key_length, new_size); + const_cast(prev_value.data()) - VarintLength(prev_size); WriteLock wl(GetLock(lkey.user_key())); + p = EncodeVarint32(p, new_size); memcpy(p, value.data(), value.size()); - assert((unsigned)((p + value.size()) - entry) == - (unsigned)(VarintLength(key_length) + key_length + - VarintLength(value.size()) + value.size())); RecordTick(moptions_.statistics, NUMBER_KEYS_UPDATED); if (kv_prot_info != nullptr) { ProtectionInfoKVOS64 updated_kv_prot_info(*kv_prot_info); // `seq` is swallowed and `existing_seq` prevails. updated_kv_prot_info.UpdateS(seq, existing_seq); - UpdateEntryChecksum(&updated_kv_prot_info, key, value, type, - existing_seq, p + value.size()); - Slice encoded(entry, p + value.size() - entry); - return VerifyEncodedEntry(encoded, updated_kv_prot_info); - } else { - UpdateEntryChecksum(nullptr, key, value, type, existing_seq, - p + value.size()); + Slice ikey = lkey.internal_key(); + return VerifyEncodedEntry(ikey, value, updated_kv_prot_info); } return Status::OK(); } @@ -1494,21 +1496,21 @@ Status MemTable::Update(SequenceNumber seq, ValueType value_type, Status MemTable::UpdateCallback(SequenceNumber seq, const Slice& key, const Slice& delta, const ProtectionInfoKVOS64* kv_prot_info) { + assert(moptions_.inplace_update_support); LookupKey lkey(key, seq); - Slice memkey = lkey.memtable_key(); std::unique_ptr iter( table_->GetDynamicPrefixIterator()); - iter->Seek(lkey.internal_key(), memkey.data()); + iter->Seek(lkey.internal_key(), lkey.memtable_key_data()); if (iter->Valid()) { - // Refer to comments under MemTable::Add() for entry format. // Check that it belongs to same user key. We do not check the // sequence number since the Seek() call above should have skipped // all entries with overly large sequence numbers. - const char* entry = iter->key(); - uint32_t key_length = 0; - const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + auto [internal_key, prev_value] = iter->GetKeyValue(); + size_t key_length = internal_key.size(); + const char* key_ptr = internal_key.data(); + assert(key_length >= 8); if (comparator_.comparator.user_comparator()->Equal( Slice(key_ptr, key_length - 8), lkey.user_key())) { // Correct user key @@ -1517,7 +1519,6 @@ Status MemTable::UpdateCallback(SequenceNumber seq, const Slice& key, uint64_t existing_seq; UnPackSequenceAndType(tag, &existing_seq, &type); if (type == kTypeValue) { - Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length); uint32_t prev_size = static_cast(prev_value.size()); char* prev_buffer = const_cast(prev_value.data()); @@ -1529,32 +1530,28 @@ Status MemTable::UpdateCallback(SequenceNumber seq, const Slice& key, delta, &str_value); if (status == UpdateStatus::UPDATED_INPLACE) { // Value already updated by callback. + char* p = prev_buffer - VarintLength(prev_size); assert(new_prev_size <= prev_size); if (new_prev_size < prev_size) { // overwrite the new prev_size - char* p = EncodeVarint32(const_cast(key_ptr) + key_length, - new_prev_size); - if (VarintLength(new_prev_size) < VarintLength(prev_size)) { + p = EncodeVarint32(p, new_prev_size); + if (p < prev_buffer) { // shift the value buffer as well. - memcpy(p, prev_buffer, new_prev_size); + memmove(p, prev_buffer, new_prev_size); prev_buffer = p; } } RecordTick(moptions_.statistics, NUMBER_KEYS_UPDATED); UpdateFlushState(); - Slice new_value(prev_buffer, new_prev_size); if (kv_prot_info != nullptr) { ProtectionInfoKVOS64 updated_kv_prot_info(*kv_prot_info); // `seq` is swallowed and `existing_seq` prevails. updated_kv_prot_info.UpdateS(seq, existing_seq); - updated_kv_prot_info.UpdateV(delta, new_value); - Slice encoded(entry, prev_buffer + new_prev_size - entry); - UpdateEntryChecksum(&updated_kv_prot_info, key, new_value, type, - existing_seq, prev_buffer + new_prev_size); - return VerifyEncodedEntry(encoded, updated_kv_prot_info); - } else { - UpdateEntryChecksum(nullptr, key, new_value, type, existing_seq, - prev_buffer + new_prev_size); + updated_kv_prot_info.UpdateV(delta, + Slice(prev_buffer, new_prev_size)); + Slice ikey = lkey.internal_key(); + Slice value(p, new_prev_size); // new value without size prefix + return VerifyEncodedEntry(ikey, value, updated_kv_prot_info); } return Status::OK(); } else if (status == UpdateStatus::UPDATED) { @@ -1585,21 +1582,19 @@ Status MemTable::UpdateCallback(SequenceNumber seq, const Slice& key, } size_t MemTable::CountSuccessiveMergeEntries(const LookupKey& key) { - Slice memkey = key.memtable_key(); - // A total ordered iterator is costly for some memtablerep (prefix aware // reps). By passing in the user key, we allow efficient iterator creation. // The iterator only needs to be ordered within the same user key. std::unique_ptr iter( table_->GetDynamicPrefixIterator()); - iter->Seek(key.internal_key(), memkey.data()); + iter->Seek(key.internal_key(), key.memtable_key_data()); size_t num_successive_merges = 0; for (; iter->Valid(); iter->Next()) { - const char* entry = iter->key(); - uint32_t key_length = 0; - const char* iter_key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); + Slice internal_key = iter->key(); + size_t key_length = internal_key.size(); + const char* iter_key_ptr = internal_key.data(); if (!comparator_.comparator.user_comparator()->Equal( Slice(iter_key_ptr, key_length - 8), key.user_key())) { break; @@ -1619,13 +1614,28 @@ size_t MemTable::CountSuccessiveMergeEntries(const LookupKey& key) { return num_successive_merges; } -void MemTableRep::Get(const LookupKey& k, void* callback_args, - bool (*callback_func)(void* arg, const char* entry)) { - auto iter = GetDynamicPrefixIterator(); - for (iter->Seek(k.internal_key(), k.memtable_key().data()); - iter->Valid() && callback_func(callback_args, iter->key()); - iter->Next()) { - } +MemTableRep::KeyValuePair::KeyValuePair(const char* key) { + Slice ikey = GetLengthPrefixedSlice(key); + ukey = Slice(ikey.data(), ikey.size() - 8); + tag = DecodeFixed64(ukey.end()); + value = GetLengthPrefixedSlice(ikey.end()); +} + +Slice MemTableRep::Iterator::key() const { + assert(Valid()); + return GetLengthPrefixedSlice(varlen_key()); +} + +Slice MemTableRep::Iterator::value() const { + assert(Valid()); + Slice k = GetLengthPrefixedSlice(varlen_key()); + return GetLengthPrefixedSlice(k.data() + k.size()); +} +std::pair MemTableRep::Iterator::GetKeyValue() const { + assert(Valid()); + Slice k = GetLengthPrefixedSlice(varlen_key()); + Slice v = GetLengthPrefixedSlice(k.data() + k.size()); + return {k, v}; } void MemTable::RefLogContainingPrepSection(uint64_t log) { diff --git a/db/memtable.h b/db/memtable.h index c55b34761e..c781d522a6 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -27,6 +27,7 @@ #include "options/cf_options.h" #include "rocksdb/db.h" #include "rocksdb/memtablerep.h" +#include "table/internal_iterator.h" #include "table/multiget_context.h" #include "util/dynamic_bloom.h" #include "util/hash.h" @@ -46,6 +47,7 @@ struct ImmutableMemTableOptions { size_t arena_block_size; uint32_t memtable_prefix_bloom_bits; size_t memtable_huge_page_size; + bool allow_merge_memtables; bool memtable_whole_key_filtering; bool inplace_update_support; size_t inplace_update_num_locks; @@ -58,7 +60,6 @@ struct ImmutableMemTableOptions { MergeOperator* merge_operator; Logger* info_log; bool allow_data_in_errors; - uint32_t protection_bytes_per_key; }; // Batched counters to updated when inserting keys in one write batch. @@ -69,6 +70,46 @@ struct MemTablePostProcessInfo { uint64_t num_entries = 0; uint64_t num_deletes = 0; uint64_t num_range_deletes = 0; + uint64_t num_merges = 0; + uint64_t largest_seqno = 0; + uint64_t raw_key_size = 0; // internal key + uint64_t raw_value_size = 0; +}; + +// Iteration over the contents of a skip collection +class MemTableRep::Iterator : public InternalIterator { + public: + // Returns the key at the current position. + // REQUIRES: Valid() + virtual const char* varlen_key() const = 0; + + // Returns the key at the current position. + // REQUIRES: Valid() + virtual Slice key() const override; + + // Returns the value at the current position. + // REQUIRES: Valid() + virtual Slice value() const override; + + // Returns the key & value at the current position. + // REQUIRES: Valid() + virtual std::pair GetKeyValue() const; + + void Seek(const Slice& ikey) override; + // Advance to the first entry with a key >= target + virtual void Seek(const Slice& internal_key, const char* memtable_key) = 0; + + void SeekForPrev(const Slice& ikey) override; + // retreat to the first entry with a key <= target + virtual void SeekForPrev(const Slice& internal_key, + const char* memtable_key) = 0; + + virtual void RandomSeek() {} + + // If true, this means that the Slice returned by GetKey() is always valid + virtual bool IsKeyPinned() const override { return true; } + virtual bool IsValuePinned() const override { return true; } + virtual Status status() const override; }; using MultiGetRange = MultiGetContext::Range; @@ -94,6 +135,7 @@ class MemTable { const char* prefix_len_key2) const override; virtual int operator()(const char* prefix_len_key, const DecodedType& key) const override; + virtual const InternalKeyComparator* icomparator() const override; }; // MemTables are reference counted. The initial reference count @@ -218,7 +260,7 @@ class MemTable { const ReadOptions& read_options, SequenceNumber read_seq, bool immutable_memtable); - Status VerifyEncodedEntry(Slice encoded, + Status VerifyEncodedEntry(Slice ikey, Slice value, const ProtectionInfoKVOS64& kv_prot_info); // Add an entry into memtable that maps key to value at the @@ -260,7 +302,7 @@ class MemTable { // @param immutable_memtable Whether this memtable is immutable. Used // internally by NewRangeTombstoneIterator(). See comment above // NewRangeTombstoneIterator() for more detail. - bool Get(const LookupKey& key, std::string* value, + bool Get(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, @@ -268,7 +310,7 @@ class MemTable { ReadCallback* callback = nullptr, bool* is_blob_index = nullptr, bool do_merge = true); - bool Get(const LookupKey& key, std::string* value, + bool Get(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, @@ -337,6 +379,15 @@ class MemTable { num_range_deletes_.fetch_add(update_counters.num_range_deletes, std::memory_order_relaxed); } + if (update_counters.num_merges != 0) { + num_merges_.fetch_add(update_counters.num_merges, + std::memory_order_relaxed); + } + if (largest_seqno_.load(std::memory_order_relaxed) < update_counters.largest_seqno) { + largest_seqno_.store(update_counters.largest_seqno, std::memory_order_relaxed); + } + raw_key_size_.fetch_add(update_counters.raw_key_size, std::memory_order_relaxed); + raw_value_size_.fetch_add(update_counters.raw_value_size, std::memory_order_relaxed); UpdateFlushState(); } @@ -353,6 +404,9 @@ class MemTable { uint64_t num_deletes() const { return num_deletes_.load(std::memory_order_relaxed); } + uint64_t num_merges() const { + return num_merges_.load(std::memory_order_relaxed); + } // Get total number of range deletions in the mem table. // REQUIRES: external synchronization to prevent simultaneous @@ -368,6 +422,15 @@ class MemTable { size_t write_buffer_size() const { return write_buffer_size_.load(std::memory_order_relaxed); } + uint64_t largest_seqno() const { + return largest_seqno_.load(std::memory_order_relaxed); + } + uint64_t raw_key_size() const { + return raw_key_size_.load(std::memory_order_relaxed); + } + uint64_t raw_value_size() const { + return raw_value_size_.load(std::memory_order_relaxed); + } // Dynamically change the memtable's capacity. If set below the current usage, // the next key added will trigger a flush. Can only increase size when @@ -477,6 +540,12 @@ class MemTable { return table_->IsSnapshotSupported() && !moptions_.inplace_update_support; } + void FinishHint(void* hint) const { table_->FinishHint(hint); } + bool SupportConvertToSST() const { + return table_->SupportConvertToSST() && is_range_del_table_empty_; + } + Status ConvertToSST(struct FileMetaData*, const struct TableBuilderOptions&); + struct MemTableStats { uint64_t size; uint64_t count; @@ -578,6 +647,10 @@ class MemTable { std::atomic num_entries_; std::atomic num_deletes_; std::atomic num_range_deletes_; + std::atomic num_merges_; + std::atomic largest_seqno_; + std::atomic raw_key_size_; + std::atomic raw_value_size_; // Dynamically changeable memtable option std::atomic write_buffer_size_; @@ -585,6 +658,7 @@ class MemTable { // These are used to manage memtable flushes to storage bool flush_in_progress_; // started the flush bool flush_completed_; // finished the flush + bool needs_user_key_cmp_in_get_; uint64_t file_number_; // filled up after flush is complete // The updates to be applied to the transaction log when this @@ -664,13 +738,6 @@ class MemTable { void UpdateOldestKeyTime(); - void GetFromTable(const LookupKey& key, - SequenceNumber max_covering_tombstone_seq, bool do_merge, - ReadCallback* callback, bool* is_blob_index, - std::string* value, PinnableWideColumns* columns, - std::string* timestamp, Status* s, - MergeContext* merge_context, SequenceNumber* seq, - bool* found_final_value, bool* merge_in_progress); // Always returns non-null and assumes certain pre-checks (e.g., // is_range_del_table_empty_) are done. This is only valid during the lifetime diff --git a/db/memtable_list.cc b/db/memtable_list.cc index dfa93461bb..053cfda2eb 100644 --- a/db/memtable_list.cc +++ b/db/memtable_list.cc @@ -40,7 +40,7 @@ void MemTableListVersion::UnrefMemTable(autovector* to_delete, MemTable* m) { if (m->Unref()) { to_delete->push_back(m); - assert(*parent_memtable_list_memory_usage_ >= m->ApproximateMemoryUsage()); + ROCKSDB_ASSERT_GE(*parent_memtable_list_memory_usage_, m->ApproximateMemoryUsage()); *parent_memtable_list_memory_usage_ -= m->ApproximateMemoryUsage(); } } @@ -104,7 +104,7 @@ int MemTableList::NumFlushed() const { // Search all the memtables starting from the most recent one. // Return the most recent value found, if any. // Operands stores the list of merge operations to apply, so far. -bool MemTableListVersion::Get(const LookupKey& key, std::string* value, +bool MemTableListVersion::Get(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, @@ -144,7 +144,7 @@ bool MemTableListVersion::GetMergeOperands( } bool MemTableListVersion::GetFromHistory( - const LookupKey& key, std::string* value, PinnableWideColumns* columns, + const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, const ReadOptions& read_opts, bool* is_blob_index) { @@ -154,7 +154,7 @@ bool MemTableListVersion::GetFromHistory( } bool MemTableListVersion::GetFromList( - std::list* list, const LookupKey& key, std::string* value, + std::list* list, const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, const ReadOptions& read_opts, ReadCallback* callback, @@ -419,6 +419,12 @@ void MemTableList::PickMemtablesToFlush(uint64_t max_memtable_id, std::max(m->GetNextLogNumber(), *max_next_log_number); } ret->push_back(m); + if (!m->GetImmutableMemTableOptions()->allow_merge_memtables) { + break; + } + if (m->SupportConvertToSST()) { + break; + } } else if (!ret->empty()) { // This `break` is necessary to prevent picking non-consecutive memtables // in case `memlist` has one or more entries with diff --git a/db/memtable_list.h b/db/memtable_list.h index 81b60288d8..328d160e83 100644 --- a/db/memtable_list.h +++ b/db/memtable_list.h @@ -57,14 +57,14 @@ class MemTableListVersion { // If any operation was found for this key, its most recent sequence number // will be stored in *seq on success (regardless of whether true/false is // returned). Otherwise, *seq will be set to kMaxSequenceNumber. - bool Get(const LookupKey& key, std::string* value, + bool Get(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, const ReadOptions& read_opts, ReadCallback* callback = nullptr, bool* is_blob_index = nullptr); - bool Get(const LookupKey& key, std::string* value, + bool Get(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, @@ -76,6 +76,8 @@ class MemTableListVersion { is_blob_index); } + bool IsEmpty() const { return memlist_.empty(); } + void MultiGet(const ReadOptions& read_options, MultiGetRange* range, ReadCallback* callback); @@ -90,13 +92,13 @@ class MemTableListVersion { // have already been flushed. Should only be used from in-memory only // queries (such as Transaction validation) as the history may contain // writes that are also present in the SST files. - bool GetFromHistory(const LookupKey& key, std::string* value, + bool GetFromHistory(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq, const ReadOptions& read_opts, bool* is_blob_index = nullptr); - bool GetFromHistory(const LookupKey& key, std::string* value, + bool GetFromHistory(const LookupKey& key, PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, @@ -161,7 +163,7 @@ class MemTableListVersion { bool TrimHistory(autovector* to_delete, size_t usage); bool GetFromList(std::list* list, const LookupKey& key, - std::string* value, PinnableWideColumns* columns, + PinnableSlice* value, PinnableWideColumns* columns, std::string* timestamp, Status* s, MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq, diff --git a/db/memtable_list_test.cc b/db/memtable_list_test.cc index 9a5b7557f8..a4a67814e9 100644 --- a/db/memtable_list_test.cc +++ b/db/memtable_list_test.cc @@ -21,6 +21,19 @@ namespace ROCKSDB_NAMESPACE { +static auto g_cspp_fac = []()-> std::shared_ptr { + const char* memtab_opt = getenv("MemTableRepFactory"); + if (memtab_opt && strncmp(memtab_opt, "cspp:", 5) == 0) { + #ifdef HAS_TOPLING_CSPP_MEMTABLE + extern MemTableRepFactory* NewCSPPMemTabForPlain(const std::string&); + return std::shared_ptr(NewCSPPMemTabForPlain(memtab_opt + 5)); + #else + fprintf(stderr, "env MemTableRepFactory is cspp but HAS_TOPLING_CSPP_MEMTABLE is not defined\n"); + #endif + } + return nullptr; +}(); + class MemTableListTest : public testing::Test { public: std::string dbname; @@ -238,7 +251,7 @@ TEST_F(MemTableListTest, GetTest) { max_write_buffer_size_to_maintain); SequenceNumber seq = 1; - std::string value; + PinnableSlice value; Status s; MergeContext merge_context; InternalKeyComparator ikey_cmp(options.comparator); @@ -255,6 +268,7 @@ TEST_F(MemTableListTest, GetTest) { InternalKeyComparator cmp(BytewiseComparator()); auto factory = std::make_shared(); options.memtable_factory = factory; + if (g_cspp_fac) options.memtable_factory = g_cspp_fac; ImmutableOptions ioptions(options); WriteBufferManager wb(options.db_write_buffer_size); @@ -373,7 +387,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) { max_write_buffer_size_to_maintain); SequenceNumber seq = 1; - std::string value; + PinnableSlice value; Status s; MergeContext merge_context; InternalKeyComparator ikey_cmp(options.comparator); @@ -390,6 +404,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) { InternalKeyComparator cmp(BytewiseComparator()); auto factory = std::make_shared(); options.memtable_factory = factory; + if (g_cspp_fac) options.memtable_factory = g_cspp_fac; ImmutableOptions ioptions(options); WriteBufferManager wb(options.db_write_buffer_size); @@ -596,6 +611,7 @@ TEST_F(MemTableListTest, FlushPendingTest) { auto factory = std::make_shared(); options.memtable_factory = factory; + if (g_cspp_fac) options.memtable_factory = g_cspp_fac; ImmutableOptions ioptions(options); InternalKeyComparator cmp(BytewiseComparator()); WriteBufferManager wb(options.db_write_buffer_size); @@ -895,6 +911,7 @@ TEST_F(MemTableListTest, AtomicFlushTest) { auto factory = std::make_shared(); options.memtable_factory = factory; + if (g_cspp_fac) options.memtable_factory = g_cspp_fac; ImmutableOptions ioptions(options); InternalKeyComparator cmp(BytewiseComparator()); WriteBufferManager wb(options.db_write_buffer_size); diff --git a/db/merge_context.h b/db/merge_context.h index 8a7b072902..e2bb0c90a3 100644 --- a/db/merge_context.h +++ b/db/merge_context.h @@ -9,12 +9,11 @@ #include #include +#include #include "rocksdb/slice.h" namespace ROCKSDB_NAMESPACE { -const std::vector empty_operand_list; - // The merge context for merging a user key. // When doing a Get(), DB will create such a class and pass it when // issuing Get() operation to memtables and version_set. The operands @@ -23,57 +22,47 @@ class MergeContext { public: // Clear all the operands void Clear() { - if (operand_list_) { - operand_list_->clear(); - copied_operands_->clear(); - } + operand_list_.clear(); + copied_operands_.erase_all(); } // Push a merge operand void PushOperand(const Slice& operand_slice, bool operand_pinned = false) { - Initialize(); SetDirectionBackward(); if (operand_pinned) { - operand_list_->push_back(operand_slice); + operand_list_.push_back(operand_slice); } else { // We need to have our own copy of the operand since it's not pinned - copied_operands_->emplace_back( - new std::string(operand_slice.data(), operand_slice.size())); - operand_list_->push_back(*copied_operands_->back()); + char* copy = MakeCopy(operand_slice); + copied_operands_.emplace_back(copy); + operand_list_.emplace_back(copy, operand_slice.size()); } } // Push back a merge operand void PushOperandBack(const Slice& operand_slice, bool operand_pinned = false) { - Initialize(); SetDirectionForward(); if (operand_pinned) { - operand_list_->push_back(operand_slice); + operand_list_.push_back(operand_slice); } else { // We need to have our own copy of the operand since it's not pinned - copied_operands_->emplace_back( - new std::string(operand_slice.data(), operand_slice.size())); - operand_list_->push_back(*copied_operands_->back()); + char* copy = MakeCopy(operand_slice); + copied_operands_.emplace_back(copy); + operand_list_.emplace_back(copy, operand_slice.size()); } } // return total number of operands in the list - size_t GetNumOperands() const { - if (!operand_list_) { - return 0; - } - return operand_list_->size(); - } + size_t GetNumOperands() const { return operand_list_.size(); } // Get the operand at the index. - Slice GetOperand(int index) const { - assert(operand_list_); - + Slice GetOperand(size_t index) const { + assert(index < operand_list_.size()); SetDirectionForward(); - return (*operand_list_)[index]; + return operand_list_[index]; } // Same as GetOperandsDirectionForward @@ -92,12 +81,8 @@ class MergeContext { // to this MergeContext. If the returned value is needed for longer, // a copy must be made. const std::vector& GetOperandsDirectionForward() const { - if (!operand_list_) { - return empty_operand_list; - } - SetDirectionForward(); - return *operand_list_; + return operand_list_; } // Return all the operands in the reversed order relative to how they were @@ -107,41 +92,39 @@ class MergeContext { // to this MergeContext. If the returned value is needed for longer, // a copy must be made. const std::vector& GetOperandsDirectionBackward() const { - if (!operand_list_) { - return empty_operand_list; - } - SetDirectionBackward(); - return *operand_list_; + return operand_list_; } - private: - void Initialize() { - if (!operand_list_) { - operand_list_.reset(new std::vector()); - copied_operands_.reset(new std::vector>()); - } + protected: + static char* MakeCopy(Slice src) { + char* copy = new char[src.size()]; + memcpy(copy, src.data(), src.size()); + return copy; } void SetDirectionForward() const { if (operands_reversed_ == true) { - std::reverse(operand_list_->begin(), operand_list_->end()); + std::reverse(operand_list_.begin(), operand_list_.end()); operands_reversed_ = false; } } void SetDirectionBackward() const { if (operands_reversed_ == false) { - std::reverse(operand_list_->begin(), operand_list_->end()); + std::reverse(operand_list_.begin(), operand_list_.end()); operands_reversed_ = true; } } // List of operands - mutable std::unique_ptr> operand_list_; + mutable std::vector operand_list_; // Copy of operands that are not pinned. - std::unique_ptr>> copied_operands_; + terark::valvec32 > copied_operands_; mutable bool operands_reversed_ = true; + mutable bool ext_bool_ = false; + mutable uint16_t ext_uint16_ = 0; + mutable uint32_t ext_flags_ = 0; // for use by derived class }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/merge_helper.cc b/db/merge_helper.cc index 703909010e..871836eb12 100644 --- a/db/merge_helper.cc +++ b/db/merge_helper.cc @@ -668,7 +668,7 @@ CompactionFilter::Decision MergeHelper::FilterMerge(const Slice& user_key, &value_slice, /* existing_columns */ nullptr, &compaction_filter_value_, /* new_columns */ nullptr, compaction_filter_skip_until_.rep()); if (ret == CompactionFilter::Decision::kRemoveAndSkipUntil) { - if (user_comparator_->Compare(*compaction_filter_skip_until_.rep(), + if (user_comparator_->Compare(compaction_filter_skip_until_.Encode(), user_key) <= 0) { // Invalid skip_until returned from compaction filter. // Keep the key as per FilterV2/FilterV3 documentation. diff --git a/db/output_validator.cc b/db/output_validator.cc index e93e2d68c4..86cd389740 100644 --- a/db/output_validator.cc +++ b/db/output_validator.cc @@ -7,9 +7,29 @@ #include "test_util/sync_point.h" #include "util/hash.h" +#include +#include namespace ROCKSDB_NAMESPACE { -Status OutputValidator::Add(const Slice& key, const Slice& value) { + +using terark::fstring; +static bool g_full_check = terark::getEnvBool("OutputValidator_full_check"); + +void OutputValidator::Init() { + full_check_ = g_full_check; + if (full_check_) { + kv_vec_.reserve(32 << 20); // 32M + } + if (icmp_.IsForwardBytewise()) + m_add = &OutputValidator::Add_tpl; + else if (icmp_.IsReverseBytewise()) + m_add = &OutputValidator::Add_tpl; + else + m_add = &OutputValidator::Add_tpl; +} + +template +Status OutputValidator::Add_tpl(const Slice key, const Slice value) { if (enable_hash_) { // Generate a rolling 64-bit hash of the key and values paranoid_hash_ = NPHash64(key.data(), key.size(), paranoid_hash_); @@ -23,11 +43,64 @@ Status OutputValidator::Add(const Slice& key, const Slice& value) { "Compaction tries to write a key without internal bytes."); } // prev_key_ starts with empty. - if (!prev_key_.empty() && icmp_.Compare(key, prev_key_) < 0) { + if (!prev_key_.empty() && Cmp{&icmp_}(key, SliceOf(prev_key_))) { return Status::Corruption("Compaction sees out-of-order keys."); } + #if 0 prev_key_.assign(key.data(), key.size()); + #else + // faster + prev_key_.resize_no_init(key.size()); + memcpy(prev_key_.data(), key.data(), key.size()); + #endif + } + if (full_check_) { + auto WriteSlice = [this](Slice s) { + unsigned char buf[16]; + size_t len = terark::save_var_uint64(buf, s.size_) - buf; + kv_vec_.append(buf, len); + kv_vec_.append(s.data_, s.size_); + }; + WriteSlice(key); + WriteSlice(value); } + num_kv_++; return Status::OK(); } + +static Slice ReadSlice(const unsigned char** ptr) { + size_t len = (size_t)terark::load_var_uint64(*ptr, ptr); + auto data = (const char*)(*ptr); + *ptr += len; + return Slice(data, len); +} + +bool OutputValidator::CompareValidator(const OutputValidator& other) { + if (full_check_) { + long long file_number = m_file_number ? m_file_number : other.m_file_number; + if (kv_vec_.size() != other.kv_vec_.size()) { + fprintf(stderr, + "FATAL: OutputValidator::CompareValidator: kv_vec_.size: %zd != %zd\n", + kv_vec_.size(), other.kv_vec_.size()); + } + ROCKSDB_VERIFY_EQ(num_kv_, other.num_kv_); + const unsigned char* x_reader = kv_vec_.begin(); + const unsigned char* y_reader = other.kv_vec_.begin(); + for (size_t i = 0, n = num_kv_; i < n; i++) { + Slice kx = ReadSlice(&x_reader); + Slice vx = ReadSlice(&x_reader); + Slice ky = ReadSlice(&y_reader); + Slice vy = ReadSlice(&y_reader); + #define HexKey(key) ParsedInternalKey(key).DebugString(true, true).c_str() + ROCKSDB_VERIFY_F(kx == ky, "%06lld.sst[%zd]: %zd(%s) %zd(%s)", file_number, i, kx.size_, HexKey(kx), ky.size_, HexKey(ky)); + ROCKSDB_VERIFY_F(vx == vy, "%06lld.sst[%zd]: %zd(%s) %zd(%s)", file_number, i, vx.size_, vx.hex().c_str(), vy.size_, vy.hex().c_str()); + } + ROCKSDB_VERIFY_EQ(x_reader, kv_vec_.end()); + ROCKSDB_VERIFY_EQ(y_reader, other.kv_vec_.end()); + ROCKSDB_VERIFY_EQ(GetHash(), other.GetHash()); + } + return GetHash() == other.GetHash(); +} + + } // namespace ROCKSDB_NAMESPACE diff --git a/db/output_validator.h b/db/output_validator.h index 40635f9c44..d331957703 100644 --- a/db/output_validator.h +++ b/db/output_validator.h @@ -7,6 +7,8 @@ #include "db/dbformat.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" +#include +#include namespace ROCKSDB_NAMESPACE { // A class that validates key/value that is inserted to an SST file. @@ -21,28 +23,39 @@ class OutputValidator { : icmp_(icmp), paranoid_hash_(precalculated_hash), enable_order_check_(enable_order_check), - enable_hash_(enable_hash) {} + enable_hash_(enable_hash) { + Init(); + } // Add a key to the KV sequence, and return whether the key follows // criteria, e.g. key is ordered. - Status Add(const Slice& key, const Slice& value); + inline Status Add(const Slice& key, const Slice& value) { + return (this->*m_add)(key, value); + } // Compare result of two key orders are the same. It can be used // to compare the keys inserted into a file, and what is read back. // Return true if the validation passes. - bool CompareValidator(const OutputValidator& other_validator) { - return GetHash() == other_validator.GetHash(); - } + bool CompareValidator(const OutputValidator& other_validator); // Not (yet) intended to be persisted, so subject to change // without notice between releases. uint64_t GetHash() const { return paranoid_hash_; } + uint64_t m_file_number = 0; // just a patch + private: + void Init(); + Status (OutputValidator::*m_add)(const Slice key, const Slice value); + template Status Add_tpl(const Slice key, const Slice value); + const InternalKeyComparator& icmp_; - std::string prev_key_; + terark::valvec32 prev_key_; uint64_t paranoid_hash_ = 0; bool enable_order_check_; bool enable_hash_; + bool full_check_ = false; + size_t num_kv_ = 0; + terark::valvec kv_vec_; }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/perf_context_test.cc b/db/perf_context_test.cc index 2666b8733c..43ea2f2415 100644 --- a/db/perf_context_test.cc +++ b/db/perf_context_test.cc @@ -188,8 +188,8 @@ TEST_F(PerfContextTest, StopWatchOverhead) { uint64_t elapsed = 0; std::vector timings(kTotalIterations); - StopWatch timer(SystemClock::Default().get(), nullptr, 0, - Histograms::HISTOGRAM_ENUM_MAX, &elapsed); + StopWatchEx timer(SystemClock::Default().get(), nullptr, 0, + Histograms::HISTOGRAM_ENUM_MAX, &elapsed); for (auto& timing : timings) { timing = elapsed; } @@ -395,7 +395,7 @@ void ProfileQueries(bool enabled_time = false) { EXPECT_GT(hist_write_scheduling_time.Average(), 0); #ifndef NDEBUG - ASSERT_LT(total_db_mutex_nanos, 100U); + //ASSERT_LT(total_db_mutex_nanos, 100U); // ToplingDB, ignore #endif } @@ -596,12 +596,11 @@ TEST_F(PerfContextTest, SeekKeyComparison) { } TEST_F(PerfContextTest, DBMutexLockCounter) { - int stats_code[] = {0, static_cast(DB_MUTEX_WAIT_MICROS)}; + int stats_code[] = {static_cast(DB_MUTEX_WAIT_NANOS)}; for (PerfLevel perf_level_test : {PerfLevel::kEnableTimeExceptForMutex, PerfLevel::kEnableTime}) { - for (int c = 0; c < 2; ++c) { - InstrumentedMutex mutex(nullptr, SystemClock::Default().get(), - stats_code[c]); + for (int c = 0; c < 1; ++c) { + InstrumentedMutex mutex(nullptr, SystemClock::Default().get()); mutex.Lock(); ROCKSDB_NAMESPACE::port::Thread child_thread([&] { SetPerfLevel(perf_level_test); @@ -610,7 +609,7 @@ TEST_F(PerfContextTest, DBMutexLockCounter) { mutex.Lock(); mutex.Unlock(); if (perf_level_test == PerfLevel::kEnableTimeExceptForMutex || - stats_code[c] != DB_MUTEX_WAIT_MICROS) { + stats_code[c] != DB_MUTEX_WAIT_NANOS) { ASSERT_EQ(get_perf_context()->db_mutex_lock_nanos, 0); } else { // increment the counter only when it's a DB Mutex @@ -626,16 +625,15 @@ TEST_F(PerfContextTest, DBMutexLockCounter) { TEST_F(PerfContextTest, FalseDBMutexWait) { SetPerfLevel(kEnableTime); - int stats_code[] = {0, static_cast(DB_MUTEX_WAIT_MICROS)}; - for (int c = 0; c < 2; ++c) { - InstrumentedMutex mutex(nullptr, SystemClock::Default().get(), - stats_code[c]); + int stats_code[] = {static_cast(DB_MUTEX_WAIT_NANOS)}; + for (int c = 0; c < 1; ++c) { + InstrumentedMutex mutex(nullptr, SystemClock::Default().get()); InstrumentedCondVar lock(&mutex); get_perf_context()->Reset(); mutex.Lock(); lock.TimedWait(100); mutex.Unlock(); - if (stats_code[c] == static_cast(DB_MUTEX_WAIT_MICROS)) { + if (stats_code[c] == static_cast(DB_MUTEX_WAIT_NANOS)) { // increment the counter only when it's a DB Mutex ASSERT_GT(get_perf_context()->db_condition_wait_nanos, 0); } else { @@ -715,17 +713,17 @@ TEST_F(PerfContextTest, CopyAndMove) { PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); ASSERT_EQ( 1, - (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + get_perf_context()->level_to_perf_context[5].bloom_filter_useful); PerfContext perf_context_assign; perf_context_assign = *get_perf_context(); ASSERT_EQ( 1, - (*(perf_context_assign.level_to_perf_context))[5].bloom_filter_useful); + perf_context_assign.level_to_perf_context[5].bloom_filter_useful); get_perf_context()->ClearPerLevelPerfContext(); get_perf_context()->Reset(); ASSERT_EQ( 1, - (*(perf_context_assign.level_to_perf_context))[5].bloom_filter_useful); + perf_context_assign.level_to_perf_context[5].bloom_filter_useful); perf_context_assign.ClearPerLevelPerfContext(); perf_context_assign.Reset(); } @@ -736,14 +734,14 @@ TEST_F(PerfContextTest, CopyAndMove) { PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); ASSERT_EQ( 1, - (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + get_perf_context()->level_to_perf_context[5].bloom_filter_useful); PerfContext perf_context_copy(*get_perf_context()); ASSERT_EQ( - 1, (*(perf_context_copy.level_to_perf_context))[5].bloom_filter_useful); + 1, perf_context_copy.level_to_perf_context[5].bloom_filter_useful); get_perf_context()->ClearPerLevelPerfContext(); get_perf_context()->Reset(); ASSERT_EQ( - 1, (*(perf_context_copy.level_to_perf_context))[5].bloom_filter_useful); + 1, perf_context_copy.level_to_perf_context[5].bloom_filter_useful); perf_context_copy.ClearPerLevelPerfContext(); perf_context_copy.Reset(); } @@ -754,14 +752,14 @@ TEST_F(PerfContextTest, CopyAndMove) { PERF_COUNTER_BY_LEVEL_ADD(bloom_filter_useful, 1, 5); ASSERT_EQ( 1, - (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + get_perf_context()->level_to_perf_context[5].bloom_filter_useful); PerfContext perf_context_move = std::move(*get_perf_context()); ASSERT_EQ( - 1, (*(perf_context_move.level_to_perf_context))[5].bloom_filter_useful); + 1, perf_context_move.level_to_perf_context[5].bloom_filter_useful); get_perf_context()->ClearPerLevelPerfContext(); get_perf_context()->Reset(); ASSERT_EQ( - 1, (*(perf_context_move.level_to_perf_context))[5].bloom_filter_useful); + 1, perf_context_move.level_to_perf_context[5].bloom_filter_useful); perf_context_move.ClearPerLevelPerfContext(); perf_context_move.Reset(); } @@ -777,13 +775,13 @@ TEST_F(PerfContextTest, PerfContextDisableEnable) { PERF_COUNTER_BY_LEVEL_ADD(block_cache_hit_count, 1, 0); get_perf_context()->DisablePerLevelPerfContext(); PerfContext perf_context_copy(*get_perf_context()); - ASSERT_EQ(1, (*(perf_context_copy.level_to_perf_context))[0] + ASSERT_EQ(1, perf_context_copy.level_to_perf_context[0] .bloom_filter_full_positive); // this was set when per level perf context is disabled, should not be copied ASSERT_NE( - 1, (*(perf_context_copy.level_to_perf_context))[5].bloom_filter_useful); + 1, perf_context_copy.level_to_perf_context[5].bloom_filter_useful); ASSERT_EQ( - 1, (*(perf_context_copy.level_to_perf_context))[0].block_cache_hit_count); + 1, perf_context_copy.level_to_perf_context[0].block_cache_hit_count); perf_context_copy.ClearPerLevelPerfContext(); perf_context_copy.Reset(); get_perf_context()->ClearPerLevelPerfContext(); @@ -803,27 +801,23 @@ TEST_F(PerfContextTest, PerfContextByLevelGetSet) { PERF_COUNTER_BY_LEVEL_ADD(block_cache_miss_count, 2, 3); PERF_COUNTER_BY_LEVEL_ADD(block_cache_miss_count, 4, 1); ASSERT_EQ( - 0, (*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful); + 0, get_perf_context()->level_to_perf_context[0].bloom_filter_useful); ASSERT_EQ( - 1, (*(get_perf_context()->level_to_perf_context))[5].bloom_filter_useful); + 1, get_perf_context()->level_to_perf_context[5].bloom_filter_useful); ASSERT_EQ( - 2, (*(get_perf_context()->level_to_perf_context))[7].bloom_filter_useful); - ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[0] + 2, get_perf_context()->level_to_perf_context[7].bloom_filter_useful); + ASSERT_EQ(1, get_perf_context()->level_to_perf_context[0] .bloom_filter_full_positive); - ASSERT_EQ(1, (*(get_perf_context()->level_to_perf_context))[2] + ASSERT_EQ(1, get_perf_context()->level_to_perf_context[2] .bloom_filter_full_true_positive); - ASSERT_EQ( - 1, - (*(get_perf_context()->level_to_perf_context))[0].block_cache_hit_count); - ASSERT_EQ( - 5, - (*(get_perf_context()->level_to_perf_context))[2].block_cache_hit_count); - ASSERT_EQ( - 2, - (*(get_perf_context()->level_to_perf_context))[3].block_cache_miss_count); - ASSERT_EQ( - 4, - (*(get_perf_context()->level_to_perf_context))[1].block_cache_miss_count); + ASSERT_EQ(1, get_perf_context()->level_to_perf_context[0] + .block_cache_hit_count); + ASSERT_EQ(5, get_perf_context()->level_to_perf_context[2] + .block_cache_hit_count); + ASSERT_EQ(2, get_perf_context()->level_to_perf_context[3] + .block_cache_miss_count); + ASSERT_EQ(4, get_perf_context()->level_to_perf_context[1] + .block_cache_miss_count); std::string zero_excluded = get_perf_context()->ToString(true); ASSERT_NE(std::string::npos, zero_excluded.find("bloom_filter_useful = 1@level5, 2@level7")); diff --git a/db/pinned_iterators_manager.h b/db/pinned_iterators_manager.h index 0fcf231dad..3eb32d04f5 100644 --- a/db/pinned_iterators_manager.h +++ b/db/pinned_iterators_manager.h @@ -16,9 +16,20 @@ namespace ROCKSDB_NAMESPACE { // PinnedIteratorsManager will be notified whenever we need to pin an Iterator // and it will be responsible for deleting pinned Iterators when they are // not needed anymore. -class PinnedIteratorsManager : public Cleanable { + +class PinIterMgrBase { + // used for dummy PinnedIteratorsManager +protected: + bool pinning_enabled = false; // first field of PinnedIteratorsManager + +public: + // Is pinning enabled ? + bool PinningEnabled() { return pinning_enabled; } +}; + +class PinnedIteratorsManager : public PinIterMgrBase, public Cleanable { public: - PinnedIteratorsManager() : pinning_enabled(false) {} + PinnedIteratorsManager() = default; ~PinnedIteratorsManager() { if (pinning_enabled) { ReleasePinnedData(); @@ -36,9 +47,6 @@ class PinnedIteratorsManager : public Cleanable { pinning_enabled = true; } - // Is pinning enabled ? - bool PinningEnabled() { return pinning_enabled; } - // Take ownership of iter and delete it when ReleasePinnedData() is called void PinIterator(InternalIterator* iter, bool arena = false) { if (arena) { @@ -85,7 +93,6 @@ class PinnedIteratorsManager : public Cleanable { reinterpret_cast(ptr)->~InternalIterator(); } - bool pinning_enabled; std::vector> pinned_ptrs_; }; diff --git a/db/snapshot_impl.h b/db/snapshot_impl.h index 23e5e98cd2..0611bd33c0 100644 --- a/db/snapshot_impl.h +++ b/db/snapshot_impl.h @@ -84,15 +84,27 @@ class SnapshotList { SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq, uint64_t unix_time, bool is_write_conflict_boundary, uint64_t ts = std::numeric_limits::max()) { + // snapshots in list_ was ordered by number_, but now we may create snapshots + // by specify seqnum in ArenaWrappedDBIter::Refresh() for pinning, which + // seqnum maybe smaller than the largest seqnum in list_, so the newly created + // snapshot can not be put to list_ tail, we should find the insert position. + // it is lucky that list_ is short, and the target should be near list tail, + // the search should be fast. + SnapshotImpl* s_prev = list_.prev_; // init to tail + for (; s_prev != &list_; s_prev = s_prev->prev_) { + if (s_prev->number_ <= seq) + break; + } + SnapshotImpl* s_next = s_prev->next_; s->number_ = seq; s->unix_time_ = unix_time; s->timestamp_ = ts; s->is_write_conflict_boundary_ = is_write_conflict_boundary; s->list_ = this; - s->next_ = &list_; - s->prev_ = list_.prev_; - s->prev_->next_ = s; - s->next_->prev_ = s; + s->next_ = s_next; + s->prev_ = s_prev; + s_prev->next_ = s; + s_next->prev_ = s; count_++; return s; } diff --git a/db/table_cache.cc b/db/table_cache.cc index 2c0092e7d4..54c0dd1e33 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -109,7 +109,9 @@ Status TableCache::GetTableReader( } if (s.ok()) { RecordTick(ioptions_.stats, NO_FILE_OPENS); - } else if (s.IsPathNotFound()) { + } +#ifdef ROCKSDB_SUPPORT_LEVELDB_FILE_LDB + if (s.IsPathNotFound()) { fname = Rocks2LevelTableFileName(fname); // If this file is also not found, we want to use the error message // that contains the table file name which is less confusing. @@ -124,6 +126,7 @@ Status TableCache::GetTableReader( s = temp_s; } } +#endif // ROCKSDB_SUPPORT_LEVELDB_FILE_LDB if (s.ok()) { if (!sequential_mode && ioptions_.advise_random_on_open) { @@ -251,7 +254,7 @@ InternalIterator* TableCache::NewIterator( InternalIterator* result = nullptr; if (s.ok()) { if (options.table_filter && - !options.table_filter(*table_reader->GetTableProperties())) { + !options.table_filter(*table_reader->GetTableProperties(), file_meta)) { result = NewEmptyInternalIterator(arena); } else { result = table_reader->NewIterator( diff --git a/db/table_cache.h b/db/table_cache.h index ae3fc93c37..14fdd6e7a6 100644 --- a/db/table_cache.h +++ b/db/table_cache.h @@ -240,6 +240,14 @@ class TableCache { } } + // Get TableReader from a cache handle. + TableReader* GetTableReaderFromHandle(TypedHandle* handle) { + return cache_.Value(handle); + } + + // Release the handle from a cache + void ReleaseHandle(TypedHandle* handle) { cache_.Release(handle); } + private: // Build a table reader Status GetTableReader( diff --git a/db/version_edit.cc b/db/version_edit.cc index 482aa65a75..b0af0d87aa 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -27,10 +27,11 @@ uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) { return number | (path_id * (kFileNumberMask + 1)); } +ROCKSDB_FLATTEN Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, SequenceNumber seqno, ValueType value_type) { - if (value_type == kTypeBlobIndex) { + if (UNLIKELY(value_type == kTypeBlobIndex)) { BlobIndex blob_index; const Status s = blob_index.DecodeFrom(value); if (!s.ok()) { @@ -49,7 +50,7 @@ Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, } } - if (smallest.size() == 0) { + if (smallest.empty()) { smallest.DecodeFrom(key); } largest.DecodeFrom(key); diff --git a/db/version_edit.h b/db/version_edit.h index 8e14e76da9..210ccbd0f1 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -197,6 +197,7 @@ struct FileMetaData { 0; // The number of entries, including deletions and range deletions. // The number of deletion entries, including range deletions. uint64_t num_deletions = 0; + uint64_t num_merges = 0; uint64_t raw_key_size = 0; // total uncompressed key size. uint64_t raw_value_size = 0; // total uncompressed value size. uint64_t num_range_deletions = 0; @@ -207,6 +208,9 @@ struct FileMetaData { int refs = 0; // Reference count + int job_id = -1; + int job_attempt = -1; + bool being_compacted = false; // Is this file undergoing compaction? bool init_stats_from_file = false; // true if the data-entry stats of this // file has initialized from file. @@ -377,11 +381,28 @@ struct FdWithKeyRange { struct LevelFilesBrief { size_t num_files; FdWithKeyRange* files; + std::shared_ptr udfa = nullptr; + uint64_t* prefix_cache = nullptr; LevelFilesBrief() { num_files = 0; files = nullptr; } }; +inline uint64_t HostPrefixCache(const Slice& ikey) { + ROCKSDB_ASSERT_GE(ikey.size_, 8); + ROCKSDB_ASSUME(ikey.size_ >= 8); + uint64_t data; + if (LIKELY(ikey.size_ >= 16)) { + memcpy(&data, ikey.data_, 8); + } else { + data = 0; + memcpy(&data, ikey.data_, ikey.size_ - 8); + } + if (port::kLittleEndian) + return __bswap_64(data); + else + return data; +} // The state of a DB at any given time is referred to as a Version. // Any modification to the Version is considered a Version Edit. A Version is diff --git a/db/version_edit_handler.cc b/db/version_edit_handler.cc index 90afc0938c..3f7af7fc0f 100644 --- a/db/version_edit_handler.cc +++ b/db/version_edit_handler.cc @@ -103,10 +103,10 @@ Status ListColumnFamiliesHandler::ApplyVersionEdit( Status s; uint32_t cf_id = edit.GetColumnFamily(); if (edit.IsColumnFamilyAdd()) { - if (column_family_names_.find(cf_id) != column_family_names_.end()) { + auto [iter, success] = column_family_names_.insert( + {cf_id, edit.GetColumnFamilyName()}); + if (!success) { s = Status::Corruption("Manifest adding the same column family twice"); - } else { - column_family_names_.insert({cf_id, edit.GetColumnFamilyName()}); } } else if (edit.IsColumnFamilyDrop()) { if (column_family_names_.find(cf_id) == column_family_names_.end()) { @@ -905,12 +905,10 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion( version->PrepareAppend( *cfd->GetLatestMutableCFOptions(), read_options_, !version_set_->db_options_->skip_stats_update_on_db_open); - auto v_iter = versions_.find(cfd->GetID()); - if (v_iter != versions_.end()) { + auto [v_iter, success ] = versions_.emplace(cfd->GetID(), version); + if (!success) { delete v_iter->second; v_iter->second = version; - } else { - versions_.emplace(cfd->GetID(), version); } } else { delete version; diff --git a/db/version_set.cc b/db/version_set.cc index 335d492bc3..c4afd65f22 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -92,18 +92,103 @@ namespace ROCKSDB_NAMESPACE { +__attribute__((weak)) void +InitUdfa(LevelFilesBrief*, const Comparator* user_cmp); +__attribute__((weak)) int +FindFileInRangeUdfa(const LevelFilesBrief&, const Slice& key); + namespace { +#if defined(_MSC_VER) /* Visual Studio */ +#define __attribute_noinline__ +#define __builtin_prefetch(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +#elif defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wattributes" +#else +#define __attribute_noinline__ +#define __builtin_prefetch(ptr) +#endif + +template +size_t FindFileInRangeTmpl(Cmp cmp, const LevelFilesBrief& brief, + Slice key, size_t lo, size_t hi) { + const uint64_t* pxcache = brief.prefix_cache; + const uint64_t key_prefix = HostPrefixCache(key); + const FdWithKeyRange* a = brief.files; + size_t mid; + while (lo < hi) { + mid = (lo + hi) / 2; + if (cmp(pxcache[mid], key_prefix)) + lo = mid + 1; + else if (cmp(key_prefix, pxcache[mid])) + hi = mid; + else + goto exact_search; + } + return lo; + + while (lo < hi) { + mid = (lo + hi) / 2; + exact_search: + __builtin_prefetch(a[mid].largest_key.data_); + if (cmp(a[mid].largest_key, key)) + lo = mid + 1; + else + hi = mid; + } + return lo; +} + +static +size_t FindFileInRangeTmpl(FallbackVirtCmp cmp, const LevelFilesBrief& brief, + Slice key, size_t lo, size_t hi) { + const FdWithKeyRange* a = brief.files; + while (lo < hi) { + size_t mid = (lo + hi) / 2; + if (cmp(a[mid].largest_key, key)) + lo = mid + 1; + else + hi = mid; + } + return lo; +} + // Find File in LevelFilesBrief data structure // Within an index range defined by left and right +#ifdef TOPLINGDB_NO_OPT_FindFileInRange +__attribute_noinline__ +#endif int FindFileInRange(const InternalKeyComparator& icmp, const LevelFilesBrief& file_level, const Slice& key, uint32_t left, uint32_t right) { +#ifdef TOPLINGDB_NO_OPT_FindFileInRange + #pragma message "TOPLINGDB_NO_OPT_FindFileInRange is defined, intended for benchmark baseline" + // here is upstream rocksdb code auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool { return icmp.InternalKeyComparator::Compare(f.largest_key, k) < 0; }; const auto& b = file_level.files; return static_cast(std::lower_bound(b + left, b + right, key, cmp) - b); +#else // ToplingDB Devirtualization and Key Prefix Cache optimization + if (icmp.IsForwardBytewise()) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + if (file_level.udfa) { + assert(&FindFileInRangeUdfa != nullptr); + return FindFileInRangeUdfa(file_level, key); + } + BytewiseCompareInternalKey cmp; + return (int)FindFileInRangeTmpl(cmp, file_level, key, left, right); + } + else if (icmp.IsReverseBytewise()) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + RevBytewiseCompareInternalKey cmp; + return (int)FindFileInRangeTmpl(cmp, file_level, key, left, right); + } + else { + FallbackVirtCmp cmp{&icmp}; + return (int)FindFileInRangeTmpl(cmp, file_level, key, left, right); + } +#endif } Status OverlapWithIterator(const Comparator* ucmp, @@ -177,6 +262,16 @@ class FilePicker { int GetCurrentLevel() const { return curr_level_; } FdWithKeyRange* GetNextFile() { + auto ucmp = user_comparator_; + if (IsForwardBytewiseComparator(ucmp)) + return GetNextFileTmpl(ForwardBytewiseCompareUserKeyNoTS()); + else if (IsReverseBytewiseComparator(ucmp)) + return GetNextFileTmpl(ReverseBytewiseCompareUserKeyNoTS()); + else + return GetNextFileTmpl(VirtualFunctionCompareUserKeyNoTS{ucmp}); + } + template + FdWithKeyRange* GetNextFileTmpl(Compare cmp) { while (!search_ended_) { // Loops over different levels. while (curr_index_in_curr_level_ < curr_file_level_->num_files) { // Loops over all files in current level. @@ -200,14 +295,11 @@ class FilePicker { // range. assert(curr_level_ == 0 || curr_index_in_curr_level_ == start_index_in_curr_level_ || - user_comparator_->CompareWithoutTimestamp( - user_key_, ExtractUserKey(f->smallest_key)) <= 0); + cmp(user_key_, ExtractUserKey(f->smallest_key)) <= 0); - int cmp_smallest = user_comparator_->CompareWithoutTimestamp( - user_key_, ExtractUserKey(f->smallest_key)); + int cmp_smallest = cmp(user_key_, ExtractUserKey(f->smallest_key)); if (cmp_smallest >= 0) { - cmp_largest = user_comparator_->CompareWithoutTimestamp( - user_key_, ExtractUserKey(f->largest_key)); + cmp_largest = cmp(user_key_, ExtractUserKey(f->largest_key)); } // Setup file search bound for the next level based on the @@ -860,11 +952,14 @@ void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, size_t num = files.size(); file_level->num_files = num; char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange)); + auto pxcache = (uint64_t*)arena->AllocateAligned(num * sizeof(uint64_t)); file_level->files = new (mem) FdWithKeyRange[num]; + file_level->prefix_cache = pxcache; for (size_t i = 0; i < num; i++) { Slice smallest_key = files[i]->smallest.Encode(); Slice largest_key = files[i]->largest.Encode(); + pxcache[i] = HostPrefixCache(largest_key); // Copy key slice to sequential memory size_t smallest_size = smallest_key.size(); @@ -973,6 +1068,7 @@ class LevelIterator final : public InternalIterator { should_sample_(should_sample), skip_filters_(skip_filters), allow_unprepared_value_(allow_unprepared_value), + opt_cmp_type_(icomparator.user_comparator()->opt_cmp_type()), is_next_read_sequential_(false), to_return_sentinel_(false) { // Empty level is not supported. @@ -980,9 +1076,25 @@ class LevelIterator final : public InternalIterator { if (range_tombstone_iter_ptr_) { *range_tombstone_iter_ptr_ = &range_tombstone_iter_; } + if (read_options.cache_sst_file_iter) { + file_iter_cache_ = new InternalIterator*[flevel->num_files](); + } else { + file_iter_cache_ = nullptr; + } } - ~LevelIterator() override { delete file_iter_.Set(nullptr); } + ~LevelIterator() override { + if (file_iter_cache_) { + for (size_t i = 0, n = flevel_->num_files; i < n; i++) { + auto iter = file_iter_cache_[i]; + if (UNLIKELY(nullptr != iter)) + delete iter; + } + delete file_iter_cache_; + } else { + delete file_iter_.Set(nullptr); + } + } // Seek to the first file with a key >= target. // If range_tombstone_iter_ is not nullptr, then we pretend that file @@ -1028,6 +1140,9 @@ class LevelIterator final : public InternalIterator { } bool PrepareValue() override { return file_iter_.PrepareValue(); } + bool PrepareAndGetValue(Slice* v) override { + return file_iter_.PrepareAndGetValue(v); + } inline bool MayBeOutOfLowerBound() override { assert(Valid()); @@ -1072,7 +1187,7 @@ class LevelIterator final : public InternalIterator { void SetFileIterator(InternalIterator* iter); void InitFileIterator(size_t new_file_index); - const Slice& file_smallest_key(size_t file_index) { + const Slice& file_smallest_key(size_t file_index) const { assert(file_index < flevel_->num_files); return flevel_->files[file_index].smallest_key; } @@ -1101,7 +1216,7 @@ class LevelIterator final : public InternalIterator { // into the new file. Old range tombstone iterator is cleared. InternalIterator* NewFileIterator() { assert(file_index_ < flevel_->num_files); - auto file_meta = flevel_->files[file_index_]; + const auto& file_meta = flevel_->files[file_index_]; if (should_sample_) { sample_file_read_inc(file_meta.file_metadata); } @@ -1113,15 +1228,30 @@ class LevelIterator final : public InternalIterator { largest_compaction_key = (*compaction_boundaries_)[file_index_].largest; } CheckMayBeOutOfLowerBound(); + ClearRangeTombstoneIter(); - return table_cache_->NewIterator( + InternalIterator* iter = nullptr; + if (file_iter_cache_) { + iter = file_iter_cache_[file_index_]; + } + if (!iter) { + iter = table_cache_->NewIterator( read_options_, file_options_, icomparator_, *file_meta.file_metadata, range_del_agg_, prefix_extractor_, nullptr /* don't need reference to table */, file_read_hist_, caller_, /*arena=*/nullptr, skip_filters_, level_, /*max_file_size_for_l0_meta_pin=*/0, smallest_compaction_key, largest_compaction_key, allow_unprepared_value_, +<<<<<<< HEAD block_protection_bytes_per_key_, &read_seq_, range_tombstone_iter_); +======= + block_protection_bytes_per_key_, range_tombstone_iter_); + if (file_iter_cache_) { + file_iter_cache_[file_index_] = iter; + } + } + return iter; +>>>>>>> sideplugin-8.04.0-2023-06-20-2926e071 } // Check if current file being fully within iterate_lower_bound. @@ -1131,10 +1261,36 @@ class LevelIterator final : public InternalIterator { void CheckMayBeOutOfLowerBound() { if (read_options_.iterate_lower_bound != nullptr && file_index_ < flevel_->num_files) { - may_be_out_of_lower_bound_ = - user_comparator_.CompareWithoutTimestamp( - ExtractUserKey(file_smallest_key(file_index_)), /*a_has_ts=*/true, - *read_options_.iterate_lower_bound, /*b_has_ts=*/false) < 0; + switch (opt_cmp_type_) { + case 0: // IsForwardBytewise() + may_be_out_of_lower_bound_ = + ExtractUserKey(file_smallest_key(file_index_)) < + *read_options_.iterate_lower_bound; + break; + case 1: // IsReverseBytewise() + may_be_out_of_lower_bound_ = + ExtractUserKey(file_smallest_key(file_index_)) > + *read_options_.iterate_lower_bound; + break; + default: + may_be_out_of_lower_bound_ = + user_comparator_.CompareWithoutTimestamp( + ExtractUserKey(file_smallest_key(file_index_)), /*a_has_ts=*/true, + *read_options_.iterate_lower_bound, /*b_has_ts=*/false) < 0; + } + } + } + bool FileIsOutOfUpperBound(size_t file_index) const { + Slice file_smallest_ukey = ExtractUserKey(file_smallest_key(file_index)); + switch (opt_cmp_type_) { + case 0: // IsForwardBytewise() + return !(file_smallest_ukey < *read_options_.iterate_upper_bound); + case 1: // IsReverseBytewise() + return !(file_smallest_ukey > *read_options_.iterate_upper_bound); + default: + return user_comparator_.CompareWithoutTimestamp( + *read_options_.iterate_upper_bound, /*b_has_ts=*/false, + file_smallest_ukey, /*a_has_ts=*/true) <= 0; } } @@ -1154,8 +1310,9 @@ class LevelIterator final : public InternalIterator { TableReaderCaller caller_; size_t file_index_; RangeDelAggregator* range_del_agg_; - IteratorWrapper file_iter_; // May be nullptr + ThinIteratorWrapper file_iter_; // May be nullptr PinnedIteratorsManager* pinned_iters_mgr_; + InternalIterator** file_iter_cache_; // To be propagated to RangeDelAggregator in order to safely truncate range // tombstones. @@ -1186,6 +1343,7 @@ class LevelIterator final : public InternalIterator { bool should_sample_; bool skip_filters_; bool allow_unprepared_value_; + uint8_t opt_cmp_type_; bool may_be_out_of_lower_bound_ = true; bool is_next_read_sequential_; // Set in Seek() when a prefix seek reaches end of the current file, @@ -1218,18 +1376,37 @@ void LevelIterator::Seek(const Slice& target) { bool need_to_reseek = true; if (file_iter_.iter() != nullptr && file_index_ < flevel_->num_files) { const FdWithKeyRange& cur_file = flevel_->files[file_index_]; - if (icomparator_.InternalKeyComparator::Compare( - target, cur_file.largest_key) <= 0 && - icomparator_.InternalKeyComparator::Compare( - target, cur_file.smallest_key) >= 0) { - need_to_reseek = false; - assert(static_cast(FindFile(icomparator_, *flevel_, target)) == - file_index_); + auto check_need_to_reseek = [&](auto cmp) { + if (!cmp(cur_file.largest_key, target) && + !cmp(target, cur_file.smallest_key)) { + need_to_reseek = false; + assert(static_cast(FindFile(icomparator_, *flevel_, target)) == + file_index_); + } + }; + switch (opt_cmp_type_) { + case 0: // IsForwardBytewise() + check_need_to_reseek(BytewiseCompareInternalKey()); + break; + case 1: // IsReverseBytewise() + check_need_to_reseek(RevBytewiseCompareInternalKey()); + default: + check_need_to_reseek(FallbackVirtCmp{&icomparator_}); + break; } } if (need_to_reseek) { TEST_SYNC_POINT("LevelIterator::Seek:BeforeFindFile"); size_t new_file_index = FindFile(icomparator_, *flevel_, target); + if (UNLIKELY(new_file_index >= flevel_->num_files)) { + SetFileIterator(nullptr); + return; + } + if (read_options_.iterate_upper_bound != nullptr && + FileIsOutOfUpperBound(new_file_index)) { + SetFileIterator(nullptr); + return; + } InitFileIterator(new_file_index); } @@ -1239,7 +1416,7 @@ void LevelIterator::Seek(const Slice& target) { // blocks has been submitted. So it should return at this point and Seek // should be called again to retrieve the requested block and execute the // remaining code. - if (file_iter_.status() == Status::TryAgain()) { + if (UNLIKELY(file_iter_.status().IsTryAgain())) { return; } if (!file_iter_.Valid() && file_iter_.status().ok() && @@ -1389,7 +1566,8 @@ bool LevelIterator::NextAndGetResult(IterateResult* result) { assert(Valid()); // file_iter_ is at EOF already when to_return_sentinel_ bool is_valid = !to_return_sentinel_ && file_iter_.NextAndGetResult(result); - if (!is_valid) { + result->is_valid = is_valid; + if (UNLIKELY(!is_valid)) { if (to_return_sentinel_) { ClearSentinel(); } else if (range_tombstone_iter_) { @@ -1399,15 +1577,16 @@ bool LevelIterator::NextAndGetResult(IterateResult* result) { SkipEmptyFileForward(); is_next_read_sequential_ = false; is_valid = Valid(); + result->is_valid = is_valid; if (is_valid) { // This could be set in TrySetDeleteRangeSentinel() or // SkipEmptyFileForward() above. if (to_return_sentinel_) { - result->key = sentinel_; + result->SetKey(sentinel_); result->bound_check_result = IterBoundCheck::kUnknown; result->value_prepared = true; } else { - result->key = key(); + result->SetKey(this->key()); result->bound_check_result = file_iter_.UpperBoundCheckResult(); // Ideally, we should return the real file_iter_.value_prepared but the // information is not here. It would casue an extra PrepareValue() @@ -1512,6 +1691,10 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) { file_iter_.UpdateReadaheadState(old_iter); } + if (file_iter_cache_) { + return; // don't PinIterator or delete old_iter + } + if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) { pinned_iters_mgr_->PinIterator(old_iter); } else { @@ -1728,6 +1911,24 @@ Status Version::GetPropertiesOfTablesInRange( return Status::OK(); } +std::string AggregateNames(const std::map& map, const char* delim) { + std::string str; + size_t dlen = strlen(delim); + for (auto& kv : map) { + str.append(kv.first.empty() ? "N/A" : kv.first); + if (map.size() > 1) { + char buf[32]; + auto len = snprintf(buf, sizeof(buf), "=%d", kv.second); + str.append(buf, len); + str.append(delim, dlen); + } + } + if (map.size() > 1) { + str.resize(str.size()-dlen); // trailing delim + } + return str; +} + Status Version::GetAggregatedTableProperties( const ReadOptions& read_options, std::shared_ptr* tp, int level) { @@ -1743,9 +1944,14 @@ Status Version::GetAggregatedTableProperties( } auto* new_tp = new TableProperties(); + new_tp->column_family_id = cfd_->GetID(); + new_tp->column_family_name = cfd_->GetName(); + std::map algos; for (const auto& item : props) { new_tp->Add(*item.second); + algos[item.second->compression_name]++; } + new_tp->compression_name = AggregateNames(algos, ","); tp->reset(new_tp); return Status::OK(); } @@ -1808,6 +2014,11 @@ void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) { files.back().num_deletions = file->num_deletions; files.back().smallest = file->smallest.Encode().ToString(); files.back().largest = file->largest.Encode().ToString(); + files.back().smallest_ikey = file->smallest.Encode().ToString(); + files.back().largest_ikey = file->largest.Encode().ToString(); + files.back().num_deletions = file->num_deletions; + files.back().job_id = file->job_id; + files.back().job_attempt = file->job_attempt; level_size += file->fd.GetFileSize(); } cf_meta->levels.emplace_back(level, level_size, std::move(files)); @@ -3054,6 +3265,8 @@ void VersionStorageInfo::GenerateLevelFilesBrief() { for (int level = 0; level < num_non_empty_levels_; level++) { DoGenerateLevelFilesBrief(&level_files_brief_[level], files_[level], &arena_); + if (InitUdfa) + InitUdfa(&level_files_brief_[level], user_comparator_); } } @@ -3395,6 +3608,46 @@ bool ShouldChangeFileTemperature(const ImmutableOptions& ioptions, } return false; } + + + +#ifndef __attribute_const__ +#define __attribute_const__ +#endif + +__attribute_const__ inline auto GetProps(const TableReader* rd) { + return rd->GetTableProperties().get(); +} +__attribute_const__ +inline uint64_t FileSizeForScore(const FileMetaData* f) { + auto fsize = f->fd.GetFileSize(); + #if !defined(ROCKSDB_UNIT_TEST) + if (auto rd = f->fd.table_reader) { + // 1. raw size is stable between compressed level and uncompressed level + // 2. We plan to mmap WAL log file and extract abstract interface for WAL + // and realize mmap WAL as BlobFile to be ref'ed by L0 sst, in this + // case, L0 FileSize maybe much smaller than raw kv size, so we need + // to use raw kv as FileSize + auto props = GetProps(rd); + return std::max(fsize, props->raw_key_size + props->raw_value_size); + } + #endif + return fsize; +} +__attribute_const__ +inline uint64_t CompensatedFileSizeForScore(const FileMetaData* f) { + #if !defined(ROCKSDB_UNIT_TEST) + if (auto rd = f->fd.table_reader) { + // raw size is stable between compressed level and uncompressed level + auto fsize = f->fd.GetFileSize(); + auto props = GetProps(rd); + auto bytes = std::max(fsize, props->raw_key_size + props->raw_value_size); + return uint64_t(f->compensated_file_size * double(bytes) / fsize); + } + #endif + return f->compensated_file_size; +} + } // anonymous namespace void VersionStorageInfo::ComputeCompactionScore( @@ -3428,9 +3681,9 @@ void VersionStorageInfo::ComputeCompactionScore( int num_sorted_runs = 0; uint64_t total_size = 0; for (auto* f : files_[level]) { - total_downcompact_bytes += static_cast(f->fd.GetFileSize()); + total_downcompact_bytes += static_cast(FileSizeForScore(f)); if (!f->being_compacted) { - total_size += f->compensated_file_size; + total_size += CompensatedFileSizeForScore(f); num_sorted_runs++; } } @@ -3505,7 +3758,7 @@ void VersionStorageInfo::ComputeCompactionScore( // over LBase -> LBase+1. uint64_t base_level_size = 0; for (auto f : files_[base_level_]) { - base_level_size += f->compensated_file_size; + base_level_size += CompensatedFileSizeForScore(f); } score = std::max(score, static_cast(total_size) / static_cast(std::max( @@ -3515,6 +3768,20 @@ void VersionStorageInfo::ComputeCompactionScore( if (score > 1.0) { score *= kScoreScale; } +#if !defined(ROCKSDB_UNIT_TEST) + } else if (total_size > + mutable_cf_options.write_buffer_size * num_sorted_runs / 2 && + mutable_cf_options.write_buffer_size >= + mutable_cf_options.max_bytes_for_level_base / 2) { + uint64_t base_level_bytes = 0; + for (auto f : files_[1]) { // base level is 1 + base_level_bytes += FileSizeForScore(f); + } + // do not consider level0_file_num_compaction_trigger + score = static_cast(total_size) / std::max + (base_level_bytes, mutable_cf_options.max_bytes_for_level_base); + //score = std::max(score, 1.01); // worst case protect +#endif // ROCKSDB_UNIT_TEST } else { score = std::max(score, static_cast(total_size) / @@ -3527,9 +3794,9 @@ void VersionStorageInfo::ComputeCompactionScore( uint64_t level_bytes_no_compacting = 0; uint64_t level_total_bytes = 0; for (auto f : files_[level]) { - level_total_bytes += f->fd.GetFileSize(); + level_total_bytes += FileSizeForScore(f); if (!f->being_compacted) { - level_bytes_no_compacting += f->compensated_file_size; + level_bytes_no_compacting += CompensatedFileSizeForScore(f); } } if (!immutable_options.level_compaction_dynamic_level_bytes) { @@ -3920,11 +4187,12 @@ void VersionStorageInfo::UpdateNumNonEmptyLevels() { namespace { // Sort `temp` based on ratio of overlapping size over file size -void SortFileByOverlappingRatio( +void SortFileByOverlapping(CompactionPri pri, const InternalKeyComparator& icmp, const std::vector& files, const std::vector& next_level_files, SystemClock* clock, int level, int num_non_empty_levels, uint64_t ttl, std::vector* temp) { + // exactly file_to_order should be file_to_score std::unordered_map file_to_order; auto next_level_it = next_level_files.begin(); @@ -3961,7 +4229,7 @@ void SortFileByOverlappingRatio( assert(ttl_boost_score > 0); assert(file->compensated_file_size != 0); file_to_order[file->fd.GetNumber()] = overlapping_bytes * 1024U / - file->compensated_file_size / + (pri == kMinOverlappingBytes ? 1 : file->compensated_file_size) / ttl_boost_score; } @@ -3975,16 +4243,33 @@ void SortFileByOverlappingRatio( // This makes the algorithm more deterministic, and also // help the trivial move case to have more files to // extend. - if (file_to_order[f1.file->fd.GetNumber()] == - file_to_order[f2.file->fd.GetNumber()]) { + auto score1 = file_to_order[f1.file->fd.GetNumber()]; + auto score2 = file_to_order[f2.file->fd.GetNumber()]; + if (score1 == score2) { return icmp.Compare(f1.file->smallest, f2.file->smallest) < 0; } - return file_to_order[f1.file->fd.GetNumber()] < - file_to_order[f2.file->fd.GetNumber()]; + return score1 < score2; }); } +void SortFileByOverlappingRatio( + const InternalKeyComparator& icmp, const std::vector& files, + const std::vector& next_level_files, SystemClock* clock, + int level, int num_non_empty_levels, uint64_t ttl, + std::vector* temp) { + SortFileByOverlapping(kMinOverlappingRatio, icmp, files, next_level_files, + clock, level, num_non_empty_levels, ttl, temp); +} +void SortFileByOverlappingBytes( + const InternalKeyComparator& icmp, const std::vector& files, + const std::vector& next_level_files, SystemClock* clock, + int level, int num_non_empty_levels, uint64_t ttl, + std::vector* temp) { + SortFileByOverlapping(kMinOverlappingBytes, icmp, files, next_level_files, + clock, level, num_non_empty_levels, ttl, temp); +} + void SortFileByRoundRobin(const InternalKeyComparator& icmp, std::vector* compact_cursor, bool level0_non_overlapping, int level, @@ -4095,6 +4380,11 @@ void VersionStorageInfo::UpdateFilesByCompactionPri( SortFileByRoundRobin(*internal_comparator_, &compact_cursor_, level0_non_overlapping_, level, &temp); break; + case kMinOverlappingBytes: + SortFileByOverlappingBytes(*internal_comparator_, files_[level], + files_[level + 1], ioptions.clock, level, + num_non_empty_levels_, options.ttl, &temp); + break; default: assert(false); } @@ -4499,6 +4789,18 @@ uint64_t VersionStorageInfo::NumLevelBytes(int level) const { return TotalFileSize(files_[level]); } +uint64_t VersionStorageInfo::NumLevelRawKV(int level) const { + assert(level >= 0); + assert(level < num_levels()); + return TotalFileRawKV(files_[level]); +} + +int VersionStorageInfo::FindFileInRange(int level, const Slice& key, + uint32_t left, uint32_t right) const { + return ROCKSDB_NAMESPACE::FindFileInRange(*internal_comparator_, + level_files_brief_[level], key, left, right); +} + const char* VersionStorageInfo::LevelSummary( LevelSummaryStorage* scratch) const { int len = 0; @@ -6643,9 +6945,32 @@ uint64_t VersionSet::ApproximateSize(const SizeApproximationOptions& options, const Slice& end, int start_level, int end_level, TableReaderCaller caller) { const auto& icmp = v->cfd_->internal_comparator(); - + if (icmp.IsForwardBytewise()) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + BytewiseCompareInternalKey cmp; + return ApproximateSizeTmpl(options, read_options, v, start, end, start_level, end_level, caller, cmp); + } + else if (icmp.IsReverseBytewise()) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + RevBytewiseCompareInternalKey cmp; + return ApproximateSizeTmpl(options, read_options, v, start, end, start_level, end_level, caller, cmp); + } + else { + FallbackVirtCmp cmp{&icmp}; + return ApproximateSizeTmpl(options, read_options, v, start, end, start_level, end_level, caller, cmp); + } +} + +template +uint64_t +VersionSet::ApproximateSizeTmpl(const SizeApproximationOptions& options, + const ReadOptions& read_options, + Version* v, const Slice& start, + const Slice& end, int start_level, + int end_level, TableReaderCaller caller, + InternalCmp cmp) { // pre-condition - assert(icmp.Compare(start, end) <= 0); + assert(!cmp(end, start)); uint64_t total_full_size = 0; const auto* vstorage = v->storage_info(); @@ -6697,16 +7022,16 @@ uint64_t VersionSet::ApproximateSize(const SizeApproximationOptions& options, // identify the file position for start key const int idx_start = - FindFileInRange(icmp, files_brief, start, 0, - static_cast(files_brief.num_files - 1)); + (int)FindFileInRangeTmpl(cmp, files_brief, start, 0, + static_cast(files_brief.num_files - 1)); assert(static_cast(idx_start) < files_brief.num_files); // identify the file position for end key int idx_end = idx_start; - if (icmp.Compare(files_brief.files[idx_end].largest_key, end) < 0) { + if (cmp(files_brief.files[idx_end].largest_key, end)) { idx_end = - FindFileInRange(icmp, files_brief, end, idx_start, - static_cast(files_brief.num_files - 1)); + (int)FindFileInRangeTmpl(cmp, files_brief, end, idx_start, + static_cast(files_brief.num_files - 1)); } assert(idx_end >= idx_start && static_cast(idx_end) < files_brief.num_files); @@ -6754,8 +7079,7 @@ uint64_t VersionSet::ApproximateSize(const SizeApproximationOptions& options, // Estimate for all the first files (might also be last files), at each // level for (const auto file_ptr : first_files) { - total_full_size += - ApproximateSize(read_options, v, *file_ptr, start, end, caller); + total_full_size += ApproximateSizeTmpl(read_options, v, *file_ptr, start, end, caller, cmp); } // Estimate for all the last files, at each level @@ -6777,12 +7101,34 @@ uint64_t VersionSet::ApproximateOffsetOf(const ReadOptions& read_options, // pre-condition assert(v); const auto& icmp = v->cfd_->internal_comparator(); + if (IsForwardBytewiseComparator(icmp.user_comparator())) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + BytewiseCompareInternalKey cmp; + return ApproximateOffsetOfTmpl(read_options, v, f, key, caller, cmp); + } + else if (IsReverseBytewiseComparator(icmp.user_comparator())) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + RevBytewiseCompareInternalKey cmp; + return ApproximateOffsetOfTmpl(read_options, v, f, key, caller, cmp); + } + else { + FallbackVirtCmp cmp{&icmp}; + return ApproximateOffsetOfTmpl(read_options, v, f, key, caller, cmp); + } +} +template +uint64_t VersionSet::ApproximateOffsetOfTmpl(const ReadOptions& read_options, + Version* v, + const FdWithKeyRange& f, + const Slice& key, + TableReaderCaller caller, + InternalCmp cmp) { uint64_t result = 0; - if (icmp.Compare(f.largest_key, key) <= 0) { + if (!cmp(key, f.largest_key)) { // Entire file is before "key", so just add the file size result = f.fd.GetFileSize(); - } else if (icmp.Compare(f.smallest_key, key) > 0) { + } else if (cmp(key, f.smallest_key)) { // Entire file is after "key", so ignore result = 0; } else { @@ -6791,6 +7137,7 @@ uint64_t VersionSet::ApproximateOffsetOf(const ReadOptions& read_options, TableCache* table_cache = v->cfd_->table_cache(); const MutableCFOptions& cf_opts = v->GetMutableCFOptions(); if (table_cache != nullptr) { + const auto& icmp = v->cfd_->internal_comparator(); result = table_cache->ApproximateOffsetOf( read_options, key, *f.file_metadata, caller, icmp, cf_opts.block_protection_bytes_per_key, cf_opts.prefix_extractor); @@ -6806,24 +7153,43 @@ uint64_t VersionSet::ApproximateSize(const ReadOptions& read_options, // pre-condition assert(v); const auto& icmp = v->cfd_->internal_comparator(); - assert(icmp.Compare(start, end) <= 0); + if (icmp.IsForwardBytewise()) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + BytewiseCompareInternalKey cmp; + return ApproximateSizeTmpl(read_options, v, f, start, end, caller, cmp); + } + else if (icmp.IsReverseBytewise()) { + ROCKSDB_ASSERT_EQ(icmp.user_comparator()->timestamp_size(), 0); + RevBytewiseCompareInternalKey cmp; + return ApproximateSizeTmpl(read_options, v, f, start, end, caller, cmp); + } + else { + FallbackVirtCmp cmp{&icmp}; + return ApproximateSizeTmpl(read_options, v, f, start, end, caller, cmp); + } +} - if (icmp.Compare(f.largest_key, start) <= 0 || - icmp.Compare(f.smallest_key, end) > 0) { +template +uint64_t VersionSet::ApproximateSizeTmpl(const ReadOptions& read_options, + Version* v, const FdWithKeyRange& f, + const Slice& start, const Slice& end, + TableReaderCaller caller, InternalCmp cmp) { + assert(!cmp(end, start)); + + if (!cmp(start, f.largest_key) || cmp(end, f.smallest_key)) { // Entire file is before or after the start/end keys range return 0; } - if (icmp.Compare(f.smallest_key, start) >= 0) { + if (!cmp(f.smallest_key, start)) { // Start of the range is before the file start - approximate by end offset - return ApproximateOffsetOf(read_options, v, f, end, caller); + return ApproximateOffsetOfTmpl(read_options, v, f, end, caller, cmp); } - if (icmp.Compare(f.largest_key, end) < 0) { + if (cmp(f.largest_key, end)) { // End of the range is after the file end - approximate by subtracting // start offset from the file size - uint64_t start_offset = - ApproximateOffsetOf(read_options, v, f, start, caller); + uint64_t start_offset = ApproximateOffsetOfTmpl(read_options, v, f, start, caller, cmp); assert(f.fd.GetFileSize() >= start_offset); return f.fd.GetFileSize() - start_offset; } @@ -6834,6 +7200,7 @@ uint64_t VersionSet::ApproximateSize(const ReadOptions& read_options, return 0; } const MutableCFOptions& cf_opts = v->GetMutableCFOptions(); + const auto& icmp = v->cfd_->internal_comparator(); return table_cache->ApproximateSize( read_options, start, end, *f.file_metadata, caller, icmp, cf_opts.block_protection_bytes_per_key, cf_opts.prefix_extractor); @@ -7186,9 +7553,7 @@ uint64_t VersionSet::GetTotalSstFilesSize(Version* dummy_versions) { VersionStorageInfo* storage_info = v->storage_info(); for (int level = 0; level < storage_info->num_levels_; level++) { for (const auto& file_meta : storage_info->LevelFiles(level)) { - if (unique_files.find(file_meta->fd.packed_number_and_path_id) == - unique_files.end()) { - unique_files.insert(file_meta->fd.packed_number_and_path_id); + if (unique_files.insert(file_meta->fd.packed_number_and_path_id).second) { total_files_size += file_meta->fd.GetFileSize(); } } @@ -7214,9 +7579,8 @@ uint64_t VersionSet::GetTotalBlobFileSize(Version* dummy_versions) { const uint64_t blob_file_number = meta->GetBlobFileNumber(); - if (unique_blob_files.find(blob_file_number) == unique_blob_files.end()) { + if (unique_blob_files.insert(blob_file_number).second) { // find Blob file that has not been counted - unique_blob_files.insert(blob_file_number); all_versions_blob_file_size += meta->GetBlobFileSize(); } } diff --git a/db/version_set.h b/db/version_set.h index 5ccb69771f..873b8fb35f 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -327,6 +327,9 @@ class VersionStorageInfo { // Return the combined file size of all files at the specified level. uint64_t NumLevelBytes(int level) const; + // Return the combined raw kv size of all files at the specified level. + uint64_t NumLevelRawKV(int level) const; + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) const std::vector& LevelFiles(int level) const { return files_[level]; @@ -343,6 +346,8 @@ class VersionStorageInfo { } void RecoverEpochNumbers(ColumnFamilyData* cfd); + int FindFileInRange(int level, const Slice& key, uint32_t left, uint32_t right) const; + class FileLocation { public: FileLocation() = default; @@ -616,7 +621,7 @@ class VersionStorageInfo { const Slice& largest_user_key, int last_level, int last_l0_idx); - private: + protected: void ComputeCompensatedSizes(); void UpdateNumNonEmptyLevels(); void CalculateBaseBytes(const ImmutableOptions& ioptions, @@ -999,6 +1004,8 @@ class Version { size_t GetMemoryUsageByTableReaders(const ReadOptions& read_options); + Env* env() const { return env_; } + ColumnFamilyData* cfd() const { return cfd_; } // Return the next Version in the linked list. @@ -1442,6 +1449,7 @@ class VersionSet { // The caller should delete the iterator when no longer needed. // @param read_options Must outlive the returned iterator. // @param start, end indicates compaction range + static InternalIterator* MakeInputIterator( const ReadOptions& read_options, const Compaction* c, RangeDelAggregator* range_del_agg, @@ -1467,6 +1475,12 @@ class VersionSet { const Slice& start, const Slice& end, int start_level, int end_level, TableReaderCaller caller); + template + uint64_t ApproximateSizeTmpl(const SizeApproximationOptions& options, + const ReadOptions& read_options, Version* v, + const Slice& start, const Slice& end, + int start_level, int end_level, + TableReaderCaller, InternalCmp); // Return the size of the current manifest file uint64_t manifest_file_size() const { return manifest_file_size_; } @@ -1576,12 +1590,23 @@ class VersionSet { const FdWithKeyRange& f, const Slice& key, TableReaderCaller caller); + template + uint64_t ApproximateOffsetOfTmpl(const ReadOptions& read_options, Version* v, + const FdWithKeyRange& f, const Slice& key, + TableReaderCaller, InternalCmp); + // Returns approximated data size between start and end keys in a file // for a given version. uint64_t ApproximateSize(const ReadOptions& read_options, Version* v, const FdWithKeyRange& f, const Slice& start, const Slice& end, TableReaderCaller caller); + template + uint64_t ApproximateSizeTmpl(const ReadOptions& read_options, Version* v, + const FdWithKeyRange& f, + const Slice& start, const Slice& end, + TableReaderCaller, InternalCmp); + struct MutableCFState { uint64_t log_number; std::string full_history_ts_low; diff --git a/db/version_set_test.cc b/db/version_set_test.cc index 5def229257..4703b1f44c 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -1037,6 +1037,8 @@ class FindLevelFileTest : public testing::Test { char* mem = arena_.AllocateAligned(num * sizeof(FdWithKeyRange)); file_level_.files = new (mem) FdWithKeyRange[num]; file_level_.num_files = 0; + file_level_.prefix_cache = + (uint64_t*)arena_.AllocateAligned(num * sizeof(uint64_t)); } void Add(const char* smallest, const char* largest, @@ -1060,6 +1062,7 @@ class FindLevelFileTest : public testing::Test { file.fd = FileDescriptor(num + 1, 0, 0); file.smallest_key = Slice(mem, smallest_slice.size()); file.largest_key = Slice(mem + smallest_slice.size(), largest_slice.size()); + file_level_.prefix_cache[num] = HostPrefixCache(largest_slice); file_level_.num_files++; } diff --git a/db/write_batch.cc b/db/write_batch.cc index 75f6e1eb48..47d983f163 100644 --- a/db/write_batch.cc +++ b/db/write_batch.cc @@ -71,6 +71,8 @@ #include "util/duplicate_detector.h" #include "util/string_util.h" +#include + namespace ROCKSDB_NAMESPACE { // anon namespace for file-local types @@ -245,6 +247,9 @@ void WriteBatch::Handler::LogData(const Slice& /*blob*/) { bool WriteBatch::Handler::Continue() { return true; } void WriteBatch::Clear() { + if (rep_.capacity() > 512*1024) { + std::string().swap(rep_); // free memory + } rep_.clear(); rep_.resize(WriteBatchInternal::kHeader); @@ -469,7 +474,8 @@ Status ReadRecordFromWriteBatch(Slice* input, char* tag, } break; default: - return Status::Corruption("unknown WriteBatch tag"); + return Status::Corruption("bad WriteBatch tag = " + + enum_stdstr(ValueType(*tag))); } return Status::OK(); } @@ -811,9 +817,7 @@ Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id, } PutLengthPrefixedSlice(&b->rep_, key); PutLengthPrefixedSlice(&b->rep_, value); - b->content_flags_.store( - b->content_flags_.load(std::memory_order_relaxed) | ContentFlags::HAS_PUT, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_PUT, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // Technically the optype could've been `kTypeColumnFamilyValue` with the // CF ID encoded in the `WriteBatch`. That distinction is unimportant @@ -905,9 +909,7 @@ Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id, } PutLengthPrefixedSliceParts(&b->rep_, key); PutLengthPrefixedSliceParts(&b->rep_, value); - b->content_flags_.store( - b->content_flags_.load(std::memory_order_relaxed) | ContentFlags::HAS_PUT, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_PUT, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the // `ValueType` argument passed to `ProtectKVO()`. @@ -976,9 +978,7 @@ Status WriteBatchInternal::PutEntity(WriteBatch* b, uint32_t column_family_id, PutLengthPrefixedSlice(&b->rep_, key); PutLengthPrefixedSlice(&b->rep_, entity); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_PUT_ENTITY, - std::memory_order_relaxed); + b->content_flags_.fetch_or(HAS_PUT_ENTITY, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { b->prot_info_->entries_.emplace_back( @@ -1058,14 +1058,16 @@ Status WriteBatchInternal::MarkEndPrepare(WriteBatch* b, const Slice& xid, : kTypeBeginPersistedPrepareXID)); b->rep_.push_back(static_cast(kTypeEndPrepareXID)); PutLengthPrefixedSlice(&b->rep_, xid); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_END_PREPARE | - ContentFlags::HAS_BEGIN_PREPARE, - std::memory_order_relaxed); if (unprepared_batch) { - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_BEGIN_UNPREPARE, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_END_PREPARE | + ContentFlags::HAS_BEGIN_PREPARE | + ContentFlags::HAS_BEGIN_UNPREPARE, + std::memory_order_relaxed); + } + else { + b->content_flags_.fetch_or(ContentFlags::HAS_END_PREPARE | + ContentFlags::HAS_BEGIN_PREPARE, + std::memory_order_relaxed); } return Status::OK(); } @@ -1073,9 +1075,8 @@ Status WriteBatchInternal::MarkEndPrepare(WriteBatch* b, const Slice& xid, Status WriteBatchInternal::MarkCommit(WriteBatch* b, const Slice& xid) { b->rep_.push_back(static_cast(kTypeCommitXID)); PutLengthPrefixedSlice(&b->rep_, xid); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_COMMIT, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_COMMIT, + std::memory_order_relaxed); return Status::OK(); } @@ -1086,8 +1087,7 @@ Status WriteBatchInternal::MarkCommitWithTimestamp(WriteBatch* b, b->rep_.push_back(static_cast(kTypeCommitXIDAndTimestamp)); PutLengthPrefixedSlice(&b->rep_, commit_ts); PutLengthPrefixedSlice(&b->rep_, xid); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_COMMIT, + b->content_flags_.fetch_or(ContentFlags::HAS_COMMIT, std::memory_order_relaxed); return Status::OK(); } @@ -1095,9 +1095,8 @@ Status WriteBatchInternal::MarkCommitWithTimestamp(WriteBatch* b, Status WriteBatchInternal::MarkRollback(WriteBatch* b, const Slice& xid) { b->rep_.push_back(static_cast(kTypeRollbackXID)); PutLengthPrefixedSlice(&b->rep_, xid); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_ROLLBACK, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_ROLLBACK, + std::memory_order_relaxed); return Status::OK(); } @@ -1112,9 +1111,8 @@ Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id, PutVarint32(&b->rep_, column_family_id); } PutLengthPrefixedSlice(&b->rep_, key); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_DELETE, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_DELETE, + std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the // `ValueType` argument passed to `ProtectKVO()`. @@ -1176,9 +1174,8 @@ Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id, PutVarint32(&b->rep_, column_family_id); } PutLengthPrefixedSliceParts(&b->rep_, key); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_DELETE, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_DELETE, + std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the // `ValueType` argument passed to `ProtectKVO()`. @@ -1226,9 +1223,8 @@ Status WriteBatchInternal::SingleDelete(WriteBatch* b, PutVarint32(&b->rep_, column_family_id); } PutLengthPrefixedSlice(&b->rep_, key); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_SINGLE_DELETE, - std::memory_order_relaxed); + b->content_flags_.fetch_or(ContentFlags::HAS_SINGLE_DELETE, + std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the // `ValueType` argument passed to `ProtectKVO()`. @@ -1292,8 +1288,7 @@ Status WriteBatchInternal::SingleDelete(WriteBatch* b, PutVarint32(&b->rep_, column_family_id); } PutLengthPrefixedSliceParts(&b->rep_, key); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_SINGLE_DELETE, + b->content_flags_.fetch_or(ContentFlags::HAS_SINGLE_DELETE, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the @@ -1344,8 +1339,7 @@ Status WriteBatchInternal::DeleteRange(WriteBatch* b, uint32_t column_family_id, } PutLengthPrefixedSlice(&b->rep_, begin_key); PutLengthPrefixedSlice(&b->rep_, end_key); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_DELETE_RANGE, + b->content_flags_.fetch_or(ContentFlags::HAS_DELETE_RANGE, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the @@ -1417,8 +1411,7 @@ Status WriteBatchInternal::DeleteRange(WriteBatch* b, uint32_t column_family_id, } PutLengthPrefixedSliceParts(&b->rep_, begin_key); PutLengthPrefixedSliceParts(&b->rep_, end_key); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_DELETE_RANGE, + b->content_flags_.fetch_or(ContentFlags::HAS_DELETE_RANGE, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the @@ -1474,8 +1467,7 @@ Status WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id, } PutLengthPrefixedSlice(&b->rep_, key); PutLengthPrefixedSlice(&b->rep_, value); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_MERGE, + b->content_flags_.fetch_or(ContentFlags::HAS_MERGE, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the @@ -1546,8 +1538,7 @@ Status WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id, } PutLengthPrefixedSliceParts(&b->rep_, key); PutLengthPrefixedSliceParts(&b->rep_, value); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_MERGE, + b->content_flags_.fetch_or(ContentFlags::HAS_MERGE, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the @@ -1594,8 +1585,7 @@ Status WriteBatchInternal::PutBlobIndex(WriteBatch* b, } PutLengthPrefixedSlice(&b->rep_, key); PutLengthPrefixedSlice(&b->rep_, value); - b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) | - ContentFlags::HAS_BLOB_INDEX, + b->content_flags_.fetch_or(ContentFlags::HAS_BLOB_INDEX, std::memory_order_relaxed); if (b->prot_info_ != nullptr) { // See comment in first `WriteBatchInternal::Put()` overload concerning the @@ -1774,13 +1764,10 @@ class MemTableInserter : public WriteBatch::Handler { ColumnFamilyMemTables* const cf_mems_; FlushScheduler* const flush_scheduler_; TrimHistoryScheduler* const trim_history_scheduler_; - const bool ignore_missing_column_families_; const uint64_t recovering_log_number_; // log number that all Memtables inserted into should reference uint64_t log_number_ref_; DBImpl* db_; - const bool concurrent_memtable_writes_; - bool post_info_created_; const WriteBatch::ProtectionInfo* prot_info_; size_t prot_info_idx_; @@ -1790,12 +1777,13 @@ class MemTableInserter : public WriteBatch::Handler { // cause memory allocations though unused. // Make creation optional but do not incur // std::unique_ptr additional allocation - using MemPostInfoMap = std::map; - using PostMapType = std::aligned_storage::type; - PostMapType mem_post_info_map_; + using MemPostInfoMap = terark::SmartMap; + MemPostInfoMap mem_post_info_map_; // current recovered transaction we are rebuilding (recovery) WriteBatch* rebuilding_trx_; SequenceNumber rebuilding_trx_seq_; + const bool ignore_missing_column_families_; + const bool concurrent_memtable_writes_; // Increase seq number once per each write batch. Otherwise increase it once // per key. bool seq_per_batch_; @@ -1805,32 +1793,24 @@ class MemTableInserter : public WriteBatch::Handler { bool write_before_prepare_; // Whether this batch was unprepared or not bool unprepared_batch_; - using DupDetector = std::aligned_storage::type; - DupDetector duplicate_detector_; bool dup_dectector_on_; bool hint_per_batch_; - bool hint_created_; + // Hints for this batch - using HintMap = std::unordered_map; - using HintMapType = std::aligned_storage::type; - HintMapType hint_; + using HintMap = terark::SmartMap; + HintMap hint_; + uint32_t curr_cf_id_ = UINT32_MAX; + + union { DuplicateDetector duplicate_detector_; }; HintMap& GetHintMap() { - assert(hint_per_batch_); - if (!hint_created_) { - new (&hint_) HintMap(); - hint_created_ = true; - } + assert(hint_per_batch_ || hint_.empty()); return *reinterpret_cast(&hint_); } MemPostInfoMap& GetPostMap() { - assert(concurrent_memtable_writes_); - if (!post_info_created_) { - new (&mem_post_info_map_) MemPostInfoMap(); - post_info_created_ = true; - } + assert(concurrent_memtable_writes_ || mem_post_info_map_.empty()); return *reinterpret_cast(&mem_post_info_map_); } @@ -1891,17 +1871,16 @@ class MemTableInserter : public WriteBatch::Handler { cf_mems_(cf_mems), flush_scheduler_(flush_scheduler), trim_history_scheduler_(trim_history_scheduler), - ignore_missing_column_families_(ignore_missing_column_families), recovering_log_number_(recovering_log_number), log_number_ref_(0), db_(static_cast_with_check(db)), - concurrent_memtable_writes_(concurrent_memtable_writes), - post_info_created_(false), prot_info_(prot_info), prot_info_idx_(0), has_valid_writes_(has_valid_writes), rebuilding_trx_(nullptr), rebuilding_trx_seq_(0), + ignore_missing_column_families_(ignore_missing_column_families), + concurrent_memtable_writes_(concurrent_memtable_writes), seq_per_batch_(seq_per_batch), // Write after commit currently uses one seq per key (instead of per // batch). So seq_per_batch being false indicates write_after_commit @@ -1911,10 +1890,8 @@ class MemTableInserter : public WriteBatch::Handler { // batch_per_txn being false indicates write_before_prepare. write_before_prepare_(!batch_per_txn), unprepared_batch_(false), - duplicate_detector_(), dup_dectector_on_(false), - hint_per_batch_(hint_per_batch), - hint_created_(false) { + hint_per_batch_(hint_per_batch) { assert(cf_mems_); } @@ -1923,15 +1900,11 @@ class MemTableInserter : public WriteBatch::Handler { reinterpret_cast(&duplicate_detector_) ->~DuplicateDetector(); } - if (post_info_created_) { - reinterpret_cast(&mem_post_info_map_)->~MemPostInfoMap(); - } - if (hint_created_) { - for (auto iter : GetHintMap()) { - delete[] reinterpret_cast(iter.second); - } - reinterpret_cast(&hint_)->~HintMap(); - } + GetHintMap().for_each([](auto& iter) { + // In base MemTableRep, FinishHint do delete [] (char*)(hint). + // In ToplingDB CSPP PatriciaTrie, FinishHint idle/release token. + iter.first->FinishHint(iter.second); + }); delete rebuilding_trx_; } @@ -1966,14 +1939,13 @@ class MemTableInserter : public WriteBatch::Handler { assert(concurrent_memtable_writes_); // If post info was not created there is nothing // to process and no need to create on demand - if (post_info_created_) { - for (auto& pair : GetPostMap()) { - pair.first->BatchPostProcess(pair.second); - } - } + GetPostMap().for_each([](auto& pair) { + pair.first->BatchPostProcess(pair.second); + }); } bool SeekToColumnFamily(uint32_t column_family_id, Status* s) { + if (UNLIKELY(curr_cf_id_ != column_family_id)) { // If we are in a concurrent mode, it is the caller's responsibility // to clone the original ColumnFamilyMemTables so that each thread // has its own instance. Otherwise, it must be guaranteed that there @@ -1986,8 +1958,11 @@ class MemTableInserter : public WriteBatch::Handler { *s = Status::InvalidArgument( "Invalid column family specified in write batch"); } + curr_cf_id_ = UINT32_MAX; // invalidate is required return false; } + curr_cf_id_ = column_family_id; + } if (recovering_log_number_ != 0 && recovering_log_number_ < cf_mems_->GetLogNumber()) { // This is true only in recovery environment (recovering_log_number_ is @@ -2047,11 +2022,14 @@ class MemTableInserter : public WriteBatch::Handler { // inplace_update_support is inconsistent with snapshots, and therefore with // any kind of transactions including the ones that use seq_per_batch assert(!seq_per_batch_ || !moptions->inplace_update_support); - if (!moptions->inplace_update_support) { - ret_status = + if (LIKELY(!moptions->inplace_update_support)) { + Status add_status = mem->Add(sequence_, value_type, key, value, kv_prot_info, concurrent_memtable_writes_, get_post_process_info(mem), hint_per_batch_ ? &GetHintMap()[mem] : nullptr); + if (UNLIKELY(!add_status.ok())) { + ret_status = add_status; + } } else if (moptions->inplace_callback == nullptr || value_type != kTypeValue) { assert(!concurrent_memtable_writes_); @@ -2931,11 +2909,12 @@ Status WriteBatchInternal::InsertInto( TrimHistoryScheduler* trim_history_scheduler, bool ignore_missing_column_families, uint64_t recovery_log_number, DB* db, bool concurrent_memtable_writes, bool seq_per_batch, bool batch_per_txn) { + bool hint = true; MemTableInserter inserter( sequence, memtables, flush_scheduler, trim_history_scheduler, ignore_missing_column_families, recovery_log_number, db, concurrent_memtable_writes, nullptr /* prot_info */, - nullptr /*has_valid_writes*/, seq_per_batch, batch_per_txn); + nullptr /*has_valid_writes*/, seq_per_batch, batch_per_txn, hint); for (auto w : write_group) { if (w->CallbackFailed()) { continue; @@ -3144,9 +3123,7 @@ Status WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src, SetCount(dst, Count(dst) + src_count); assert(src->rep_.size() >= WriteBatchInternal::kHeader); dst->rep_.append(src->rep_.data() + WriteBatchInternal::kHeader, src_len); - dst->content_flags_.store( - dst->content_flags_.load(std::memory_order_relaxed) | src_flags, - std::memory_order_relaxed); + dst->content_flags_.fetch_or(src_flags, std::memory_order_relaxed); return Status::OK(); } diff --git a/db/write_batch_test.cc b/db/write_batch_test.cc index 00faea4ce4..2cd02f3bc3 100644 --- a/db/write_batch_test.cc +++ b/db/write_batch_test.cc @@ -20,6 +20,7 @@ #include "rocksdb/env.h" #include "rocksdb/memtablerep.h" #include "rocksdb/utilities/write_batch_with_index.h" +#include "utilities/write_batch_with_index/write_batch_with_index_internal.h" #include "rocksdb/write_buffer_manager.h" #include "table/scoped_arena_iterator.h" #include "test_util/testharness.h" diff --git a/db/write_thread.cc b/db/write_thread.cc index 7987007752..8342642ee0 100644 --- a/db/write_thread.cc +++ b/db/write_thread.cc @@ -13,6 +13,22 @@ #include "port/port.h" #include "test_util/sync_point.h" #include "util/random.h" +#if defined(OS_LINUX) + #include + #include /* For SYS_xxx definitions */ + #include +//template +inline long //typename std::enable_if::type +futex(void* uaddr, uint32_t op, uint32_t val, const timespec* timeout = NULL, + void* uaddr2 = NULL, uint32_t val3 = 0) { + return syscall(SYS_futex, uaddr, (unsigned long)op, (unsigned long)val, + timeout, uaddr2, (unsigned long)val3); +} + #define TOPLINGDB_HAS_FUTEX 1 +#else + #define TOPLINGDB_HAS_FUTEX 0 + #define futex(...) +#endif namespace ROCKSDB_NAMESPACE { @@ -63,6 +79,22 @@ uint8_t WriteThread::BlockingAwaitState(Writer* w, uint8_t goal_mask) { uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask, AdaptationContext* ctx) { + if (TOPLINGDB_HAS_FUTEX && w->reduce_cpu_usage) { + uint32_t state = w->state.load(std::memory_order_acquire); + while (!(state & goal_mask)) { + if (w->state.compare_exchange_weak(state, STATE_LOCKED_WAITING, std::memory_order_acq_rel)) { + TEST_SYNC_POINT_CALLBACK("WriteThread::AwaitState:BlockingWaiting", w); + if (futex(&w->state, FUTEX_WAIT_PRIVATE, STATE_LOCKED_WAITING) < 0) { + int err = errno; + if (!(EINTR == err || EAGAIN == err)) + ROCKSDB_DIE("futex(WAIT) = %d: %s", err, strerror(err)); + } + state = w->state.load(std::memory_order_acquire); + } + } + return (uint8_t)state; + } + else { uint8_t state = 0; // 1. Busy loop using "pause" for 1 micro sec @@ -207,10 +239,21 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask, assert((state & goal_mask) != 0); return state; + } } void WriteThread::SetState(Writer* w, uint8_t new_state) { assert(w); + if (TOPLINGDB_HAS_FUTEX && w->reduce_cpu_usage) { + uint32_t state = w->state.load(std::memory_order_acquire); + while (state != new_state && +!w->state.compare_exchange_weak(state,new_state,std::memory_order_acq_rel)){ + // w->state may have been updated by other threads + } + if (STATE_LOCKED_WAITING == state) + futex(&w->state, FUTEX_WAKE_PRIVATE, INT_MAX); + } + else { auto state = w->state.load(std::memory_order_acquire); if (state == STATE_LOCKED_WAITING || !w->state.compare_exchange_strong(state, new_state)) { @@ -221,6 +264,7 @@ void WriteThread::SetState(Writer* w, uint8_t new_state) { w->state.store(new_state, std::memory_order_relaxed); w->StateCV().notify_one(); } + } } bool WriteThread::LinkOne(Writer* w, std::atomic* newest_writer) { @@ -415,9 +459,9 @@ void WriteThread::JoinBatchGroup(Writer* w) { /** * Wait util: * 1) An existing leader pick us as the new leader when it finishes - * 2) An existing leader pick us as its follewer and + * 2) An existing leader pick us as its follower and * 2.1) finishes the memtable writes on our behalf - * 2.2) Or tell us to finish the memtable writes in pralallel + * 2.2) Or tell us to finish the memtable writes in parallel * 3) (pipelined write) An existing leader pick us as its follower and * finish book-keeping and WAL write for us, enqueue us as pending * memtable writer, and @@ -631,8 +675,16 @@ static WriteThread::AdaptationContext cpmtw_ctx( bool WriteThread::CompleteParallelMemTableWriter(Writer* w) { auto* write_group = w->write_group; if (!w->status.ok()) { + if (TOPLINGDB_HAS_FUTEX && w->reduce_cpu_usage) { + static std::mutex mtx; + auto tmp = w->status; + std::lock_guard guard(mtx); + write_group->status = std::move(tmp); + } + else { std::lock_guard guard(write_group->leader->StateMutex()); write_group->status = w->status; + } } if (write_group->running-- > 1) { diff --git a/db/write_thread.h b/db/write_thread.h index 6e5805e376..bbb17792f6 100644 --- a/db/write_thread.h +++ b/db/write_thread.h @@ -118,6 +118,7 @@ class WriteThread { bool sync; bool no_slowdown; bool disable_wal; + bool reduce_cpu_usage; Env::IOPriority rate_limiter_priority; bool disable_memtable; size_t batch_cnt; // if non-zero, number of sub-batches in the write batch @@ -128,7 +129,7 @@ class WriteThread { uint64_t log_ref; // log number that memtable insert should reference WriteCallback* callback; bool made_waitable; // records lazy construction of mutex and cv - std::atomic state; // write under StateMutex() or pre-link + std::atomic state; // write under StateMutex() or pre-link WriteGroup* write_group; SequenceNumber sequence; // the sequence number to use for the first key Status status; @@ -144,6 +145,7 @@ class WriteThread { sync(false), no_slowdown(false), disable_wal(false), + reduce_cpu_usage(true), rate_limiter_priority(Env::IOPriority::IO_TOTAL), disable_memtable(false), batch_cnt(0), @@ -169,6 +171,7 @@ class WriteThread { sync(write_options.sync), no_slowdown(write_options.no_slowdown), disable_wal(write_options.disableWAL), + reduce_cpu_usage(write_options.reduce_cpu_usage), rate_limiter_priority(write_options.rate_limiter_priority), disable_memtable(_disable_memtable), batch_cnt(_batch_cnt), diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index adbc554ab0..e7ee57897d 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -110,6 +110,7 @@ DECLARE_int32(max_write_buffer_number_to_maintain); DECLARE_int64(max_write_buffer_size_to_maintain); DECLARE_bool(use_write_buffer_manager); DECLARE_double(memtable_prefix_bloom_size_ratio); +DECLARE_bool(allow_merge_memtables); DECLARE_bool(memtable_whole_key_filtering); DECLARE_int32(open_files); DECLARE_uint64(compressed_secondary_cache_size); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index c6ffbc93e5..b6c9b52a2e 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -193,6 +193,10 @@ DEFINE_double(memtable_prefix_bloom_size_ratio, "creates prefix blooms for memtables, each with size " "`write_buffer_size * memtable_prefix_bloom_size_ratio`."); +DEFINE_bool(allow_merge_memtables, + ROCKSDB_NAMESPACE::Options().allow_merge_memtables, + "allow merge memtables on flush."); + DEFINE_bool(memtable_whole_key_filtering, ROCKSDB_NAMESPACE::Options().memtable_whole_key_filtering, "Enable whole key filtering in memtables."); diff --git a/db_stress_tool/db_stress_shared_state.h b/db_stress_tool/db_stress_shared_state.h index bad6a77e1f..ff466da929 100644 --- a/db_stress_tool/db_stress_shared_state.h +++ b/db_stress_tool/db_stress_shared_state.h @@ -47,7 +47,7 @@ class SharedState { // local variable updated via sync points to keep track of errors injected // while reading filter blocks in order to ignore the Get/MultiGet result // for those calls - static thread_local bool ignore_read_error; + static thread_local bool ignore_read_error ROCKSDB_STATIC_TLS; SharedState(Env* /*env*/, StressTest* stress_test) : cv_(&mu_), diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index e79a711272..8744d556f3 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -27,6 +27,7 @@ #include "rocksdb/types.h" #include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/write_batch_with_index.h" +#include "utilities/write_batch_with_index/write_batch_with_index_internal.h" #include "test_util/testutil.h" #include "util/cast_util.h" #include "utilities/backup/backup_engine_impl.h" diff --git a/docs/_includes/footer.html b/docs/_includes/footer.html index f5b78babd3..6fd4ad858f 100644 --- a/docs/_includes/footer.html +++ b/docs/_includes/footer.html @@ -13,7 +13,7 @@

Meta Open Source
GitHub - Twitter + X Terms of Use Privacy Policy
diff --git a/docs/_includes/plugins/post_social_plugins.html b/docs/_includes/plugins/post_social_plugins.html index a2ecb90eeb..b13020d1b7 100644 --- a/docs/_includes/plugins/post_social_plugins.html +++ b/docs/_includes/plugins/post_social_plugins.html @@ -1,6 +1,6 @@
+