Fix folly build (#12795)

Summary:
- Updated pinned folly version to the latest
- gcc/g++ 10 is required since facebook/folly@2c1c617e9e so we had to modify the tests using gcc/g++ 7
- libsodium 1.0.17 is no longer downloadable from GitHub so I found it elsewhere. I will submit a PR for that upstream to folly
- USE_FOLLY_LITE changes
  - added boost header dependency instead of commenting out the `#include`s since that approach stopped working
  - added "folly/lang/Exception.cpp" to the compilation

Pull Request resolved: facebook/rocksdb#12795

Reviewed By: hx235

Differential Revision: D58916693

Pulled By: ajkr

fbshipit-source-id: b5f9bca2d929825846ac898b785972b071db62b1

(cherry picked from commit 40944cbbdbdcfac694fc3b291ba1838e943a789b)

.circleci/config.yml

@@ -473,23 +473,29 @@ jobs:
       - run: make V=1 -j8 -k check-headers # could be moved to a different build
       - post-steps
 
-  build-linux-gcc-7-with-folly:
+  build-linux-make-with-folly:
     executor: linux-docker
     resource_class: xlarge
+    environment:
+      CC: gcc-10
+      CXX: g++-10
     steps:
       - pre-steps
       - setup-folly
       - build-folly
-      - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures
+      - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures
       - post-steps
 
-  build-linux-gcc-7-with-folly-lite-no-test:
+  build-linux-make-with-folly-lite-no-test:
     executor: linux-docker
     resource_class: xlarge
+    environment:
+      CC: gcc-10
+      CXX: g++-10
     steps:
       - pre-steps
       - setup-folly
-      - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j8 all
+      - run: USE_FOLLY_LITE=1 V=1 make -j8 all
       - post-steps
 
   build-linux-gcc-8-no_test_run:
@@ -903,8 +909,8 @@ workflows:
       - build-linux
       - build-linux-cmake-with-folly
       - build-linux-cmake-with-folly-lite-no-test
-      - build-linux-gcc-7-with-folly
-      - build-linux-gcc-7-with-folly-lite-no-test
+      - build-linux-make-with-folly
+      - build-linux-make-with-folly-lite-no-test
       - build-linux-cmake-with-folly-coroutines
       - build-linux-cmake-with-benchmark
       - build-linux-encrypted_env-no_compression
@@ -979,7 +985,7 @@ workflows:
       - build-linux-arm-test-full
       - build-linux-run-microbench
       - build-linux-non-shm
-      - build-linux-clang-13-asan-ubsan-with-folly
+#      - build-linux-clang-13-asan-ubsan-with-folly
       - build-linux-valgrind
       - build-windows-vs2022-avx2
       - build-windows-vs2022

CMakeLists.txt

@@ -596,7 +596,7 @@ if(USE_FOLLY)
     FMT_INST_PATH)
     exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE
     GFLAGS_INST_PATH)
-    set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0)
+    set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.83.0)
     if(EXISTS ${FMT_INST_PATH}/lib64)
       set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt)
     else()
@@ -1026,13 +1026,20 @@ if(USE_FOLLY_LITE)
   list(APPEND SOURCES
     third-party/folly/folly/container/detail/F14Table.cpp
     third-party/folly/folly/detail/Futex.cpp
+    third-party/folly/folly/lang/Exception.cpp
     third-party/folly/folly/lang/SafeAssert.cpp
     third-party/folly/folly/lang/ToAscii.cpp
     third-party/folly/folly/ScopeGuard.cpp
     third-party/folly/folly/synchronization/AtomicNotification.cpp
     third-party/folly/folly/synchronization/DistributedMutex.cpp
     third-party/folly/folly/synchronization/ParkingLot.cpp)
   include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
+  exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS
+  build/fbcode_builder/getdeps.py show-source-dir boost OUTPUT_VARIABLE
+  BOOST_SOURCE_PATH)
+  exec_program(ls ARGS -d ${BOOST_SOURCE_PATH}/boost* OUTPUT_VARIABLE
+  BOOST_INCLUDE_DIR)
+  include_directories(${BOOST_INCLUDE_DIR})
   add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG)
   list(APPEND THIRDPARTY_LIBS glog)
 endif()

Makefile

@@ -105,6 +105,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
                   export LIB_MODE="$(LIB_MODE)"; \
 		  export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \
 		  export USE_FOLLY="$(USE_FOLLY)"; \
+		  export USE_FOLLY_LITE="$(USE_FOLLY_LITE)"; \
                   "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
 # this file is generated by the previous line to set build flags and sources
 include make_config.mk
@@ -502,6 +503,17 @@ endif
 ifeq ($(USE_FOLLY_LITE),1)
 	# Path to the Folly source code and include files
 	FOLLY_DIR = ./third-party/folly
+ifneq ($(strip $(BOOST_SOURCE_PATH)),)
+	BOOST_INCLUDE = $(shell (ls -d $(BOOST_SOURCE_PATH)/boost*/))
+	# AIX: pre-defined system headers are surrounded by an extern "C" block
+	ifeq ($(PLATFORM), OS_AIX)
+		PLATFORM_CCFLAGS += -I$(BOOST_INCLUDE)
+		PLATFORM_CXXFLAGS += -I$(BOOST_INCLUDE)
+	else
+		PLATFORM_CCFLAGS += -isystem $(BOOST_INCLUDE)
+		PLATFORM_CXXFLAGS += -isystem $(BOOST_INCLUDE)
+	endif
+endif  # BOOST_SOURCE_PATH
 	# AIX: pre-defined system headers are surrounded by an extern "C" block
 	ifeq ($(PLATFORM), OS_AIX)
 		PLATFORM_CCFLAGS += -I$(FOLLY_DIR)
@@ -2482,14 +2494,15 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba
-	@# A hack to remove boost dependency.
-	@# NOTE: this hack is only needed if building using USE_FOLLY_LITE
-	perl -pi -e 's/^(#include <boost)/\/\/$$1/' third-party/folly/folly/functional/Invoke.h
+	cd third-party/folly && git reset --hard 17054c85c2dee0e84cbee7ef28d7bae7d0b1630e
 	@# NOTE: this hack is required for clang in some cases
 	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
 	@# NOTE: this hack is required for gcc in some cases
 	perl -pi -e 's/(__has_include.<experimental.memory_resource>.)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h
+	@# NOTE: this hack is only needed if building target build_folly in the future
+	perl -pi -e 's,https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz,https://download.libsodium.org/libsodium/releases/old/unsupported/libsodium-1.0.17.tar.gz,' third-party/folly/build/fbcode_builder/manifests/libsodium
+	@# NOTE: boost source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on boost headers
+	cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py fetch boost
 
 CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS))
 
@@ -2501,8 +2514,6 @@ build_folly:
 		echo "Please run checkout_folly first"; \
 		false; \
 	fi
-	# Restore the original version of Invoke.h with boost dependency
-	cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h
 	cd third-party/folly && \
 		CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests
 

build_tools/build_detect_platform

@@ -603,7 +603,7 @@ EOF
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark"
         fi
     fi
-    if test $USE_FOLLY; then
+    if test $USE_FOLLY || test $USE_FOLLY_LITE; then
         # Test whether libfolly library is installed
         $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <folly/synchronization/DistributedMutex.h>
@@ -751,6 +751,11 @@ if [ "$USE_FOLLY" ]; then
     FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly`
   fi
 fi
+if [ "$USE_FOLLY_LITE" ]; then
+  if [ "$FOLLY_DIR" ]; then
+    BOOST_SOURCE_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-source-dir boost`
+  fi
+fi
 
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
@@ -792,6 +797,7 @@ echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
 echo "FIND=$FIND" >> "$OUTPUT"
 echo "WATCH=$WATCH" >> "$OUTPUT"
 echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT"
+echo "BOOST_SOURCE_PATH=$BOOST_SOURCE_PATH" >> "$OUTPUT"
 
 # This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then

src.mk

@@ -401,6 +401,7 @@ TEST_LIB_SOURCES =                                              \
 FOLLY_SOURCES =                                                 \
   $(FOLLY_DIR)/folly/container/detail/F14Table.cpp              \
   $(FOLLY_DIR)/folly/detail/Futex.cpp                           \
+  $(FOLLY_DIR)/folly/lang/Exception.cpp                         \
   $(FOLLY_DIR)/folly/lang/SafeAssert.cpp                        \
   $(FOLLY_DIR)/folly/lang/ToAscii.cpp                           \
   $(FOLLY_DIR)/folly/ScopeGuard.cpp                             \