From 204a4c1c54cb48d857eb9c6c9349decb5716d55e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Szab=C3=B3?= Date: Sat, 21 Dec 2024 04:33:53 +0100 Subject: [PATCH] Fix autotools test harness --- .github/workflows/build.yml | 10 ++++ mcrouter/.gitignore | 6 +++ mcrouter/configure.ac | 4 +- mcrouter/lib/Makefile.am | 24 ++++++++- mcrouter/lib/TestMain.cpp | 22 ++++++++ mcrouter/lib/carbon/Makefile.am | 8 ++- mcrouter/lib/carbon/example/gen/Makefile.am | 25 +++++++++ mcrouter/lib/fbi/cpp/test/Makefile.am | 4 +- mcrouter/lib/network/gen/Makefile.am | 16 ++++-- mcrouter/lib/network/test/Makefile.am | 44 +++++++++++++-- mcrouter/lib/network/test/gen/Makefile.am | 13 +++++ mcrouter/lib/test/Makefile.am | 15 +++++- mcrouter/routes/test/Makefile.am | 52 +++++++++++++++++- mcrouter/test/MCProcess.py | 11 ++-- mcrouter/test/Makefile.am | 60 +++++++++++++++++---- mcrouter/test/cpp_unit_tests/Makefile.am | 21 +++++++- mcrouter/test/mcrouter_config.py | 2 + 17 files changed, 305 insertions(+), 32 deletions(-) create mode 100644 mcrouter/lib/TestMain.cpp create mode 100644 mcrouter/lib/carbon/example/gen/Makefile.am create mode 100644 mcrouter/lib/network/test/gen/Makefile.am diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 837c51605..8c0542b41 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,3 +22,13 @@ jobs: run: | mkdir -p "$(pwd)"/mcrouter-install/install ./mcrouter/scripts/install_ubuntu_24.04.sh "$(pwd)"/mcrouter-install mcrouter + - name: Test mcrouter + run: | + cd mcrouter + make check -s -j$(nproc) || (cd test && make recheck -s j$(nproc)) + - name: Upload test logs + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: test-suite.log + path: mcrouter/test/test-suite.log diff --git a/mcrouter/.gitignore b/mcrouter/.gitignore index 749dde82d..e3fda6a22 100644 --- a/mcrouter/.gitignore +++ b/mcrouter/.gitignore @@ -21,10 +21,15 @@ Makefile.in /depcomp /install-sh /lib/carbon/gen-cpp2/ +/lib/carbon/example/gen/gen-cpp2/ /lib/network/CaretSerializedMessage.h /lib/network/McAsciiParser-gen.cpp /lib/network/gen-cpp2/* /lib/network/gen/gen-cpp2/ +/lib/network/test/gen/gen-cpp2/ +/lib/network/test/mock_mc_server +/lib/network/test/mock_mc_server_dual +/lib/network/test/mock_mc_thrift_server /libtool /m4/libtool.m4 /m4/ltoptions.m4 @@ -44,3 +49,4 @@ Makefile.in *.o *.a *.la +*.lo diff --git a/mcrouter/configure.ac b/mcrouter/configure.ac index a84c8b4a0..478c4efe6 100644 --- a/mcrouter/configure.ac +++ b/mcrouter/configure.ac @@ -198,7 +198,7 @@ AC_CHECK_FUNCS([gettimeofday \ LIBS="$LIBS $BOOST_LDFLAGS $BOOST_CONTEXT_LIB $BOOST_FILESYSTEM_LIB \ $BOOST_PROGRAM_OPTIONS_LIB $BOOST_SYSTEM_LIB $BOOST_REGEX_LIB \ - $BOOST_THREAD_LIB -lpthread -pthread -ldl -lunwind \ + $BOOST_THREAD_LIB -lfmt -lpthread -pthread -ldl -lunwind \ -lbz2 -llz4 -llzma -lsnappy -lzstd -latomic -lxxhash" AM_PATH_PYTHON([2.6],, [:]) @@ -212,6 +212,7 @@ AC_CONFIG_FILES([Makefile lib/Makefile lib/test/Makefile lib/carbon/Makefile + lib/carbon/example/gen/Makefile lib/config/Makefile lib/config/test/Makefile lib/fbi/Makefile @@ -220,6 +221,7 @@ AC_CONFIG_FILES([Makefile lib/fbi/cpp/test/Makefile lib/network/gen/Makefile lib/network/test/Makefile + lib/network/test/gen/Makefile routes/Makefile routes/test/Makefile test/Makefile diff --git a/mcrouter/lib/Makefile.am b/mcrouter/lib/Makefile.am index 01345f9c2..14431db6f 100644 --- a/mcrouter/lib/Makefile.am +++ b/mcrouter/lib/Makefile.am @@ -3,7 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -SUBDIRS = carbon network/gen . config fbi network/test test +SUBDIRS = carbon network/gen carbon/example/gen . config fbi network/test test noinst_LIBRARIES = libmcrouter.a @@ -21,9 +21,13 @@ libmcrouter_a_SOURCES = \ CompressionCodecManager.cpp \ CompressionCodecManager.h \ Crc32HashFunc.h \ + DynamicUtil.h \ + DynamicUtil.cpp \ FailoverErrorsSettingsBase.cpp \ FailoverErrorsSettingsBase.h \ FailoverErrorsSettings.h \ + FiberLocal.h \ + FiberLocalInternal.cpp \ HashUtil.h \ IOBufUtil.cpp \ IOBufUtil.h \ @@ -87,16 +91,26 @@ libmcrouter_a_SOURCES = \ carbon/TypeList.h \ carbon/Util.h \ carbon/Variant.h \ + carbon/connection/CarbonConnectionUtil.h \ + carbon/connection/ExternalCarbonConnectionImpl.cpp \ + carbon/connection/ExternalCarbonConnectionImpl.h \ + carbon/connection/ExternalCarbonConnectionImpl-inl.h \ + carbon/connection/InternalCarbonConnectionImpl.h \ + carbon/connection/PooledCarbonConnectionImpl.h \ carbon/gen-cpp2/carbon_result_types.cpp \ carbon/gen-cpp2/carbon_result_types.h \ carbon/gen-cpp2/carbon_result_types.tcc \ carbon/gen-cpp2/carbon_result_data.cpp \ carbon/gen-cpp2/carbon_result_data.h \ + carbon/gen-cpp2/carbon_result_metadata.cpp \ + carbon/gen-cpp2/carbon_result_metadata.h \ carbon/gen-cpp2/carbon_types.cpp \ carbon/gen-cpp2/carbon_types.h \ carbon/gen-cpp2/carbon_types.tcc \ carbon/gen-cpp2/carbon_data.cpp \ carbon/gen-cpp2/carbon_data.h \ + carbon/gen-cpp2/carbon_metadata.cpp \ + carbon/gen-cpp2/carbon_metadata.h \ config/ConfigPreprocessor.cpp \ config/ConfigPreprocessor.h \ config/ImportResolverIf.h \ @@ -191,6 +205,10 @@ libmcrouter_a_SOURCES = \ network/gen/gen-cpp2/Common_types.tcc \ network/gen/gen-cpp2/Common_data.cpp \ network/gen/gen-cpp2/Common_data.h \ + network/gen/gen-cpp2/Common_metadata.cpp \ + network/gen/gen-cpp2/Common_metadata.h \ + network/gen/gen-cpp2/Memcache.cpp \ + network/gen/gen-cpp2/Memcache.h \ network/gen/gen-cpp2/MemcacheAsyncClient.cpp \ network/gen/gen-cpp2/MemcacheAsyncClient.h \ network/gen/gen-cpp2/MemcacheService_constants.cpp \ @@ -198,6 +216,8 @@ libmcrouter_a_SOURCES = \ network/gen/gen-cpp2/MemcacheService_types.cpp \ network/gen/gen-cpp2/MemcacheService_types.h \ network/gen/gen-cpp2/MemcacheService_types.tcc \ + network/gen/gen-cpp2/MemcacheService_metadata.cpp \ + network/gen/gen-cpp2/MemcacheService_metadata.h \ network/gen/gen-cpp2/MemcacheService_data.cpp \ network/gen/gen-cpp2/MemcacheService_data.h \ network/gen/gen-cpp2/Memcache_types.cpp \ @@ -205,6 +225,8 @@ libmcrouter_a_SOURCES = \ network/gen/gen-cpp2/Memcache_types.tcc \ network/gen/gen-cpp2/Memcache_data.cpp \ network/gen/gen-cpp2/Memcache_data.h \ + network/gen/gen-cpp2/Memcache_metadata.cpp \ + network/gen/gen-cpp2/Memcache_metadata.h \ network/FizzContextProvider.cpp \ network/FizzContextProvider.h \ network/McAsciiParser-gen.cpp \ diff --git a/mcrouter/lib/TestMain.cpp b/mcrouter/lib/TestMain.cpp new file mode 100644 index 000000000..b5ae12ba5 --- /dev/null +++ b/mcrouter/lib/TestMain.cpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +#include + +/** + * Main entry point shared by several test suites + * in the mcrouter OSS build. + */ +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + folly::Init init( + &argc, &argv, folly::InitOptions{}.removeFlags(false).useGFlags(false)); + + return RUN_ALL_TESTS(); +} diff --git a/mcrouter/lib/carbon/Makefile.am b/mcrouter/lib/carbon/Makefile.am index deba6c205..21b19fa8a 100644 --- a/mcrouter/lib/carbon/Makefile.am +++ b/mcrouter/lib/carbon/Makefile.am @@ -17,14 +17,18 @@ BUILT_SOURCES = \ gen-cpp2/carbon_result_types.cpp: gen-cpp2/carbon_result_types.h gen-cpp2/carbon_result_types.h: gen-cpp2/carbon_result_types.tcc -gen-cpp2/carbon_result_types.tcc: gen-cpp2/carbon_result_data.cpp +gen-cpp2/carbon_result_types.tcc: gen-cpp2/carbon_result_metadata.cpp +gen-cpp2/carbon_result_metadata.cpp: gen-cpp2/carbon_result_metadata.h +gen-cpp2/carbon_result_metadata.h: gen-cpp2/carbon_result_data.cpp gen-cpp2/carbon_result_data.cpp: gen-cpp2/carbon_result_data.h gen-cpp2/carbon_result_data.h: carbon_result.thrift @FBTHRIFT@ -gen mstch_cpp2:stack_arguments,sync_methods_return_try,include_prefix=mcrouter/lib/carbon/ carbon_result.thrift gen-cpp2/carbon_types.cpp: gen-cpp2/carbon_types.h gen-cpp2/carbon_types.h: gen-cpp2/carbon_types.tcc -gen-cpp2/carbon_types.tcc: gen-cpp2/carbon_data.cpp +gen-cpp2/carbon_types.tcc: gen-cpp2/carbon_metadata.cpp +gen-cpp2/carbon_metadata.cpp: gen-cpp2/carbon_metadata.h +gen-cpp2/carbon_metadata.h: gen-cpp2/carbon_data.cpp gen-cpp2/carbon_data.cpp: gen-cpp2/carbon_data.h gen-cpp2/carbon_data.h: carbon.thrift @FBTHRIFT@ -gen mstch_cpp2:stack_arguments,sync_methods_return_try,include_prefix=mcrouter/lib/carbon/ -I $(INSTALL_DIR)/include/ carbon.thrift diff --git a/mcrouter/lib/carbon/example/gen/Makefile.am b/mcrouter/lib/carbon/example/gen/Makefile.am new file mode 100644 index 000000000..a28adbb13 --- /dev/null +++ b/mcrouter/lib/carbon/example/gen/Makefile.am @@ -0,0 +1,25 @@ +noinst_LIBRARIES = libcarbon_hello_goodbye.a + +thrift_sources = \ + gen-cpp2/HelloGoodbye_types.cpp \ + gen-cpp2/HelloGoodbye_types.h \ + gen-cpp2/HelloGoodbye_types.tcc \ + gen-cpp2/HelloGoodbye_data.cpp \ + gen-cpp2/HelloGoodbye_data.h + +BUILT_SOURCES = $(thrift_sources) + +gen-cpp2/HelloGoodbye_types.cpp: gen-cpp2/HelloGoodbye_types.h $(top_srcdir)/lib/network/gen/gen-cpp2/Common_types.h +gen-cpp2/HelloGoodbye_types.h: gen-cpp2/HelloGoodbye_types.tcc +gen-cpp2/HelloGoodbye_types.tcc: gen-cpp2/HelloGoodbye_data.cpp +gen-cpp2/HelloGoodbye_data.cpp: gen-cpp2/HelloGoodbye_data.h +gen-cpp2/HelloGoodbye_data.h: HelloGoodbye.thrift + @FBTHRIFT@ -gen mstch_cpp2:stack_arguments,sync_methods_return_try,deprecated_terse_writes,include_prefix=mcrouter/lib/carbon/example/gen/ -I $(top_srcdir)/.. -I $(INSTALL_DIR)/include/ HelloGoodbye.thrift + +libcarbon_hello_goodbye_a_SOURCES = \ + $(thrift_sources) \ + HelloGoodbyeMessages.h \ + HelloGoodbyeMessages-inl.h \ + HelloGoodbyeMessagesThrift.cpp + +libcarbon_hello_goodbye_a_CPPFLAGS = -I$(top_srcdir)/.. diff --git a/mcrouter/lib/fbi/cpp/test/Makefile.am b/mcrouter/lib/fbi/cpp/test/Makefile.am index b12cc25f5..03b38f954 100644 --- a/mcrouter/lib/fbi/cpp/test/Makefile.am +++ b/mcrouter/lib/fbi/cpp/test/Makefile.am @@ -6,7 +6,9 @@ check_PROGRAMS = mcrouter_fbi_cpp_test mcrouter_fbi_cpp_test_SOURCES = \ - main.cpp + main.cpp \ + LowerBoundPrefixMapTest.cpp \ + ObjectPoolTests.cpp mcrouter_fbi_cpp_test_CPPFLAGS = \ -I$(top_srcdir)/.. \ diff --git a/mcrouter/lib/network/gen/Makefile.am b/mcrouter/lib/network/gen/Makefile.am index b456c64a4..d20ee3f53 100644 --- a/mcrouter/lib/network/gen/Makefile.am +++ b/mcrouter/lib/network/gen/Makefile.am @@ -14,6 +14,8 @@ BUILT_SOURCES = \ gen-cpp2/Memcache_types.tcc \ gen-cpp2/Memcache_data.cpp \ gen-cpp2/Memcache_data.h \ + gen-cpp2/Memcache.cpp \ + gen-cpp2/Memcache.h \ gen-cpp2/MemcacheAsyncClient.cpp \ gen-cpp2/MemcacheAsyncClient.h \ gen-cpp2/MemcacheService_constants.cpp \ @@ -26,25 +28,33 @@ BUILT_SOURCES = \ gen-cpp2/Common_types.cpp: gen-cpp2/Common_types.h gen-cpp2/Common_types.h: gen-cpp2/Common_types.tcc -gen-cpp2/Common_types.tcc: gen-cpp2/Common_data.cpp +gen-cpp2/Common_types.tcc: gen-cpp2/Common_metadata.cpp +gen-cpp2/Common_metadata.cpp: gen-cpp2/Common_metadata.h +gen-cpp2/Common_metadata.h: gen-cpp2/Common_data.cpp gen-cpp2/Common_data.cpp: gen-cpp2/Common_data.h gen-cpp2/Common_data.h: Common.thrift @FBTHRIFT@ -gen mstch_cpp2:stack_arguments,sync_methods_return_try,deprecated_terse_writes,include_prefix=mcrouter/lib/network/gen -I $(top_srcdir)/.. -I $(INSTALL_DIR)/include/ Common.thrift gen-cpp2/Memcache_types.cpp: gen-cpp2/Memcache_types.h gen-cpp2/Memcache_types.h: gen-cpp2/Memcache_types.tcc -gen-cpp2/Memcache_types.tcc: gen-cpp2/Memcache_data.cpp +gen-cpp2/Memcache_types.tcc: gen-cpp2/Memcache_metadata.cpp +gen-cpp2/Memcache_metadata.cpp: gen-cpp2/Memcache_metadata.h +gen-cpp2/Memcache_metadata.h: gen-cpp2/Memcache_data.cpp gen-cpp2/Memcache_data.cpp: gen-cpp2/Memcache_data.h gen-cpp2/Memcache_data.h: Memcache.thrift @FBTHRIFT@ -gen mstch_cpp2:stack_arguments,sync_methods_return_try,deprecated_terse_writes,include_prefix=mcrouter/lib/network/gen -I $(top_srcdir)/.. -I $(INSTALL_DIR)/include/ Memcache.thrift +gen-cpp2/Memcache.cpp: gen-cpp2/Memcache.h +gen-cpp2/Memcache.h: gen-cpp2/MemcacheAsyncClient.cpp gen-cpp2/MemcacheAsyncClient.cpp: gen-cpp2/MemcacheAsyncClient.h gen-cpp2/MemcacheAsyncClient.h: gen-cpp2/MemcacheService_constants.cpp gen-cpp2/MemcacheService_constants.cpp: gen-cpp2/MemcacheService_constants.h gen-cpp2/MemcacheService_constants.h: gen-cpp2/MemcacheService_types.cpp gen-cpp2/MemcacheService_types.cpp: gen-cpp2/MemcacheService_types.h gen-cpp2/MemcacheService_types.h: gen-cpp2/MemcacheService_types.tcc -gen-cpp2/MemcacheService_types.tcc: gen-cpp2/MemcacheService_data.cpp +gen-cpp2/MemcacheService_types.tcc: gen-cpp2/MemcacheService_metadata.cpp +gen-cpp2/MemcacheService_metadata.cpp: gen-cpp2/MemcacheService_metadata.h +gen-cpp2/MemcacheService_metadata.h: gen-cpp2/MemcacheService_data.cpp gen-cpp2/MemcacheService_data.cpp: gen-cpp2/MemcacheService_data.h gen-cpp2/MemcacheService_data.h: MemcacheService.thrift @FBTHRIFT@ -gen mstch_cpp2:stack_arguments,sync_methods_return_try,deprecated_terse_writes,include_prefix=mcrouter/lib/network/gen -I $(top_srcdir)/.. -I $(INSTALL_DIR)/include/ MemcacheService.thrift diff --git a/mcrouter/lib/network/test/Makefile.am b/mcrouter/lib/network/test/Makefile.am index 8f9106c22..163573fe7 100644 --- a/mcrouter/lib/network/test/Makefile.am +++ b/mcrouter/lib/network/test/Makefile.am @@ -3,7 +3,9 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -noinst_PROGRAMS = mock_mc_server +SUBDIRS = gen + +noinst_PROGRAMS = mock_mc_server mock_mc_thrift_server mock_mc_server_dual noinst_LIBRARIES = libtest_util.a check_PROGRAMS = mcrouter_network_test @@ -44,11 +46,17 @@ mock_mc_thrift_server_CPPFLAGS = -I$(top_srcdir)/.. mock_mc_thrift_server_LDADD = \ $(top_builddir)/lib/libmcrouter.a \ -lthriftcpp2 \ + -lserverdbginfo \ -ltransport \ + -lthriftanyrep \ + -lthrifttype \ + -lthrifttyperep \ -lthriftprotocol \ -lrpcmetadata \ + -lthriftmetadata \ -lasync \ -lconcurrency \ + -lruntime \ -lthrift-core \ -lfmt \ -lfizz \ @@ -65,11 +73,17 @@ mock_mc_server_dual_CPPFLAGS = -I$(top_srcdir)/.. mock_mc_server_dual_LDADD = \ $(top_builddir)/lib/libmcrouter.a \ -lthriftcpp2 \ + -lserverdbginfo \ -ltransport \ + -lthriftanyrep \ + -lthrifttype \ + -lthrifttyperep \ -lthriftprotocol \ -lrpcmetadata \ + -lthriftmetadata \ -lasync \ -lconcurrency \ + -lruntime \ -lthrift-core \ -lfmt \ -lfizz \ @@ -86,15 +100,25 @@ libtest_util_a_CPPFLAGS = -I$(top_srcdir)/.. mcrouter_network_test_SOURCES = \ AccessPointTest.cpp \ AsyncMcClientTestSync.cpp \ + AsyncMcServerShutdownTest.cpp \ + AsyncMcServerTest.cpp \ CarbonMessageDispatcherTest.cpp \ CarbonMockMcTest.cpp \ CarbonQueueAppenderTest.cpp \ + gen/gen-cpp2/CarbonTest_types.cpp \ + gen/gen-cpp2/CarbonTest_types.h \ + gen/gen-cpp2/CarbonTest_types.tcc \ + gen/gen-cpp2/CarbonTest_data.cpp \ + gen/gen-cpp2/CarbonTest_data.h \ gen/CarbonTestMessages.cpp \ + gen/CarbonTestMessagesThrift.cpp \ McAsciiParserTest.cpp \ McParserTest.cpp \ McServerAsciiParserTest.cpp \ + MemcacheConnectionTest.cpp \ + MemcacheTest.cpp \ + RequestExpiryTest.cpp \ MockMc.cpp \ - MockMcServer.cpp \ SessionTest.cpp \ SessionTestHarness.cpp \ SessionTestHarness.h \ @@ -108,13 +132,25 @@ mcrouter_network_test_CPPFLAGS = \ -isystem $(top_srcdir)/lib/gtest/include mcrouter_network_test_LDADD = \ + $(top_builddir)/libmcroutercore.a \ $(top_builddir)/lib/libmcrouter.a \ + $(top_builddir)/lib/carbon/example/gen/libcarbon_hello_goodbye.a \ $(top_builddir)/lib/libtestmain.la \ - $(top_builddir)/lib/network/libtest_util.a \ + $(top_builddir)/lib/network/test/libtest_util.a \ -lthriftcpp2 \ + -lserverdbginfo \ + -ltransport \ + -lthriftanyrep \ + -lthrifttype \ + -lthrifttyperep \ -lthriftprotocol \ -lrpcmetadata \ - -ltransport \ + -lthriftannotation \ + -lthriftmetadata \ + -lasync \ + -lconcurrency \ + -lruntime \ + -lthrift-core \ -lwangle \ -lfizz \ -lsodium \ diff --git a/mcrouter/lib/network/test/gen/Makefile.am b/mcrouter/lib/network/test/gen/Makefile.am new file mode 100644 index 000000000..53eb5e25e --- /dev/null +++ b/mcrouter/lib/network/test/gen/Makefile.am @@ -0,0 +1,13 @@ +BUILT_SOURCES = \ + gen-cpp2/CarbonTest_types.cpp \ + gen-cpp2/CarbonTest_types.h \ + gen-cpp2/CarbonTest_types.tcc \ + gen-cpp2/CarbonTest_data.cpp \ + gen-cpp2/CarbonTest_data.h + +gen-cpp2/CarbonTest_types.cpp: gen-cpp2/CarbonTest_types.h +gen-cpp2/CarbonTest_types.h: gen-cpp2/CarbonTest_types.tcc +gen-cpp2/CarbonTest_types.tcc: gen-cpp2/CarbonTest_data.cpp +gen-cpp2/CarbonTest_data.cpp: gen-cpp2/CarbonTest_data.h +gen-cpp2/CarbonTest_data.h: CarbonTest.thrift + @FBTHRIFT@ -gen mstch_cpp2:stack_arguments,sync_methods_return_try,deprecated_terse_writes,include_prefix=mcrouter/lib/network/test/gen/ -I $(top_srcdir)/.. -I $(INSTALL_DIR)/include/ CarbonTest.thrift diff --git a/mcrouter/lib/test/Makefile.am b/mcrouter/lib/test/Makefile.am index 131ac56b5..e86646862 100644 --- a/mcrouter/lib/test/Makefile.am +++ b/mcrouter/lib/test/Makefile.am @@ -6,21 +6,30 @@ check_PROGRAMS = mcrouter_lib_test mcrouter_lib_test_SOURCES = \ + AllSyncCollectionRoute.h \ Ch3HashTest.cpp \ + CompressionCodecManagerTest.cpp \ CompressionTest.cpp \ CompressionTestUtil.cpp \ CompressionTestUtil.h \ Crc32HashTest.cpp \ + DynamicUtilTest.cpp \ + FiberLocalTest.cpp \ HashTestUtil.cpp \ HashTestUtil.h \ + IovecCursorTest.cpp \ + Lz4ImmutableTest.cpp \ Main.cpp \ + McResUtilTest.cpp \ MigrateRouteTest.cpp \ RandomRouteTest.cpp \ RendezvousHashTest.cpp \ RouteHandleTest.cpp \ - WeightedChHashFuncBaseTest.cpp \ + RouteHandleTestUtil.h \ + TestRouteHandle.h \ WeightedCh3HashFuncTest.cpp \ WeightedCh4HashFuncTest.cpp \ + WeightedChHashFuncBaseTest.cpp \ WeightedRendezvousHashTest.cpp mcrouter_lib_test_CPPFLAGS = \ @@ -29,4 +38,6 @@ mcrouter_lib_test_CPPFLAGS = \ mcrouter_lib_test_LDADD = \ $(top_builddir)/lib/libmcrouter.a \ - $(top_builddir)/lib/libtestmain.la + $(top_builddir)/lib/libtestmain.la \ + -lthriftcpp2 \ + -lthriftprotocol diff --git a/mcrouter/routes/test/Makefile.am b/mcrouter/routes/test/Makefile.am index c1d31ba86..5d40c1cae 100644 --- a/mcrouter/routes/test/Makefile.am +++ b/mcrouter/routes/test/Makefile.am @@ -6,18 +6,52 @@ check_PROGRAMS = mcrouter_routes_test mcrouter_routes_test_SOURCES = \ - BigValueRouteTest.cpp \ + AllSyncCollectionRouteFactory.h \ BigValueRouteTestBase.h \ + BigValueRouteTest.cpp \ ConstShardHashFuncTest.cpp \ + DistributionRouteTest.cpp \ + FailoverRouteTest.cpp \ FailoverWithExptimeRouteTest.cpp \ + KeyParseRouteTest.cpp \ + LatestRouteTest.cpp \ + LoadBalancerRouteTest.cpp \ + LoggingRouteTest.cpp \ Main.cpp \ - PoolRouteTest.cpp \ + McBucketRouteTest.cpp \ + McRefillRouteTest.cpp \ + MissFailoverRouteTest.cpp \ + OriginalClientHashRouteTest.cpp \ + OutstandingLimitRouteTest.cpp \ + PrefixSelectorRouteTest.cpp \ RateLimitRouteTest.cpp \ + RootRouteTest.cpp \ + RouteHandleTestBase.h \ RouteHandleTestUtil.h \ + RoutePolicyMapTest.cpp \ ShadowRouteTest.cpp \ + ShadowSettingsTest.cpp \ + ShardSplitRouteTest.cpp \ + ShardSplitRouteTestUtil.h \ + ShardSplitterTest.cpp \ SlowWarmUpRouteTest.cpp \ WarmUpRouteTest.cpp +# The following tests depend on HelloGoodbyeService +# and thus need to be disabled until mcrouter OSS depends on fb303. +# +# HashStopAllowListRouteTest.cpp +# KeySplitRouteTest.cpp +# ErrorRouteTest.cpp +# EagerShardSelectionRouteTest.cpp +# EagerShardSelectionShadowRouteTest.cpp +# CollectionRouteFactoryTest.cpp +# ClientCompatibilityCheckerRouteTest.cpp +# LatencyInjectionRouteTest.cpp +# BlackholeRouteTest.cpp +# ShardSelectionRouteTest.cpp +# StagingRouteTest.cpp + mcrouter_routes_test_CPPFLAGS = \ -I$(top_srcdir)/.. \ -isystem $(top_srcdir)/lib/gtest/include @@ -26,6 +60,20 @@ mcrouter_routes_test_LDADD = \ $(top_builddir)/libmcroutercore.a \ $(top_builddir)/lib/libmcrouter.a \ $(top_builddir)/lib/libtestmain.la \ + -lthriftcpp2 \ + -lserverdbginfo \ + -ltransport \ + -lthriftanyrep \ + -lthrifttype \ + -lthrifttyperep \ + -lthriftprotocol \ + -lrpcmetadata \ + -lthriftannotation \ + -lthriftmetadata \ + -lasync \ + -lconcurrency \ + -lruntime \ + -lthrift-core \ -lwangle \ -lfizz \ -lsodium \ diff --git a/mcrouter/test/MCProcess.py b/mcrouter/test/MCProcess.py index c9a2d909c..55f924a5d 100644 --- a/mcrouter/test/MCProcess.py +++ b/mcrouter/test/MCProcess.py @@ -20,7 +20,6 @@ import tempfile import time -from carbon.carbon_result.thrift_types import Result from mcrouter.test.config import McrouterGlobals @@ -257,7 +256,7 @@ def ensure_connected(self): while True: try: res = self.thrift_client.mcVersion() - if res == Result.OK: + if res == carbon.carbon_result.thrift_types.Result.OK: return except Exception as e: print(f"Error on sending mcVersion in Thrift: {e}") @@ -1029,7 +1028,8 @@ def __init__(self, port=None, ssl_port=None, extra_args=None): pass_fds = [] # if mockmc is used here, we initialize the same way as MockMemcached - if McrouterGlobals.binPath("mockmc") == args[0]: + self.is_mock_server = McrouterGlobals.binPath("mockmc") == args[0] + if self.is_mock_server: if port is None: listen_sock = create_listen_socket() port = listen_sock.getsockname()[1] @@ -1102,6 +1102,11 @@ def __init__(self, port=None, ssl_port=None, extra_args=None): tries -= 1 self.disconnect() + def stats(self, spec=None): + if self.is_mock_server: + return super().stats('__mockmc__') + return super().stats(spec) + def getsslport(self): return self.ssl_port diff --git a/mcrouter/test/Makefile.am b/mcrouter/test/Makefile.am index 4e05073d0..4483b8c57 100644 --- a/mcrouter/test/Makefile.am +++ b/mcrouter/test/Makefile.am @@ -11,31 +11,51 @@ TESTS = \ ../lib/fbi/test/mcrouter_fbi_test \ ../lib/test/mcrouter_lib_test \ ../routes/test/mcrouter_routes_test \ - cpp_unit_tests/mcrouter_test + cpp_unit_tests/mcrouter_test \ + ../lib/network/test/mcrouter_network_test -# TODO: fix the test -# ../lib/network/test/mcrouter_network_test +XFAIL_TESTS = \ + ../lib/test/mcrouter_lib_test \ + ../lib/network/test/mcrouter_network_test \ + ../routes/test/mcrouter_routes_test \ + cpp_unit_tests/mcrouter_test if HAVE_PYTHON TESTS += \ + test_additional_fields.py \ test_allow_only_gets.py \ test_ascii_error.py \ test_ascii_multiget_mock.py \ + test_async_files_attr.py \ test_async_files.py \ - test_bad_params.py \ + test_axonlog.py \ + test_bigvalue.py \ + test_bucketized_poolroute.py \ + test_carbonlookaside_route.py \ test_config_params.py \ test_const_shard_hash.py \ test_custom_failover.py \ + test_debug_fifos.py \ + test_deterministic_failover.py \ + test_dump_config.py \ test_empty_pool.py \ test_flush_all.py \ test_largeobj.py \ + test_latency_injection_route.py \ + test_lease_pairing.py \ + test_linenumbers.py \ + test_loadbalancer_route.py \ + test_logging_route.py \ test_logical_routing_policies.py \ test_max_shadow_requests.py \ test_mcpiper.py \ - test_mcrouter.py \ - test_mcrouter_basic.py \ + test_mcrefill.py \ test_mcrouter_errors.py \ + test_mcrouter_processing_time.py \ + test_mcrouter.py \ test_mcrouter_sanity_mock.py \ + test_mcrouter_serialized.py \ + test_mcrouter_states.py \ test_mcrouter_to_mcrouter_tko.py \ test_memcache_router.py \ test_migrated_failover.py \ @@ -44,26 +64,44 @@ TESTS += \ test_modify_key.py \ test_named_handles.py \ test_noreply.py \ + test_operation_selector_route.py \ + test_poolstats.py \ test_probe_timeout.py \ - test_rates.py \ + test_rendezvous_failover.py \ + test_routing_group_route.py \ test_routing_prefixes.py \ test_send_to_all_hosts.py \ - test_server_stats.py \ test_service_info.py \ test_shadow.py \ test_shadow_route.py \ test_shadow_with_file.py \ test_shard_splits.py \ test_slow_warmup.py \ + test_stats.py \ test_tko_inactive.py \ test_tko_reconfigure.py \ test_validate_config.py \ + test_warmup2.py \ test_warmup.py \ test_wch3.py -endif -# TODO fix the test -# test_warmup2.py +# Integration tests that need to be fixed or skipped on OSS. +# Note we exclude test_bad_params.py and test_mcrouter_basic.py +# rather than listing them here because they currently time out. +XFAIL_TESTS += \ + test_axonlog.py \ + test_mcrefill.py \ + test_dump_config.py \ + test_mcrouter_serialized.py \ + test_carbonlookaside_route.py \ + test_mcrouter_processing_time.py \ + test_named_handles.py \ + test_async_files_attr.py \ + test_warmup2.py \ + test_mcrouter_basic.py \ + test_poolstats.py \ + test_async_files.py +endif TEST_EXTENSIONS = .py PY_LOG_COMPILER = ./run_python_test.sh diff --git a/mcrouter/test/cpp_unit_tests/Makefile.am b/mcrouter/test/cpp_unit_tests/Makefile.am index e3d25f1c5..d1db7e43c 100644 --- a/mcrouter/test/cpp_unit_tests/Makefile.am +++ b/mcrouter/test/cpp_unit_tests/Makefile.am @@ -14,7 +14,6 @@ mcrouter_test_SOURCES = \ flavor_test.cpp \ LeaseTokenMapTest.cpp \ mc_route_handle_provider_test.cpp \ - McrouterClientUsage.cpp \ observable_test.cpp \ options_test.cpp \ pool_factory_test.cpp \ @@ -22,6 +21,9 @@ mcrouter_test_SOURCES = \ route_test.cpp \ runtime_vars_data_test.cpp +# Note: McrouterClientUsage.cpp is excluded +# since it depends on HelloGoodbyeService and fb303. + mcrouter_test_CPPFLAGS = \ -I$(top_srcdir)/.. \ -isystem $(top_srcdir)/lib/gtest/include @@ -30,7 +32,22 @@ mcrouter_test_LDADD = \ $(top_builddir)/libmcroutercore.a \ $(top_builddir)/lib/libmcrouter.a \ $(top_builddir)/lib/libtestmain.la \ - $(top_builddir)/lib/network/libtest_util.a \ + $(top_builddir)/lib/network/test/libtest_util.a \ + $(top_builddir)/lib/carbon/example/gen/libcarbon_hello_goodbye.a \ + -lthriftcpp2 \ + -lserverdbginfo \ + -ltransport \ + -lthriftanyrep \ + -lthrifttype \ + -lthrifttyperep \ + -lthriftprotocol \ + -lrpcmetadata \ + -lthriftannotation \ + -lthriftmetadata \ + -lasync \ + -lconcurrency \ + -lruntime \ + -lthrift-core \ -lwangle \ -lfizz \ -lsodium \ diff --git a/mcrouter/test/mcrouter_config.py b/mcrouter/test/mcrouter_config.py index f04d07192..6e134d375 100644 --- a/mcrouter/test/mcrouter_config.py +++ b/mcrouter/test/mcrouter_config.py @@ -21,6 +21,8 @@ def binPath(name): "mcrouter": "./mcrouter/mcrouter", "mcpiper": "./mcrouter/tools/mcpiper/mcpiper", "mockmc": "./mcrouter/lib/network/test/mock_mc_server", + "mockmcthrift": "./mcrouter/lib/network/test/mock_mc_thrift_server", + "mockmcdual": "./mcrouter/lib/network/test/mock_mc_server_dual", "prodmc": "./mcrouter/lib/network/test/mock_mc_server", } return bins[name]