Skip to content

Commit

Permalink
Refactor: arrange and document dynamic config usages in functional te…
Browse files Browse the repository at this point in the history
…sts (temporalio#7096)

## What changed?
<!-- Describe what has changed in this PR -->
- Consolidated all global DCs in one file with clear name for easy
discovery.
- Documented how to override DCs properly. 
- Cleaned up duplicated configs.

## Why?
<!-- Tell your future self why have you made these changes -->
Quality of life for function test writers.

## How did you test it?
<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->
Run.

## Potential risks
<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->
No risks.

## Documentation
<!-- Have you made sure this change doesn't falsify anything currently
stated in `docs/`? If significant
new behavior is added, have you described that in `docs/`? -->
No.

## Is hotfix candidate?
<!-- Is this PR a hotfix candidate or does it require a notification to
be sent to the broader community? (Yes/No) -->
No.

---------

Co-authored-by: Stephan Behnke <[email protected]>
  • Loading branch information
alexshtin and stephanos authored Jan 17, 2025
1 parent 72fa968 commit 38903d3
Show file tree
Hide file tree
Showing 14 changed files with 147 additions and 160 deletions.
15 changes: 5 additions & 10 deletions tests/activity_api_pause_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ import (
sdkclient "go.temporal.io/sdk/client"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
"go.temporal.io/server/common/dynamicconfig"
"go.temporal.io/server/common/testing/testvars"
"go.temporal.io/server/common/util"
"go.temporal.io/server/tests/testcore"
Expand All @@ -52,15 +51,16 @@ type ActivityApiPauseClientTestSuite struct {
activityRetryPolicy *temporal.RetryPolicy
}

func (s *ActivityApiPauseClientTestSuite) SetupSuite() {
s.FunctionalTestSdkSuite.SetupSuite()
s.OverrideDynamicConfig(dynamicconfig.ActivityAPIsEnabled, true)
s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
func TestActivityApiPauseClientTestSuite(t *testing.T) {
s := new(ActivityApiPauseClientTestSuite)
suite.Run(t, s)
}

func (s *ActivityApiPauseClientTestSuite) SetupTest() {
s.FunctionalTestSdkSuite.SetupTest()

s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())

s.initialRetryInterval = 1 * time.Second
s.scheduleToCloseTimeout = 30 * time.Minute
s.startToCloseTimeout = 15 * time.Minute
Expand All @@ -71,11 +71,6 @@ func (s *ActivityApiPauseClientTestSuite) SetupTest() {
}
}

func TestActivityApiPauseClientTestSuite(t *testing.T) {
s := new(ActivityApiPauseClientTestSuite)
suite.Run(t, s)
}

func (s *ActivityApiPauseClientTestSuite) makeWorkflowFunc(activityFunction ActivityFunctions) WorkflowFunction {
return func(ctx workflow.Context) error {

Expand Down
15 changes: 5 additions & 10 deletions tests/activity_api_reset_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ import (
sdkclient "go.temporal.io/sdk/client"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
"go.temporal.io/server/common/dynamicconfig"
"go.temporal.io/server/common/testing/testvars"
"go.temporal.io/server/common/util"
"go.temporal.io/server/tests/testcore"
Expand All @@ -54,15 +53,16 @@ type ActivityApiResetClientTestSuite struct {
activityRetryPolicy *temporal.RetryPolicy
}

func (s *ActivityApiResetClientTestSuite) SetupSuite() {
s.FunctionalTestSdkSuite.SetupSuite()
s.OverrideDynamicConfig(dynamicconfig.ActivityAPIsEnabled, true)
s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
func TestActivityApiResetClientTestSuite(t *testing.T) {
s := new(ActivityApiResetClientTestSuite)
suite.Run(t, s)
}

func (s *ActivityApiResetClientTestSuite) SetupTest() {
s.FunctionalTestSdkSuite.SetupTest()

s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())

s.initialRetryInterval = 1 * time.Second
s.scheduleToCloseTimeout = 30 * time.Minute
s.startToCloseTimeout = 15 * time.Minute
Expand All @@ -73,11 +73,6 @@ func (s *ActivityApiResetClientTestSuite) SetupTest() {
}
}

func TestActivityApiResetClientTestSuite(t *testing.T) {
s := new(ActivityApiResetClientTestSuite)
suite.Run(t, s)
}

func (s *ActivityApiResetClientTestSuite) makeWorkflowFunc(activityFunction ActivityFunctions) WorkflowFunction {
return func(ctx workflow.Context) error {

Expand Down
14 changes: 4 additions & 10 deletions tests/activity_api_update_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import (
sdkclient "go.temporal.io/sdk/client"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
"go.temporal.io/server/common/dynamicconfig"
"go.temporal.io/server/common/testing/testvars"
"go.temporal.io/server/tests/testcore"
"google.golang.org/protobuf/types/known/durationpb"
Expand All @@ -52,19 +51,14 @@ type ActivityApiUpdateClientTestSuite struct {
tv *testvars.TestVars
}

func (s *ActivityApiUpdateClientTestSuite) SetupSuite() {
s.FunctionalTestSdkSuite.SetupSuite()
s.OverrideDynamicConfig(dynamicconfig.ActivityAPIsEnabled, true)
s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
func TestActivityApiUpdateClientTestSuite(t *testing.T) {
s := new(ActivityApiUpdateClientTestSuite)
suite.Run(t, s)
}

func (s *ActivityApiUpdateClientTestSuite) SetupTest() {
s.FunctionalTestSdkSuite.SetupTest()
}

func TestActivityApiUpdateClientTestSuite(t *testing.T) {
s := new(ActivityApiUpdateClientTestSuite)
suite.Run(t, s)
s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
}

type (
Expand Down
5 changes: 3 additions & 2 deletions tests/archival_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,10 @@ func TestArchivalSuite(t *testing.T) {
}

func (s *ArchivalSuite) SetupSuite() {
s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(map[dynamicconfig.Key]any{
dynamicConfigOverrides := map[dynamicconfig.Key]any{
dynamicconfig.ArchivalProcessorArchiveDelay.Key(): time.Duration(0),
}))
}
s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides))
}

func (s *ArchivalSuite) TestArchival_TimerQueueProcessor() {
Expand Down
6 changes: 4 additions & 2 deletions tests/deployment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func TestDeploymentSuite(t *testing.T) {
}

func (s *DeploymentSuite) SetupSuite() {
s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(map[dynamicconfig.Key]any{
dynamicConfigOverrides := map[dynamicconfig.Key]any{
dynamicconfig.EnableDeployments.Key(): true,
dynamicconfig.FrontendEnableWorkerVersioningDataAPIs.Key(): true,
dynamicconfig.FrontendEnableWorkerVersioningWorkflowAPIs.Key(): true,
Expand All @@ -99,7 +99,9 @@ func (s *DeploymentSuite) SetupSuite() {

// Reduce the chance of hitting max batch job limit in tests
dynamicconfig.FrontendMaxConcurrentBatchOperationPerNamespace.Key(): maxConcurrentBatchOps,
}))
}

s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides))
}

func (s *DeploymentSuite) SetupTest() {
Expand Down
2 changes: 1 addition & 1 deletion tests/http_api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ func (s *HttpApiTestSuite) runHTTPAPIBasicsTest_Shorthand(contentType string, pr
}

func (s *HttpApiTestSuite) TestHTTPHostValidation() {
s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.FrontendHTTPAllowedHosts, []string{"allowed"})
s.OverrideDynamicConfig(dynamicconfig.FrontendHTTPAllowedHosts, []string{"allowed"})
{
req, err := http.NewRequest("GET", "/system-info", nil)
s.Require().NoError(err)
Expand Down
13 changes: 8 additions & 5 deletions tests/namespace_delete_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ func (s *namespaceTestSuite) SetupSuite() {
dynamicConfigOverrides := map[dynamicconfig.Key]any{
// Run tests at full speed.
dynamicconfig.DeleteNamespaceDeleteActivityRPS.Key(): 1000000,

dynamicconfig.TransferProcessorUpdateAckInterval.Key(): 1 * time.Second,
dynamicconfig.VisibilityProcessorUpdateAckInterval.Key(): 1 * time.Second,
}

if testcore.UsingSQLAdvancedVisibility() {
Expand All @@ -82,9 +85,9 @@ func (s *namespaceTestSuite) SetupSuite() {

func (s *namespaceTestSuite) Test_NamespaceDelete_InvalidUTF8() {
// don't fail for this test, we're testing this behavior specifically
s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ValidateUTF8FailRPCRequest, false)
s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ValidateUTF8FailRPCResponse, false)
s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ValidateUTF8FailPersistence, false)
s.OverrideDynamicConfig(dynamicconfig.ValidateUTF8FailRPCRequest, false)
s.OverrideDynamicConfig(dynamicconfig.ValidateUTF8FailRPCResponse, false)
s.OverrideDynamicConfig(dynamicconfig.ValidateUTF8FailPersistence, false)

capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture()
defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture)
Expand Down Expand Up @@ -184,7 +187,7 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_OverrideDelay() {
ctx, cancel := rpc.NewContextWithTimeoutAndVersionHeaders(10000 * time.Second)
defer cancel()

s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.DeleteNamespaceNamespaceDeleteDelay, time.Hour)
s.OverrideDynamicConfig(dynamicconfig.DeleteNamespaceNamespaceDeleteDelay, time.Hour)

retention := 24 * time.Hour
_, err := s.FrontendClient().RegisterNamespace(ctx, &workflowservice.RegisterNamespaceRequest{
Expand Down Expand Up @@ -522,7 +525,7 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_Protected() {
})
s.NoError(err)

s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ProtectedNamespaces, []string{tv.NamespaceName().String()})
s.OverrideDynamicConfig(dynamicconfig.ProtectedNamespaces, []string{tv.NamespaceName().String()})

delResp, err := s.OperatorClient().DeleteNamespace(ctx, &operatorservice.DeleteNamespaceRequest{
Namespace: tv.NamespaceName().String(),
Expand Down
14 changes: 12 additions & 2 deletions tests/sizelimit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"go.temporal.io/api/workflowservice/v1"
"go.temporal.io/server/common"
"go.temporal.io/server/common/convert"
"go.temporal.io/server/common/dynamicconfig"
"go.temporal.io/server/common/log/tag"
"go.temporal.io/server/common/payloads"
"go.temporal.io/server/service/history/consts"
Expand All @@ -59,9 +60,18 @@ func TestSizeLimitFunctionalSuite(t *testing.T) {
suite.Run(t, new(SizeLimitFunctionalSuite))
}

// This cluster use customized threshold for history config
func (s *SizeLimitFunctionalSuite) SetupSuite() {
s.FunctionalTestSuite.SetupSuiteWithCluster("testdata/sizelimit_cluster.yaml")
dynamicConfigOverrides := map[dynamicconfig.Key]any{
dynamicconfig.HistoryCountLimitWarn.Key(): 10,
dynamicconfig.HistoryCountLimitError.Key(): 20,
dynamicconfig.HistorySizeLimitWarn.Key(): 5000,
dynamicconfig.HistorySizeLimitError.Key(): 9000,
dynamicconfig.BlobSizeLimitWarn.Key(): 1,
dynamicconfig.BlobSizeLimitError.Key(): 1000,
dynamicconfig.MutableStateSizeLimitWarn.Key(): 200,
dynamicconfig.MutableStateSizeLimitError.Key(): 1100,
}
s.FunctionalTestBase.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides))
}

func (s *SizeLimitFunctionalSuite) TestTerminateWorkflowCausedByHistoryCountLimit() {
Expand Down
85 changes: 85 additions & 0 deletions tests/testcore/dynamic_config_overrides.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// The MIT License
//
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
//
// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package testcore

import (
"time"

"go.temporal.io/server/common/dynamicconfig"
"go.temporal.io/server/common/persistence/visibility"
)

var (
// Functional tests don't use dynamic config files. All settings get their default values
// defined in common/dynamicconfig/constants.go.
//
// There are 4 ways to override a setting:
// 1. Globally using this file. Every test suite creates a new test cluster using this overrides.
// 2. Per test suite using FunctionalTestBase.SetupSuiteWithCluster() and WithDynamicConfigOverrides() option.
// 3. Per test using FunctionalTestBase.OverrideDynamicConfig() method.
// 4. Per specific cluster per test (if test has more than one cluster) using TestCluster.OverrideDynamicConfig() method.
//
// NOTE1: settings which are not really dynamic (requires server restart to take effect) can't be overridden on test level,
// i.e., must be overridden globally (1) or per test suite (2).
// NOTE2: per test overrides change the value for the cluster, therefore, it affects not only a specific test, but
// all tests for that suite. The automatic cleanup reverts to the previous value and tests don't affect each other.
// But that means tests in the same suite can't be run in parallel. This is not a problem because testify
// doesn't allow parallel execution of tests in the same suite anyway. If one day, it will be allowed,
// unique namespaces with overrides per namespace should be used for tests that require overrides.
dynamicConfigOverrides = map[dynamicconfig.Key]any{
dynamicconfig.FrontendRPS.Key(): 3000,
dynamicconfig.FrontendMaxNamespaceVisibilityRPSPerInstance.Key(): 50,
dynamicconfig.FrontendMaxNamespaceVisibilityBurstRatioPerInstance.Key(): 1,
dynamicconfig.ReplicationTaskProcessorErrorRetryMaxAttempts.Key(): 1,
dynamicconfig.SecondaryVisibilityWritingMode.Key(): visibility.SecondaryVisibilityWritingModeOff,
dynamicconfig.WorkflowTaskHeartbeatTimeout.Key(): 5 * time.Second,
dynamicconfig.ReplicationTaskFetcherAggregationInterval.Key(): 200 * time.Millisecond,
dynamicconfig.ReplicationTaskFetcherErrorRetryWait.Key(): 50 * time.Millisecond,
dynamicconfig.ReplicationTaskProcessorErrorRetryWait.Key(): time.Millisecond,
dynamicconfig.ClusterMetadataRefreshInterval.Key(): 100 * time.Millisecond,
dynamicconfig.NamespaceCacheRefreshInterval.Key(): NamespaceCacheRefreshInterval,
dynamicconfig.ReplicationEnableUpdateWithNewTaskMerge.Key(): true,
dynamicconfig.ValidateUTF8SampleRPCRequest.Key(): 1.0,
dynamicconfig.ValidateUTF8SampleRPCResponse.Key(): 1.0,
dynamicconfig.ValidateUTF8SamplePersistence.Key(): 1.0,
dynamicconfig.ValidateUTF8FailRPCRequest.Key(): true,
dynamicconfig.ValidateUTF8FailRPCResponse.Key(): true,
dynamicconfig.ValidateUTF8FailPersistence.Key(): true,
dynamicconfig.EnableWorkflowExecutionTimeoutTimer.Key(): true,
dynamicconfig.FrontendMaskInternalErrorDetails.Key(): false,
dynamicconfig.HistoryScannerEnabled.Key(): false,
dynamicconfig.TaskQueueScannerEnabled.Key(): false,
dynamicconfig.ExecutionsScannerEnabled.Key(): false,
dynamicconfig.BuildIdScavengerEnabled.Key(): false,

// Better to read through in tests than add artificial sleeps (which is what we previously had).
dynamicconfig.ForceSearchAttributesCacheRefreshOnRead.Key(): true,

dynamicconfig.RetentionTimerJitterDuration.Key(): time.Second,
dynamicconfig.EnableEagerWorkflowStart.Key(): true,
dynamicconfig.FrontendEnableExecuteMultiOperation.Key(): true,
dynamicconfig.ActivityAPIsEnabled.Key(): true,
}
)
25 changes: 6 additions & 19 deletions tests/testcore/functional_test_base.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ import (
"go.temporal.io/server/common/namespace"
"go.temporal.io/server/common/payloads"
"go.temporal.io/server/common/persistence"
"go.temporal.io/server/common/persistence/visibility"
"go.temporal.io/server/common/primitives"
"go.temporal.io/server/common/primitives/timestamp"
"go.temporal.io/server/common/rpc"
Expand Down Expand Up @@ -207,23 +208,12 @@ func (s *FunctionalTestBase) SetupSuiteWithCluster(clusterConfigFile string, opt
s.Require().Empty(s.testClusterConfig.DeprecatedFrontendAddress, "Functional tests against external frontends are not supported")
s.Require().Empty(s.testClusterConfig.DeprecatedClusterNo, "ClusterNo should not be present in cluster config files")

if s.testClusterConfig.DynamicConfigOverrides == nil {
s.testClusterConfig.DynamicConfigOverrides = make(map[dynamicconfig.Key]any)
}

// TODO (alex): clusterConfig shouldn't have DC at all.
maps.Copy(s.testClusterConfig.DynamicConfigOverrides, map[dynamicconfig.Key]any{
dynamicconfig.HistoryScannerEnabled.Key(): false,
dynamicconfig.TaskQueueScannerEnabled.Key(): false,
dynamicconfig.ExecutionsScannerEnabled.Key(): false,
dynamicconfig.BuildIdScavengerEnabled.Key(): false,
// Better to read through in tests than add artificial sleeps (which is what we previously had).
dynamicconfig.ForceSearchAttributesCacheRefreshOnRead.Key(): true,
dynamicconfig.RetentionTimerJitterDuration.Key(): time.Second,
dynamicconfig.EnableEagerWorkflowStart.Key(): true,
dynamicconfig.FrontendEnableExecuteMultiOperation.Key(): true,
})
s.testClusterConfig.DynamicConfigOverrides = make(map[dynamicconfig.Key]any)
maps.Copy(s.testClusterConfig.DynamicConfigOverrides, params.DynamicConfigOverrides)
// TODO (alex): is it needed?
if s.testClusterConfig.ESConfig != nil {
s.testClusterConfig.DynamicConfigOverrides[dynamicconfig.SecondaryVisibilityWritingMode.Key()] = visibility.SecondaryVisibilityWritingModeDual
}

s.testClusterConfig.ServiceFxOptions = params.ServiceOptions
s.testClusterConfig.EnableMetricsCapture = true
Expand Down Expand Up @@ -259,15 +249,12 @@ func (s *FunctionalTestBase) SetupSuiteWithCluster(clusterConfigFile string, opt
s.adminClient = s.testCluster.AdminClient()
s.operatorClient = s.testCluster.OperatorClient()
s.httpAPIAddress = s.testCluster.Host().FrontendHTTPAddress()

}

// All test suites that inherit FunctionalTestBase and overwrite SetupTest must
// call this testcore FunctionalTestBase.SetupTest function to distribute the tests
// into partitions. Otherwise, the test suite will be executed multiple times
// in each partition.
// Furthermore, all test suites in the "tests/" directory that don't inherit
// from FunctionalTestBase must implement SetupTest that calls checkTestShard.
func (s *FunctionalTestBase) SetupTest() {
s.checkTestShard()
s.initAssertions()
Expand Down
Loading

0 comments on commit 38903d3

Please sign in to comment.