Load yaml front config independently

cache_test.go

@@ -22,8 +22,8 @@ func TestCaching(t *testing.T) {
 	cloudsackID := "cloudsack"
 
 	providers := map[string]*Provider{
-		testProviderID: NewProvider(nil, "", nil, nil, nil, nil, nil),
-		cloudsackID:    NewProvider(nil, "", nil, nil, nil, nil, nil),
+		testProviderID: NewProvider(nil, "", nil, nil, nil, nil, nil, ""),
+		cloudsackID:    NewProvider(nil, "", nil, nil, nil, nil, nil, ""),
 	}
 
 	log.Debug("Creating fronted")

front.go

@@ -59,7 +59,7 @@ type Front interface {
 	// Accessor for the domain of the masquerade
 	getDomain() string
 
-	//Accessor for the IP address of the masquerade
+	// Accessor for the IP address of the masquerade
 	getIpAddress() string
 
 	markSucceeded()
@@ -238,19 +238,12 @@ type Provider struct {
 	TestURL     string
 	Masquerades []*Masquerade
 
-	// SNIConfig has the configuration that sets if we should or not use arbitrary SNIs
-	// and which SNIs to use.
-	SNIConfig *SNIConfig
-
-	// Optional response validator used to determine whether
-	// fronting succeeded for this provider. If the validator
-	// detects a failure for a given masquerade, it is discarded.
-	// The default validator is used if nil.
-	Validator ResponseValidator
-
 	// VerifyHostname is used for checking if the certificate for a given hostname is valid.
 	// This attribute is only being defined here so it can be sent to the masquerade struct later.
 	VerifyHostname *string
+
+	// FrontingSNIs is a map of country code the the SNI config to use for that country.
+	FrontingSNIs map[string]*SNIConfig
 }
 
 type SNIConfig struct {
@@ -259,25 +252,33 @@ type SNIConfig struct {
 }
 
 // Create a Provider with the given details
-func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, validator ResponseValidator, passthrough []string, sniConfig *SNIConfig, verifyHostname *string) *Provider {
-	d := &Provider{
+func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, validator ResponseValidator, passthrough []string, frontingSNIs map[string]*SNIConfig, verifyHostname *string, countryCode string) *Provider {
+	p := &Provider{
 		HostAliases:         make(map[string]string),
 		TestURL:             testURL,
 		Masquerades:         make([]*Masquerade, 0, len(masquerades)),
-		Validator:           validator,
 		PassthroughPatterns: make([]string, 0, len(passthrough)),
-		SNIConfig:           sniConfig,
+		VerifyHostname:      verifyHostname,
+		FrontingSNIs:        frontingSNIs,
 	}
 	for k, v := range hosts {
-		d.HostAliases[strings.ToLower(k)] = v
+		p.HostAliases[strings.ToLower(k)] = v
 	}
 
+	var config *SNIConfig
+	if countryCode != "" {
+		var ok bool
+		config, ok = frontingSNIs[countryCode]
+		if !ok {
+			config = frontingSNIs["default"]
+		}
+	}
 	for _, m := range masquerades {
-		sni := generateSNI(d.SNIConfig, m)
-		d.Masquerades = append(d.Masquerades, &Masquerade{Domain: m.Domain, IpAddress: m.IpAddress, SNI: sni, VerifyHostname: verifyHostname})
+		sni := generateSNI(config, m)
+		p.Masquerades = append(p.Masquerades, &Masquerade{Domain: m.Domain, IpAddress: m.IpAddress, SNI: sni, VerifyHostname: verifyHostname})
 	}
-	d.PassthroughPatterns = append(d.PassthroughPatterns, passthrough...)
-	return d
+	p.PassthroughPatterns = append(p.PassthroughPatterns, passthrough...)
+	return p
 }
 
 // generateSNI generates a SNI for the given domain and ip address
@@ -323,11 +324,7 @@ func (p *Provider) Lookup(hostname string) string {
 // response failed to reach the origin, eg if the request
 // was rejected by the provider.
 func (p *Provider) ValidateResponse(res *http.Response) error {
-	if p.Validator != nil {
-		return p.Validator(res)
-	} else {
-		return defaultValidator(res)
-	}
+	return defaultValidator(res)
 }
 
 // A validator for fronted responses.  Returns an error if the

front_test.go

@@ -10,13 +10,14 @@ import (
 func TestNewProvider(t *testing.T) {
 	verifyHostname := "verifyHostname.com"
 	var tests = []struct {
-		name                string
-		givenHosts          map[string]string
-		givenTestURL        string
-		givenMasquerades    []*Masquerade
-		givenValidator      ResponseValidator
-		givenPassthrough    []string
-		givenSNIConfig      *SNIConfig
+		name             string
+		givenHosts       map[string]string
+		givenTestURL     string
+		givenMasquerades []*Masquerade
+		givenValidator   ResponseValidator
+		givenPassthrough []string
+		//givenSNIConfig      *SNIConfig
+		givenFrontingSNIs   map[string]*SNIConfig
 		givenVerifyHostname *string
 		assert              func(t *testing.T, actual *Provider)
 	}{
@@ -29,8 +30,8 @@ func TestNewProvider(t *testing.T) {
 				assert.Empty(t, actual.Masquerades)
 				assert.Empty(t, actual.PassthroughPatterns)
 				assert.Equal(t, "http://test.com", actual.TestURL)
-				assert.Nil(t, actual.Validator)
-				assert.Nil(t, actual.SNIConfig)
+				//assert.Nil(t, actual.Validator)
+				assert.Nil(t, actual.FrontingSNIs)
 			},
 		},
 		{
@@ -40,9 +41,11 @@ func TestNewProvider(t *testing.T) {
 			givenMasquerades: []*Masquerade{{Domain: "domain1", IpAddress: "127.0.0.1"}},
 			givenValidator:   func(*http.Response) error { return nil },
 			givenPassthrough: []string{"passthrough1", "passthrough2"},
-			givenSNIConfig: &SNIConfig{
-				UseArbitrarySNIs: true,
-				ArbitrarySNIs:    []string{"sni1.com", "sni2.com"},
+			givenFrontingSNIs: map[string]*SNIConfig{
+				"test": &SNIConfig{
+					UseArbitrarySNIs: true,
+					ArbitrarySNIs:    []string{"sni1.com", "sni2.com"},
+				},
 			},
 			givenVerifyHostname: &verifyHostname,
 			assert: func(t *testing.T, actual *Provider) {
@@ -61,7 +64,7 @@ func TestNewProvider(t *testing.T) {
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			actual := NewProvider(tt.givenHosts, tt.givenTestURL, tt.givenMasquerades, tt.givenValidator, tt.givenPassthrough, tt.givenSNIConfig, tt.givenVerifyHostname)
+			actual := NewProvider(tt.givenHosts, tt.givenTestURL, tt.givenMasquerades, tt.givenValidator, tt.givenPassthrough, tt.givenFrontingSNIs, tt.givenVerifyHostname, "test")
 			tt.assert(t, actual)
 		})
 	}

fronted.go

@@ -64,7 +64,7 @@ type Fronted interface {
 	http.RoundTripper
 
 	// OnNewFronts updates the set of domain fronts to try.
-	OnNewFronts(pool *x509.CertPool, providers map[string]*Provider)
+	OnNewFronts(pool *x509.CertPool, providers map[string]*Provider, countryCode string)
 
 	// Close closes any resources, such as goroutines that are testing fronts.
 	Close()
@@ -101,15 +101,15 @@ func NewFronted(cacheFile string) Fronted {
 
 // OnNewFronts sets the domain fronts to use, the trusted root CAs and the fronting providers
 // (such as Akamai, Cloudfront, etc)
-func (f *fronted) OnNewFronts(pool *x509.CertPool, providers map[string]*Provider) {
+func (f *fronted) OnNewFronts(pool *x509.CertPool, providers map[string]*Provider, countryCode string) {
 	// Make copies just to avoid any concurrency issues with access that may be happening on the
 	// caller side.
 	log.Debug("Updating fronted configuration")
 	if len(providers) == 0 {
 		log.Errorf("No providers configured")
 		return
 	}
-	providersCopy := copyProviders(providers)
+	providersCopy := copyProviders(providers, countryCode)
 	f.addProviders(providersCopy)
 	f.addFronts(loadFronts(providersCopy))
 	f.certPool.Store(pool)
@@ -559,10 +559,10 @@ func (f *fronted) isStopped() bool {
 	return f.stopped.Load()
 }
 
-func copyProviders(providers map[string]*Provider) map[string]*Provider {
+func copyProviders(providers map[string]*Provider, countryCode string) map[string]*Provider {
 	providersCopy := make(map[string]*Provider, len(providers))
 	for key, p := range providers {
-		providersCopy[key] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.Validator, p.PassthroughPatterns, p.SNIConfig, p.VerifyHostname)
+		providersCopy[key] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, nil, p.PassthroughPatterns, p.FrontingSNIs, p.VerifyHostname, countryCode)
 	}
 	return providersCopy
 }

fronted_test.go

@@ -19,11 +19,29 @@ import (
 	"time"
 
 	. "github.com/getlantern/waitforserver"
+	"github.com/goccy/go-yaml"
 	tls "github.com/refraction-networking/utls"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func TestYamlParsing(t *testing.T) {
+	// Read fronted.yaml into a map of Provider structs
+	f, err := os.ReadFile("fronted.yaml")
+	require.NoError(t, err)
+	providers := make(map[string]*Provider)
+	err = yaml.Unmarshal(f, &providers)
+	require.NoError(t, err)
+
+	// Now write out the struct to a new yaml file.
+	f2, err := os.Create("fronted2.yaml")
+	require.NoError(t, err)
+	defer f2.Close()
+	err = yaml.NewEncoder(f2).Encode(providers)
+	require.NoError(t, err)
+
+}
+
 func TestDirectDomainFrontingWithoutSNIConfig(t *testing.T) {
 	dir, err := os.MkdirTemp("", "direct_test")
 	require.NoError(t, err, "Unable to create temp dir")
@@ -59,7 +77,7 @@ func TestDirectDomainFrontingWithSNIConfig(t *testing.T) {
 	})
 	defaultFrontedProviderID = "akamai"
 	transport := NewFronted(cacheFile)
-	transport.OnNewFronts(certs, p)
+	transport.OnNewFronts(certs, p, "")
 
 	client := &http.Client{
 		Transport: transport,
@@ -87,7 +105,7 @@ func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtE
 	p := testProvidersWithHosts(hosts)
 	defaultFrontedProviderID = testProviderID
 	transport := NewFronted(cacheFile)
-	transport.OnNewFronts(certs, p)
+	transport.OnNewFronts(certs, p, "")
 
 	client := &http.Client{
 		Transport: transport,
@@ -97,7 +115,7 @@ func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtE
 
 	defaultFrontedProviderID = testProviderID
 	transport = NewFronted(cacheFile)
-	transport.OnNewFronts(certs, p)
+	transport.OnNewFronts(certs, p, "")
 	client = &http.Client{
 		Transport: transport,
 	}
@@ -205,15 +223,15 @@ func TestHostAliasesBasic(t *testing.T) {
 		"abc.forbidden.com": "abc.cloudsack.biz",
 		"def.forbidden.com": "def.cloudsack.biz",
 	}
-	p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil, nil, nil)
+	p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil, nil, nil, "")
 
 	certs := x509.NewCertPool()
 	certs.AddCert(cloudSack.Certificate())
 
 	defaultFrontedProviderID = "cloudsack"
 	rt := NewFronted("")
 
-	rt.OnNewFronts(certs, map[string]*Provider{"cloudsack": p})
+	rt.OnNewFronts(certs, map[string]*Provider{"cloudsack": p}, "")
 
 	client := &http.Client{Transport: rt}
 	for _, test := range tests {
@@ -303,14 +321,14 @@ func TestHostAliasesMulti(t *testing.T) {
 		"abc.forbidden.com": "abc.cloudsack.biz",
 		"def.forbidden.com": "def.cloudsack.biz",
 	}
-	p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil, nil, nil)
+	p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil, nil, nil, "")
 
 	masq2 := []*Masquerade{{Domain: "example.com", IpAddress: sadCloudAddr}}
 	alias2 := map[string]string{
 		"abc.forbidden.com": "abc.sadcloud.io",
 		"def.forbidden.com": "def.sadcloud.io",
 	}
-	p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil, nil, nil)
+	p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil, nil, nil, "")
 
 	certs := x509.NewCertPool()
 	certs.AddCert(cloudSack.Certificate())
@@ -323,7 +341,7 @@ func TestHostAliasesMulti(t *testing.T) {
 
 	defaultFrontedProviderID = "cloudsack"
 	rt := NewFronted("")
-	rt.OnNewFronts(certs, providers)
+	rt.OnNewFronts(certs, providers, "")
 
 	client := &http.Client{Transport: rt}
 
@@ -441,14 +459,14 @@ func TestPassthrough(t *testing.T) {
 	masq := []*Masquerade{{Domain: "example.com", IpAddress: cloudSackAddr}}
 	alias := map[string]string{}
 	passthrough := []string{"*.ok.cloudsack.biz", "abc.cloudsack.biz"}
-	p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, passthrough, nil, nil)
+	p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, passthrough, nil, nil, "")
 
 	certs := x509.NewCertPool()
 	certs.AddCert(cloudSack.Certificate())
 
 	defaultFrontedProviderID = "cloudsack"
 	rt := NewFronted("")
-	rt.OnNewFronts(certs, map[string]*Provider{"cloudsack": p})
+	rt.OnNewFronts(certs, map[string]*Provider{"cloudsack": p}, "")
 
 	client := &http.Client{Transport: rt}
 	for _, test := range tests {
@@ -507,7 +525,7 @@ func TestCustomValidators(t *testing.T) {
 		alias := map[string]string{
 			"abc.forbidden.com": "abc.sadcloud.io",
 		}
-		p := NewProvider(alias, "https://ttt.sadcloud.io/ping", masq, validator, nil, nil, nil)
+		p := NewProvider(alias, "https://ttt.sadcloud.io/ping", masq, validator, nil, nil, nil, "")
 
 		certs := x509.NewCertPool()
 		certs.AddCert(sadCloud.Certificate())
@@ -518,7 +536,7 @@ func TestCustomValidators(t *testing.T) {
 
 		defaultFrontedProviderID = "sadcloud"
 		f := NewFronted("")
-		f.OnNewFronts(certs, providers)
+		f.OnNewFronts(certs, providers, "")
 		return f, nil
 	}
 
@@ -848,7 +866,7 @@ func TestFindWorkingMasquerades(t *testing.T) {
 				stopCh:           make(chan interface{}, 10),
 			}
 			f.providers = make(map[string]*Provider)
-			f.providers["testProviderId"] = NewProvider(nil, "", nil, nil, nil, nil, nil)
+			f.providers["testProviderId"] = NewProvider(nil, "", nil, nil, nil, nil, nil, "")
 			f.fronts = make(sortedFronts, len(tt.masquerades))
 			for i, m := range tt.masquerades {
 				f.fronts[i] = m

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/getlantern/ops v0.0.0-20231025133620-f368ab734534
 	github.com/getlantern/tlsdialer/v3 v3.0.3
 	github.com/getlantern/waitforserver v1.0.1
+	github.com/goccy/go-yaml v1.15.13
 	github.com/refraction-networking/utls v1.6.7
 	github.com/stretchr/testify v1.8.4
 )

go.sum

@@ -64,6 +64,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
 github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
+github.com/goccy/go-yaml v1.15.13 h1:Xd87Yddmr2rC1SLLTm2MNDcTjeO/GYo0JGiww6gSTDg=
+github.com/goccy/go-yaml v1.15.13/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=

test_support.go

@@ -25,7 +25,7 @@ func ConfigureCachingForTest(t *testing.T, cacheFile string) Fronted {
 	p := testProviders()
 	defaultFrontedProviderID = testProviderID
 	f := NewFronted(cacheFile)
-	f.OnNewFronts(certs, p)
+	f.OnNewFronts(certs, p, "")
 	return f
 }
 
@@ -34,7 +34,7 @@ func ConfigureHostAlaisesForTest(t *testing.T, hosts map[string]string) Fronted
 	p := testProvidersWithHosts(hosts)
 	defaultFrontedProviderID = testProviderID
 	f := NewFronted("")
-	f.OnNewFronts(certs, p)
+	f.OnNewFronts(certs, p, "")
 	return f
 }
 
@@ -53,17 +53,20 @@ func trustedCACerts(t *testing.T) *x509.CertPool {
 
 func testProviders() map[string]*Provider {
 	return map[string]*Provider{
-		testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil, nil, nil),
+		testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil, nil, nil, ""),
 	}
 }
 
 func testProvidersWithHosts(hosts map[string]string) map[string]*Provider {
 	return map[string]*Provider{
-		testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil, nil, nil),
+		testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil, nil, nil, ""),
 	}
 }
 func testAkamaiProvidersWithHosts(hosts map[string]string, sniConfig *SNIConfig) map[string]*Provider {
+	frontingSNIs := map[string]*SNIConfig{
+		"test": sniConfig,
+	}
 	return map[string]*Provider{
-		"akamai": NewProvider(hosts, "https://fronted-ping.dsa.akamai.getiantem.org/ping", DefaultAkamaiMasquerades, nil, nil, sniConfig, nil),
+		"akamai": NewProvider(hosts, "https://fronted-ping.dsa.akamai.getiantem.org/ping", DefaultAkamaiMasquerades, nil, nil, frontingSNIs, nil, "test"),
 	}
 }