Add custom domain verification

.env.example

@@ -12,7 +12,11 @@ PAGESHIP_STORAGE_KEY_PREFIX=pageship/
 PAGESHIP_MAX_DEPLOYMENT_SIZE=1G
 PAGESHIP_TOKEN_AUTHORITY=http://api.localtest.me:8001
 # PAGESHIP_APP=test
-PAGESHIP_CLEANUP_EXPIRED_CRONTAB=* * * * *
+PAGESHIP_DOMAIN_VERIFICATION_ENABLED=true
+PAGESHIP_CLEANUP_EXPIRED_CRONTAB="* * * * *"
+PAGESHIP_KEEP_AFTER_EXPIRED="1h"
+PAGESHIP_VERIFY_DOMAIN_OWNERSHIP_CRONTAB="* * * * *"
+PAGESHIP_DOMAIN_VERIFICATION_INTERVAL="1h"
 # PAGESHIP_HOST_ID_SCHEME=suffix
 
 # PAGESHIP_CUSTOM_DOMAIN_MESSAGE=

.github/workflows/test.yaml

@@ -0,0 +1,37 @@
+name: Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:11.5
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - name: Test with sqlite
+        run: go test ./...
+      - name: Test with postgres
+        run: go test ./...
+        env:
+          TEST_PAGESHIP_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable
+

cmd/controller/app/start.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"errors"
+	"net"
 	"net/http"
 	"os"
 	"time"
@@ -64,6 +65,9 @@ func init() {
 
 	startCmd.PersistentFlags().String("cleanup-expired-crontab", "", "cleanup expired schedule")
 	startCmd.PersistentFlags().Duration("keep-after-expired", time.Hour*24, "keep-after-expired")
+	startCmd.PersistentFlags().String("verify-domain-ownership-crontab", "", "verify domain ownership schedule")
+	startCmd.PersistentFlags().Bool("domain-verification-enabled", false, "enable/disable domain verification")
+	startCmd.PersistentFlags().Duration("domain-verification-interval", time.Hour, "duration before next domain verification start for a verified domain")
 
 	startCmd.PersistentFlags().Bool("controller", true, "run controller server")
 	startCmd.PersistentFlags().Bool("cron", true, "run cron jobs")
@@ -105,12 +109,16 @@ type StartControllerConfig struct {
 	ReservedApps      []string `mapstructure:"reserved-apps"`
 	APIACLFile        string   `mapstructure:"api-acl" validate:"omitempty,filepath"`
 
-	CustomDomainMessage string `mapstructure:"custom-domain-message"`
+	CustomDomainMessage       string `mapstructure:"custom-domain-message"`
+	DomainVerificationEnabled bool   `mapstructure:"domain-verification-enabled" validate:"omitempty"`
 }
 
 type StartCronConfig struct {
-	CleanupExpiredCrontab string        `mapstructure:"cleanup-expired-crontab" validate:"omitempty,cron"`
-	KeepAfterExpired      time.Duration `mapstructure:"keep-after-expired" validate:"min=0"`
+	CleanupExpiredCrontab        string        `mapstructure:"cleanup-expired-crontab" validate:"omitempty,cron"`
+	KeepAfterExpired             time.Duration `mapstructure:"keep-after-expired" validate:"min=0"`
+	VerifyDomainOwnershipCrontab string        `mapstructure:"verify-domain-ownership-crontab" validate:"omitempty,cron"`
+	DomainVerificationEnabled    bool          `mapstructure:"domain-verification-enabled" validate:"omitempty"`
+	DomainVerificationInterval   time.Duration `mapstructure:"domain-verification-interval" validate:"min=1"`
 }
 
 type setup struct {
@@ -176,15 +184,16 @@ func (s *setup) controller(domain string, conf StartControllerConfig, sitesConf
 	}
 
 	controllerConf := controller.Config{
-		MaxDeploymentSize:   int64(maxDeploymentSize),
-		StorageKeyPrefix:    conf.StorageKeyPrefix,
-		HostIDScheme:        sitesConf.HostIDScheme,
-		HostPattern:         config.NewHostPattern(sitesConf.HostPattern),
-		ReservedApps:        reservedApps,
-		TokenSigningKey:     []byte(tokenSigningKey),
-		TokenAuthority:      conf.TokenAuthority,
-		ServerVersion:       versioninfo.Short(),
-		CustomDomainMessage: conf.CustomDomainMessage,
+		MaxDeploymentSize:         int64(maxDeploymentSize),
+		StorageKeyPrefix:          conf.StorageKeyPrefix,
+		HostIDScheme:              sitesConf.HostIDScheme,
+		HostPattern:               config.NewHostPattern(sitesConf.HostPattern),
+		ReservedApps:              reservedApps,
+		TokenSigningKey:           []byte(tokenSigningKey),
+		TokenAuthority:            conf.TokenAuthority,
+		ServerVersion:             versioninfo.Short(),
+		CustomDomainMessage:       conf.CustomDomainMessage,
+		DomainVerificationEnabled: conf.DomainVerificationEnabled,
 	}
 
 	if conf.APIACLFile != "" {
@@ -245,15 +254,28 @@ func (s *setup) controller(domain string, conf StartControllerConfig, sitesConf
 }
 
 func (s *setup) cron(conf StartCronConfig) error {
+	cronjobs := []command.CronJob{
+		&cron.CleanupExpired{
+			Schedule:         conf.CleanupExpiredCrontab,
+			KeepAfterExpired: conf.KeepAfterExpired,
+			DB:               s.database,
+		},
+	}
+	if conf.DomainVerificationEnabled {
+		cronjobs = append(cronjobs,
+			&cron.VerifyDomainOwnership{
+				Schedule:                     conf.VerifyDomainOwnershipCrontab,
+				DB:                           s.database,
+				MaxConsumeActiveDomainCount:  10,
+				MaxConsumePendingDomainCount: 10,
+				Resolver:                     net.DefaultResolver,
+				VerificationInterval:         conf.DomainVerificationInterval,
+			},
+		)
+	}
 	cronr := command.CronRunner{
 		Logger: logger.Named("cron"),
-		Jobs: []command.CronJob{
-			&cron.CleanupExpired{
-				Schedule:         conf.CleanupExpiredCrontab,
-				KeepAfterExpired: conf.KeepAfterExpired,
-				DB:               s.database,
-			},
-		},
+		Jobs:   cronjobs,
 	}
 
 	s.works = append(s.works, cronr.Run)

cmd/pageship/app/domains.go

@@ -46,16 +46,18 @@ var domainsCmd = &cobra.Command{
 		}
 
 		type domainEntry struct {
-			name  string
-			site  string
-			model *api.APIDomain
+			name         string
+			site         string
+			model        *models.Domain
+			verification *models.DomainVerification
 		}
 		domains := map[string]domainEntry{}
 		for _, dconf := range app.Config.Domains {
 			domains[dconf.Domain] = domainEntry{
-				name:  dconf.Domain,
-				site:  dconf.Site,
-				model: nil,
+				name:         dconf.Domain,
+				site:         dconf.Site,
+				model:        nil,
+				verification: nil,
 			}
 		}
 
@@ -66,36 +68,63 @@ var domainsCmd = &cobra.Command{
 
 		for _, d := range apiDomains {
 			dd := d
-			domains[d.Domain.Domain] = domainEntry{
-				name:  d.Domain.Domain,
-				site:  d.Domain.SiteName,
-				model: &dd,
+			domain := dd.Domain
+			verification := dd.DomainVerification
+			if domain != nil {
+				domains[domain.Domain] = domainEntry{
+					name:         domain.Domain,
+					site:         domain.SiteName,
+					model:        domain,
+					verification: verification,
+				}
+			} else if verification != nil {
+				if record, ok := domains[verification.Domain]; ok {
+					domains[verification.Domain] = domainEntry{
+						name:         verification.Domain,
+						site:         record.site,
+						model:        nil,
+						verification: verification,
+					}
+				}
 			}
 		}
 
 		w := tabwriter.NewWriter(os.Stdout, 1, 4, 4, ' ', 0)
-		fmt.Fprintln(w, "NAME\tSITE\tCREATED AT\tSTATUS")
+		fmt.Fprintln(w, "NAME\tSITE\tCREATED AT\tSTATUS\tLAST CHECKED AT\tNOTE")
 		for _, domain := range domains {
 			createdAt := "-"
+			lastCheckedAt := "-"
 			site := "-"
+			note := "-"
 			if domain.model != nil {
 				createdAt = domain.model.CreatedAt.Local().Format(time.DateTime)
 				site = fmt.Sprintf("%s/%s", domain.model.AppID, domain.model.SiteName)
 			} else {
 				site = fmt.Sprintf("%s/%s", app.ID, domain.site)
 			}
+			if domain.verification != nil && domain.verification.LastCheckedAt != nil {
+				lastCheckedAt = domain.verification.LastCheckedAt.Local().Format(time.DateTime)
+			}
 
 			var status string
 			switch {
 			case domain.model != nil && domain.model.AppID != app.ID:
 				status = "IN_USE"
 			case domain.model != nil && domain.model.AppID == app.ID:
 				status = "ACTIVE"
+			case domain.verification != nil:
+				if domain.verification.WillCheckAt == nil {
+					status = "INACTIVE"
+				} else {
+					status = "PENDING"
+				}
+				key, value := domain.verification.GetTxtRecord()
+				note = fmt.Sprintf("Add TXT record with domain \"%s\" and value \"%s\" to your DNS server", key, value)
 			default:
 				status = "INACTIVE"
 			}
 
-			fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", domain.name, site, createdAt, status)
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", domain.name, site, createdAt, status, lastCheckedAt, note)
 		}
 		w.Flush()
 
@@ -116,8 +145,8 @@ func promptDomainReplaceApp(ctx context.Context, appID string, domainName string
 
 	appID = ""
 	for _, d := range domains {
-		if d.Domain.Domain == domainName {
-			appID = d.AppID
+		if d.Domain != nil && d.Domain.Domain == domainName {
+			appID = d.Domain.AppID
 		}
 	}
 
@@ -160,21 +189,38 @@ var domainsActivateCmd = &cobra.Command{
 			return fmt.Errorf("undefined domain")
 		}
 
-		_, err = API().CreateDomain(cmd.Context(), appID, domainName, "")
+		var result *api.APIDomain = nil
+		result, err = API().CreateDomain(cmd.Context(), appID, domainName, "")
 		if code, ok := api.ErrorStatusCode(err); ok && code == http.StatusConflict {
 			var replaceApp string
 			replaceApp, err = promptDomainReplaceApp(cmd.Context(), appID, domainName)
 			if err != nil {
 				return err
 			}
-			_, err = API().CreateDomain(cmd.Context(), appID, domainName, replaceApp)
+			result, err = API().CreateDomain(cmd.Context(), appID, domainName, replaceApp)
 		}
 
 		if err != nil {
 			return fmt.Errorf("failed to create domain: %w", err)
 		}
 
-		Info("Domain %q activated.", domainName)
+		if result != nil {
+			if result.Domain != nil {
+				Info("Domain %q activated.", domainName)
+				return nil
+			}
+			domainVerification := result.DomainVerification
+			if domainVerification != nil {
+				Info("To activate the domain, please add a TXT record into your DNS server:")
+
+				w := tabwriter.NewWriter(os.Stdout, 1, 4, 4, ' ', 0)
+				fmt.Fprintln(w, "DOMAIN\tVALUE")
+				domain, value := domainVerification.GetTxtRecord()
+				fmt.Fprintf(w, "%s\t%s\n\n", domain, value)
+				fmt.Fprintf(w, "The activation may take few minutes, run \"pageship domains\" to check latest activation status.")
+				w.Flush()
+			}
+		}
 		return nil
 	},
 }

docker-compose.yaml

@@ -21,6 +21,8 @@ services:
     image: postgres:11.5
     volumes:
       - data:/data
+    ports:
+      - "5432:5432"
 
   controller:
     image: ghcr.io/oursky/pageship-controller

docs/development/getting-started.md

@@ -12,9 +12,25 @@ Only creating new database migrations requires installing the tool.
 ## Setup environment
 
 Copy `.env.example` to `.env` and adjust as needed.
-We used [`direnv`](direnv.net) to help setup required environment variables.
+We recommend to use [`direnv`](direnv.net) to help setup required environment variables.
 
-By default, the local data is stored in `data.local`.
+Once you've setup direnv, please ensure to have following config to load dotenv.
+
+```
+# ~/.config/direnv/direnv.toml
+[global]
+load_dotenv = true
+```
+
+Load environment variables:
+
+```
+direnv allow .
+```
+
+## Database
+
+By default, the database is `sqlite` and data is stored in `data.local` directory, please create the folder `./data.local/storage` before start development.
 
 ## Running in single site mode
 
@@ -35,7 +51,7 @@ Open the sites at `http://localtest.me:8000/` or `http://dev.localtest.me:8000/`
 ## Running in managed sites mode
 
 ```sh
-go run ./cmd/controller start
+go run ./cmd/controller start --migrate
 ```
 
 Setup pageship command to use `http://api.localtest.me:8001` as the API server.

examples/dev/pageship.toml

@@ -10,7 +10,7 @@ name = "main"
 name = "dev"
 
 [[app.domains]]
-domain="example.com:8001"
+domain="example.com"
 site="dev"
 
 [site]

go.mod

@@ -72,6 +72,7 @@ require (
 	github.com/aws/smithy-go v1.14.0 // indirect
 	github.com/chzyer/readline v1.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/foxcpp/go-mockdns v1.0.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect