diff --git a/mock/src/index.js b/mock/src/index.js index c9386ab..3bbe995 100644 --- a/mock/src/index.js +++ b/mock/src/index.js @@ -7,7 +7,10 @@ app.use(express.json()); const port = process.env.PORT || 9000; app.post("/auth", (req, res) => { - res.status(200).send({ token: "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJPbmxpbmUgSldUIEJ1aWxkZXIiLCJpYXQiOjE2NzYxNjQwMjEsImV4cCI6MTcwNzcwMDAyMSwiYXVkIjoid3d3LmV4YW1wbGUuY29tIiwic3ViIjoianJvY2tldEBleGFtcGxlLmNvbSIsIkdpdmVuTmFtZSI6IkpvaG5ueSIsIlN1cm5hbWUiOiJSb2NrZXQiLCJFbWFpbCI6Impyb2NrZXRAZXhhbXBsZS5jb20iLCJSb2xlIjpbIk1hbmFnZXIiLCJQcm9qZWN0IEFkbWluaXN0cmF0b3IiXX0.7D8EeZkX8uNKMiMuUXpd0isKQwvFK3c3BTiMMnQvzZY"}); + res.status(200).send({ + token: + "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJPbmxpbmUgSldUIEJ1aWxkZXIiLCJpYXQiOjE2NzYxNjQwMjEsImV4cCI6MTcwNzcwMDAyMSwiYXVkIjoid3d3LmV4YW1wbGUuY29tIiwic3ViIjoianJvY2tldEBleGFtcGxlLmNvbSIsIkdpdmVuTmFtZSI6IkpvaG5ueSIsIlN1cm5hbWUiOiJSb2NrZXQiLCJFbWFpbCI6Impyb2NrZXRAZXhhbXBsZS5jb20iLCJSb2xlIjpbIk1hbmFnZXIiLCJQcm9qZWN0IEFkbWluaXN0cmF0b3IiXX0.7D8EeZkX8uNKMiMuUXpd0isKQwvFK3c3BTiMMnQvzZY", + }); }); app.get("/likes", async (req, res) => { @@ -20,6 +23,16 @@ app.get("/posts", async (req, res) => { res.status(200).send({ posts: posts }); }); +app.get("/hashes", async (req, res) => { + res.header("Content-Type", "application/json"); + res.status(200).send({ keys: { somekey: "somevalue" } }); +}); + +app.post("/hashes", async (req, res) => { + res.header("Content-Type", "application/json"); + res.status(201).send(); +}); + // Requests to R2 cloud storage app.put("/", async (req, res) => { res.status(200).send(); diff --git a/pkg/repo/api.go b/pkg/repo/api.go index 2468bc4..da1b7be 100644 --- a/pkg/repo/api.go +++ b/pkg/repo/api.go @@ -1,6 +1,7 @@ package repo import ( + "bytes" "encoding/json" "fmt" "net/http" @@ -110,3 +111,56 @@ func (api *APIService) GetPublishedPosts() (types.Posts, error) { return posts, nil } + +func (api *APIService) GetHashbrown() (types.Hashbrown, error) { + // Use empty struct as default + hashbrown := types.Hashbrown{ + Keys: make(map[string]string), + } + + client := &http.Client{} + url := fmt.Sprintf("%s/hashes", api.Config.APIEndpoint) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return hashbrown, types.WrapErr(err, "failed to build http request") + } + req.Header.Set("Authorization", api.AuthToken) + + resp, err := client.Do(req) + if err != nil { + return hashbrown, types.WrapErr(err, "http request failed") + } + defer resp.Body.Close() + + err = json.NewDecoder(resp.Body).Decode(&hashbrown) + if err != nil { + return hashbrown, types.WrapErr(err, "failed to decode http response body") + } + + return hashbrown, nil +} + +func (api *APIService) PostHashbrown(hashbrown types.Hashbrown) error { + client := &http.Client{} + url := fmt.Sprintf("%s/hashes", api.Config.APIEndpoint) + + hashbrownBytes, err := json.Marshal(hashbrown) + if err != nil { + return types.WrapErr(err, "failed to marshal hashbrown") + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(hashbrownBytes)) + if err != nil { + return types.WrapErr(err, "failed to build http request") + } + req.Header.Set("Authorization", api.AuthToken) + + resp, err := client.Do(req) + if err != nil { + return types.WrapErr(err, "http request failed") + } + defer resp.Body.Close() + + return nil +} diff --git a/pkg/repo/r2.go b/pkg/repo/r2.go index 29d8137..76ae151 100644 --- a/pkg/repo/r2.go +++ b/pkg/repo/r2.go @@ -1,7 +1,6 @@ package repo import ( - "encoding/json" "fmt" "io" "net/http" @@ -58,29 +57,6 @@ func (r2 *R2Service) WriteString(key string, content string) error { return r2.Put(key, strings.NewReader(content)) } -// List returns a full list of keys available in the R2 storage bucket. -func (r2 *R2Service) List() (ListResponse, error) { - client := &http.Client{} - req, err := http.NewRequest("GET", r2.Config.R2Endpoint, nil) - if err != nil { - return ListResponse{}, types.WrapErr(err, "failed to build http request") - } - req.Header.Set("X-Access-Token", r2.Config.R2AccessToken) - resp, err := client.Do(req) - if err != nil { - return ListResponse{}, types.WrapErr(err, "http request failed") - } - defer resp.Body.Close() - - var result ListResponse - err = json.NewDecoder(resp.Body).Decode(&result) - if err != nil { - return ListResponse{}, types.WrapErr(err, "failed to decode response body") - } - - return result, nil -} - // Put writes a single object to the R2 storage bucket. func (r2 *R2Service) Put(key string, object io.Reader) error { client := &http.Client{} diff --git a/pkg/types/types.go b/pkg/types/types.go index b54950a..4ecb3f7 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -115,7 +115,7 @@ type JSONFeedItem struct { // Hashbrown represents a set of hashes for given keys in R2. // Hashbrown is just a fun name. type Hashbrown struct { - Hashes map[string]string `json:"hashes"` + Keys map[string]string `json:"keys"` } // WrapErr wraps an error and returns a new one diff --git a/pkg/web/main.go b/pkg/web/main.go index 4ec28a5..03bb766 100644 --- a/pkg/web/main.go +++ b/pkg/web/main.go @@ -5,7 +5,6 @@ import ( "context" "crypto/md5" "encoding/hex" - "encoding/json" "fmt" "log/slog" "strconv" @@ -206,16 +205,21 @@ func Build(options Options) (string, error) { return "", types.WrapErr(err, "failed one or more build steps") } + // Fetch set of hashes for existing files in R2 + // This will be used to determine if a file has changed, and to determine file deletions + logger.Info("fetching hashes for existing files") + hashbrown, err := api.GetHashbrown() + if err != nil { + logger.Warn("failed to fetch hashbrown, assuming all files are new; " + err.Error()) + } + logger.Info("found " + strconv.Itoa(len(hashbrown.Keys)) + " existing hash(es)") + // Find difference between new and existing files // Mark unused files for deletion logger.Info("finding diff between new and existing files") - existingKeys, err := r2.List() - if err != nil { - return "", types.WrapErr(err, "failed to list existing files") - } keysToDelete := []string{} - for _, existingKey := range existingKeys.Keys { + for existingKey, _ := range hashbrown.Keys { found := false for _, file := range files { if file.GetKey() == existingKey { @@ -224,38 +228,18 @@ func Build(options Options) (string, error) { } } - // TODO: Refactor, make this cleaner - if !found && existingKey != "hashbrown.json" { + if !found { keysToDelete = append(keysToDelete, existingKey) } } logger.Info("marking the following files for deletion: " + fmt.Sprint(keysToDelete)) - // Fetch set of hashes for existing files in R2 - // This will be used to determine if a file has changed - // TODO: Debug - logger.Info("fetching hashes for existing files") - hashbrown := types.Hashbrown{ - Hashes: make(map[string]string), - } - responseBody, err := r2.Get("hashbrown.json") - - if err == nil { - err = json.Unmarshal(responseBody, &hashbrown) - if err != nil { - logger.Warn("failed to unmarshal hashbrown, assuming all files are new; " + err.Error()) - } - } else { - logger.Warn("failed to fetch hashbrown, assuming all files are new; " + err.Error()) - } - logger.Info("found " + strconv.Itoa(len(hashbrown.Hashes)) + " existing hash(es)") - // Write all site files to destination (as well as backup location). // Calculate hashes for each site file to determine whether or not to write, as well as to build a new hashbrown. logger.Info("writing files to destination") newHashbrown := types.Hashbrown{ - Hashes: make(map[string]string), + Keys: make(map[string]string), } maxParallel := 35 @@ -286,13 +270,13 @@ func Build(options Options) (string, error) { hash := hex.EncodeToString(hashBytes) // Add hash to new hashbrown - newHashbrown.Hashes[key] = hash + newHashbrown.Keys[key] = hash writeToDestination := true writeToArchive := options.Archive // Check if file has changed - if existingHash, ok := hashbrown.Hashes[key]; ok && existingHash == hash { + if existingHash, ok := hashbrown.Keys[key]; ok && existingHash == hash { logger.Info("file has not changed, skipping: " + key) writeToDestination = false } @@ -336,11 +320,7 @@ func Build(options Options) (string, error) { } logger.Info("writing new hashbrown") - hashbrownJSON, err := json.Marshal(newHashbrown) - if err != nil { - return "", types.WrapErr(err, "failed to marshal new hashbrown") - } - err = r2.WriteString("hashbrown.json", string(hashbrownJSON)) + err = api.PostHashbrown(newHashbrown) if err != nil { return "", types.WrapErr(err, "failed to write new hashbrown") }