From 060b221088db1a14405f175e6f7ca175e859934b Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Thu, 28 Nov 2024 12:40:14 +0000 Subject: [PATCH 1/4] fixed dubious ownership error --- Dockerfile.local | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.local b/Dockerfile.local index 520c169d..808e9ac2 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,9 +1,11 @@ -FROM golang:1.22.6-bullseye as build +FROM golang:1.23-bullseye as build ENV GOCACHE=/go/.go/cache GOPATH=/go/.go/path TZ=Europe/London RUN GOBIN=/bin go install github.com/cespare/reflex@latest +RUN git config --global --add safe.directory /go + # Map between the working directories of dev and live RUN ln -s /go /dp-dataset-api WORKDIR /dp-dataset-api From 7c00181a8304943196c15e0963be3f67e4a198de Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Thu, 28 Nov 2024 12:41:36 +0000 Subject: [PATCH 2/4] url rewriting for getDatasets, getDataset and addDataset --- api/dataset.go | 170 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 143 insertions(+), 27 deletions(-) diff --git a/api/dataset.go b/api/dataset.go index 8fcc2acf..6bac467d 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -5,16 +5,24 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "strings" "time" errs "github.com/ONSdigital/dp-dataset-api/apierrors" + "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/models" dphttp "github.com/ONSdigital/dp-net/v2/http" "github.com/ONSdigital/log.go/v2/log" "github.com/gorilla/mux" ) +type contextKey string + +const ( + ctxPathPrefix contextKey = "pathPrefix" +) + var ( // errors that should return a 403 status datasetsForbidden = map[error]bool{ @@ -79,11 +87,15 @@ func (api *DatasetAPI) getDatasets(w http.ResponseWriter, r *http.Request, limit return nil, 0, err } - if authorised { - return datasets, totalCount, nil + pathPrefix := r.Header.Get("X-Forwarded-Path-Prefix") + ctx = context.WithValue(ctx, ctxPathPrefix, pathPrefix) + datasetsResponse, err := mapResultsAndRewriteLinks(ctx, datasets, authorised) + if err != nil { + log.Error(ctx, "Error mapping results and rewriting links", err) + return nil, 0, err } - return mapResults(datasets), totalCount, nil + return datasetsResponse, totalCount, nil } func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { @@ -102,27 +114,13 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { authorised := api.authenticate(r, logData) var b []byte - var datasetResponse interface{} - - if !authorised { - // User is not authenticated and hence has only access to current sub document - if dataset.Current == nil { - log.Info(ctx, "getDataset endpoint: published dataset not found", logData) - return nil, errs.ErrDatasetNotFound - } - log.Info(ctx, "getDataset endpoint: caller not authorised returning dataset", logData) - - dataset.Current.ID = dataset.ID - datasetResponse = dataset.Current - } else { - // User has valid authentication to get raw dataset document - if dataset == nil { - log.Info(ctx, "getDataset endpoint: published or unpublished dataset not found", logData) - return nil, errs.ErrDatasetNotFound - } - log.Info(ctx, "getDataset endpoint: caller authorised returning dataset current sub document", logData) - datasetResponse = dataset + pathPrefix := r.Header.Get("X-Forwarded-Path-Prefix") + ctx = context.WithValue(ctx, ctxPathPrefix, pathPrefix) + datasetResponse, err := mapResultsAndRewriteLinks(ctx, []*models.DatasetUpdate{dataset}, authorised) + if err != nil { + log.Error(ctx, "Error mapping results and rewriting links", err) + return nil, err } b, err = json.Marshal(datasetResponse) @@ -203,11 +201,11 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) { } dataset.Links.Editions = &models.LinkObject{ - HRef: fmt.Sprintf("%s/datasets/%s/editions", api.host, datasetID), + HRef: fmt.Sprintf("/datasets/%s/editions", datasetID), } dataset.Links.Self = &models.LinkObject{ - HRef: fmt.Sprintf("%s/datasets/%s", api.host, datasetID), + HRef: fmt.Sprintf("/datasets/%s", datasetID), } // Remove latest version from new dataset resource, this cannot be added at this point @@ -499,16 +497,134 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) { log.Info(ctx, "delete dataset", logData) } -func mapResults(results []*models.DatasetUpdate) []*models.Dataset { +func mapResultsAndRewriteLinks(ctx context.Context, results []*models.DatasetUpdate, authorised bool) ([]*models.Dataset, error) { items := []*models.Dataset{} for _, item := range results { + if authorised && item.Current == nil && item.Next != nil { + item.Next.ID = item.ID + err := rewriteAllLinks(ctx, item.Next.Links) + if err != nil { + log.Error(ctx, "unable to rewrite 'next' links", err) + return nil, err + } + items = append(items, item.Next) + continue + } + if item.Current == nil { continue } + item.Current.ID = item.ID + err := rewriteAllLinks(ctx, item.Current.Links) + if err != nil { + log.Error(ctx, "unable to rewrite 'current' links", err) + return nil, err + } items = append(items, item.Current) + + if authorised && item.Next != nil { + item.Next.ID = item.ID + err := rewriteAllLinks(ctx, item.Next.Links) + if err != nil { + log.Error(ctx, "unable to rewrite 'next' links", err) + return nil, err + } + items = append(items, item.Next) + } } - return items + + return items, nil + +} + +func rewriteAllLinks(ctx context.Context, oldLinks *models.DatasetLinks) error { + if oldLinks.AccessRights != nil && oldLinks.AccessRights.HRef != "" { + accessRights, err := URLBuild(ctx, oldLinks.AccessRights.HRef) + if err != nil { + log.Error(ctx, "error rewriting AccessRights link", err) + return err + } + oldLinks.AccessRights.HRef = accessRights + } + + if oldLinks.Editions != nil && oldLinks.Editions.HRef != "" { + editions, err := URLBuild(ctx, oldLinks.Editions.HRef) + if err != nil { + log.Error(ctx, "error rewriting Editions link", err) + return err + } + oldLinks.Editions.HRef = editions + } + + if oldLinks.LatestVersion != nil && oldLinks.LatestVersion.HRef != "" { + latestVersion, err := URLBuild(ctx, oldLinks.LatestVersion.HRef) + if err != nil { + log.Error(ctx, "error rewriting LatestVersion link", err) + return err + } + oldLinks.LatestVersion.HRef = latestVersion + } + + if oldLinks.Self != nil && oldLinks.Self.HRef != "" { + self, err := URLBuild(ctx, oldLinks.Self.HRef) + if err != nil { + log.Error(ctx, "error rewriting Self link", err) + return err + } + oldLinks.Self.HRef = self + } + + if oldLinks.Taxonomy != nil && oldLinks.Taxonomy.HRef != "" { + taxonomy, err := URLBuild(ctx, oldLinks.Taxonomy.HRef) + if err != nil { + log.Error(ctx, "error rewriting Taxonomy link", err) + return err + } + oldLinks.Taxonomy.HRef = taxonomy + } + + return nil +} + +func URLBuild(ctx context.Context, oldURL string) (string, error) { + cfg, err := config.Get() + if err != nil { + log.Error(ctx, "unable to retrieve config", err) + return "", err + } + + parsedURL, err := url.Parse(oldURL) + if err != nil { + return "", fmt.Errorf("error parsing old URL: %v", err) + } + + parsedAPIDomainUrl, err := url.Parse(cfg.APIDomainUrl) + if err != nil { + return "", fmt.Errorf("error parsing old URL: %v", err) + } + + fmt.Printf("\nOld URL: %s\n", parsedURL.String()) + + newPathPrefix := ctx.Value(ctxPathPrefix).(string) + + var newUrl string + + if newPathPrefix != "" { + newUrl, err = url.JoinPath(parsedAPIDomainUrl.String(), newPathPrefix, parsedURL.Path) + if err != nil { + return "", fmt.Errorf("error joining paths: %v", err) + } + } else { + newUrl, err = url.JoinPath(parsedAPIDomainUrl.String(), parsedURL.Path) + if err != nil { + return "", fmt.Errorf("error joining paths: %v", err) + } + } + + fmt.Printf("New URL: %s\n", newUrl) + + return newUrl, nil } func handleDatasetAPIErr(ctx context.Context, err error, w http.ResponseWriter, data log.Data) { From 88ed9020a6e46dd01cf416b8dd8f18f9229d7e71 Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Thu, 28 Nov 2024 12:42:50 +0000 Subject: [PATCH 3/4] added API_DOMAIN_URL config --- config/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/config.go b/config/config.go index 84091429..c7be6dd7 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,7 @@ type Configuration struct { CantabularExportStartTopic string `envconfig:"CANTABULAR_EXPORT_START"` CodeListAPIURL string `envconfig:"CODE_LIST_API_URL"` DatasetAPIURL string `envconfig:"DATASET_API_URL"` + APIDomainUrl string `envconfig:"API_DOMAIN_URL"` WebsiteURL string `envconfig:"WEBSITE_URL"` ZebedeeURL string `envconfig:"ZEBEDEE_URL"` DownloadServiceSecretKey string `envconfig:"DOWNLOAD_SERVICE_SECRET_KEY" json:"-"` @@ -83,6 +84,7 @@ func Get() (*Configuration, error) { CantabularExportStartTopic: "cantabular-export-start", CodeListAPIURL: "http://localhost:22400", DatasetAPIURL: "http://localhost:22000", + APIDomainUrl: "https://api.beta.ons.gov.uk", WebsiteURL: "http://localhost:20000", ZebedeeURL: "http://localhost:8082", ServiceAuthToken: "FD0108EA-825D-411C-9B1D-41EF7727F465", From cd380ecaf802c6c77cc149624141e65e65e559eb Mon Sep 17 00:00:00 2001 From: aryan-patel Date: Thu, 28 Nov 2024 15:02:32 +0000 Subject: [PATCH 4/4] removed need for path prefix --- api/dataset.go | 21 +++------------------ config/config.go | 2 +- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/api/dataset.go b/api/dataset.go index 6bac467d..c91fd994 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -87,8 +87,6 @@ func (api *DatasetAPI) getDatasets(w http.ResponseWriter, r *http.Request, limit return nil, 0, err } - pathPrefix := r.Header.Get("X-Forwarded-Path-Prefix") - ctx = context.WithValue(ctx, ctxPathPrefix, pathPrefix) datasetsResponse, err := mapResultsAndRewriteLinks(ctx, datasets, authorised) if err != nil { log.Error(ctx, "Error mapping results and rewriting links", err) @@ -115,8 +113,6 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { var b []byte - pathPrefix := r.Header.Get("X-Forwarded-Path-Prefix") - ctx = context.WithValue(ctx, ctxPathPrefix, pathPrefix) datasetResponse, err := mapResultsAndRewriteLinks(ctx, []*models.DatasetUpdate{dataset}, authorised) if err != nil { log.Error(ctx, "Error mapping results and rewriting links", err) @@ -606,20 +602,9 @@ func URLBuild(ctx context.Context, oldURL string) (string, error) { fmt.Printf("\nOld URL: %s\n", parsedURL.String()) - newPathPrefix := ctx.Value(ctxPathPrefix).(string) - - var newUrl string - - if newPathPrefix != "" { - newUrl, err = url.JoinPath(parsedAPIDomainUrl.String(), newPathPrefix, parsedURL.Path) - if err != nil { - return "", fmt.Errorf("error joining paths: %v", err) - } - } else { - newUrl, err = url.JoinPath(parsedAPIDomainUrl.String(), parsedURL.Path) - if err != nil { - return "", fmt.Errorf("error joining paths: %v", err) - } + newUrl, err := url.JoinPath(parsedAPIDomainUrl.String(), parsedURL.Path) + if err != nil { + return "", fmt.Errorf("error joining paths: %v", err) } fmt.Printf("New URL: %s\n", newUrl) diff --git a/config/config.go b/config/config.go index c7be6dd7..1455d3e0 100644 --- a/config/config.go +++ b/config/config.go @@ -84,7 +84,7 @@ func Get() (*Configuration, error) { CantabularExportStartTopic: "cantabular-export-start", CodeListAPIURL: "http://localhost:22400", DatasetAPIURL: "http://localhost:22000", - APIDomainUrl: "https://api.beta.ons.gov.uk", + APIDomainUrl: "https://api.beta.ons.gov.uk/v1", WebsiteURL: "http://localhost:20000", ZebedeeURL: "http://localhost:8082", ServiceAuthToken: "FD0108EA-825D-411C-9B1D-41EF7727F465",