From 6636d1f3a970a5b1b68a6a0c378052f5559b756d Mon Sep 17 00:00:00 2001 From: kumarashit Date: Thu, 25 Feb 2021 22:38:26 +0530 Subject: [PATCH 1/2] Adding support for Azure and GCP Archival and retrieval. Sample code changes required. --- .../multicloud/ObjectArchivalRestore.md | 86 ++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/arch-design/multicloud/ObjectArchivalRestore.md b/arch-design/multicloud/ObjectArchivalRestore.md index 4e27a165..1907e8d7 100644 --- a/arch-design/multicloud/ObjectArchivalRestore.md +++ b/arch-design/multicloud/ObjectArchivalRestore.md @@ -99,6 +99,36 @@ TBD [https://docs.microsoft.com/en-in/azure/storage/blobs/storage-blob-storage-tiers?tabs=azure-portal](https://docs.microsoft.com/en-in/azure/storage/blobs/storage-blob-storage-tiers?tabs=azure-portal) +###### Changes in SODA multi-cloud: +Check the value of request header X-Amz-Storage-Class, set the Storage class in call to SetTier() + +``` +// If the Storage Class is defined from the API request Header, set it else define defaults + var storClass string + + if storageClass != "" { + storClass = storageClass + } else { + if object.Tier == 0 { + // default + object.Tier = utils.Tier1 + } + storClass, err = osdss3.GetNameFromTier(object.Tier, utils.OSTYPE_Azure) + if err != nil { + log.Infof("translate tier[%d] to azure storage class failed", object.Tier) + return result, ErrInternalError + } +} +.... +.... +_, err = blobURL.SetTier(ctx, azblob.AccessTierType(storClass), azblob.LeaseAccessConditions{}) + if err != nil { + log.Errorf("set azure blob tier[%s] failed:%v\n", object.Tier, err) + return result, ErrPutToBackendFailed + } + +``` + #### Retrieval in Azure: Archive Storage currently supports 2 rehydrate priorities, High and Standard, that offers different retrieval latencies ![Retrieval in Azure](resources/RehydrateAzure.png) @@ -111,7 +141,14 @@ Archive Storage currently supports 2 rehydrate priorities, High and Standard, th - Standard priority: The rehydration request will be processed in the order it was received and may take up to 15 hours. - High priority: The rehydration request will be prioritized over Standard requests and may finish in under 1 hour for objects under ten GB in size. - + +###### Changes in SODA multi-cloud +SODA uses azure-storage-blob-go lib to connect to Azure object/blob service +This lib has been updated with version 0.10.x to support **rehydratePriority** +So this lib version need to be updated in go mod + +This also, changes signature of SetTier(). This need to be updated + #### Archival in Google Cloud Storage: ![Archival in GCS](resources/ArchivalGCS.png) @@ -121,6 +158,52 @@ Unlike the "coldest" storage services offered by other Cloud providers, data is [https://cloud.google.com/storage/docs/storage-classes#descriptions](https://cloud.google.com/storage/docs/storage-classes#descriptions) +##### Changes required in SODA multi-cloud +Currently SODA used webrtcn/s3client to connect to GCP object services. This will not help to update storage class. SODA need to use the cloud.google..com/go/storage lib. +GCP support changing storage class of object within a bucket through rwriting the object. To retrive archived object, GCP supports just changing the storage class + +``` +import ( + "context" + "fmt" + "io" + "time" + + "cloud.google.com/go/storage" +) + +// changeObjectStorageClass changes the storage class of a single object. +func changeObjectStorageClass(w io.Writer, bucket, object string) error { + // bucket := "bucket-name" + // object := "object-name" + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("storage.NewClient: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + bkt := client.Bucket(bucket) + obj := bkt.Object(object) + // See the StorageClass documentation for other valid storage classes: + // https://cloud.google.com/storage/docs/storage-classes + newStorageClass := "COLDLINE" + // You can't change an object's storage class directly, the only way is + // to rewrite the object with the desired storage class. + copier := obj.CopierFrom(obj) + copier.StorageClass = newStorageClass + if _, err := copier.Run(ctx); err != nil { + return fmt.Errorf("copier.Run: %v", err) + } + fmt.Fprintf(w, "Object %v in bucket %v had its storage class set to %v\n", object, bucket, newStorageClass) + return nil +} + +From GCP storage doc: https://cloud.google.com/storage/docs/changing-storage-classes#storage-change-object-storage-class-go +``` ### Sample Code @@ -129,6 +212,7 @@ Azure: [https://github.com/Azure/azure-sdk-for-go/](https://github.com/Azure/azu AWS: [https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#PutObjectInput](https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#PutObjectInput) GCP: [https://cloud.google.com/storage/docs/json_api/v1/objects](https://cloud.google.com/storage/docs/json_api/v1/objects) + Sample code for directly uploading into AWS Glacier Storage Class ``` From 84bc08b554a62586eac25a5e999bcc755a6927af Mon Sep 17 00:00:00 2001 From: kumarashit Date: Mon, 8 Mar 2021 18:00:50 +0530 Subject: [PATCH 2/2] Adding details for Azure --- arch-design/multicloud/ObjectArchivalRestore.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch-design/multicloud/ObjectArchivalRestore.md b/arch-design/multicloud/ObjectArchivalRestore.md index 1907e8d7..66ea65c9 100644 --- a/arch-design/multicloud/ObjectArchivalRestore.md +++ b/arch-design/multicloud/ObjectArchivalRestore.md @@ -99,6 +99,10 @@ TBD [https://docs.microsoft.com/en-in/azure/storage/blobs/storage-blob-storage-tiers?tabs=azure-portal](https://docs.microsoft.com/en-in/azure/storage/blobs/storage-blob-storage-tiers?tabs=azure-portal) +1. Value of X-Amz-Storage-Class in the API for archining into Azure is 'Archive' +2. For rehydrating or restoring an archived blob from Azure container, user can provide: a) Tier/Priority i.e. Hot or Standard b) Storage class i.e. the storage class to which the object will be moved to Possible values of Storage classes to which an Archived object can be restored is "Hot" or "Cold". Currently the golang SDK doesn't support RehydratePriority so it will supported in furture. + https://github.com/Azure/azure-storage-blob-go/issues/245 + ###### Changes in SODA multi-cloud: Check the value of request header X-Amz-Storage-Class, set the Storage class in call to SetTier()