From f671975b2b44f63eecd55adfbbfe581d7daa4fe4 Mon Sep 17 00:00:00 2001 From: Virtanen Riku Date: Fri, 24 Jan 2025 10:39:00 +0200 Subject: [PATCH 1/4] Initial implementation --- .../WriteBlob_build_and_test_on_main.yml | 27 ++ .../WriteBlob_build_and_test_on_push.yml | 28 ++ .github/workflows/WriteBlob_release.yml | 12 + .../CHANGELOG.md | 5 + ...ds.AzureBlobStorage.WriteBlob.Tests.csproj | 24 ++ .../UnitTests.cs | 254 ++++++++++++++++++ .../Frends.AzureBlobStorage.WriteBlob.sln | 40 +++ .../Definitions/Destination.cs | 108 ++++++++ .../Definitions/Options.cs | 18 ++ .../Definitions/Result.cs | 29 ++ .../Definitions/Source.cs | 55 ++++ .../Definitions/Tag.cs | 23 ++ .../Enums/Enums.cs | 92 +++++++ .../Frends.AzureBlobStorage.WriteBlob.csproj | 31 +++ .../FrendsTaskMetadata.json | 7 + .../WriteBlob.cs | 145 ++++++++++ Frends.AzureBlobStorage.WriteBlob/README.md | 26 ++ 17 files changed, 924 insertions(+) create mode 100644 .github/workflows/WriteBlob_build_and_test_on_main.yml create mode 100644 .github/workflows/WriteBlob_build_and_test_on_push.yml create mode 100644 .github/workflows/WriteBlob_release.yml create mode 100644 Frends.AzureBlobStorage.WriteBlob/CHANGELOG.md create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/Frends.AzureBlobStorage.WriteBlob.Tests.csproj create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.sln create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Options.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Source.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Tag.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Enums/Enums.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/FrendsTaskMetadata.json create mode 100644 Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs create mode 100644 Frends.AzureBlobStorage.WriteBlob/README.md diff --git a/.github/workflows/WriteBlob_build_and_test_on_main.yml b/.github/workflows/WriteBlob_build_and_test_on_main.yml new file mode 100644 index 0000000..309d9b0 --- /dev/null +++ b/.github/workflows/WriteBlob_build_and_test_on_main.yml @@ -0,0 +1,27 @@ +name: WriteBlob_build_main + +on: + push: + branches: + - main + paths: + - 'Frends.AzureBlobStorage.WriteBlob/**' + workflow_dispatch: + +jobs: + build: + uses: FrendsPlatform/FrendsTasks/.github/workflows/build_main.yml@main + with: + workdir: Frends.AzureBlobStorage.WriteBlob + env_var_name_1: Frends_AzureBlobStorage_ConnString + env_var_name_2: Frends_AzureBlobStorage_AppID + env_var_name_3: Frends_AzureBlobStorage_ClientSecret + env_var_name_4: Frends_AzureBlobStorage_TenantID + env_var_name_5: Frends_AzureBlobStorage_SASToken + secrets: + badge_service_api_key: ${{ secrets.BADGE_SERVICE_API_KEY }} + env_var_value_1: ${{ secrets.Frends_AZUREBLOBSTORAGE_CONNSTRING }} + env_var_value_2: ${{ secrets.Frends_AZUREBLOBSTORAGE_APPID }} + env_var_value_3: ${{ secrets.Frends_AZUREBLOBSTORAGE_CLIENTSECRET }} + env_var_value_4: ${{ secrets.Frends_AZUREBLOBSTORAGE_TENANTID }} + env_var_value_5: ${{ secrets.Frends_AZUREBLOBSTORAGE_SASTOKEN }} diff --git a/.github/workflows/WriteBlob_build_and_test_on_push.yml b/.github/workflows/WriteBlob_build_and_test_on_push.yml new file mode 100644 index 0000000..5bd76d4 --- /dev/null +++ b/.github/workflows/WriteBlob_build_and_test_on_push.yml @@ -0,0 +1,28 @@ +name: WriteBlob_push + +on: + push: + branches-ignore: + - main + paths: + - 'Frends.AzureBlobStorage.WriteBlob/**' + workflow_dispatch: + +jobs: + build: + uses: FrendsPlatform/FrendsTasks/.github/workflows/build_test.yml@main + with: + workdir: Frends.AzureBlobStorage.WriteBlob + env_var_name_1: Frends_AzureBlobStorage_ConnString + env_var_name_2: Frends_AzureBlobStorage_AppID + env_var_name_3: Frends_AzureBlobStorage_ClientSecret + env_var_name_4: Frends_AzureBlobStorage_TenantID + env_var_name_5: Frends_AzureBlobStorage_SASToken + secrets: + badge_service_api_key: ${{ secrets.BADGE_SERVICE_API_KEY }} + test_feed_api_key: ${{ secrets.TASKS_TEST_FEED_API_KEY }} + env_var_value_1: ${{ secrets.Frends_AZUREBLOBSTORAGE_CONNSTRING }} + env_var_value_2: ${{ secrets.Frends_AZUREBLOBSTORAGE_APPID }} + env_var_value_3: ${{ secrets.Frends_AZUREBLOBSTORAGE_CLIENTSECRET }} + env_var_value_4: ${{ secrets.Frends_AZUREBLOBSTORAGE_TENANTID }} + env_var_value_5: ${{ secrets.Frends_AZUREBLOBSTORAGE_SASTOKEN }} \ No newline at end of file diff --git a/.github/workflows/WriteBlob_release.yml b/.github/workflows/WriteBlob_release.yml new file mode 100644 index 0000000..992ad35 --- /dev/null +++ b/.github/workflows/WriteBlob_release.yml @@ -0,0 +1,12 @@ +name: WriteBlob_release + +on: + workflow_dispatch: + +jobs: + build: + uses: FrendsPlatform/FrendsTasks/.github/workflows/release.yml@main + with: + workdir: Frends.AzureBlobStorage.WriteBlob + secrets: + feed_api_key: ${{ secrets.TASKS_FEED_API_KEY }} diff --git a/Frends.AzureBlobStorage.WriteBlob/CHANGELOG.md b/Frends.AzureBlobStorage.WriteBlob/CHANGELOG.md new file mode 100644 index 0000000..c690e62 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## [1.0.0] - 2025-01-24 +### Added +- Initial implementation of Frends.AzureBlobStorage.WriteBlob. diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/Frends.AzureBlobStorage.WriteBlob.Tests.csproj b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/Frends.AzureBlobStorage.WriteBlob.Tests.csproj new file mode 100644 index 0000000..d2a0686 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/Frends.AzureBlobStorage.WriteBlob.Tests.csproj @@ -0,0 +1,24 @@ + + + + net6.0 + false + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs new file mode 100644 index 0000000..16be2d2 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs @@ -0,0 +1,254 @@ +using NUnit.Framework; +using Azure.Storage.Blobs; +using Frends.AzureBlobStorage.WriteBlob.Definitions; +using Frends.AzureBlobStorage.WriteBlob.Enums; +using System; +using System.Globalization; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Azure.Storage.Blobs.Models; +using System.Collections.Generic; +using Azure.Identity; + +namespace Frends.AzureBlobStorage.WriteBlob.Tests; + +[TestFixture] +public class UnitTests +{ + private readonly string _connectionString = Environment.GetEnvironmentVariable("Frends_AzureBlobStorage_ConnString"); + private string _containerName; + private readonly string _appID = Environment.GetEnvironmentVariable("Frends_AzureBlobStorage_AppID"); + private readonly string _clientSecret = Environment.GetEnvironmentVariable("Frends_AzureBlobStorage_ClientSecret"); + private readonly string _tenantID = Environment.GetEnvironmentVariable("Frends_AzureBlobStorage_TenantID"); + private readonly string _uri = "https://stataskdevelopment.blob.core.windows.net"; + private readonly string _sasToken = Environment.GetEnvironmentVariable("Frends_AzureBlobStorage_SASToken"); + private readonly Tag[] _tags = new[] { new Tag { Name = "TagName", Value = "TagValue" } }; + private readonly string _container = "const-test-container"; + private Destination _destination; + private Source _source; + private Options _options; + private readonly string _testContent = "This is test data"; + + [SetUp] + public async Task TestSetup() + { + _containerName = $"test-container{DateTime.Now.ToString("mmssffffff", CultureInfo.InvariantCulture)}"; + + await CreateBlobContainer(_connectionString, _containerName); + + _source = new Source + { + SourceType = SourceType.String, + ContentString = _testContent, + ContentBytes = Encoding.UTF8.GetBytes(_testContent), + Encoding = FileEncoding.UTF8 + }; + + _destination = new Destination + { + ConnectionMethod = ConnectionMethod.ConnectionString, + ContainerName = _containerName, + ConnectionString = _connectionString, + CreateContainerIfItDoesNotExist = false, + BlobName = $"testblob_{Guid.NewGuid()}", + Tags = null, + HandleExistingFile = HandleExistingFile.Overwrite, + TenantID = _tenantID, + ApplicationID = _appID, + Uri = _uri, + ClientSecret = _clientSecret, + }; + + _options = new Options() { ThrowErrorOnFailure = true }; + } + + [TearDown] + public async Task CleanUp() + { + await DeleteBlobContainer(_containerName); + } + + [Test] + public async Task WriteBlob_TestWriteFromString() + { + // Connection string + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + + // OAuth + _destination.BlobName = $"testblob_{Guid.NewGuid()}"; + _destination.ConnectionMethod = ConnectionMethod.OAuth2; + result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + } + + [Test] + public async Task WriteBlob_TestWriteFromByteArray() + { + _source.SourceType = SourceType.Bytes; + + // Connection string + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + + // OAuth + _destination.BlobName = $"testblob_{Guid.NewGuid()}"; + _destination.ConnectionMethod = ConnectionMethod.OAuth2; + result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + } + + [Test] + public async Task WriteBlob_TestFolderBlobName() + { + // Connection string + _destination.BlobName = $"C:\\folder\\testBlob_{Guid.NewGuid()}"; + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + + // OAuth + _destination.BlobName = $"C:\\folder\\testBlob_{Guid.NewGuid()}"; + _destination.ConnectionMethod = ConnectionMethod.OAuth2; + result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + } + + [Test] + public async Task WriteBlob_TestEncoding() + { + var encodings = new List() + { + FileEncoding.UTF8, + FileEncoding.Default, + FileEncoding.ASCII, + FileEncoding.WINDOWS1252, + FileEncoding.Other + }; + + _source.FileEncodingString = "windows-1251"; + + foreach (var encoding in encodings) + { + _source.Encoding = encoding; + + // Connection string + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success, $"Encoding: {encoding}"); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + + // OAuth + _destination.BlobName = $"testblob_{Guid.NewGuid()}"; + _destination.ConnectionMethod = ConnectionMethod.OAuth2; + result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success, $"Encoding: {encoding}"); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + } + } + + [Test] + public async Task WriteBlob_TestCreateContainer() + { + _destination.CreateContainerIfItDoesNotExist = true; + + // Connection string + _destination.ContainerName = $"test-container{DateTime.Now.ToString("mmssffffff", CultureInfo.InvariantCulture)}"; + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + + var blobServiceClient = new BlobServiceClient(_destination.ConnectionString); + var containerClient = blobServiceClient.GetBlobContainerClient(_destination.ContainerName); + Assert.IsTrue(containerClient.Exists()); + + await DeleteBlobContainer(_destination.ContainerName); + + // OAuth + _destination.ConnectionString = null; + _destination.ContainerName = $"test-container{DateTime.Now.ToString("mmssffffff", CultureInfo.InvariantCulture)}"; + _destination.BlobName = $"testblob_{Guid.NewGuid()}"; + _destination.ConnectionMethod = ConnectionMethod.OAuth2; + result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + + containerClient = blobServiceClient.GetBlobContainerClient(_destination.ContainerName); + Assert.IsTrue(containerClient.Exists()); + + await DeleteBlobContainer(_destination.ContainerName); + } + + [Test] + public void WriteBlob_InvalidConnectionString_ShouldThrowException() + { + _destination.ConnectionString = "DefaultEndpointsProtocol=https;AccountName=invalid;AccountKey=InvalidAccountKey;EndpointSuffix=core.windows.net"; // Simulate an invalid connection string + + var ex = Assert.ThrowsAsync(async () => await AzureBlobStorage.WriteBlob(_source, _destination, _options, default)); + Assert.AreEqual("No valid combination of account information found.", ex.Message); + } + + [Test] + public void WriteBlob_InvalidOAuth2_ShouldThrowException() + { + _destination.ConnectionMethod = ConnectionMethod.OAuth2; + _destination.ClientSecret = "InvalidClientSecret"; + + var ex = Assert.ThrowsAsync(async () => await AzureBlobStorage.WriteBlob(_source, _destination, _options, default)); + Assert.IsTrue(ex.Message.Contains("ClientSecretCredential authentication failed")); + } + + [Test] + public async Task WriteBlob_Tags() + { + _destination.Tags = _tags; + + // Connection string + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + } + + [Test] + public async Task WriteBlob_SasToken() + { + _destination.ConnectionMethod = ConnectionMethod.SASToken; + _destination.SASToken = _sasToken; + _destination.ContainerName = _container; + + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); + } + + private async static Task CreateBlobContainer(string connectionString, string containerName) + { + var blobServiceClient = new BlobServiceClient(connectionString); + var container = blobServiceClient.GetBlobContainerClient(containerName); + await container.CreateIfNotExistsAsync(PublicAccessType.None, null, null); + } + + private async Task DeleteBlobContainer(string containerName) + { + var blobServiceClient = new BlobServiceClient(_connectionString); + var container = blobServiceClient.GetBlobContainerClient(containerName); + await container.DeleteIfExistsAsync(); + } + + private async Task BlobExists(string containerName, string blobName, string expected) + { + var blobServiceClient = new BlobServiceClient(_connectionString); + var container = blobServiceClient.GetBlobContainerClient(containerName); + var blob = container.GetBlobClient(blobName); + if (!blob.Exists()) + return false; + + var blobClient = new BlobClient(_connectionString, _destination.ContainerName, _destination.BlobName); + var blobDownload = await blobClient.DownloadAsync(); + + using var reader = new StreamReader(blobDownload.Value.Content); + var content = await reader.ReadToEndAsync(); + return content == expected; + } +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.sln b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.sln new file mode 100644 index 0000000..356c649 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.sln @@ -0,0 +1,40 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29613.14 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Frends.AzureBlobStorage.WriteBlob", "Frends.AzureBlobStorage.WriteBlob\Frends.AzureBlobStorage.WriteBlob.csproj", "{35C305C0-8108-4A98-BB1D-AFE5C926239E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Frends.AzureBlobStorage.WriteBlob.Tests", "Frends.AzureBlobStorage.WriteBlob.Tests\Frends.AzureBlobStorage.WriteBlob.Tests.csproj", "{8CA92187-8E4F-4414-803B-EC899479022E}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{78F7F22E-6E20-4BCE-8362-0C558568B729}" + ProjectSection(SolutionItems) = preProject + CHANGELOG.md = CHANGELOG.md + ..\.github\workflows\WriteBlob_build_and_test_on_main.yml = ..\.github\workflows\WriteBlob_build_and_test_on_main.yml + ..\.github\workflows\WriteBlob_build_and_test_on_push.yml = ..\.github\workflows\WriteBlob_build_and_test_on_push.yml + ..\.github\workflows\WriteBlob_release.yml = ..\.github\workflows\WriteBlob_release.yml + README.md = README.md + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {35C305C0-8108-4A98-BB1D-AFE5C926239E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {35C305C0-8108-4A98-BB1D-AFE5C926239E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {35C305C0-8108-4A98-BB1D-AFE5C926239E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {35C305C0-8108-4A98-BB1D-AFE5C926239E}.Release|Any CPU.Build.0 = Release|Any CPU + {8CA92187-8E4F-4414-803B-EC899479022E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8CA92187-8E4F-4414-803B-EC899479022E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8CA92187-8E4F-4414-803B-EC899479022E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8CA92187-8E4F-4414-803B-EC899479022E}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {55BC6629-85C9-48D8-8CA2-B0046AF1AF4B} + EndGlobalSection +EndGlobal diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs new file mode 100644 index 0000000..bcb24f0 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs @@ -0,0 +1,108 @@ +using System.ComponentModel; +using System.ComponentModel.DataAnnotations; +using Frends.AzureBlobStorage.WriteBlob.Enums; + +namespace Frends.AzureBlobStorage.WriteBlob.Definitions; + +/// +/// Destination parameters. +/// +public class Destination +{ + /// + /// Connection method to be used to connect to Azure Blob Storage. + /// + /// ConnectionMethod.ConnectionString + [DefaultValue(ConnectionMethod.ConnectionString)] + public ConnectionMethod ConnectionMethod { get; set; } + + /// + /// Connection string for Azure blob storage. + /// + /// DefaultEndpointsProtocol=https;AccountName=accountname;AccountKey=Pdlrxyz==;EndpointSuffix=core.windows.net + [UIHint(nameof(ConnectionMethod), "", ConnectionMethod.ConnectionString)] + [PasswordPropertyText] + [DisplayFormat(DataFormatString = "Text")] + public string ConnectionString { get; set; } + + /// + /// The base URI for the Azure Storage container. + /// Required for SAS and OAuth 2 authentication methods. + /// + /// https://{account_name}.blob.core.windows.net + [UIHint(nameof(ConnectionMethod), "", ConnectionMethod.OAuth2, ConnectionMethod.SASToken)] + public string Uri { get; set; } + + /// + /// Application (Client) ID of Azure AD Application. + /// + /// Y6b1hf2a-80e2-xyz2-qwer3h-3a7c3a8as4b7f + [UIHint(nameof(ConnectionMethod), "", ConnectionMethod.OAuth2)] + public string ApplicationID { get; set; } + + /// + /// Tenant ID of Azure Tenant. + /// + /// Y6b1hf2a-80e2-xyz2-qwer3h-3a7c3a8as4b7f + [UIHint(nameof(ConnectionMethod), "", ConnectionMethod.OAuth2)] + public string TenantID { get; set; } + + /// + /// Client Secret of Azure AD Application. + /// + /// Password! + [UIHint(nameof(ConnectionMethod), "", ConnectionMethod.OAuth2)] + [PasswordPropertyText] + public string ClientSecret { get; set; } + + /// + /// A shared access signature to use when connecting to Azure storage container. + /// Grants restricted access rights to Azure Storage resources when combined with URI. + /// + /// sv=2021-04-10&se=2022-04-10T10%3A431Z&sr=c&sp=l&sig=ZJg983RovE%2BZXI + [UIHint(nameof(ConnectionMethod), "", ConnectionMethod.SASToken)] + [PasswordPropertyText] + public string SASToken { get; set; } + + /// + /// Name of the Azure Blob Storage container. + /// Task will convert all letters to lowercase. + /// See more info: https://learn.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names + /// + /// examplecontainer + [DisplayFormat(DataFormatString = "Text")] + public string ContainerName { get; set; } + + /// + /// Name of the blob. Blob name can also be folder structure and folders will be created to Blob Storage. + /// See more info: https://learn.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names + /// + /// BlobName.txt; C:\folder\blobName.txt + [DisplayFormat(DataFormatString = "Text")] + public string BlobName { get; set; } + + /// + /// Tags for the block or append blob. + /// + /// {name, value} + public Tag[] Tags { get; set; } + + /// + /// Determines if the container should be created if it does not exist. + /// See https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata for naming rules. + /// + /// false + [UIHint(nameof(ConnectionMethod), "", ConnectionMethod.ConnectionString)] + [DefaultValue(false)] + public bool CreateContainerIfItDoesNotExist { get; set; } + + /// + /// How the existing blob will be handled. + /// Append: Append the blob with Source.SourceFile. Block and Page blobs will be downloaded as a temp file which will be deleted after local append and upload processes are complete. No downloading needed for Append Blob. + /// Overwrite: The original blob will be deleted before uploading the new one. + /// Error: Depending on Options.ThrowErrorOnFailure, throw an exception or Result will contain an error message instead of the blob's URL. + /// + /// HandleExistingFile.Error + [DefaultValue(HandleExistingFile.Error)] + public HandleExistingFile HandleExistingFile { get; set; } +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Options.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Options.cs new file mode 100644 index 0000000..5e28cc2 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Options.cs @@ -0,0 +1,18 @@ +using System.ComponentModel; + +namespace Frends.AzureBlobStorage.WriteBlob.Definitions; + +/// +/// Optional parameters. +/// +public class Options +{ + + /// + /// True: Throw an exception. + /// False: If the error is ignorable, such as when a Blob already exists, the error will be added to the Result.ErrorMessages list instead of stopping the Task. + /// + /// true + [DefaultValue(true)] + public bool ThrowErrorOnFailure { get; set; } +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs new file mode 100644 index 0000000..3acfd43 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs @@ -0,0 +1,29 @@ +using Azure.Storage.Blobs.Models; + +namespace Frends.AzureBlobStorage.WriteBlob.Definitions; + +/// +/// Task's result. +/// +public class Result +{ + /// + /// Operation complete. + /// Operation is seens as completed if an ignorable error has occured and Options.ThrowErrorOnFailure is set to false. + /// + /// true + public bool Success { get; private set; } + + /// + /// This object contains the source file path and the URL of the blob. + /// If an ignorable error occurs, such as when a blob already exists and Options.ThrowErrorOnFailure is set to false, the URL will be replaced with the corresponding error message.age. + /// + /// { { c:\temp\examplefile.txt, https://storage.blob.core.windows.net/container/examplefile.txt }, { c:\temp\examplefile2.txt, Blob examplefile2 already exists. } } + public BlobContentInfo Info { get; private set; } + + internal Result(bool success, BlobContentInfo info) + { + Success = success; + Info = info; + } +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Source.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Source.cs new file mode 100644 index 0000000..3ccd71d --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Source.cs @@ -0,0 +1,55 @@ +using System.ComponentModel; +using System.ComponentModel.DataAnnotations; +using Frends.AzureBlobStorage.WriteBlob.Enums; + +namespace Frends.AzureBlobStorage.WriteBlob.Definitions; + +/// +/// Source parameters. +/// +public class Source +{ + /// + /// Selection of source types. + /// + /// SourceType.String + [DefaultValue(SourceType.Bytes)] + public SourceType SourceType { get; set; } + + /// + /// Source content in string format. + /// + /// This is test + [UIHint(nameof(SourceType), "", SourceType.String)] + public string ContentString { get; set; } + + /// + /// Source content in byte array. + /// + /// VGhpcyBpcyB0ZXN0 + [DisplayFormat(DataFormatString = "Expression")] + [UIHint(nameof(SourceType), "", SourceType.Bytes)] + public byte[] ContentBytes { get; set; } + + /// + /// Set desired content-encoding. + /// Defaults to UTF8 BOM. + /// + /// utf8 + [DefaultValue(FileEncoding.UTF8)] + public FileEncoding Encoding { get; set; } + + /// + /// Enables BOM for UTF-8. + /// + [UIHint(nameof(Encoding), "", FileEncoding.UTF8)] + [DefaultValue(true)] + public bool EnableBOM { get; set; } + + /// + /// Content encoding as string. A partial list of possible encodings: https://en.wikipedia.org/wiki/Windows_code_page#List. + /// + /// windows-1252 + [UIHint(nameof(Encoding), "", FileEncoding.Other)] + public string FileEncodingString { get; set; } +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Tag.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Tag.cs new file mode 100644 index 0000000..ee87694 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Tag.cs @@ -0,0 +1,23 @@ +using System.ComponentModel.DataAnnotations; + +namespace Frends.AzureBlobStorage.WriteBlob.Definitions; + +/// +/// Tag parameters. +/// +public class Tag +{ + /// + /// Name of the tag. + /// + /// Name + [DisplayFormat(DataFormatString = "Text")] + public string Name { get; set; } + + /// + /// Value of the tag. + /// + /// Value + [DisplayFormat(DataFormatString = "Text")] + public string Value { get; set; } +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Enums/Enums.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Enums/Enums.cs new file mode 100644 index 0000000..fb207d8 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Enums/Enums.cs @@ -0,0 +1,92 @@ +namespace Frends.AzureBlobStorage.WriteBlob.Enums; + +/// +/// Upload a single file or entire directory. +/// +public enum SourceType +{ +#pragma warning disable CS1591 // self explanatory + String, + Bytes +#pragma warning restore CS1591 // self explanatory +} + +/// +/// Blob types. +/// +public enum AzureBlobType +{ + /// + /// Made up of blocks like block blobs, but are optimized for append operations. Append blobs are ideal for scenarios such as logging data from virtual machines. + /// + Append, + + /// + /// Store text and binary data. Block blobs are made up of blocks of data that can be managed individually. Block blobs can store up to about 190.7 TiB. + /// + Block, + + /// + /// Store random access files up to 8 TiB in size. Page blobs store virtual hard drive (VHD) files and serve as disks for Azure virtual machines. + /// + Page +} + +/// +/// How to handle an existing blob. +/// +public enum HandleExistingFile +{ + /// + /// An error. + /// + Error, + + /// + /// Overwrite with source file. + /// + Overwrite, + + /// + /// Append blob with 'Source File'. Block and Page blob will be downloaded as temp file which will be deleted after local append and reupload processes are complete. No downloading needed for Append Blob. + /// + Append +} + +/// +/// Connection methods. +/// +public enum ConnectionMethod +{ + /// + /// Authenticate with connectiong string. + /// + ConnectionString, + + /// + /// Authenticate with SAS Token. Requires Storage URI. + /// + SASToken, + + /// + /// OAuth2. + /// + OAuth2 +} + +/// +/// Content encoding. +/// +public enum FileEncoding +{ +#pragma warning disable CS1591 // self explanatory + UTF8, + Default, + ASCII, + WINDOWS1252, +#pragma warning restore CS1591 // self explanatory + /// + /// Other enables users to add other encoding options as string. + /// + Other, +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj new file mode 100644 index 0000000..6328b15 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj @@ -0,0 +1,31 @@ + + + + net6.0 + 1.0.0 + Frends + Frends + Frends + Frends + Frends + MIT + true + Frends Task to write content to Azure Blob Storage. + https://frends.com/ + https://github.com/FrendsPlatform/Frends.AzureBlobStorage/tree/main/Frends.AzureBlobStorage.WriteBlob + + + + + PreserveNewest + + + + + + + + + + + diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/FrendsTaskMetadata.json b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/FrendsTaskMetadata.json new file mode 100644 index 0000000..cf0ec24 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/FrendsTaskMetadata.json @@ -0,0 +1,7 @@ +{ + "Tasks": [ + { + "TaskMethod": "Frends.AzureBlobStorage.WriteBlob.AzureBlobStorage.WriteBlob" + } + ] +} diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs new file mode 100644 index 0000000..cdcd670 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs @@ -0,0 +1,145 @@ +using Azure; +using Azure.Identity; +using Azure.Storage; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Frends.AzureBlobStorage.WriteBlob.Definitions; +using Frends.AzureBlobStorage.WriteBlob.Enums; +using MimeMapping; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace Frends.AzureBlobStorage.WriteBlob; + +/// +/// Azure Blob Storage Task. +/// +public class AzureBlobStorage +{ + /// + /// Frends Task to write content to Azure Blob Storage. + /// [Documentation](https://tasks.frends.com/tasks/frends-tasks/Frends.AzureBlobStorage.WriteBlob) + /// + /// Source parameters. + /// Destination parameters. + /// Optional parameters. + /// Token generated by Frends to stop this Task. + /// Object { bool Success, BlobContentInfo Info } + public static async Task WriteBlob([PropertyTab] Source source, [PropertyTab] Destination destination, [PropertyTab] Options options, CancellationToken cancellationToken) + { + CheckDestinationParameters(destination); + + try + { + BlobClient blobClient; + BlobServiceClient blobServiceClient = null; + + switch (destination.ConnectionMethod) + { + case ConnectionMethod.ConnectionString: + blobServiceClient = new BlobServiceClient(destination.ConnectionString); + blobClient = new BlobClient(destination.ConnectionString, destination.ContainerName.ToLower(), destination.BlobName); + break; + case ConnectionMethod.SASToken: + var blobContainerClient = new BlobContainerClient(new Uri($"{destination.Uri}/{destination.ContainerName}?"), new AzureSasCredential(destination.SASToken)); + blobClient = blobContainerClient.GetBlobClient(destination.BlobName); + break; + case ConnectionMethod.OAuth2: + var serviceURI = new Uri($"{destination.Uri}"); + var credentials = new ClientSecretCredential(destination.TenantID, destination.ApplicationID, destination.ClientSecret, new ClientSecretCredentialOptions()); + blobServiceClient = new BlobServiceClient(serviceURI, credentials); + var uri = new Uri($"{destination.Uri}/{destination.ContainerName.ToLower()}/{destination.BlobName}"); + blobClient = new BlobClient(uri, credentials); + break; + default: throw new NotSupportedException(); + } + + if (destination.CreateContainerIfItDoesNotExist && (destination.ConnectionMethod is ConnectionMethod.ConnectionString || destination.ConnectionMethod is ConnectionMethod.OAuth2)) + await CreateContainerIfItDoesNotExist(blobServiceClient, destination.ContainerName.ToLower(), cancellationToken); + + var info = await HandleWrite(blobClient, source, destination, cancellationToken); + return new Result(true, info); + } + catch (Exception) + { + if (options.ThrowErrorOnFailure) + throw; + + return new Result(false, null); + } + } + + private static async Task HandleWrite(BlobClient blobClient, Source source, Destination destination, CancellationToken cancellationToken) + { + var overwrite = destination.HandleExistingFile == HandleExistingFile.Overwrite; + var encoding = GetEncoding(source.Encoding, source.FileEncodingString, source.EnableBOM) ?? throw new Exception("Encoding was invalid."); + byte[] bytes = source.SourceType == SourceType.String ? encoding.GetBytes(source.ContentString) : source.ContentBytes; + var tags = destination.Tags != null ? destination.Tags.ToDictionary(tag => tag.Name, tag => tag.Value) : new Dictionary(); + + var uploadOptions = new BlobUploadOptions + { + Conditions = overwrite ? null : new BlobRequestConditions { IfNoneMatch = new ETag("*") }, + HttpHeaders = new BlobHttpHeaders { ContentType = MimeUtility.GetMimeMapping(destination.BlobName), ContentEncoding = encoding.WebName }, + Tags = tags.Count > 0 ? tags : null + }; + + return await blobClient.UploadAsync(BinaryData.FromBytes(bytes), uploadOptions, cancellationToken).ConfigureAwait(false); + } + + private static async Task CreateContainerIfItDoesNotExist(BlobServiceClient blobServiceClient, string containerName, CancellationToken cancellationToken) + { + try + { + var container = blobServiceClient.GetBlobContainerClient(containerName); + await container.CreateIfNotExistsAsync(PublicAccessType.None, null, null, cancellationToken); + } + catch (Exception ex) + { + throw new Exception($"An error occured while checking if container exists or while creating a new container. {ex}"); + } + } + + private static Encoding GetEncoding(FileEncoding encoding, string encodingString, bool enableBom) + { + return encoding switch + { + FileEncoding.UTF8 => enableBom ? new UTF8Encoding(true) : new UTF8Encoding(false), + FileEncoding.ASCII => new ASCIIEncoding(), + FileEncoding.Default => Encoding.Default, + FileEncoding.WINDOWS1252 => CodePagesEncodingProvider.Instance.GetEncoding("windows-1252"), + FileEncoding.Other => CodePagesEncodingProvider.Instance.GetEncoding(encodingString), + _ => throw new ArgumentOutOfRangeException($"Unknown Encoding type: '{encoding}'."), + }; + } + + private static void CheckDestinationParameters(Destination destination) + { + if (destination.ConnectionMethod is ConnectionMethod.OAuth2 && (string.IsNullOrEmpty(destination.ApplicationID) || string.IsNullOrEmpty(destination.ClientSecret) || string.IsNullOrEmpty(destination.TenantID) || string.IsNullOrEmpty(destination.Uri))) + throw new Exception("Destination.StorageAccountName, Destination.ClientSecret, Destination.ApplicationID and Destination.TenantID parameters can't be empty when Destination.ConnectionMethod = OAuth."); + if (destination.ConnectionMethod is ConnectionMethod.ConnectionString && string.IsNullOrEmpty(destination.ConnectionString)) + throw new Exception("Destination.ConnectionString parameter can't be empty when Destination.ConnectionMethod = ConnectionString."); + if (destination.ConnectionMethod is ConnectionMethod.SASToken && (string.IsNullOrEmpty(destination.SASToken) || string.IsNullOrEmpty(destination.Uri))) + throw new Exception("Destination.SASToken and Destination.URI parameters can't be empty when Destination.ConnectionMethod = SASToken."); + if (string.IsNullOrEmpty(destination.ContainerName)) + throw new Exception("Destination.ContainerName parameter can't be empty."); + if (string.IsNullOrEmpty(destination.BlobName)) + throw new Exception("Blob name parameter can't be empty."); + ValidateContainerName(destination.ContainerName); + } + + private static void ValidateContainerName(string container) + { + if (container.Length < 3) + throw new Exception("Container name is too short. Name needs to be between 3 and 63 characters."); + if (container.Length > 63) + throw new Exception("Container name is too long. Name needs to be between 3 and 63 characters."); + if (!Regex.IsMatch(container, @"^[a-z0-9]+(-[a-z0-9]+)*$")) + throw new Exception("Container name includes invalid characters. See more information: https://learn.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names"); + } +} \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/README.md b/Frends.AzureBlobStorage.WriteBlob/README.md new file mode 100644 index 0000000..0f6e1b9 --- /dev/null +++ b/Frends.AzureBlobStorage.WriteBlob/README.md @@ -0,0 +1,26 @@ +# Frends.AzureBlobStorage.WriteBlob +Frends Task to write blobs to Azure Blob Storage asynchronously. + +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) +[![Build](https://github.com/FrendsPlatform/Frends.AzureBlobStorage/actions/workflows/WriteBlob_build_and_test_on_main.yml/badge.svg)](https://github.com/FrendsPlatform/Frends.AzureBlobStorage/actions) +![Coverage](https://app-github-custom-badges.azurewebsites.net/Badge?key=FrendsPlatform/Frends.AzureBlobStorage/Frends.AzureBlobStorage.WriteBlob|main) + +# Installing + +You can install the Task via Frends UI Task View. + +## Building + + +Rebuild the project + +`dotnet build` + +Run tests + +`dotnet test` + + +Create a NuGet package + +`dotnet pack --configuration Release` From 3176be1f1ed5a42f4085114cafb22cf13f810fe6 Mon Sep 17 00:00:00 2001 From: Virtanen Riku Date: Fri, 24 Jan 2025 10:55:04 +0200 Subject: [PATCH 2/4] Lint and CodeRabbit fixes --- .../Frends.AzureBlobStorage.WriteBlob.csproj | 6 +++--- .../Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj index 6328b15..3e973f0 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.csproj @@ -22,9 +22,9 @@ - - - + + + diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs index cdcd670..fe33eb1 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs @@ -62,7 +62,7 @@ public static async Task WriteBlob([PropertyTab] Source source, [Propert if (destination.CreateContainerIfItDoesNotExist && (destination.ConnectionMethod is ConnectionMethod.ConnectionString || destination.ConnectionMethod is ConnectionMethod.OAuth2)) await CreateContainerIfItDoesNotExist(blobServiceClient, destination.ContainerName.ToLower(), cancellationToken); - + var info = await HandleWrite(blobClient, source, destination, cancellationToken); return new Result(true, info); } @@ -101,7 +101,7 @@ private static async Task CreateContainerIfItDoesNotExist(BlobServiceClient blob } catch (Exception ex) { - throw new Exception($"An error occured while checking if container exists or while creating a new container. {ex}"); + throw new Exception($"An error occurred while checking if container exists or while creating a new container.", ex); } } @@ -121,7 +121,7 @@ private static Encoding GetEncoding(FileEncoding encoding, string encodingString private static void CheckDestinationParameters(Destination destination) { if (destination.ConnectionMethod is ConnectionMethod.OAuth2 && (string.IsNullOrEmpty(destination.ApplicationID) || string.IsNullOrEmpty(destination.ClientSecret) || string.IsNullOrEmpty(destination.TenantID) || string.IsNullOrEmpty(destination.Uri))) - throw new Exception("Destination.StorageAccountName, Destination.ClientSecret, Destination.ApplicationID and Destination.TenantID parameters can't be empty when Destination.ConnectionMethod = OAuth."); + throw new Exception("Destination.Uri, Destination.ClientSecret, Destination.ApplicationID, and Destination.TenantID parameters can't be empty when Destination.ConnectionMethod = OAuth2."); if (destination.ConnectionMethod is ConnectionMethod.ConnectionString && string.IsNullOrEmpty(destination.ConnectionString)) throw new Exception("Destination.ConnectionString parameter can't be empty when Destination.ConnectionMethod = ConnectionString."); if (destination.ConnectionMethod is ConnectionMethod.SASToken && (string.IsNullOrEmpty(destination.SASToken) || string.IsNullOrEmpty(destination.Uri))) From 9267b6750b875c9c76738c2147ee1d99a0cec54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksi=20Hyt=C3=B6nen?= Date: Thu, 30 Jan 2025 14:02:28 +0200 Subject: [PATCH 3/4] ISSUE-97: Added compression option and URI to result --- .../UnitTests.cs | 10 +++++ .../Definitions/Destination.cs | 6 +++ .../Definitions/Result.cs | 8 +++- .../WriteBlob.cs | 44 +++++++++++++++---- 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs index 16be2d2..5be4d36 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs @@ -58,6 +58,7 @@ public async Task TestSetup() ApplicationID = _appID, Uri = _uri, ClientSecret = _clientSecret, + Compress = false }; _options = new Options() { ThrowErrorOnFailure = true }; @@ -222,6 +223,15 @@ public async Task WriteBlob_SasToken() Assert.IsTrue(await BlobExists(_destination.ContainerName, _destination.BlobName, _testContent)); } + [Test] + public async Task WriteBlob_Compress() + { + _destination.Compress = true; + var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); + Assert.IsTrue(result.Success); + Assert.AreEqual($"https://stataskdevelopment.blob.core.windows.net/{_destination.ContainerName}/{_destination.BlobName}.gz", result.Uri); + } + private async static Task CreateBlobContainer(string connectionString, string containerName) { var blobServiceClient = new BlobServiceClient(connectionString); diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs index bcb24f0..e65bd02 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Destination.cs @@ -105,4 +105,10 @@ public class Destination /// HandleExistingFile.Error [DefaultValue(HandleExistingFile.Error)] public HandleExistingFile HandleExistingFile { get; set; } + + /// + /// Should the string be compressed before sending? + /// + [DefaultValue(false)] + public bool Compress { get; set; } } \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs index 3acfd43..6dc9411 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/Definitions/Result.cs @@ -21,9 +21,15 @@ public class Result /// { { c:\temp\examplefile.txt, https://storage.blob.core.windows.net/container/examplefile.txt }, { c:\temp\examplefile2.txt, Blob examplefile2 already exists. } } public BlobContentInfo Info { get; private set; } - internal Result(bool success, BlobContentInfo info) + /// + /// URI of uploaded file. + /// + public string Uri { get; private set; } + + internal Result(bool success, BlobContentInfo info, string uri) { Success = success; Info = info; + Uri = uri; } } \ No newline at end of file diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs index fe33eb1..70cc3ce 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs @@ -1,6 +1,5 @@ using Azure; using Azure.Identity; -using Azure.Storage; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Frends.AzureBlobStorage.WriteBlob.Definitions; @@ -9,6 +8,8 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.IO.Compression; +using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; @@ -30,7 +31,7 @@ public class AzureBlobStorage /// Destination parameters. /// Optional parameters. /// Token generated by Frends to stop this Task. - /// Object { bool Success, BlobContentInfo Info } + /// Object { bool Success, BlobContentInfo Info, string Uri } public static async Task WriteBlob([PropertyTab] Source source, [PropertyTab] Destination destination, [PropertyTab] Options options, CancellationToken cancellationToken) { CheckDestinationParameters(destination); @@ -39,22 +40,23 @@ public static async Task WriteBlob([PropertyTab] Source source, [Propert { BlobClient blobClient; BlobServiceClient blobServiceClient = null; + string blobName = destination.Compress ? destination.BlobName + ".gz" : destination.BlobName; switch (destination.ConnectionMethod) { case ConnectionMethod.ConnectionString: blobServiceClient = new BlobServiceClient(destination.ConnectionString); - blobClient = new BlobClient(destination.ConnectionString, destination.ContainerName.ToLower(), destination.BlobName); + blobClient = new BlobClient(destination.ConnectionString, destination.ContainerName.ToLower(), blobName); break; case ConnectionMethod.SASToken: var blobContainerClient = new BlobContainerClient(new Uri($"{destination.Uri}/{destination.ContainerName}?"), new AzureSasCredential(destination.SASToken)); - blobClient = blobContainerClient.GetBlobClient(destination.BlobName); + blobClient = blobContainerClient.GetBlobClient(blobName); break; case ConnectionMethod.OAuth2: var serviceURI = new Uri($"{destination.Uri}"); var credentials = new ClientSecretCredential(destination.TenantID, destination.ApplicationID, destination.ClientSecret, new ClientSecretCredentialOptions()); blobServiceClient = new BlobServiceClient(serviceURI, credentials); - var uri = new Uri($"{destination.Uri}/{destination.ContainerName.ToLower()}/{destination.BlobName}"); + var uri = new Uri($"{destination.Uri}/{destination.ContainerName.ToLower()}/{blobName}"); blobClient = new BlobClient(uri, credentials); break; default: throw new NotSupportedException(); @@ -64,14 +66,14 @@ public static async Task WriteBlob([PropertyTab] Source source, [Propert await CreateContainerIfItDoesNotExist(blobServiceClient, destination.ContainerName.ToLower(), cancellationToken); var info = await HandleWrite(blobClient, source, destination, cancellationToken); - return new Result(true, info); + return new Result(true, info, blobClient.Uri.ToString()); } catch (Exception) { if (options.ThrowErrorOnFailure) throw; - return new Result(false, null); + return new Result(false, null, null); } } @@ -80,12 +82,14 @@ private static async Task HandleWrite(BlobClient blobClient, So var overwrite = destination.HandleExistingFile == HandleExistingFile.Overwrite; var encoding = GetEncoding(source.Encoding, source.FileEncodingString, source.EnableBOM) ?? throw new Exception("Encoding was invalid."); byte[] bytes = source.SourceType == SourceType.String ? encoding.GetBytes(source.ContentString) : source.ContentBytes; + if (destination.Compress) + bytes = Compress(bytes); var tags = destination.Tags != null ? destination.Tags.ToDictionary(tag => tag.Name, tag => tag.Value) : new Dictionary(); var uploadOptions = new BlobUploadOptions { Conditions = overwrite ? null : new BlobRequestConditions { IfNoneMatch = new ETag("*") }, - HttpHeaders = new BlobHttpHeaders { ContentType = MimeUtility.GetMimeMapping(destination.BlobName), ContentEncoding = encoding.WebName }, + HttpHeaders = new BlobHttpHeaders { ContentType = MimeUtility.GetMimeMapping(destination.Compress ? "gz" : destination.BlobName), ContentEncoding = encoding.WebName }, Tags = tags.Count > 0 ? tags : null }; @@ -142,4 +146,28 @@ private static void ValidateContainerName(string container) if (!Regex.IsMatch(container, @"^[a-z0-9]+(-[a-z0-9]+)*$")) throw new Exception("Container name includes invalid characters. See more information: https://learn.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names"); } + + private static byte[] Compress(byte[] original) + { + using var msi = new MemoryStream(original); + using var mso = new MemoryStream(); + using (var gs = new GZipStream(mso, CompressionMode.Compress)) + { + CopyTo(msi, gs); + } + + return mso.ToArray(); + } + + private static void CopyTo(Stream src, Stream dest) + { + byte[] bytes = new byte[4096]; + + int cnt; + + while ((cnt = src.Read(bytes, 0, bytes.Length)) != 0) + { + dest.Write(bytes, 0, cnt); + } + } } \ No newline at end of file From 8413020bcd51654f9630c9a4c9954a61051569b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksi=20Hyt=C3=B6nen?= Date: Fri, 31 Jan 2025 12:25:57 +0200 Subject: [PATCH 4/4] ISSUE-97: Removed automatic file extension handling from compression --- .../Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs | 2 +- .../Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs index 5be4d36..6328642 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob.Tests/UnitTests.cs @@ -229,7 +229,7 @@ public async Task WriteBlob_Compress() _destination.Compress = true; var result = await AzureBlobStorage.WriteBlob(_source, _destination, _options, default); Assert.IsTrue(result.Success); - Assert.AreEqual($"https://stataskdevelopment.blob.core.windows.net/{_destination.ContainerName}/{_destination.BlobName}.gz", result.Uri); + Assert.AreEqual($"https://stataskdevelopment.blob.core.windows.net/{_destination.ContainerName}/{_destination.BlobName}", result.Uri); } private async static Task CreateBlobContainer(string connectionString, string containerName) diff --git a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs index 70cc3ce..2c67dac 100644 --- a/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs +++ b/Frends.AzureBlobStorage.WriteBlob/Frends.AzureBlobStorage.WriteBlob/WriteBlob.cs @@ -40,23 +40,22 @@ public static async Task WriteBlob([PropertyTab] Source source, [Propert { BlobClient blobClient; BlobServiceClient blobServiceClient = null; - string blobName = destination.Compress ? destination.BlobName + ".gz" : destination.BlobName; switch (destination.ConnectionMethod) { case ConnectionMethod.ConnectionString: blobServiceClient = new BlobServiceClient(destination.ConnectionString); - blobClient = new BlobClient(destination.ConnectionString, destination.ContainerName.ToLower(), blobName); + blobClient = new BlobClient(destination.ConnectionString, destination.ContainerName.ToLower(), destination.BlobName); break; case ConnectionMethod.SASToken: var blobContainerClient = new BlobContainerClient(new Uri($"{destination.Uri}/{destination.ContainerName}?"), new AzureSasCredential(destination.SASToken)); - blobClient = blobContainerClient.GetBlobClient(blobName); + blobClient = blobContainerClient.GetBlobClient(destination.BlobName); break; case ConnectionMethod.OAuth2: var serviceURI = new Uri($"{destination.Uri}"); var credentials = new ClientSecretCredential(destination.TenantID, destination.ApplicationID, destination.ClientSecret, new ClientSecretCredentialOptions()); blobServiceClient = new BlobServiceClient(serviceURI, credentials); - var uri = new Uri($"{destination.Uri}/{destination.ContainerName.ToLower()}/{blobName}"); + var uri = new Uri($"{destination.Uri}/{destination.ContainerName.ToLower()}/{destination.BlobName}"); blobClient = new BlobClient(uri, credentials); break; default: throw new NotSupportedException();