Change the value of the multipart upload storage override to medibytes instead of bytes

env variable is now called GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES

Author: tspascoal

README.md

@@ -95,6 +95,24 @@ When the CLI is launched, it logs if a newer version of the CLI is available. Yo
 
 When the CLI is launched, it logs a warning if there are any ongoing [GitHub incidents](https://www.githubstatus.com/) that might affect your use of the CLI. You can skip this check by setting the `GEI_SKIP_STATUS_CHECK` environment variable to `true`. 
 
+### Configuring multipart upload chunk size
+
+Set the `GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES` environment variable to change the archive upload part size. Provide the value in mebibytes (MiB); For example:
+
+```powershell
+# Windows PowerShell
+$env:GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES = "10"
+```
+
+```bash
+# macOS/Linux
+export GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES=10
+```
+
+This sets the chunk size to 10 MiB (10,485,760 bytes). The minimum supported value is 5 MiB, and the default remains 100 MiB.
+
+This might be needed to improve upload reliability in environments with proxies or very slow connections.
+
 ## Contributions
 
 See [Contributing](CONTRIBUTING.md) for more info on how to get involved.

RELEASENOTES.md

@@ -1 +1 @@
-- Added support for configurable multipart upload chunk size for GitHub-owned storage uploads via `GITHUB_OWNED_STORAGE_MULTIPART_BYTES` environment variable (minimum 5 MiB, default 100 MiB) to improve upload reliability in environments with proxies or slow connections
+- Added support for configurable multipart upload chunk size for GitHub-owned storage uploads via `GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES` environment variable (minimum 5 MiB, default 100 MiB) to improve upload reliability in environments with proxies or slow connections

src/Octoshift/Services/ArchiveUploader.cs

@@ -11,13 +11,15 @@ namespace OctoshiftCLI.Services;
 
 public class ArchiveUploader
 {
-    private const int MIN_MULTIPART_BYTES = 5 * 1024 * 1024; // 5 MiB minimum size for multipart upload. Don't allow overrides smaller than this.
+    private const int BYTES_PER_MEBIBYTE = 1024 * 1024;
+    private const int MIN_MULTIPART_MEBIBYTES = 5; // 5 MiB minimum size for multipart upload. Don't allow overrides smaller than this.
+    private const int DEFAULT_MULTIPART_MEBIBYTES = 100;
 
     private readonly GithubClient _client;
     private readonly string _uploadsUrl;
     private readonly OctoLogger _log;
     private readonly EnvironmentVariableProvider _environmentVariableProvider;
-    internal int _streamSizeLimit = 100 * 1024 * 1024; // 100 MiB
+    internal int _streamSizeLimit = DEFAULT_MULTIPART_MEBIBYTES * BYTES_PER_MEBIBYTE; // 100 MiB stored in bytes
     private readonly RetryPolicy _retryPolicy;
 
     public ArchiveUploader(GithubClient client, string uploadsUrl, OctoLogger log, RetryPolicy retryPolicy, EnvironmentVariableProvider environmentVariableProvider)
@@ -169,19 +171,20 @@ private Uri GetNextUrl(IEnumerable<KeyValuePair<string, IEnumerable<string>>> he
 
     private void SetStreamSizeLimitFromEnvironment()
     {
-        var envValue = _environmentVariableProvider.GithubOwnedStorageMultipartBytes();
-        if (!int.TryParse(envValue, out var limit) || limit <= 0)
+        var envValue = _environmentVariableProvider.GithubOwnedStorageMultipartMebibytes();
+        if (!int.TryParse(envValue, out var limitInMebibytes) || limitInMebibytes <= 0)
         {
             return;
         }
 
-        if (limit < MIN_MULTIPART_BYTES)
+        if (limitInMebibytes < MIN_MULTIPART_MEBIBYTES)
         {
-            _log.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_BYTES is set to {limit} bytes, but the minimum value is {MIN_MULTIPART_BYTES} bytes. Using default value of {_streamSizeLimit} bytes.");
+            _log.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES is set to {limitInMebibytes} MiB, but the minimum value is {MIN_MULTIPART_MEBIBYTES} MiB. Using default value of {DEFAULT_MULTIPART_MEBIBYTES} MiB.");
             return;
         }
 
-        _streamSizeLimit = limit;
-        _log.LogInformation($"Multipart upload part size set to {((long)_streamSizeLimit).ToLogFriendlySize()}.");
+        var limitBytes = (int)((long)limitInMebibytes * BYTES_PER_MEBIBYTE);
+        _streamSizeLimit = limitBytes;
+        _log.LogInformation($"Multipart upload part size set to {limitInMebibytes} MB.");
     }
 }

src/Octoshift/Services/EnvironmentVariableProvider.cs

@@ -18,7 +18,7 @@ public class EnvironmentVariableProvider
     private const string SMB_PASSWORD = "SMB_PASSWORD";
     private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK";
     private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK";
-    private const string GITHUB_OWNED_STORAGE_MULTIPART_BYTES = "GITHUB_OWNED_STORAGE_MULTIPART_BYTES";
+    private const string GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES = "GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES";
 
     private readonly OctoLogger _logger;
 
@@ -66,8 +66,8 @@ public virtual string SkipStatusCheck(bool throwIfNotFound = false) =>
     public virtual string SkipVersionCheck(bool throwIfNotFound = false) =>
         GetValue(GEI_SKIP_VERSION_CHECK, throwIfNotFound);
 
-    public virtual string GithubOwnedStorageMultipartBytes(bool throwIfNotFound = false) =>
-        GetValue(GITHUB_OWNED_STORAGE_MULTIPART_BYTES, throwIfNotFound);
+    public virtual string GithubOwnedStorageMultipartMebibytes(bool throwIfNotFound = false) =>
+        GetValue(GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES, throwIfNotFound);
 
     private string GetValue(string name, bool throwIfNotFound)
     {

src/OctoshiftCLI.Tests/Octoshift/Services/ArchiveUploadersTests.cs

@@ -46,21 +46,22 @@ public async Task Upload_Should_Throw_ArgumentNullException_When_Archive_Content
     public void Constructor_Should_Use_Valid_Environment_Variable_Value()
     {
         // Arrange
-        var customSize = 10 * 1024 * 1024; // 10 MiB
+        var customSizeMiB = 10; // 10 MiB
+        var customSizeBytes = customSizeMiB * 1024 * 1024;
         var logMock = TestHelpers.CreateMock<OctoLogger>();
         var githubClientMock = TestHelpers.CreateMock<GithubClient>();
         var environmentVariableProviderMock = TestHelpers.CreateMock<EnvironmentVariableProvider>();
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
-            .Returns(customSize.ToString());
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
+            .Returns(customSizeMiB.ToString());
 
         // Act
         var archiveUploader = new ArchiveUploader(_githubClientMock.Object, UPLOADS_URL, logMock.Object, retryPolicy, environmentVariableProviderMock.Object);
 
         // Assert
-        archiveUploader._streamSizeLimit.Should().Be(customSize);
+        archiveUploader._streamSizeLimit.Should().Be(customSizeBytes);
         logMock.Verify(x => x.LogInformation($"Multipart upload part size set to 10 MB."), Times.Once);
     }
 
@@ -75,7 +76,7 @@ public void Constructor_Should_Use_Default_When_Environment_Variable_Not_Set()
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
             .Returns(() => null);
 
         // Act
@@ -96,7 +97,7 @@ public void Constructor_Should_Use_Default_When_Environment_Variable_Is_Invalid(
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
             .Returns("invalid_value");
 
         // Act
@@ -117,7 +118,7 @@ public void Constructor_Should_Use_Default_When_Environment_Variable_Is_Zero()
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
             .Returns("0");
 
         // Act
@@ -138,7 +139,7 @@ public void Constructor_Should_Use_Default_When_Environment_Variable_Is_Negative
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
             .Returns("-1000");
 
         // Act
@@ -152,67 +153,70 @@ public void Constructor_Should_Use_Default_When_Environment_Variable_Is_Negative
     public void Constructor_Should_Use_Default_And_Log_Warning_When_Environment_Variable_Below_Minimum()
     {
         // Arrange
-        var belowMinimumSize = 1024 * 1024; // 1 MiB (below 5 MiB minimum)
-        var defaultSize = 100 * 1024 * 1024; // 100 MiB
-        var minSize = 5 * 1024 * 1024; // 5 MiB minimum
+        var belowMinimumSizeMiB = 1; // below 5 MiB minimum
+        var defaultSizeMiB = 100;
+        var defaultSizeBytes = defaultSizeMiB * 1024 * 1024;
+        var minSizeMiB = 5; // 5 MiB minimum
         var logMock = TestHelpers.CreateMock<OctoLogger>();
         var githubClientMock = TestHelpers.CreateMock<GithubClient>();
         var environmentVariableProviderMock = TestHelpers.CreateMock<EnvironmentVariableProvider>();
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
-            .Returns(belowMinimumSize.ToString());
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
+            .Returns(belowMinimumSizeMiB.ToString());
 
         // Act
         var archiveUploader = new ArchiveUploader(_githubClientMock.Object, UPLOADS_URL, logMock.Object, retryPolicy, environmentVariableProviderMock.Object);
 
         // Assert
-        archiveUploader._streamSizeLimit.Should().Be(defaultSize);
-        logMock.Verify(x => x.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_BYTES is set to {belowMinimumSize} bytes, but the minimum value is {minSize} bytes. Using default value of {defaultSize} bytes."), Times.Once);
+        archiveUploader._streamSizeLimit.Should().Be(defaultSizeBytes);
+        logMock.Verify(x => x.LogWarning($"GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES is set to {belowMinimumSizeMiB} MiB, but the minimum value is {minSizeMiB} MiB. Using default value of {defaultSizeMiB} MiB."), Times.Once);
     }
 
     [Fact]
     public void Constructor_Should_Accept_Value_Equal_To_Minimum()
     {
         // Arrange
-        var minimumSize = 5 * 1024 * 1024; // 5 MiB minimum
+        var minimumSizeMiB = 5; // 5 MiB minimum
+        var minimumSizeBytes = minimumSizeMiB * 1024 * 1024;
         var logMock = TestHelpers.CreateMock<OctoLogger>();
         var githubClientMock = TestHelpers.CreateMock<GithubClient>();
         var environmentVariableProviderMock = TestHelpers.CreateMock<EnvironmentVariableProvider>();
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
-            .Returns(minimumSize.ToString());
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
+            .Returns(minimumSizeMiB.ToString());
 
         // Act
         var archiveUploader = new ArchiveUploader(_githubClientMock.Object, UPLOADS_URL, logMock.Object, retryPolicy, environmentVariableProviderMock.Object);
 
         // Assert
-        archiveUploader._streamSizeLimit.Should().Be(minimumSize);
+        archiveUploader._streamSizeLimit.Should().Be(minimumSizeBytes);
         logMock.Verify(x => x.LogInformation($"Multipart upload part size set to 5 MB."), Times.Once);
     }
 
     [Fact]
     public void Constructor_Should_Accept_Large_Valid_Value()
     {
         // Arrange
-        var largeSize = 500 * 1024 * 1024; // 500 MiB
+        var largeSizeMiB = 500; // 500 MiB
+        var largeSizeBytes = largeSizeMiB * 1024 * 1024;
         var logMock = TestHelpers.CreateMock<OctoLogger>();
         var githubClientMock = TestHelpers.CreateMock<GithubClient>();
         var environmentVariableProviderMock = TestHelpers.CreateMock<EnvironmentVariableProvider>();
         var retryPolicy = new RetryPolicy(logMock.Object);
 
         environmentVariableProviderMock
-            .Setup(x => x.GithubOwnedStorageMultipartBytes(false))
-            .Returns(largeSize.ToString());
+            .Setup(x => x.GithubOwnedStorageMultipartMebibytes(false))
+            .Returns(largeSizeMiB.ToString());
 
         // Act
         var archiveUploader = new ArchiveUploader(githubClientMock.Object, UPLOADS_URL, logMock.Object, retryPolicy, environmentVariableProviderMock.Object);
 
         // Assert
-        archiveUploader._streamSizeLimit.Should().Be(largeSize);
+        archiveUploader._streamSizeLimit.Should().Be(largeSizeBytes);
         logMock.Verify(x => x.LogInformation($"Multipart upload part size set to 500 MB."), Times.Once);
     }