Merge release branch 4.18 to main

* 4.18: test,refactor: fix test_project_resources cleanup (apache#8097) kvm: fix direct download template size (apache#8093)
gzhao9 · Oct 16, 2023 · c9d2303 · c9d2303
2 parents e437d10 + 065abe2
commit c9d2303
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 78 deletions.
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
@@ -19,29 +19,6 @@
 
 package org.apache.cloudstack.direct.download;
 
-import com.cloud.utils.Pair;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.script.Script;
-import com.cloud.utils.storage.QCOW2Utils;
-import org.apache.cloudstack.utils.security.SSLUtils;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.io.IOUtils;
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.commons.collections.MapUtils;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -60,6 +37,32 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+
+import org.apache.cloudstack.utils.security.SSLUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+
+import com.cloud.utils.Pair;
+import com.cloud.utils.UriUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.QCOW2Utils;
+
 public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
 
     protected CloseableHttpClient httpsClient;
@@ -183,8 +186,7 @@ public Long getRemoteFileSize(String url, String format) {
                 SSLContext context = getSSLContext();
                 urlConnection.setSSLSocketFactory(context.getSocketFactory());
                 urlConnection.connect();
-                boolean isCompressed = !url.endsWith("qcow2");
-                return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
+                return QCOW2Utils.getVirtualSize(urlObj.openStream(), UriUtils.isUrlForCompressedFile(url));
             } catch (IOException e) {
                 throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
             }

diff --git a/test/integration/component/test_project_resources.py b/test/integration/component/test_project_resources.py
@@ -551,66 +551,59 @@ def setUpClass(cls):
         cls.hypervisor = cls.testClient.getHypervisorInfo()
         if cls.hypervisor.lower() in ['lxc']:
             raise unittest.SkipTest("create template from volume is not supported on %s" % cls.hypervisor.lower())
+        cls._cleanup = []
 
         cls.template = get_template(
-                            cls.api_client,
-                            cls.zone.id,
-                            cls.services["ostype"]
-                            )
+            cls.api_client,
+            cls.zone.id,
+            cls.services["ostype"]
+        )
         cls.services["server"]["zoneid"] = cls.zone.id
 
         # Create Domains, Account etc
         cls.domain = Domain.create(
-                                   cls.api_client,
-                                   cls.services["domain"]
-                                   )
+            cls.api_client,
+            cls.services["domain"]
+        )
+        cls._cleanup.append(cls.domain)
 
         cls.account = Account.create(
-                            cls.api_client,
-                            cls.services["account"],
-                            domainid=cls.domain.id
-                            )
+            cls.api_client,
+            cls.services["account"],
+            domainid=cls.domain.id
+        )
+        cls._cleanup.append(cls.account)
         cls.user = Account.create(
-                            cls.api_client,
-                            cls.services["account"],
-                            domainid=cls.domain.id
-                            )
+            cls.api_client,
+            cls.services["account"],
+            domainid=cls.domain.id
+        )
+        cls._cleanup.append(cls.user)
         # Create project as a domain admin
         cls.project = Project.create(
-                                 cls.api_client,
-                                 cls.services["project"],
-                                 account=cls.account.name,
-                                 domainid=cls.account.domainid
-                                 )
+            cls.api_client,
+            cls.services["project"],
+            account=cls.account.name,
+            domainid=cls.account.domainid
+        )
+        cls._cleanup.append(cls.project)
         cls.services["account"] = cls.account.name
 
         # Create Service offering and disk offerings etc
         cls.service_offering = ServiceOffering.create(
-                                            cls.api_client,
-                                            cls.services["service_offering"]
-                                            )
+            cls.api_client,
+            cls.services["service_offering"]
+        )
+        cls._cleanup.append(cls.service_offering)
         cls.userapiclient = cls.testClient.getUserApiClient(
-                                UserName=cls.account.name,
-                                DomainName=cls.domain.name
-				)
-
-        cls._cleanup = [
-                        cls.project,
-                        cls.service_offering,
-                        cls.account,
-                        cls.user,
-                        cls.domain
-                        ]
+            UserName=cls.account.name,
+            DomainName=cls.domain.name
+        )
         return
 
     @classmethod
     def tearDownClass(cls):
-        try:
-            #Cleanup resources used
-            cleanup_resources(cls.api_client, cls._cleanup)
-        except Exception as e:
-            raise Exception("Warning: Exception during cleanup : %s" % e)
-        return
+        super(TestTemplates, cls).tearDownClass()
 
     def setUp(self):
         self.apiclient = self.testClient.getApiClient()
@@ -619,12 +612,7 @@ def setUp(self):
         return
 
     def tearDown(self):
-        try:
-            #Clean up, terminate the created instance, volumes and snapshots
-            cleanup_resources(self.apiclient, self.cleanup)
-        except Exception as e:
-            raise Exception("Warning: Exception during cleanup : %s" % e)
-        return
+        super(TestTemplates, self).tearDown()
 
     @attr(tags=["advanced", "basic", "sg", "eip", "advancedns"], required_hardware="false")
     def test_04_public_private_template_use_in_project(self):

diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java
@@ -408,14 +408,14 @@ public static List<String> getMetalinkUrls(String metalinkUrl) {
         return urls;
     }
 
-    public static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
+    public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
 
     public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {
         final ImmutableSet.Builder<String> builder = ImmutableSet.builder();
 
         for (String baseExtension : baseExtensions) {
             builder.add("." + baseExtension);
-            for (String format : COMMPRESSION_FORMATS) {
+            for (String format : COMPRESSION_FORMATS) {
                 builder.add("." + baseExtension + "." + format);
             }
         }
@@ -647,4 +647,8 @@ private static UriInfo getRbdUrlInfo(String url) {
             throw new CloudRuntimeException(url + " is not a valid uri for RBD");
         }
     }
+
+    public static boolean isUrlForCompressedFile(String url) {
+        return UriUtils.COMPRESSION_FORMATS.stream().anyMatch(f -> url.toLowerCase().endsWith(f));
+    }
 }
diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
@@ -32,6 +32,7 @@
 import org.apache.log4j.Logger;
 
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.UriUtils;
 
 public final class QCOW2Utils {
     public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName());
@@ -114,7 +115,7 @@ private static long getVirtualSizeFromInputStream(InputStream inputStream) throw
     public static long getVirtualSize(String urlStr) {
         try {
             URL url = new URL(urlStr);
-            return getVirtualSizeFromInputStream(url.openStream());
+            return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
         } catch (MalformedURLException e) {
             LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
             throw new IllegalArgumentException("Invalid URL: " + urlStr);

diff --git a/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java
@@ -101,7 +101,7 @@ public static boolean isCorrectExtension(String path, String format) {
 
     public static boolean isCompressedExtension(String path) {
         final String lowerCasePath = path.toLowerCase();
-        return UriUtils.COMMPRESSION_FORMATS
+        return UriUtils.COMPRESSION_FORMATS
                        .stream()
                        .map(extension -> "." + extension)
                        .anyMatch(lowerCasePath::endsWith);

diff --git a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java
@@ -19,13 +19,13 @@
 
 package com.cloud.utils;
 
-import org.junit.Assert;
-import org.junit.Test;
-
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import org.junit.Assert;
+import org.junit.Test;
+
 public class UriUtilsTest {
     @Test
     public void encodeURIComponent() {
@@ -265,4 +265,12 @@ public void testGetUriInfoIpv6() {
         testGetUriInfoInternal(url11, host);
         testGetUriInfoInternal(url12, host);
     }
+
+    @Test
+    public void testIsUrlForCompressedFile() {
+        Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2"));
+        Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip"));
+        Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz"));
+        Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2"));
+    }
 }