diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java index 1bee45c477d2..70a3eb29bc7a 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -19,29 +19,6 @@ package org.apache.cloudstack.direct.download; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; -import com.cloud.utils.storage.QCOW2Utils; -import org.apache.cloudstack.utils.security.SSLUtils; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.io.IOUtils; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.commons.collections.MapUtils; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -60,6 +37,32 @@ import java.util.List; import java.util.Map; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; + +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.utils.storage.QCOW2Utils; + public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected CloseableHttpClient httpsClient; @@ -183,8 +186,7 @@ public Long getRemoteFileSize(String url, String format) { SSLContext context = getSSLContext(); urlConnection.setSSLSocketFactory(context.getSocketFactory()); urlConnection.connect(); - boolean isCompressed = !url.endsWith("qcow2"); - return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed); + return QCOW2Utils.getVirtualSize(urlObj.openStream(), UriUtils.isUrlForCompressedFile(url)); } catch (IOException e) { throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e); } diff --git a/test/integration/component/test_project_resources.py b/test/integration/component/test_project_resources.py index f767af1788b9..16d21dfd7cba 100644 --- a/test/integration/component/test_project_resources.py +++ b/test/integration/component/test_project_resources.py @@ -551,66 +551,59 @@ def setUpClass(cls): cls.hypervisor = cls.testClient.getHypervisorInfo() if cls.hypervisor.lower() in ['lxc']: raise unittest.SkipTest("create template from volume is not supported on %s" % cls.hypervisor.lower()) + cls._cleanup = [] cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) cls.services["server"]["zoneid"] = cls.zone.id # Create Domains, Account etc cls.domain = Domain.create( - cls.api_client, - cls.services["domain"] - ) + cls.api_client, + cls.services["domain"] + ) + cls._cleanup.append(cls.domain) cls.account = Account.create( - cls.api_client, - cls.services["account"], - domainid=cls.domain.id - ) + cls.api_client, + cls.services["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) cls.user = Account.create( - cls.api_client, - cls.services["account"], - domainid=cls.domain.id - ) + cls.api_client, + cls.services["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.user) # Create project as a domain admin cls.project = Project.create( - cls.api_client, - cls.services["project"], - account=cls.account.name, - domainid=cls.account.domainid - ) + cls.api_client, + cls.services["project"], + account=cls.account.name, + domainid=cls.account.domainid + ) + cls._cleanup.append(cls.project) cls.services["account"] = cls.account.name # Create Service offering and disk offerings etc cls.service_offering = ServiceOffering.create( - cls.api_client, - cls.services["service_offering"] - ) + cls.api_client, + cls.services["service_offering"] + ) + cls._cleanup.append(cls.service_offering) cls.userapiclient = cls.testClient.getUserApiClient( - UserName=cls.account.name, - DomainName=cls.domain.name - ) - - cls._cleanup = [ - cls.project, - cls.service_offering, - cls.account, - cls.user, - cls.domain - ] + UserName=cls.account.name, + DomainName=cls.domain.name + ) return @classmethod def tearDownClass(cls): - try: - #Cleanup resources used - cleanup_resources(cls.api_client, cls._cleanup) - except Exception as e: - raise Exception("Warning: Exception during cleanup : %s" % e) - return + super(TestTemplates, cls).tearDownClass() def setUp(self): self.apiclient = self.testClient.getApiClient() @@ -619,12 +612,7 @@ def setUp(self): return def tearDown(self): - try: - #Clean up, terminate the created instance, volumes and snapshots - cleanup_resources(self.apiclient, self.cleanup) - except Exception as e: - raise Exception("Warning: Exception during cleanup : %s" % e) - return + super(TestTemplates, self).tearDown() @attr(tags=["advanced", "basic", "sg", "eip", "advancedns"], required_hardware="false") def test_04_public_private_template_use_in_project(self): diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index dffc0106e8a8..a2bfa9eaa6fa 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -408,14 +408,14 @@ public static List getMetalinkUrls(String metalinkUrl) { return urls; } - public static final Set COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); + public static final Set COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); public static final Set buildExtensionSet(boolean metalink, String... baseExtensions) { final ImmutableSet.Builder builder = ImmutableSet.builder(); for (String baseExtension : baseExtensions) { builder.add("." + baseExtension); - for (String format : COMMPRESSION_FORMATS) { + for (String format : COMPRESSION_FORMATS) { builder.add("." + baseExtension + "." + format); } } @@ -647,4 +647,8 @@ private static UriInfo getRbdUrlInfo(String url) { throw new CloudRuntimeException(url + " is not a valid uri for RBD"); } } + + public static boolean isUrlForCompressedFile(String url) { + return UriUtils.COMPRESSION_FORMATS.stream().anyMatch(f -> url.toLowerCase().endsWith(f)); + } } diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index 32a54722aaeb..4daf138bb8c8 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -32,6 +32,7 @@ import org.apache.log4j.Logger; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.UriUtils; public final class QCOW2Utils { public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName()); @@ -114,7 +115,7 @@ private static long getVirtualSizeFromInputStream(InputStream inputStream) throw public static long getVirtualSize(String urlStr) { try { URL url = new URL(urlStr); - return getVirtualSizeFromInputStream(url.openStream()); + return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr)); } catch (MalformedURLException e) { LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage()); throw new IllegalArgumentException("Invalid URL: " + urlStr); diff --git a/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java index 02878cf29606..2b37ce5fb72f 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java @@ -101,7 +101,7 @@ public static boolean isCorrectExtension(String path, String format) { public static boolean isCompressedExtension(String path) { final String lowerCasePath = path.toLowerCase(); - return UriUtils.COMMPRESSION_FORMATS + return UriUtils.COMPRESSION_FORMATS .stream() .map(extension -> "." + extension) .anyMatch(lowerCasePath::endsWith); diff --git a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java index 1a3ae17f584e..4ec1f9a9bd92 100644 --- a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java @@ -19,13 +19,13 @@ package com.cloud.utils; -import org.junit.Assert; -import org.junit.Test; - import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.junit.Assert; +import org.junit.Test; + public class UriUtilsTest { @Test public void encodeURIComponent() { @@ -265,4 +265,12 @@ public void testGetUriInfoIpv6() { testGetUriInfoInternal(url11, host); testGetUriInfoInternal(url12, host); } + + @Test + public void testIsUrlForCompressedFile() { + Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2")); + Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip")); + Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz")); + Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2")); + } }