diff --git a/depends/libhdfs3/CMake/FindCurl.cmake b/depends/libhdfs3/CMake/FindCurl.cmake deleted file mode 100644 index e93b01de8c..0000000000 --- a/depends/libhdfs3/CMake/FindCurl.cmake +++ /dev/null @@ -1,26 +0,0 @@ -# - Try to find the CURL library (curl) -# -# Once done this will define -# -# CURL_FOUND - System has gnutls -# CURL_INCLUDE_DIR - The gnutls include directory -# CURL_LIBRARIES - The libraries needed to use gnutls -# CURL_DEFINITIONS - Compiler switches required for using gnutls - - -IF (CURL_INCLUDE_DIR AND CURL_LIBRARIES) - # in cache already - SET(CURL_FIND_QUIETLY TRUE) -ENDIF (CURL_INCLUDE_DIR AND CURL_LIBRARIES) - -FIND_PATH(CURL_INCLUDE_DIR curl/curl.h) - -FIND_LIBRARY(CURL_LIBRARIES curl) - -INCLUDE(FindPackageHandleStandardArgs) - -# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARIES CURL_INCLUDE_DIR) - -MARK_AS_ADVANCED(CURL_INCLUDE_DIR CURL_LIBRARIES) diff --git a/depends/libhdfs3/CMake/FindSSL.cmake b/depends/libhdfs3/CMake/FindSSL.cmake deleted file mode 100644 index bcbc5d8965..0000000000 --- a/depends/libhdfs3/CMake/FindSSL.cmake +++ /dev/null @@ -1,26 +0,0 @@ -# - Try to find the Open ssl library (ssl) -# -# Once done this will define -# -# SSL_FOUND - System has gnutls -# SSL_INCLUDE_DIR - The gnutls include directory -# SSL_LIBRARIES - The libraries needed to use gnutls -# SSL_DEFINITIONS - Compiler switches required for using gnutls - - -IF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) - # in cache already - SET(SSL_FIND_QUIETLY TRUE) -ENDIF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) - -FIND_PATH(SSL_INCLUDE_DIR openssl/opensslv.h) - -FIND_LIBRARY(SSL_LIBRARIES crypto) - -INCLUDE(FindPackageHandleStandardArgs) - -# handle the QUIETLY and REQUIRED arguments and set SSL_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(SSL DEFAULT_MSG SSL_LIBRARIES SSL_INCLUDE_DIR) - -MARK_AS_ADVANCED(SSL_INCLUDE_DIR SSL_LIBRARIES) \ No newline at end of file diff --git a/depends/libhdfs3/mock/MockCryptoCodec.h b/depends/libhdfs3/mock/MockCryptoCodec.h deleted file mode 100644 index a9a220e58f..0000000000 --- a/depends/libhdfs3/mock/MockCryptoCodec.h +++ /dev/null @@ -1,38 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _HDFS_LIBHDFS3_MOCK_CRYPTOCODEC_H_ -#define _HDFS_LIBHDFS3_MOCK_CRYPTOCODEC_H_ - -#include "gmock/gmock.h" - -#include "client/CryptoCodec.h" -#include "client/KmsClientProvider.h" - -class MockCryptoCodec: public Hdfs::CryptoCodec { -public: - MockCryptoCodec(FileEncryptionInfo *encryptionInfo, shared_ptr kcp, int32_t bufSize) : CryptoCodec(encryptionInfo, kcp, bufSize) {} - - MOCK_METHOD2(init, int(CryptoMethod crypto_method, int64_t stream_offset)); - MOCK_METHOD2(cipher_wrap, std::string(const char * buffer,int64_t size)); -}; - -#endif /* _HDFS_LIBHDFS3_MOCK_CRYPTOCODEC_H_ */ diff --git a/depends/libhdfs3/mock/MockHttpClient.h b/depends/libhdfs3/mock/MockHttpClient.h deleted file mode 100644 index 9da11869d3..0000000000 --- a/depends/libhdfs3/mock/MockHttpClient.h +++ /dev/null @@ -1,52 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _HDFS_LIBHDFS3_MOCK_HTTPCLIENT_H_ -#define _HDFS_LIBHDFS3_MOCK_HTTPCLIENT_H_ - -#include "gmock/gmock.h" - -#include "client/HttpClient.h" -#include "client/KmsClientProvider.h" -#include - -using boost::property_tree::ptree; - -class MockHttpClient: public Hdfs::HttpClient { -public: - MOCK_METHOD0(post, std::string()); - MOCK_METHOD0(del, std::string()); - MOCK_METHOD0(put, std::string()); - MOCK_METHOD0(get, std::string()); - - std::string getPostResult(FileEncryptionInfo &encryptionInfo) { - ptree map; - map.put("name", encryptionInfo.getKeyName()); - map.put("iv", encryptionInfo.getIv()); - map.put("material", encryptionInfo.getKey()); - - std::string json = KmsClientProvider::toJson(map); - return json; - } - -}; - -#endif /* _HDFS_LIBHDFS3_MOCK_HTTPCLIENT_H_ */ diff --git a/depends/libhdfs3/mock/MockKmsClientProvider.h b/depends/libhdfs3/mock/MockKmsClientProvider.h deleted file mode 100644 index 81fb8f364a..0000000000 --- a/depends/libhdfs3/mock/MockKmsClientProvider.h +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _HDFS_LIBHDFS3_MOCK_KMSCLIENTPROVIDER_H_ -#define _HDFS_LIBHDFS3_MOCK_KMSCLIENTPROVIDER_H_ - -#include "gmock/gmock.h" - -#include "client/KmsClientProvider.h" - -using namespace Hdfs::Internal; - -class MockKmsClientProvider: public Hdfs::KmsClientProvider { -public: - MockKmsClientProvider(shared_ptr auth, shared_ptr conf) : KmsClientProvider(auth, conf) {} - MOCK_METHOD1(setHttpClient, void(shared_ptr hc)); - MOCK_METHOD1(getKeyMetadata, ptree(const FileEncryptionInfo &encryptionInfo)); - MOCK_METHOD1(deleteKey, void(const FileEncryptionInfo &encryptionInfo)); - MOCK_METHOD1(decryptEncryptedKey, ptree(const FileEncryptionInfo &encryptionInfo)); - MOCK_METHOD5(createKey, void(const std::string &keyName, const std::string &cipher, const int length, const std::string &material, const std::string &description)); - - ptree getEDKResult(FileEncryptionInfo &encryptionInfo) { - ptree map; - map.put("name", encryptionInfo.getKeyName()); - map.put("iv", encryptionInfo.getIv()); - map.put("material", KmsClientProvider::base64Encode(encryptionInfo.getKey())); - return map; - } - -}; - -#endif /* _HDFS_LIBHDFS3_MOCK_KMSCLIENTPROVIDER_H_ */ diff --git a/depends/libhdfs3/src/client/CryptoCodec.cpp b/depends/libhdfs3/src/client/CryptoCodec.cpp deleted file mode 100644 index bd4443f971..0000000000 --- a/depends/libhdfs3/src/client/CryptoCodec.cpp +++ /dev/null @@ -1,216 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "CryptoCodec.h" -#include "Logger.h" - -using namespace Hdfs::Internal; - - -namespace Hdfs { - - //copy from java HDFS code - std::string CryptoCodec::calculateIV(const std::string& initIV, unsigned long counter) { - char IV[initIV.length()]; - - int i = initIV.length(); // IV length - int j = 0; // counter bytes index - unsigned int sum = 0; - while (i-- > 0) { - // (sum >>> Byte.SIZE) is the carry for addition - sum = (initIV[i] & 0xff) + (sum >> 8); - if (j++ < 8) { // Big-endian, and long is 8 bytes length - sum += (char) counter & 0xff; - counter >>= 8; - } - IV[i] = (char) sum; - } - - return std::string(IV, initIV.length()); - } - - CryptoCodec::CryptoCodec(FileEncryptionInfo *encryptionInfo, shared_ptr kcp, int32_t bufSize) : - encryptionInfo(encryptionInfo), kcp(kcp), bufSize(bufSize) - { - - // Init global status - ERR_load_crypto_strings(); - OpenSSL_add_all_algorithms(); - OPENSSL_config(NULL); - - // Create cipher context - cipherCtx = EVP_CIPHER_CTX_new(); - cipher = NULL; - - padding = 0; - counter = 0; - is_init = false; - } - - CryptoCodec::~CryptoCodec() - { - if (cipherCtx) - EVP_CIPHER_CTX_free(cipherCtx); - } - - std::string CryptoCodec::getDecryptedKeyFromKms() - { - ptree map = kcp->decryptEncryptedKey(*encryptionInfo); - std::string key; - try { - key = map.get < std::string > ("material"); - } catch (...) { - THROW(HdfsIOException, "CryptoCodec : Can not get key from kms."); - } - - int rem = key.length() % 4; - if (rem) { - rem = 4 - rem; - while (rem != 0) { - key = key + "="; - rem--; - } - } - - std::replace(key.begin(), key.end(), '-', '+'); - std::replace(key.begin(), key.end(), '_', '/'); - - LOG(DEBUG3, "CryptoCodec : getDecryptedKeyFromKms material is :%s", key.c_str()); - - key = KmsClientProvider::base64Decode(key); - return key; - } - - int CryptoCodec::init(CryptoMethod crypto_method, int64_t stream_offset) { - // Check CryptoCodec init or not. - if (is_init) - return 0; - - // Get decrypted key from KMS. - decryptedKey = getDecryptedKeyFromKms(); - - // Select cipher method based on the decrypted key length. - AlgorithmBlockSize = decryptedKey.length(); - if (AlgorithmBlockSize == KEY_LENGTH_256) { - cipher = EVP_aes_256_ctr(); - } else if (AlgorithmBlockSize == KEY_LENGTH_128) { - cipher = EVP_aes_128_ctr(); - } else { - LOG(WARNING, "CryptoCodec : Invalid key length."); - return -1; - } - - is_init = true; - // Calculate iv and counter in order to init cipher context with cipher method. Default value is 0. - if ((resetStreamOffset(crypto_method, stream_offset)) < 0) { - is_init = false; - return -1; - } - - LOG(DEBUG3, "CryptoCodec init success, length of the decrypted key is : %llu, crypto method is : %d", AlgorithmBlockSize, crypto_method); - return 1; - - } - - int CryptoCodec::resetStreamOffset(CryptoMethod crypto_method, int64_t stream_offset) { - // Check CryptoCodec init or not. - if (is_init == false) - return -1; - // Calculate new IV when appending an existed file. - std::string iv = encryptionInfo->getIv(); - if (stream_offset > 0) { - counter = stream_offset / AlgorithmBlockSize; - padding = stream_offset % AlgorithmBlockSize; - iv = this->calculateIV(iv, counter); - } - - // Judge the crypto method is encrypt or decrypt. - int enc = (method == CryptoMethod::ENCRYPT) ? 1 : 0; - - // Init cipher context with cipher method. - if (!EVP_CipherInit_ex(cipherCtx, cipher, NULL, - (const unsigned char *) decryptedKey.c_str(), (const unsigned char *) iv.c_str(), - enc)) { - LOG(WARNING, "EVP_CipherInit_ex failed"); - return -1; - } - - // AES/CTR/NoPadding, set padding to 0. - EVP_CIPHER_CTX_set_padding(cipherCtx, 0); - - return 1; - } - - std::string CryptoCodec::cipher_wrap(const char * buffer, int64_t size) { - if (!is_init) - THROW(InvalidParameter, "CryptoCodec isn't init"); - - int offset = 0; - int remaining = size; - int len = 0; - int ret = 0; - - std::string in_buf(buffer,size); - std::string out_buf(size, 0); - //set necessary padding when appending a existed file - if (padding > 0) { - in_buf.insert(0, padding, 0); - out_buf.resize(padding+size); - remaining += padding; - } - - // If the encode/decode buffer size larger than crypto buffer size, encode/decode buffer one by one - while (remaining > bufSize) { - ret = EVP_CipherUpdate(cipherCtx, (unsigned char *) &out_buf[offset], &len, - (const unsigned char *)in_buf.data() + offset, bufSize); - - if (!ret) { - std::string err = ERR_lib_error_string(ERR_get_error()); - THROW(HdfsIOException, "CryptoCodec : cipher_wrap AES data failed:%s, crypto_method:%d", err.c_str(), method); - } - offset += len; - remaining -= len; - LOG(DEBUG3, "CryptoCodec : EVP_CipherUpdate successfully, len:%d", len); - } - - if (remaining) { - ret = EVP_CipherUpdate(cipherCtx, (unsigned char *) &out_buf[offset], &len, - (const unsigned char *) in_buf.data() + offset, remaining); - - if (!ret) { - std::string err = ERR_lib_error_string(ERR_get_error()); - THROW(HdfsIOException, "CryptoCodec : cipher_wrap AES data failed:%s, crypto_method:%d", err.c_str(), method); - } - - } - - //cut off padding when necessary - if (padding > 0) { - out_buf.erase(0, padding); - padding = 0; - } - - return out_buf; - } - -} - diff --git a/depends/libhdfs3/src/client/CryptoCodec.h b/depends/libhdfs3/src/client/CryptoCodec.h deleted file mode 100644 index f5070fe6ec..0000000000 --- a/depends/libhdfs3/src/client/CryptoCodec.h +++ /dev/null @@ -1,112 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _HDFS_LIBHDFS3_CLIENT_CRYPTOCODEC_H_ -#define _HDFS_LIBHDFS3_CLIENT_CRYPTOCODEC_H_ - -#include - -#include "openssl/conf.h" -#include "openssl/evp.h" -#include "openssl/err.h" -#include "FileEncryptionInfo.h" -#include "KmsClientProvider.h" - -#define KEY_LENGTH_256 32 -#define KEY_LENGTH_128 16 - -namespace Hdfs { - - enum CryptoMethod { - DECRYPT = 0, - ENCRYPT = 1 - }; - - class CryptoCodec { - public: - /** - * Construct a CryptoCodec instance. - * @param encryptionInfo the encryption info of file. - * @param kcp a KmsClientProvider instance to get key from kms server. - * @param bufSize crypto buffer size. - */ - CryptoCodec(FileEncryptionInfo *encryptionInfo, shared_ptr kcp, int32_t bufSize); - - /** - * Destroy a CryptoCodec instance. - */ - virtual ~CryptoCodec(); - - /** - * encrypt/decrypt(depends on init()) buffer data - * @param buffer - * @param size - * @return encrypt/decrypt result string - */ - virtual std::string cipher_wrap(const char * buffer, int64_t size); - - /** - * init CryptoCodec - * @param method CryptoMethod - * @param stream_offset 0 when open a new file; file_lenght when append a existed file - * @return 1 success; 0 no need(already inited); -1 failed - */ - virtual int init(CryptoMethod crypto_method, int64_t stream_offset = 0); - - /** - * Reset iv and padding value when seek file. - * @param crypto_method do encrypt/decrypt work according to crypto_method. - * @param stream_offset the offset of the current file. - * @return 1 sucess; -1 failed. - */ - virtual int resetStreamOffset(CryptoMethod crypto_method, int64_t stream_offset); - - private: - - /** - * Get decrypted key from kms. - */ - std::string getDecryptedKeyFromKms(); - - /** - * calculate new IV for appending a existed file - * @param initIV - * @param counter - * @return new IV string - */ - std::string calculateIV(const std::string& initIV, unsigned long counter); - - shared_ptr kcp; - FileEncryptionInfo* encryptionInfo; - EVP_CIPHER_CTX* cipherCtx; - const EVP_CIPHER* cipher; - CryptoMethod method; - - bool is_init; - int32_t bufSize; - int64_t padding; - int64_t counter; - std::string decryptedKey; - uint64_t AlgorithmBlockSize; - }; - -} -#endif diff --git a/depends/libhdfs3/src/client/HttpClient.cpp b/depends/libhdfs3/src/client/HttpClient.cpp deleted file mode 100644 index 09a74a672a..0000000000 --- a/depends/libhdfs3/src/client/HttpClient.cpp +++ /dev/null @@ -1,349 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "HttpClient.h" -#include "Logger.h" - -using namespace Hdfs::Internal; - -namespace Hdfs { - -#define CURL_SETOPT(handle, option, optarg, fmt, ...) \ - res = curl_easy_setopt(handle, option, optarg); \ - if (res != CURLE_OK) { \ - THROW(HdfsIOException, fmt, ##__VA_ARGS__); \ - } - -#define CURL_SETOPT_ERROR1(handle, option, optarg, fmt) \ - CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res)); - -#define CURL_SETOPT_ERROR2(handle, option, optarg, fmt) \ - CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res), \ - errorString().c_str()) - -#define CURL_PERFORM(handle, fmt) \ - res = curl_easy_perform(handle); \ - if (res != CURLE_OK) { \ - THROW(HdfsIOException, fmt, curl_easy_strerror(res), errorString().c_str()); \ - } - -#define CURL_GETOPT_ERROR2(handle, option, optarg, fmt) \ - res = curl_easy_getinfo(handle, option, optarg); \ - if (res != CURLE_OK) { \ - THROW(HdfsIOException, fmt, curl_easy_strerror(res), errorString().c_str()); \ - } - -#define CURL_GET_RESPONSE(handle, code, fmt) \ - CURL_GETOPT_ERROR2(handle, CURLINFO_RESPONSE_CODE, code, fmt); - -HttpClient::HttpClient() : curl(NULL), list(NULL) { -} - -/** - * Construct a HttpClient instance. - * @param url a url which is the address to send the request to the corresponding http server. - */ -HttpClient::HttpClient(const std::string &url) { - curl = NULL; - list = NULL; - this->url = url; -} - -/** - * Destroy a HttpClient instance. - */ -HttpClient::~HttpClient() -{ - destroy(); -} - -/** - * Receive error string from curl. - */ -std::string HttpClient::errorString() { - if (strlen(errbuf) == 0) { - return ""; - } - return errbuf; -} - -/** - * Curl call back function to receive the reponse messages. - * @return return the size of reponse messages. - */ -size_t HttpClient::CurlWriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) -{ - size_t realsize = size * nmemb; - if (userp == NULL || contents == NULL) { - return 0; - } - ((std::string *) userp)->append((const char *) contents, realsize); - LOG(DEBUG3, "HttpClient : Http response is : %s", ((std::string * )userp)->c_str()); - return realsize; -} - -/** - * Init curl handler and set curl options. - */ -void HttpClient::init() { - if (!initialized) { - initialized = true; - if (curl_global_init (CURL_GLOBAL_ALL)) { - THROW(HdfsIOException, "Cannot initialize curl client for KMS"); - } - } - - curl = curl_easy_init(); - if (!curl) { - THROW(HdfsIOException, "Cannot initialize curl handle for KMS"); - } - - CURL_SETOPT_ERROR1(curl, CURLOPT_ERRORBUFFER, errbuf, - "Cannot initialize curl error buffer for KMS: %s"); - - errbuf[0] = 0; - - CURL_SETOPT_ERROR2(curl, CURLOPT_NOPROGRESS, 1, - "Cannot initialize no progress in HttpClient: %s: %s"); - - CURL_SETOPT_ERROR2(curl, CURLOPT_VERBOSE, 0, - "Cannot initialize no verbose in HttpClient: %s: %s"); - - CURL_SETOPT_ERROR2(curl, CURLOPT_COOKIEFILE, "", - "Cannot initialize cookie behavior in HttpClient: %s: %s"); - - CURL_SETOPT_ERROR2(curl, CURLOPT_HTTPHEADER, list, - "Cannot initialize headers in HttpClient: %s: %s"); - - CURL_SETOPT_ERROR2(curl, CURLOPT_WRITEFUNCTION, HttpClient::CurlWriteMemoryCallback, - "Cannot initialize body reader in HttpClient: %s: %s"); - - CURL_SETOPT_ERROR2(curl, CURLOPT_WRITEDATA, (void *)&response, - "Cannot initialize body reader data in HttpClient: %s: %s"); - - - /* Some servers don't like requests that are made without a user-agent - * field, so we provide one - */ - CURL_SETOPT_ERROR2(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0", - "Cannot initialize user agent in HttpClient: %s: %s"); - list = NULL; -} - -/** - * Do clean up for curl. - */ -void HttpClient::destroy() { - if (curl) { - curl_easy_cleanup(curl); - curl = NULL; - } - if (list) { - curl_slist_free_all(list); - list = NULL; - } - initialized = false; -} - -/** - * Set url for http client. - */ -void HttpClient::setURL(const std::string &url) { - this->url = url; -} - -/** - * Set retry times for http request which can be configured in config file. - */ -void HttpClient::setRequestRetryTimes(int request_retry_times) { - if (request_retry_times < 0) { - THROW(InvalidParameter, "HttpClient : Invalid value for request_retry_times."); - } - this->request_retry_times = request_retry_times; -} - -/** - * Set request timeout which can be configured in config file. - */ -void HttpClient::setRequestTimeout(int64_t curl_timeout) { - if (curl_timeout < 0) { - THROW(InvalidParameter, "HttpClient : Invalid value for curl_timeout."); - } - this->curl_timeout = curl_timeout; -} - -/** - * Set headers for http client. - */ -void HttpClient::setHeaders(const std::vector &headers) { - if (!headers.empty()) { - this->headers = headers; - for (std::string header : headers) { - list = curl_slist_append(list, header.c_str()); - if (!list) { - THROW(HdfsIOException, "Cannot add header in HttpClient."); - } - } - } else { - LOG(DEBUG3, "HttpClient : Header is empty."); - } -} - - -/** - * Set body for http client. - */ -void HttpClient::setBody(const std::string &body) { - this->body = body; -} - -/** - * Set expected response code. - */ -void HttpClient::setExpectedResponseCode(int64_t response_code_ok) { - this->response_code_ok = response_code_ok; -} - -/** - * Http common method to get response info by sending request to http server. - * @param method : define different http methods. - * @return return response info. - */ -std::string HttpClient::httpCommon(httpMethod method) { - /* Set headers and url. */ - if (list != NULL) { - CURL_SETOPT_ERROR2(curl, CURLOPT_HTTPHEADER, list, - "Cannot initialize headers in HttpClient: %s: %s"); - } else { - LOG(DEBUG3, "HttpClient : Http Header is NULL"); - } - - if (curl != NULL) { - CURL_SETOPT_ERROR2(curl, CURLOPT_URL, url.c_str(), - "Cannot initialize url in HttpClient: %s: %s"); - } else { - LOG(LOG_ERROR, "HttpClient : Http URL is NULL"); - } - - /* Set body based on different http method. */ - switch (method) { - case HTTP_GET: - { - break; - } - case HTTP_POST: - { - CURL_SETOPT_ERROR2(curl, CURLOPT_COPYPOSTFIELDS, body.c_str(), - "Cannot initialize post data in HttpClient: %s: %s"); - break; - } - case HTTP_DELETE: - { - CURL_SETOPT_ERROR2(curl, CURLOPT_CUSTOMREQUEST, "DELETE", - "Cannot initialize set customer request in HttpClient: %s: %s"); - break; - } - case HTTP_PUT: - { - CURL_SETOPT_ERROR2(curl, CURLOPT_CUSTOMREQUEST, "PUT", - "Cannot initialize set customer request in HttpClient: %s: %s"); - CURL_SETOPT_ERROR2(curl, CURLOPT_COPYPOSTFIELDS, body.c_str(), - "Cannot initialize post data in HttpClient: %s: %s"); - break; - } - default: - { - LOG(LOG_ERROR, "HttpClient : unknown method: %d", method); - } - } - - /* Do several http request try according to request_retry_times - * until got the right response code. - */ - int64_t response_code = -1; - - while (request_retry_times >= 0 && response_code != response_code_ok) { - request_retry_times -= 1; - response = ""; - CURL_SETOPT_ERROR2(curl, CURLOPT_TIMEOUT, curl_timeout, - "Send request to http server timeout: %s: %s"); - CURL_PERFORM(curl, "Could not send request in HttpClient: %s %s"); - CURL_GET_RESPONSE(curl, &response_code, - "Cannot get response code in HttpClient: %s: %s"); - } - LOG(DEBUG3, "HttpClient : The http method is %d. The http url is %s. The http response is %s.", - method, url.c_str(), response.c_str()); - return response; -} - -/** - * Http GET method. - */ -std::string HttpClient::get() { - return httpCommon(HTTP_GET); -} - -/** - * Http POST method. - */ -std::string HttpClient::post() { - return httpCommon(HTTP_POST); -} - -/** - * Http DELETE method. - */ -std::string HttpClient::del() { - return httpCommon(HTTP_DELETE); -} - -/** - * Http PUT method. - */ -std::string HttpClient::put() { - return httpCommon(HTTP_PUT); -} - - -/** - * URL encodes the given string. - */ -std::string HttpClient::escape(const std::string &data) { - if (curl) { - char *output = curl_easy_escape(curl, data.c_str(), data.length()); - if (output) { - std::string out(output); - return out; - } else { - THROW(HdfsIOException, "HttpClient : Curl escape failed."); - } - } else { - LOG(WARNING, "HttpClient : Curl in escape method is NULL"); - } - std::string empty; - return empty; -} -} - -/* Curl global init only can be done once. */ -bool Hdfs::HttpClient::initialized = false; - diff --git a/depends/libhdfs3/src/client/HttpClient.h b/depends/libhdfs3/src/client/HttpClient.h deleted file mode 100644 index c77789bc28..0000000000 --- a/depends/libhdfs3/src/client/HttpClient.h +++ /dev/null @@ -1,155 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _HDFS_LIBHDFS3_CLIENT_HTTPCLIENT_H_ -#define _HDFS_LIBHDFS3_CLIENT_HTTPCLIENT_H_ - -#include -#include -#include -#include "Exception.h" -#include "ExceptionInternal.h" - -typedef enum httpMethod { - HTTP_GET = 0, - HTTP_POST = 1, - HTTP_DELETE = 2, - HTTP_PUT = 3 -} httpMethod; - -namespace Hdfs { - -class HttpClient { -public: - HttpClient(); - - /** - * Construct a HttpClient instance. - * @param url a url which is the address to send the request to the corresponding http server. - */ - HttpClient(const std::string &url); - - /** - * Destroy a HttpClient instance. - */ - virtual ~HttpClient(); - - /** - * Set url for http client. - */ - void setURL(const std::string &url); - - /** - * Set headers for http client. - */ - void setHeaders(const std::vector &headers); - - /** - * Set body for http client. - */ - void setBody(const std::string &body); - - /** - * Set retry times for http request which can be configured in config file. - */ - void setRequestRetryTimes(int requst_retry_times); - - /** - * Set request timeout which can be configured in config file. - */ - void setRequestTimeout(int64_t curl_timeout); - - /** - * Set expected response code. - */ - void setExpectedResponseCode(int64_t response_code_ok); - - /** - * Init curl handler and set options for curl. - */ - void init(); - - /** - * Do clean up for curl. - */ - void destroy(); - - /** - * Http POST method. - */ - virtual std::string post(); - - /** - * Http DELETE method. - */ - virtual std::string del(); - - /** - * Http PUT method. - */ - virtual std::string put(); - - /** - * Http GET method. - */ - virtual std::string get(); - - /** - * URL encodes the given string. - */ - std::string escape(const std::string &data); - - /** - * Receive error string from curl. - */ - std::string errorString(); - -private: - - /** - * Http common method to get response info by sending request to http server. - * @param method : define different http methods. - * @return return response info. - */ - std::string httpCommon(httpMethod method); - - /** - * Curl call back function to receive the reponse messages. - * @return return the size of reponse messages. - */ - static size_t CurlWriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp); - - static bool initialized; - CURLcode res; - std::string url; - std::vector headers; - std::string body; - int64_t response_code_ok; - int request_retry_times; - int64_t curl_timeout; - CURL *curl; - struct curl_slist *list; - std::string response; - char errbuf[CURL_ERROR_SIZE] = { 0 }; -}; - -} -#endif diff --git a/depends/libhdfs3/src/client/KmsClientProvider.cpp b/depends/libhdfs3/src/client/KmsClientProvider.cpp deleted file mode 100644 index ac5957080e..0000000000 --- a/depends/libhdfs3/src/client/KmsClientProvider.cpp +++ /dev/null @@ -1,325 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "KmsClientProvider.h" -#include "Logger.h" -#include -#include -#include -using namespace Hdfs::Internal; -using boost::property_tree::read_json; -using boost::property_tree::write_json; - -namespace Hdfs { - -/** - * Convert ptree format to json format - */ -std::string KmsClientProvider::toJson(const ptree &data) { - std::ostringstream buf; - try { - write_json(buf, data, false); - std::string json = buf.str(); - return json; - } catch (...) { - THROW(HdfsIOException, "KmsClientProvider : Write json failed."); - } -} - -/** - * Convert json format to ptree format - */ -ptree KmsClientProvider::fromJson(const std::string &data) { - ptree pt2; - try { - std::istringstream is(data); - read_json(is, pt2); - return pt2; - } catch (...) { - THROW(HdfsIOException, "KmsClientProvider : Read json failed."); - } -} - -/** - * Encode string to base64. - */ -std::string KmsClientProvider::base64Encode(const std::string &data) { - char * buffer = NULL; - size_t len = 0; - int rc = 0; - std::string result; - - LOG(DEBUG3, "KmsClientProvider : Encode data is %s", data.c_str()); - - if (GSASL_OK != (rc = gsasl_base64_to(data.data(), data.size(), &buffer, &len))) { - assert(GSASL_MALLOC_ERROR == rc); - throw std::bad_alloc(); - } - - if (buffer) { - result.assign(buffer, len); - free(buffer); - } - - if (!buffer || result.length() != len) { - THROW(HdfsIOException, - "KmsClientProvider: Failed to encode string to base64"); - } - - return result; -} - -/** - * Decode base64 to string. - */ -std::string KmsClientProvider::base64Decode(const std::string &data) { - char * buffer = NULL; - size_t len = 0; - int rc = 0; - std::string result; - - if (GSASL_OK != (rc = gsasl_base64_from(data.data(), data.size(), &buffer, &len))) { - assert(GSASL_MALLOC_ERROR == rc); - throw std::bad_alloc(); - } - - if (buffer) { - result.assign(buffer, len); - free(buffer); - } - - if (!buffer || result.length() != len) { - THROW(HdfsIOException, - "KmsClientProvider: Failed to decode base64 to string"); - } - - return result; -} - -/** - * Construct a KmsClientProvider instance. - * @param auth RpcAuth to get the auth method and user info. - * @param conf a SessionConfig to get the configuration. - */ -KmsClientProvider::KmsClientProvider(shared_ptr rpcAuth, shared_ptr config) : auth(rpcAuth), conf(config) -{ - hc.reset(new HttpClient()); - method = RpcAuth::ParseMethod(conf->getKmsMethod()); -} - -/** - * Set HttpClient object. - */ -void KmsClientProvider::setHttpClient(shared_ptr hc) -{ - this->hc = hc; -} - -/** - * Parse kms url from configure file. - */ -std::string KmsClientProvider::parseKmsUrl() -{ - std::string start = "kms://"; - std::string http = "http@"; - std::string https = "https@"; - std::string urlParse = conf->getKmsUrl(); - LOG(DEBUG3, "KmsClientProvider : Get kms url from conf : %s.", - urlParse.c_str()); - if (urlParse.compare(0, start.length(), start) == 0) { - start = urlParse.substr(start.length()); - if (start.compare(0, http.length(), http) == 0) { - return "http://" + start.substr(http.length()); - } else if (start.compare(0, https.length(), https) == 0) { - return "https://" + start.substr(https.length()); - } else - THROW(HdfsIOException, "Bad KMS provider URL: %s", urlParse.c_str()); - } else - THROW(HdfsIOException, "Bad KMS provider URL: %s", urlParse.c_str()); - -} - -/** - * Build kms url based on urlSuffix and different auth method. - */ -std::string KmsClientProvider::buildKmsUrl(const std::string &url, const std::string &urlSuffix) -{ - std::string baseUrl = url; - baseUrl = url + "/v1/" + urlSuffix; - std::size_t found = urlSuffix.find('?'); - - if (method == AuthMethod::KERBEROS) { - // todo - THROW(InvalidParameter, "KmsClientProvider : Not support kerberos yet."); - } else if (method == AuthMethod::SIMPLE) { - std::string user = auth->getUser().getRealUser(); - LOG(DEBUG3, - "KmsClientProvider : Kms urlSuffix is : %s. Auth real user is : %s.", - urlSuffix.c_str(), user.c_str()); - if (user.length() == 0) - user = auth->getUser().getKrbName(); - if (found != std::string::npos) - return baseUrl + "&user.name=" + user; - else - return baseUrl + "?user.name=" + user; - } else { - return baseUrl; - } -} - -/** - * Set common headers for kms API. - */ -void KmsClientProvider::setCommonHeaders(std::vector& headers) -{ - headers.push_back("Content-Type: application/json"); - headers.push_back("Accept: *"); -} - - -/** - * Create an encryption key from kms. - * @param keyName the name of this key. - * @param cipher the ciphertext of this key. e.g. "AES/CTR/NoPadding" . - * @param length the length of this key. - * @param material will be encode to base64. - * @param description key's info. - */ -void KmsClientProvider::createKey(const std::string &keyName, const std::string &cipher, const int length, const std::string &material, const std::string &description) -{ - hc->init(); - /* Prepare url for HttpClient.*/ - url = parseKmsUrl(); - std::string urlSuffix = "keys"; - url = buildKmsUrl(url, urlSuffix); - /* Prepare headers for HttpClient.*/ - std::vector headers; - setCommonHeaders(headers); - /* Prepare body for HttpClient. */ - ptree map; - map.put("name", keyName); - map.put("cipher", cipher); - map.put("description", description); - std::string body = toJson(map); - /* Set options for HttpClient to get response. */ - hc->setURL(url); - hc->setHeaders(headers); - hc->setBody(body); - hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes()); - hc->setRequestTimeout(conf->getCurlTimeOut()); - hc->setExpectedResponseCode(201); - std::string response = hc->post(); - - LOG(DEBUG3, - "KmsClientProvider::createKey : The key name, key cipher, key length, key material, description are : %s, %s, %d, %s, %s. The kms url is : %s . The kms body is : %s. The response of kms server is : %s .", - keyName.c_str(), cipher.c_str(), length, material.c_str(), - description.c_str(), url.c_str(), body.c_str(), response.c_str()); - -} - -/** - * Get key metadata based on encrypted file's key name. - * @param encryptionInfo the encryption info of file. - * @return return response info about key metadata from kms server. - */ -ptree KmsClientProvider::getKeyMetadata(const FileEncryptionInfo &encryptionInfo) -{ - hc->init(); - url = parseKmsUrl(); - std::string urlSuffix = "key/" + hc->escape(encryptionInfo.getKeyName()) + "/_metadata"; - url = buildKmsUrl(url, urlSuffix); - - hc->setURL(url); - hc->setExpectedResponseCode(200); - hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes()); - hc->setRequestTimeout(conf->getCurlTimeOut()); - std::string response = hc->get(); - - LOG(DEBUG3, - "KmsClientProvider::getKeyMetadata : The kms url is : %s. The response of kms server is : %s .", - url.c_str(), response.c_str()); - - return fromJson(response); - -} - -/** - * Delete an encryption key from kms. - * @param encryptionInfo the encryption info of file. - */ -void KmsClientProvider::deleteKey(const FileEncryptionInfo &encryptionInfo) -{ - hc->init(); - url = parseKmsUrl(); - std::string urlSuffix = "key/" + hc->escape(encryptionInfo.getKeyName()); - url = buildKmsUrl(url, urlSuffix); - - hc->setURL(url); - hc->setExpectedResponseCode(200); - hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes()); - hc->setRequestTimeout(conf->getCurlTimeOut()); - std::string response = hc->del(); - - LOG(DEBUG3, - "KmsClientProvider::deleteKey : The kms url is : %s. The response of kms server is : %s .", - url.c_str(), response.c_str()); -} - -/** - * Decrypt an encrypted key from kms. - * @param encryptionInfo the encryption info of file. - * @return return decrypted key. - */ -ptree KmsClientProvider::decryptEncryptedKey(const FileEncryptionInfo &encryptionInfo) -{ - hc->init(); - /* Prepare HttpClient url. */ - url = parseKmsUrl(); - std::string urlSuffix = "keyversion/" + hc->escape(encryptionInfo.getEzKeyVersionName()) + "/_eek?eek_op=decrypt"; - url = buildKmsUrl(url, urlSuffix); - /* Prepare HttpClient headers. */ - std::vector headers; - setCommonHeaders(headers); - /* Prepare HttpClient body with json format. */ - ptree map; - map.put("name", encryptionInfo.getKeyName()); - map.put("iv", base64Encode(encryptionInfo.getIv())); - map.put("material", base64Encode(encryptionInfo.getKey())); - std::string body = toJson(map); - - /* Set options for HttpClient. */ - hc->setURL(url); - hc->setHeaders(headers); - hc->setBody(body); - hc->setExpectedResponseCode(200); - hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes()); - hc->setRequestTimeout(conf->getCurlTimeOut()); - std::string response = hc->post(); - - LOG(DEBUG3, - "KmsClientProvider::decryptEncryptedKey : The kms url is : %s . The kms body is : %s. The response of kms server is : %s .", - url.c_str(), body.c_str(), response.c_str()); - return fromJson(response); -} - -} - diff --git a/depends/libhdfs3/src/client/KmsClientProvider.h b/depends/libhdfs3/src/client/KmsClientProvider.h deleted file mode 100644 index a6c433600b..0000000000 --- a/depends/libhdfs3/src/client/KmsClientProvider.h +++ /dev/null @@ -1,142 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _HDFS_LIBHDFS3_CLIENT_KMSCLIENTPROVIDER_H_ -#define _HDFS_LIBHDFS3_CLIENT_KMSCLIENTPROVIDER_H_ - -#include -#include - -#include "openssl/conf.h" -#include "openssl/evp.h" -#include "openssl/err.h" -#include "FileEncryptionInfo.h" -#include "HttpClient.h" -#include -#include "common/SessionConfig.h" -#include "rpc/RpcAuth.h" -#include "common/Memory.h" -#include - -using boost::property_tree::ptree; -using namespace Hdfs::Internal; - - -namespace Hdfs { - -class KmsClientProvider { -public: - - /** - * Construct a KmsClientProvider instance. - * @param auth RpcAuth to get the auth method and user info. - * @param conf a SessionConfig to get the configuration. - */ - KmsClientProvider(shared_ptr auth, shared_ptr conf); - - /** - * Destroy a KmsClientProvider instance. - */ - virtual ~KmsClientProvider() { - } - - /** - * Set HttpClient object. - */ - void setHttpClient(shared_ptr hc); - - /** - * Create an encryption key from kms. - * @param keyName the name of this key. - * @param cipher the ciphertext of this key. e.g. "AES/CTR/NoPadding" . - * @param length the length of this key. - * @param material will be encode to base64. - * @param description key's info. - */ - virtual void createKey(const std::string &keyName, const std::string &cipher, const int length, const std::string &material, const std::string &description); - - /** - * Get key metadata based on encrypted file's key name. - * @param encryptionInfo the encryption info of file. - * @return return response info about key metadata from kms server. - */ - virtual ptree getKeyMetadata(const FileEncryptionInfo &encryptionInfo); - - /** - * Delete an encryption key from kms. - * @param encryptionInfo the encryption info of file. - */ - virtual void deleteKey(const FileEncryptionInfo &encryptionInfo); - - /** - * Decrypt an encrypted key from kms. - * @param encryptionInfo the encryption info of file. - * @return return decrypted key. - */ - virtual ptree decryptEncryptedKey(const FileEncryptionInfo &encryptionInfo); - - /** - * Encode string to base64. - */ - static std::string base64Encode(const std::string &data); - - /** - * Decode base64 to string. - */ - static std::string base64Decode(const std::string &data); - -private: - - /** - * Convert ptree format to json format. - */ - static std::string toJson(const ptree &data); - - /** - * Convert json format to ptree format. - */ - static ptree fromJson(const std::string &data); - - /** - * Parse kms url from configure file. - */ - std::string parseKmsUrl(); - - /** - * Build kms url based on urlSuffix and different auth method. - */ - std::string buildKmsUrl(const std::string &url, const std::string &urlSuffix); - /** - * Set common headers for kms API. - */ - void setCommonHeaders(std::vector& headers); - - shared_ptr hc; - std::string url; - - shared_ptr auth; - AuthMethod method; - shared_ptr conf; - -}; - -} -#endif diff --git a/depends/libhdfs3/test/function/TestKmsClient.cpp b/depends/libhdfs3/test/function/TestKmsClient.cpp deleted file mode 100644 index 0295866c8b..0000000000 --- a/depends/libhdfs3/test/function/TestKmsClient.cpp +++ /dev/null @@ -1,178 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "client/FileSystem.h" -#include "client/FileSystemInter.h" -#include "DateTime.h" -#include "Exception.h" -#include "ExceptionInternal.h" -#include "gtest/gtest.h" -#include "TestUtil.h" -#include "Thread.h" -#include "XmlConfig.h" -#include "client/KmsClientProvider.h" -#include "client/HttpClient.h" -#include "client/hdfs.h" - -#include - -#ifndef TEST_HDFS_PREFIX -#define TEST_HDFS_PREFIX "./" -#endif - -#define BASE_DIR TEST_HDFS_PREFIX"/testKmsClient/" - -using namespace Hdfs; -using namespace Hdfs::Internal; - -class TestKmsClient: public ::testing::Test { -public: - TestKmsClient() : - conf("function-test.xml") { - conf.set("hadoop.kms.authentication.type", "simple"); - conf.set("dfs.encryption.key.provider.uri", - "kms://http@0.0.0.0:16000/kms"); - sconf.reset(new SessionConfig(conf)); - userInfo.setRealUser("abai"); - auth.reset(new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod()))); - hc.reset(new HttpClient()); - kcp.reset(new KmsClientProvider(auth, sconf)); - kcp->setHttpClient(hc); - fs.reset(new FileSystem(conf)); - fs->connect(); - } - - ~TestKmsClient() { - try { - fs->disconnect(); - } catch (...) { - } - } -protected: - Config conf; - UserInfo userInfo; - shared_ptr auth; - shared_ptr hc; - shared_ptr kcp; - shared_ptr sconf; - shared_ptr fs; -}; - -TEST_F(TestKmsClient, CreateKeySuccess) { - std::string keyName = "testcreatekeyname"; - std::string cipher = "AES/CTR/NoPadding"; - int length = 128; - std::string material = "testCreateKey"; - std::string description = "Test create key success."; - ASSERT_NO_THROW( - kcp->createKey(keyName, cipher, length, material, description)); -} - -TEST_F(TestKmsClient, GetKeyMetadataSuccess) { - FileEncryptionInfo encryptionInfo; - encryptionInfo.setKeyName("testcreatekeyname"); - ptree map = kcp->getKeyMetadata(encryptionInfo); - std::string keyName = map.get < std::string > ("name"); - ASSERT_STREQ("testcreatekeyname", keyName.c_str()); -} - -TEST_F(TestKmsClient, DeleteKeySuccess) { - FileEncryptionInfo encryptionInfo; - encryptionInfo.setKeyName("testcreatekeyname"); - ASSERT_NO_THROW(kcp->deleteKey(encryptionInfo)); -} - - -TEST_F(TestKmsClient, DecryptEncryptedKeySuccess) { - hdfsFS hfs = NULL; - struct hdfsBuilder * bld = hdfsNewBuilder(); - assert(bld != NULL); - hdfsBuilderSetNameNode(bld, "default"); - hfs = hdfsBuilderConnect(bld); - - //create key - hc.reset(new HttpClient()); - kcp.reset(new KmsClientProvider(auth, sconf)); - kcp->setHttpClient(hc); - std::string keyName = "testdekeyname"; - std::string cipher = "AES/CTR/NoPadding"; - int length = 128; - std::string material = "test DEK"; - std::string description = "Test DEK create key success."; - kcp->createKey(keyName, cipher, length, material, description); - - //delete dir - hdfsDelete(hfs, BASE_DIR"/testDEKey", true); - - //create dir - EXPECT_EQ(0, hdfsCreateDirectory(hfs, BASE_DIR"/testDEKey")); - - //create encryption zone and encrypted file - ASSERT_EQ(0, - hdfsCreateEncryptionZone(hfs, BASE_DIR"/testDEKey", "testdekeyname")); - std::string hadoop_command = "hadoop fs -touchz "; - std::string tdeFile = BASE_DIR"/testDEKey/tdefile"; - std::string createFile = hadoop_command + tdeFile; - std::system(createFile.c_str()); - - //decrypt encrypted key - hc.reset(new HttpClient()); - kcp.reset(new KmsClientProvider(auth, sconf)); - kcp->setHttpClient(hc); - FileStatus fileStatus = fs->getFileStatus(tdeFile.c_str()); - FileEncryptionInfo *enInfo = fileStatus.getFileEncryption(); - ptree map = kcp->decryptEncryptedKey(*enInfo); - std::string versionName = map.get < std::string > ("versionName"); - ASSERT_STREQ("EK", versionName.c_str()); - - //delete key - hc.reset(new HttpClient()); - kcp.reset(new KmsClientProvider(auth, sconf)); - kcp->setHttpClient(hc); - FileEncryptionInfo encryptionInfo; - encryptionInfo.setKeyName("testdekeyname"); - kcp->deleteKey(encryptionInfo); - -} - -TEST_F(TestKmsClient, CreateKeyFailediBadUrl) { - std::string keyName = "testcreatekeyfailname"; - std::string cipher = "AES/CTR/NoPadding"; - std::string material = "testCreateKey"; - - std::string url[4] = { "ftp:///http@localhost:16000/kms", - "kms://htttp@localhost:16000/kms", - "kms:///httpss@localhost:16000/kms", - "kms:///http@localhost:16000/kms" }; - for (int i = 0; i < 4; i++) { - conf.set("hadoop.kms.authentication.type", "simple"); - conf.set("dfs.encryption.key.provider.uri", url[i]); - sconf.reset(new SessionConfig(conf)); - userInfo.setRealUser("abai"); - auth.reset(new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod()))); - hc.reset(new HttpClient()); - kcp.reset(new KmsClientProvider(auth, sconf)); - ASSERT_THROW(kcp->createKey("tesTdeBadUrl", "AES/CTR/NoPadding", 128, - "test DEK", "test DEK description"), HdfsIOException); - } -} - - diff --git a/depends/libhdfs3/test/unit/UnitTestCryptoCodec.cpp b/depends/libhdfs3/test/unit/UnitTestCryptoCodec.cpp deleted file mode 100644 index 92e94038a2..0000000000 --- a/depends/libhdfs3/test/unit/UnitTestCryptoCodec.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/******************************************************************** - * 2014 - - * open source under Apache License Version 2.0 - ********************************************************************/ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gtest/gtest.h" -#include "gmock/gmock.h" - -#include "client/FileSystem.h" -#include "client/FileSystemImpl.h" -#include "client/FileSystemInter.h" -#include "client/OutputStream.h" -#include "client/OutputStreamImpl.h" -#include "client/Packet.h" -#include "client/Pipeline.h" -#include "DateTime.h" -#include "MockFileSystemInter.h" -#include "MockCryptoCodec.h" -#include "MockKmsClientProvider.h" -#include "MockHttpClient.h" -#include "MockLeaseRenewer.h" -#include "MockPipeline.h" -#include "NamenodeStub.h" -#include "server/ExtendedBlock.h" -#include "TestDatanodeStub.h" -#include "TestUtil.h" -#include "Thread.h" -#include "XmlConfig.h" -#include "client/KmsClientProvider.h" -#include - -using namespace Hdfs; -using namespace Hdfs::Internal; -using namespace Hdfs::Mock; -using namespace testing; -using ::testing::AtLeast; - - -class TestCryptoCodec: public ::testing::Test { -public: - TestCryptoCodec() { - - } - - ~TestCryptoCodec() { - } - -protected: -}; - -TEST_F(TestCryptoCodec, KmsGetKey_Success) { - FileEncryptionInfo encryptionInfo; - encryptionInfo.setKeyName("KmsName"); - encryptionInfo.setIv("KmsIv"); - encryptionInfo.setEzKeyVersionName("KmsVersionName"); - encryptionInfo.setKey("KmsKey"); - Config conf; - conf.set("hadoop.kms.authentication.type", "simple"); - conf.set("dfs.encryption.key.provider.uri", "kms://http@0.0.0.0:16000/kms"); - shared_ptr sconf(new SessionConfig(conf)); - UserInfo userInfo; - userInfo.setRealUser("abai"); - shared_ptr auth(new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod()))); - - KmsClientProvider kcp(auth, sconf); - shared_ptr hc(new MockHttpClient()); - kcp.setHttpClient(hc); - - EXPECT_CALL(*hc, post()).Times(1).WillOnce( - Return(hc->getPostResult(encryptionInfo))); - - ptree map = kcp.decryptEncryptedKey(encryptionInfo); - std::string KmsKey = map.get < std::string > ("material"); - - ASSERT_STREQ("KmsKey", KmsKey.c_str()); -} - -TEST_F(TestCryptoCodec, encode_Success) { - FileEncryptionInfo encryptionInfo; - encryptionInfo.setKeyName("ESKeyName"); - encryptionInfo.setIv("ESIv"); - encryptionInfo.setEzKeyVersionName("ESVersionName"); - - Config conf; - conf.set("hadoop.kms.authentication.type", "simple"); - conf.set("dfs.encryption.key.provider.uri", "kms://http@0.0.0.0:16000/kms"); - shared_ptr sconf(new SessionConfig(conf)); - UserInfo userInfo; - userInfo.setRealUser("abai"); - shared_ptr auth( - new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod()))); - - shared_ptr kcp( - new MockKmsClientProvider(auth, sconf)); - - //char buf[1024] = "encode hello world"; - char buf[1024]; - Hdfs::FillBuffer(buf, sizeof(buf)-1, 2048); - buf[sizeof(buf)-1] = 0; - - int32_t bufSize = 1024; - - std::string Key[2] = { "012345678901234567890123456789ab", - "0123456789012345"}; - for (int i = 0; i < 2; i++) { - encryptionInfo.setKey(Key[i]); - shared_ptr hc(new MockHttpClient()); - kcp->setHttpClient(hc); - - EXPECT_CALL(*kcp, decryptEncryptedKey(_)).Times(2).WillRepeatedly( - Return(kcp->getEDKResult(encryptionInfo))); - - CryptoCodec es(&encryptionInfo, kcp, bufSize); - es.init(CryptoMethod::ENCRYPT); - CryptoCodec ds(&encryptionInfo, kcp, bufSize); - ds.init(CryptoMethod::DECRYPT); - - - std::string encodeStr = es.cipher_wrap(buf, strlen(buf)); - ASSERT_NE(0, memcmp(buf, encodeStr.c_str(), strlen(buf))); - - std::string decodeStr = ds.cipher_wrap(encodeStr.c_str(), strlen(buf)); - ASSERT_STREQ(decodeStr.c_str(), buf); - } -}