add uploading progress

qiniu · Aug 28, 2024 · e9d0833 · e9d0833
1 parent 69d57f7
commit e9d0833
Show file tree

Hide file tree

Showing 9 changed files with 326 additions and 43 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@
 - 支持缓存多个 `v4/query` 查询结果
 - 上传重试支持切换上传域名
 - 支持上传加速域名
+- 增加上传进度回调支持
 
 ## v7.7.0 (2023-12-25)
 

diff --git a/qiniu/http.c b/qiniu/http.c
@@ -553,25 +553,61 @@ static Qiniu_Error Qiniu_Client_callWithBody(
 Qiniu_Error Qiniu_Client_CallWithMethod(
     Qiniu_Client *self, Qiniu_Json **ret, const char *url,
     Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5)
+{
+    return Qiniu_Client_CallWithMethodAndProgressCallback(self, ret, url, body, bodyLen, mimeType, httpMethod, md5, NULL, NULL);
+}
+
+Qiniu_Error Qiniu_Client_CallWithMethodAndProgressCallback(
+    Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+    Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5,
+    int (*callback)(void *, double, double, double, double), void *callbackData)
 {
     CURL *curl = Qiniu_Client_initcall_withMethod(self, url, httpMethod);
 
     curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, bodyLen);
     curl_easy_setopt(curl, CURLOPT_READFUNCTION, body.Read);
     curl_easy_setopt(curl, CURLOPT_READDATA, body.self);
+    if (callback != NULL)
+    {
+        curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, callback);
+        curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, callbackData);
+        curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+    }
+    else
+    {
+        curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+    }
 
     return Qiniu_Client_callWithBody(self, ret, url, NULL, bodyLen, mimeType, md5);
 }
 
 Qiniu_Error Qiniu_Client_CallWithBinary(
     Qiniu_Client *self, Qiniu_Json **ret, const char *url,
     Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType)
+{
+    return Qiniu_Client_CallWithBinaryAndProgressCallback(self, ret, url, body, bodyLen, mimeType, NULL, NULL);
+}
+
+Qiniu_Error Qiniu_Client_CallWithBinaryAndProgressCallback(
+    Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+    Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType,
+    int (*callback)(void *, double, double, double, double), void *callbackData)
 {
     CURL *curl = Qiniu_Client_initcall(self, url);
 
     curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, bodyLen);
     curl_easy_setopt(curl, CURLOPT_READFUNCTION, body.Read);
     curl_easy_setopt(curl, CURLOPT_READDATA, body.self);
+    if (callback != NULL)
+    {
+        curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, callback);
+        curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, callbackData);
+        curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+    }
+    else
+    {
+        curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+    }
 
     return Qiniu_Client_callWithBody(self, ret, url, NULL, bodyLen, mimeType, NULL);
 }

diff --git a/qiniu/http.h b/qiniu/http.h
@@ -177,6 +177,16 @@ extern "C"
 	QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithMethod(
 		Qiniu_Client *self, Qiniu_Json **ret, const char *url,
 		Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5);
+
+	QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithBinaryAndProgressCallback(
+		Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+		Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType,
+		int (*callback)(void *, double, double, double, double), void *callbackData);
+
+	QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithMethodAndProgressCallback(
+		Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+		Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5,
+		int (*callback)(void *, double, double, double, double), void *callbackData);
 	/*============================================================================*/
 	/* func Qiniu_Client_InitNoAuth/InitMacAuth  */
 

diff --git a/qiniu/io.c b/qiniu/io.c
@@ -152,6 +152,13 @@ Qiniu_Error Qiniu_Client_config(Qiniu_Client *self)
     return err;
 }
 
+static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+    void (*uploadingProgress)(size_t, size_t) = (void (*)(size_t, size_t))clientp;
+    uploadingProgress((size_t)ultotal, (size_t)ulnow);
+    return 0;
+}
+
 static Qiniu_Error Qiniu_Io_call(
     Qiniu_Client *self, const char *accessKey, const char *bucketName,
     Qiniu_Io_PutRet *ret, struct curl_httppost *formpost, Qiniu_Io_PutExtra *extra)
@@ -187,6 +194,16 @@ static Qiniu_Error Qiniu_Io_call(
         curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries % upHostsCount]);
         curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+        if (extra->uploadingProgress != NULL)
+        {
+            curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, _Qiniu_Progress_Callback);
+            curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, extra->uploadingProgress);
+            curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+        }
+        else
+        {
+            curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+        }
 
         //// For aborting uploading file.
         if (extra->upAbortCallback)
@@ -379,6 +396,17 @@ static Qiniu_Error Qiniu_Io_call_with_callback(
         curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
         curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);
 
+        if (extra->uploadingProgress != NULL)
+        {
+            curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, _Qiniu_Progress_Callback);
+            curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, extra->uploadingProgress);
+            curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+        }
+        else
+        {
+            curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+        }
+
         err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
         if (err.code == 200)
         {

diff --git a/qiniu/io.h b/qiniu/io.h
@@ -61,6 +61,9 @@ extern "C"
 		// Specify multiple upHosts
 		const char *const *upHosts;
 		size_t upHostsCount;
+
+		// Uploading file progress
+		void (*uploadingProgress)(size_t ultotal, size_t ulnow);
 	} Qiniu_Io_PutExtra;
 
 	/*============================================================================*/

diff --git a/qiniu/multipart_upload.c b/qiniu/multipart_upload.c
@@ -50,6 +50,12 @@ typedef struct
     int totalPartNum;
 } Qiniu_UploadParts_Ret;
 
+struct _Qiniu_Progress_Callback_Data
+{
+    size_t base, totalSize, previousUlNow;
+    void (*callback)(size_t, size_t);
+};
+
 static Qiniu_Error readMedium(struct Qiniu_Record_Medium *medium, char **uploadId, Qiniu_Uint64 *expireAt, Qiniu_UploadPartResp *ret);
 static Qiniu_Error writeMedium(struct Qiniu_Record_Medium *medium, const Qiniu_InitPart_Ret *, const Qiniu_UploadPartResp *);
 static Qiniu_Error initializeRecorder(Qiniu_Multipart_PutExtra *param, const char *uptoken, const char *key, const char *fileName, Qiniu_FileInfo *fi, Qiniu_Record_Medium *medium, Qiniu_Multipart_Recorder *recorder);
@@ -166,13 +172,31 @@ static Qiniu_Error init_upload(Qiniu_Client *client, const char *bucket, const c
     return err;
 }
 
+static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+    struct _Qiniu_Progress_Callback_Data *data = (struct _Qiniu_Progress_Callback_Data *)clientp;
+    if (data->previousUlNow != (size_t)ulnow)
+    {
+        data->callback((size_t)data->totalSize, data->base + (size_t)ulnow);
+        data->previousUlNow = (size_t)ulnow;
+    }
+    return 0;
+}
+
 Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extraParam, const char *path,
                             int partNum, Qiniu_ReaderAt reader, Qiniu_Int64 partOffset, Qiniu_Int64 partSize, const char *md5str,
-                            Qiniu_UploadPartResp *ret)
+                            Qiniu_UploadPartResp *ret, struct _Qiniu_Progress_Callback_Data *progressCallback)
 {
     Qiniu_Error err = Qiniu_OK;
     const char *const *upHosts;
     size_t upHostsCount;
+    int (*callback)(void *, double, double, double, double) = NULL;
+    void *callbackData = NULL;
+    if (progressCallback != NULL && progressCallback->callback != NULL)
+    {
+        callback = _Qiniu_Progress_Callback;
+        callbackData = (void *)progressCallback;
+    }
 
     if (extraParam->upHost != NULL)
     {
@@ -192,7 +216,7 @@ Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extr
         Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call.
         const char *upHost = upHosts[tries % upHostsCount];
         const char *reqUrl = Qiniu_String_Concat(upHost, path, NULL);
-        err = Qiniu_Client_CallWithMethod(client, &callRet, reqUrl, thisPartBody, partSize, NULL, "PUT", md5str);
+        err = Qiniu_Client_CallWithMethodAndProgressCallback(client, &callRet, reqUrl, thisPartBody, partSize, NULL, "PUT", md5str, callback, callbackData);
         Qiniu_Free((void *)reqUrl);
         if (err.code == 200)
         {
@@ -229,47 +253,54 @@ static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const
     Qiniu_Error err = Qiniu_OK;
     Qiniu_Int64 partSize = extraParam->partSize;
     const int lastPart = totalPartNum - 1;
-    for (int partNum = 0; partNum < totalPartNum; partNum++)
-    {
-        if ((uploadPartsRet->PartsRet + partNum)->etag != NULL)
-        {
-            continue;
-        }
+    struct _Qiniu_Progress_Callback_Data progressCallbackData;
+    Qiniu_Zero(progressCallbackData);
 
-        const int partNumInReq = partNum + 1; // partNum start from 1
-        char partNumStr[10];            // valid partNum ={"1"~"10000"}
-        snprintf(partNumStr, 10, "%d", partNumInReq);
-        const char *path = Qiniu_String_Concat("/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL);
+    if (extraParam->uploadingProgress != NULL)
+    {
+        progressCallbackData.callback = extraParam->uploadingProgress;
+        progressCallbackData.totalSize = (size_t)fsize;
+    }
 
+    for (int partNum = 0; partNum < totalPartNum; partNum++)
+    {
         Qiniu_Int64 thisPartOffset = partNum * partSize;
         Qiniu_Int64 thisPartSize = partSize;
         if (partNum == lastPart)
         {
             thisPartSize = fsize - (totalPartNum - 1) * partSize;
         }
 
-        const char *md5str = NULL;
-        if (extraParam->enableContentMd5)
+        if ((uploadPartsRet->PartsRet + partNum)->etag == NULL)
         {
-            md5str = caculatePartMd5(*reader, thisPartOffset, thisPartSize);
-            // Qiniu_Log_Debug("partNum:%d, local Md5:%s ", partNumInReq, md5str);
-        }
-        err = upload_one_part(client, extraParam, path, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum]);
-        Qiniu_Multi_Free(2, (void *)path, (void *)md5str);
+            const int partNumInReq = partNum + 1; // partNum start from 1
+            char partNumStr[10];                  // valid partNum ={"1"~"10000"}
+            snprintf(partNumStr, 10, "%d", partNumInReq);
+            const char *path = Qiniu_String_Concat("/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL);
 
-        if (err.code != 200)
-        {
-            return err;
-        }
+            const char *md5str = NULL;
+            if (extraParam->enableContentMd5)
+            {
+                md5str = caculatePartMd5(*reader, thisPartOffset, thisPartSize);
+                // Qiniu_Log_Debug("partNum:%d, local Md5:%s ", partNumInReq, md5str);
+            }
+            err = upload_one_part(client, extraParam, path, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum], &progressCallbackData);
+            Qiniu_Multi_Free(2, (void *)path, (void *)md5str);
 
-        if (recorder != NULL && recorder->recorderMedium != NULL)
-        {
-            err = writeMedium(recorder->recorderMedium, initParts, &uploadPartsRet->PartsRet[partNum]);
             if (err.code != 200)
             {
                 return err;
             }
+            if (recorder != NULL && recorder->recorderMedium != NULL)
+            {
+                err = writeMedium(recorder->recorderMedium, initParts, &uploadPartsRet->PartsRet[partNum]);
+                if (err.code != 200)
+                {
+                    return err;
+                }
+            }
         }
+        progressCallbackData.base += thisPartSize;
     }
     return err;
 }

diff --git a/qiniu/multipart_upload.h b/qiniu/multipart_upload.h
@@ -63,6 +63,9 @@ extern "C"
         // Specify multiple upHosts, if not set explicitly, will global QINIU_UP_HOST
         const char *const *upHosts;
         size_t upHostsCount;
+
+        // Uploading file progress
+        void (*uploadingProgress)(size_t ultotal, size_t ulnow);
     } Qiniu_Multipart_PutExtra;
 
     typedef struct