On 6/1/2015 6:45 AM, Vincas.Razma_at_bentley.com wrote:
>> That is usually a sign that you haven't provided the (correct) size of the data you are to upload.
> All size fields have 4000000 correctly set and logs indicate that everything was read to CURL. Below is sample code that I brought together to reproduce this issue - only thing I changed is URL and JSON. Let me know if you need working server URL for this to reproduce in your side.
>
> size_t header_callback (char *buffer, size_t size, size_t count, std::map<std::string, std::string>* headers)
> {
> size_t bufferSize = size * count;
> std::string header ((char*)buffer, bufferSize);
>
> size_t seperatorPos = header.find_first_of (':');
>
> if (std::string::npos != seperatorPos)
> {
> std::string key = header.substr (0, seperatorPos);
> std::string value = header.substr (seperatorPos + 2, header.length () - seperatorPos - 4);
> (*headers)[key] = value;
> }
>
> return bufferSize;
> }
>
> size_t write_callback (void* buffer, size_t size, size_t count, void* data)
> {
> size_t bufferSize = size * count;
> printf (std::string ((char*)buffer, bufferSize).c_str ());
> return bufferSize;
> }
>
> struct ReadInfo
> {
> size_t bytesTotal = 0;
> size_t bytesRead = 0;
> };
>
> size_t read_callback (void* buffer, size_t size, size_t count, ReadInfo* data)
> {
> size_t bufferSize = size * count;
>
> size_t bytesToRead = data->bytesTotal - data->bytesRead;
>
> size_t bytesReadToBuffer = 0;
> while (bytesReadToBuffer < bufferSize && bytesReadToBuffer < bytesToRead)
> {
> ((char*)buffer)[bytesReadToBuffer] = 'x';
> bytesReadToBuffer++;
> }
>
> data->bytesRead += bytesReadToBuffer;
>
> printf ("BytesRead: %d\n", data->bytesRead);
> printf ("BytesRead bytesReadToBuffer: %d\n", bytesReadToBuffer);
> printf ("BytesRead buffersize: %d\n", bufferSize);
>
> if (bytesReadToBuffer < bufferSize && bytesReadToBuffer != 0)
> {
> // This fixes issue with server hanging even when using CURLM approach
> //return bytesReadToBuffer + 1;
> }
>
> return bytesReadToBuffer;
> }
>
> TEST_F (CurlTests, CURL_ResumableHugeFileUpload_Hangs)
> {
> // HANDSHAKE JSON and URL (fakes here)
> std::string handshakeBody = R"({"some":"json"})";
> char* url = "https://test.com";
>
> CURLM* multi = curl_multi_init ();
> auto perform = [&] (CURL* curl)
> {
> // This code mimics reusing CURLM in WebThread
> ASSERT_EQ (CURLM_OK, curl_multi_add_handle (multi, curl));
>
> int running = 1;
> while (running)
> {
> ASSERT_EQ (CURLM_OK, curl_multi_perform (multi, &running));
> }
>
> int messages = 0;
> CURLMsg* curlMsg = curl_multi_info_read (multi, &messages);
>
> ASSERT_EQ (CURLE_OK, curlMsg->data.result); // When failing, returns CURLE_OPERATION_TIMEDOUT (28)
> ASSERT_EQ (curl, curlMsg->easy_handle);
> ASSERT_EQ (CURLM_OK, curl_multi_remove_handle (multi, curl));
> };
>
> // This works
> //auto perform = [&] (CURL* curl)
> // {
> // ASSERT_EQ (CURLE_OK, curl_easy_perform (curl));
> // };
>
> CURL* curl = nullptr;
> int responseStatus = 0;
> curl_slist* requestHeaders = nullptr;
> std::map<std::string, std::string> responseHeaders;
>
> auto setupCurlBase = [&] (CURL* curl)
> {
> responseStatus = 0;
> responseHeaders.clear ();
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_URL, url));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_CUSTOMREQUEST, "POST"));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_USERNAME, "admin"));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_PASSWORD, "admin"));
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_VERBOSE, 1L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_SSL_VERIFYPEER, 0L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_SSL_VERIFYHOST, 0L));
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_LOW_SPEED_LIMIT, 1L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_LOW_SPEED_TIME, 30L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_CONNECTTIMEOUT, 30L));
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_HEADERFUNCTION, header_callback));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_HEADERDATA, &responseHeaders));
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_callback));
> };
>
> // UPLOAD HANDSHAKE
> curl = curl_easy_init ();
> setupCurlBase (curl);
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_POSTFIELDSIZE, handshakeBody.length ()));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_POSTFIELDS, handshakeBody.c_str ()));
>
> curl_slist_free_all (requestHeaders);
> requestHeaders = curl_slist_append (requestHeaders, "Content-Disposition: attachment; filename=TestFile.txt");
> requestHeaders = curl_slist_append (requestHeaders, "Content-Range: bytes */8000000");
> requestHeaders = curl_slist_append (requestHeaders, "Content-Type: application/json");
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_HTTPHEADER, requestHeaders));
>
> perform (curl);
> ASSERT_EQ (CURLE_OK, curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &responseStatus));
> ASSERT_EQ (308, responseStatus);
> curl_easy_cleanup (curl);
>
> std::string eTag = responseHeaders["ETag"];
> std::string ifMatchHeader = "If-Match: " + eTag;
>
> // CHUNK 1
> curl = curl_easy_init ();
> setupCurlBase (curl);
>
> ReadInfo readInfo;
> readInfo.bytesRead = 0;
> readInfo.bytesTotal = 4000000;
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_UPLOAD, 1L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_INFILESIZE_LARGE, 4000000L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_READFUNCTION, read_callback));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_READDATA, &readInfo));
>
> curl_slist_free_all (requestHeaders);
> requestHeaders = nullptr;
> requestHeaders = curl_slist_append (requestHeaders, "Content-Range: bytes 0-3999999/8000000");
> requestHeaders = curl_slist_append (requestHeaders, ifMatchHeader.c_str ());
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_HTTPHEADER, requestHeaders));
>
> perform (curl);
> ASSERT_EQ (CURLE_OK, curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &responseStatus));
> ASSERT_EQ (308, responseStatus);
> curl_easy_cleanup (curl);
>
> // CHUNK 2
> curl = curl_easy_init ();
> setupCurlBase (curl);
>
> readInfo.bytesRead = 0;
> readInfo.bytesTotal = 4000000;
>
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_UPLOAD, 1L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_INFILESIZE_LARGE, 4000000L));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_READFUNCTION, read_callback));
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_READDATA, &readInfo));
>
> curl_slist_free_all (requestHeaders);
> requestHeaders = nullptr;
> requestHeaders = curl_slist_append (requestHeaders, "Content-Range: bytes 4000000-7999999/8000000");
> requestHeaders = curl_slist_append (requestHeaders, ifMatchHeader.c_str ());
> ASSERT_EQ (CURLE_OK, curl_easy_setopt (curl, CURLOPT_HTTPHEADER, requestHeaders));
>
> perform (curl);
> ASSERT_EQ (CURLE_OK, curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &responseStatus));
> ASSERT_EQ (201, responseStatus);
>
> curl_easy_cleanup (curl);
> }

Vincas I'm curious if you've made any progress on this. Did you happen
to try what Daniel suggested [1] ?

It looks like in your unit test a server is needed that provides the
custom response and also it's not self contained. I can use the google
testing framework but I can tell it's not going to build without
CurlTests class. Also I don't know what 'R"({"some":"json"})"' is but it
looks wrong, and so does the declaration of ReadInfo. Did you actually
run this test?

What version of libcurl are you using and what SSL backend and version
are you using? It's weird that you have the problem in Server 2008 but
not Server 2012. Are you using a version of libcurl or SSL backend that
differs between the two? x86 or x64?

I have Server 2008 R2 x64 SP1 but in a VM. I can try there but I need
something that will build.

[1]: http://curl.haxx.se/mail/lib-2015-06/0088.html

-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
Received on 2015-06-26