I'm using libcurl with C++. I have made a thread-safe class for downloading webpages. Each call to static download method creates "easy" handle, performs the job and frees the handle. When I use it from the single thread - everything's fine. But when I spawn several threads to download several pages in parallel I sometimes (not fro every download, but quite often) get error saying "timeout". I have a reasonably high timeout configured (5 sec connection timeout and 25 sec global timeout).

Any ideas as to what might be the cause and how to fix it?
Thanks is advance!

Here's the code of the method in question:

void CHttpDownloaderLibcurl::downloaderThread( const CUrl& url, CThreadSafeQueue<CHtmlPage>& q)
{
ššššCHtmlPage page (url);

ššššCURL* handle = curl_easy_init();

ššššif (!handle)
šššš{
ššššššššassert(handle);
ššššššššreturn;
šššš}

ššššint curlErr = setCurlOptions(handle, url, (void*)onCurlDownloadCallback, (void*)&page.byteArray());
ššššif (CURLE_OK != curlErr)
šššš{
ššššššššassert("Error setting options" == (char*)curlErr);
ššššššššreturn;
šššš}

ššššcurlErr = curl_easy_perform(handle);

ššššpage._info = getInfo(handle);

ššššcurl_easy_cleanup(handle);
ššššif (CURLE_OK != curlErr)
šššš{
ššššššššif (curlErr == CURLE_OPERATION_TIMEDOUT)
šššššššš{
ššššššššššššCLogger() << "Curl timeout!";
šššššššš}
ššššššššelse
ššššššššššššCLogger() << url.urlString() << ": Error performing download = " << curlErr;
ššššššššreturn;
šššš}

ššššq.push(page);
}
-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
Received on 2011-12-14