Hi all,

We're investigating the multi interface provided by libcurl to enhance transfer rates. We used the examples we found to upload a number of files to one server; each file is related to an easy_handle. We observe that a huge number of connections are opened each with its own control and data channel. After a short while a significant number of connections drop after failure. Our observation is not as expected and a few queries arose:

:

1.) Is there a way to limit the number of connections?

2.) Is there a way to open one control channel with a number of associated data channels while transferring to one server?

3.) Would a handle pool be an appropriate approach?

4.) Are the errors we observe forced by the way we used the multi interface or are these likely server related issues?

Please refer to the code for upload below (Windows specific so far):

            // Initialize:

      curl_global_init( CURL_GLOBAL_WIN32 );

g_hMutex = CreateMutex( NULL, FALSE, NULL );

      CURLSH* hShare = curl_share_init();

      curl_share_setopt( hShare, CURLSHOPT_LOCKFUNC, Curl_Share_Lock );

      curl_share_setopt( hShare, CURLSHOPT_UNLOCKFUNC, Curl_Share_Unlock );

      curl_share_setopt( hShare, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS );

      CURLM* hMulti = curl_multi_init();

      std::vector<CURL*> easy_handles;

// Iterate all files:

      for( OsFileVector::iterator iter = files.begin(); iter != files.end(); iter++ )

{

            CURL* hCurl = curl_easy_init();

            easy_handles.push_back( hCurl );

            curl_easy_setopt( hCurl, CURLOPT_SHARE, hShare );

            __int64 size;

            if( (*iter)->GetSize( size ) )

                  curl_easy_setopt( hCurl, CURLOPT_INFILESIZE_LARGE, size );

            curl_easy_setopt( hCurl, CURLOPT_UPLOAD, 1 );

            curl_easy_setopt( hCurl, CURLOPT_URL, ToMbcs( to_url + to_path + files.GetRelativePath( (*iter)->GetPathname() ) ).c_str() );

            curl_easy_setopt( hCurl, CURLOPT_USERPWD, ToMbcs( user + _T(":") + password ).c_str() );

            curl_easy_setopt( hCurl, CURLOPT_READFUNCTION, UploadCallback );

            curl_easy_setopt( hCurl, CURLOPT_READDATA, *iter );

            curl_easy_setopt( hCurl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0 );

            curl_easy_setopt( hCurl, CURLOPT_NOSIGNAL, 1 );

            curl_easy_setopt( hCurl, CURLOPT_VERBOSE, 1 );

            curl_easy_setopt( hCurl, CURLOPT_FTP_CREATE_MISSING_DIRS, 1 );

curl_multi_add_handle( hMulti, hCurl );

      }

      int running_handles;

      while( curl_multi_perform( hMulti, &running_handles ) == CURLM_CALL_MULTI_PERFORM ) ;

      while( running_handles )

      {

            int num_messages;

            CURLMsg* pMsg = curl_multi_info_read( hMulti, &num_messages );

            while( pMsg ) {

                  switch( pMsg->msg )

                  {

                        case CURLMSG_DONE:

                             std::cout << "-------> Handle " << pMsg->easy_handle << ", code " << pMsg->data.result << ": " << curl_easy_strerror( pMsg->data.result ) << std::endl;

                             break;

                  }

                  pMsg = curl_multi_info_read( hMulti, &num_messages );

            }

            int fdMax;

            fd_set fdRead, fdWrite, fdException;

            FD_ZERO( &fdRead );

            FD_ZERO( &fdWrite );

            FD_ZERO( &fdException );

            curl_multi_fdset( hMulti, &fdRead, &fdWrite, &fdException, &fdMax );

            timeval timeout;

            timeout.tv_sec = 1;

            timeout.tv_usec = 0;

            if( select( fdMax + 1, &fdRead, &fdWrite, &fdException, &timeout ) < 0 )

                  break; // error

            else

                  while( curl_multi_perform( hMulti, &running_handles ) == CURLM_CALL_MULTI_PERFORM ) ;

      }

// Cleanup:

for( std::vector<CURL*>::iterator iter = easy_handles.begin(); iter != easy_handles.end(); iter++ )

            curl_easy_cleanup( *iter );

      curl_multi_cleanup( hMulti );

      curl_share_cleanup( hShare );

      CloseHandle( g_hMutex );

      curl_global_cleanup();

Thanks a lot for your hints and support.

-- Peter :-)

Peter Thiess, Software Engineer

Adobe Systems Engineering GmbH

Grosse Elbstrasse 27

22767 Hamburg, Germany

+49 40 306 36 214

pthiess_at_adobe.com

Registergericht: Hamburg HRB 745 37

Geschäftsführer: Thomas Mührke

 
Received on 2008-08-12