Hello Sir,

 I am working on developing a code, which is req. to read ~2000 urls through
its run. The code is developed to run with multiple threads. My each threads
tries to read a page from website by calling function "NEW_URL_LIB ::
read_curl()".

 I am creating CURL *curl_handle when i initialize thread.

 The code seems to working correctly for ~1000 pages from internet(~50-55
Min. of Execution.). After that my code starts firing following error
messages for some time.

 NEW_URL_LIB::read_curl(): URL=http://XXXXXXXXXXXXXX : Exception:
curl_easy_perform() failed with 55 : Failed sending HTTP request occured.

 Once i get this error for any thread, after that code starts repeating same
error for next 80-90 sites & after that it fires Signal 11 from
"curl_easy_perform(curl_handle)" call.

(gdb)bt
#0 0x281ed4f0 in _init () from /usr/local/lib/libcurl.so.2
#1 0x281fa861 in ConnectionExists () from /usr/local/lib/libcurl.so.2
#2 0x281fc66e in CreateConnection () from /usr/local/lib/libcurl.so.2
#3 0x281fcdfd in Curl_connect () from /usr/local/lib/libcurl.so.2
#4 0x28207d76 in Curl_perform () from /usr/local/lib/libcurl.so.2
#5 0x28208288 in curl_easy_perform () from /usr/local/lib/libcurl.so.2
#6 0x08079dd5 in NEW_URL_LIB::read_curl(void*, std::string const&,
std::string&, std::string const&) (this=0x80d4110, curl_handle=0x8151000,
stringURL1=@0x81ed044, urlText=@0x81ed048,
    name1=@0x81ed050) at src/new_url_lib.cpp:412

Environment:
 I have tested the code with 10 threads.
 FreeBSD - 5.1
 LibCurl 7.11.0
 g++ 3.2.2

 Once code exited, If i re run code on same 80-90 sites which were failed in
previous run, it will work properly for same sites.

 I have seen that if i decrease no. of threads, then total run time of code
will be longer. But no. of sites done are almost same.

 I have tested the code several time in last 3-4 days & not able to where to
do modification in code to prevent the error.

 I am planning to recreate CURL handle, Once i get "curl_easy_perform()
failed with 55 : Failed sending HTTP request occured.". But i am not sure
whether it will definitely solve the problem of Signal 11.

Please guide me.

Vijay Patel.

struct MemoryStruct {
 char *memory;
 size_t size;
 int max_page_size;
};

extern "C" {
size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{
 register int realsize = size * nmemb;
 struct MemoryStruct *mem = (struct MemoryStruct *)data;

 mem->memory = (char *)realloc(mem->memory, mem->size + realsize + 1);
 if (mem->memory) {
  memcpy(&(mem->memory[mem->size]), ptr, realsize);
  mem->size += realsize;
  mem->memory[mem->size] = 0;
 }
 else
  return 0;

 if(mem->size > mem->max_page_size) {
  return 0;
 }

 return realsize;
}
}

bool NEW_URL_LIB :: read_curl (CURL *curl_handle, const String & stringURL1,
String & urlText, const String & name1)
{
 //CURL *curl_handle;
 struct MemoryStruct chunk;
 chunk.memory=NULL; /* we expect realloc(NULL, size) to work */
 chunk.size = 0; /* no data at this point */
 chunk.max_page_size = var->S_7MXPS;
 //char *err_string;
 int curl_errno;

 try
 {
  CharType err_string[CURL_ERROR_SIZE+1];
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER,
err_string))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_ERRORBUFFER) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 0))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_VERBOSE) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_NOPROGRESS) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1l))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_NOSIGNAL) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT,
(var->S_PR7LT/1000)))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_TIMEOUT) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE,
var->S_7MXPS))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_MAXFILESIZE) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_URL,
stringURL1.c_str()))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_URL) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION,
WriteMemoryCallback))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_WRITEFUNCTION) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void
*)&chunk))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_WRITEDATA) failed with " +
NumUtils::ToString(curl_errno));
  }
  if((curl_errno=curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1))>0) {
   throw Exception("curl_easy_setopt(CURLOPT_FAILONERROR) failed with " +
NumUtils::ToString(curl_errno));
  }

  if((curl_errno=curl_easy_perform(curl_handle))>0) {
   throw Exception("curl_easy_perform() failed with " +
NumUtils::ToString(curl_errno) + " : " + String(err_string));
  }

  char *content_type_info;
  if((curl_errno=curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_TYPE,
&content_type_info))>0)
  {
   throw Exception("curl_easy_getinfo(CURLINFO_CONTENT_TYPE) failed with " +
NumUtils::ToString(curl_errno));
  }
  String content_type = String(content_type_info);
  String readText;
  if ( !starts_with(content_type, "text/"))
  {
   if ( !starts_with(content_type, "application/x-shockwave-flash"))
   {
    throw Exception("OTHER_CONTENTTYPE, content_type = " + content_type);
   }
   else {
    if(!convertSwf2Text(name1,chunk,readText))
    {
     throw Exception("convertSwf2Text() returned false.");
    }
   }
  } else {
   readText = String(chunk.memory);
  }

  replace_me(readText, '\n', ' ');
  urlText = readText;
  free(chunk.memory);
  chunk.size = 0;
  return true;
 }
 catch (Exception &e)
 {
  var->writeLogLDEO(0,1,1,0,"NEW_URL_LIB::read_curl(): URL=" + stringURL1 +
" : " + e.toString() + " occured.");
  free(chunk.memory);
  chunk.size = 0;
  return false;
 }
 catch (std::exception &e) {
  var->writeLogLDEO(0,1,1,0,"NEW_URL_LIB::read_curl(): URL=" + stringURL1 +
" : " + e.what() + " occured.");
  free(chunk.memory);
  chunk.size = 0;
  return false;
   }
 catch (...) {
  var->writeLogLDEO(0,1,1,0,"NEW_URL_LIB::read_curl(): URL=" + stringURL1 +
" : ... unknown exception occured.");
  free(chunk.memory);
  chunk.size = 0;
  return false;
 }
}
Received on 2004-03-26