I am a beginner in both C programming and libcurl and writing a program to
fetch 1000 data values from a website. The website provides a job number
and is redirected into another page for the results. Since, the code I have
written is almost 500 lines, I am giving a general flow of the program and
a short code which I think is the problematic area:

      for(row=0;row<1000;row++)
      {
       ------
        url = "http://xxx.com";

        curl_global_init(CURL_GLOBAL_ALL);
curlHandle = curl_easy_init();
if(curlHandle)
{
curl_easy_setopt(curlHandle, CURLOPT_TIMEOUT, 1800);
curl_easy_setopt(curlHandle, CURLOPT_ERRORBUFFER, curlErrStr);
curl_easy_setopt(curlHandle, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curlHandle, CURLOPT_URL, url);
curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_LIMIT, dl_lowspeed_bytes);

curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_TIME, dl_lowspeed_time);
curl_easy_setopt(curlHandle, CURLOPT_VERBOSE, 1L);
free(url);
curlErr = curl_easy_perform(curlHandle);
if(curlErr != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed:
%s\n",curl_easy_strerror(curlErr));
}
else
{
curlErr = curl_easy_getinfo(curlHandle, CURLINFO_EFFECTIVE_URL, &url_new);
if((CURLE_OK == curlErr) && url_new)
{
sprintf(job,"%.*s\n", 18, url_new + 28);
if((ptr1 = strchr(job, '\n')) != NULL)
*ptr1 = '\0';
init_string(&s);
curl_easy_setopt(curlHandle, CURLOPT_TIMEOUT, 1800 );
curl_easy_setopt(curlHandle, CURLOPT_URL, url_new);
curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, &s);
curlErr1 = curl_easy_perform(curlHandle);
printf("###### %lu\t%s\n",strlen(s.ptr),s.ptr);
free(s.ptr);
}
curl_easy_cleanup(curlHandle);
}
}

The functions are:

      struct string
      {
   char *ptr;
   size_t len;
      };

      void init_string(struct string *a)
      {
   a->len = 0;
   a->ptr = malloc(a->len+1);
   if (a->ptr == NULL)
   {
fprintf(stderr, "malloc() failed\n");
            exit(EXIT_FAILURE);
   }
   a->ptr[0] = '\0';
      }

      size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string
*a)
      {
   size_t new_len = a->len + size*nmemb;
   a->ptr = realloc(a->ptr, new_len+1);
   if (a->ptr == NULL)
   {
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
   }
   memcpy(a->ptr+a->len, ptr, size*nmemb);
   a->ptr[new_len] = '\0';
   a->len = new_len;
   return size*nmemb;
      }

The program shows no error of any kind. But out of the 1000 data, almost
50% couldn't be fetched due to curl_easy_perform() failed: Timeout was
reached; and 20% of them have the output of the line strlen(s.ptr),s.ptr =>
0. The rest are fetched correctly.

The verbose option for the zero output gave the following:

* Connection #0 to host www.xxx.com left intact
* getaddrinfo(3) failed for :80
* Couldn't resolve host ''
* Closing connection #1
* Couldn't resolve host name
###### 0

Please suggest the possible errors in the program.

-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
Received on 2014-05-19