Hello,

I made this little program, attached at the end of this mail. It
requests two small files from the same server, and tries to pipeline
them.
It prints the libCurl version, the header lines, a log line for every
received data buffer, and a "Task done" message.
I attach outputs for libCurl 7.22 and 7.36. As you can see, in the
older version, received data is completely serialized: nothing refers
to transfer 2 until transfer 1 is done. This is what I was expecting.
For the newer version, I SOMETIMES get this behaviour, but sometimes
data is mixed (I would say 50% of the time).

Thanks for the help!

Xavi

============================== Output for libCurl 7.22
libcurl/7.22.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3
1> HTTP/1.1 200 OK
1> Date: Mon, 31 Mar 2014 15:27:00 GMT
1> Server: Apache/2.2.15 (CentOS)
1> Last-Modified: Fri, 11 Feb 2011 12:09:01 GMT
1> ETag: "73609e0-5240-49c009041b140"
1> Accept-Ranges: bytes
1> Content-Length: 21056
1> Content-Type: image/tiff
1>
1> Received 1207 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1025 bytes
1> Task done
2> HTTP/1.1 200 OK
2> Date: Mon, 31 Mar 2014 15:27:00 GMT
2> Server: Apache/2.2.15 (CentOS)
2> Last-Modified: Mon, 19 Feb 2007 20:40:21 GMT
2> ETag: "735e98d-6384-429da55817740"
2> Accept-Ranges: bytes
2> Content-Length: 25476
2> Content-Type: application/x-shockwave-flash
2>
2> Received 1188 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1120 bytes
2> Task done

============================== Output for libCurl 7.36
libcurl/7.36.0 OpenSSL/1.0.1 zlib/1.2.3.4 libidn/1.23 librtmp/2.3
2> HTTP/1.1 200 OK
2> Date: Mon, 31 Mar 2014 15:27:30 GMT
2> Server: Apache/2.2.15 (CentOS)
2> Last-Modified: Mon, 19 Feb 2007 20:40:21 GMT
2> ETag: "735e98d-6384-429da55817740"
2> Accept-Ranges: bytes
2> Content-Length: 25476
2> Content-Type: application/x-shockwave-flash
2>
2> Received 1188 bytes
2> Received 1448 bytes
1> HTTP/1.1 200 OK
1> Date: Mon, 31 Mar 2014 15:27:30 GMT
1> Server: Apache/2.2.15 (CentOS)
1> Last-Modified: Fri, 11 Feb 2011 12:09:01 GMT
1> ETag: "73609e0-5240-49c009041b140"
1> Accept-Ranges: bytes
1> Content-Length: 21056
1> Content-Type: image/tiff
1>
1> Received 1207 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1448 bytes
1> Received 1025 bytes
1> Task done
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1448 bytes
2> Received 1120 bytes
2> Task done

============================== Source code
/* gcc pipelining.c `pkg-config --cflags --libs libcurl` -o pipelining */
#include "curl/curl.h"

CURLM *mhandle;
int tasks_running = 0;

size_t write_function (void *buffer, size_t size, size_t nmemb,
    int userdata) {
  int total_size = size * nmemb;
  printf ("%d> Received %d bytes\n", userdata, total_size);

  return total_size;
}

size_t header_function (const char *line, size_t size, size_t nmemb,
    int userdata) {
  int total_size = size * nmemb;
  printf ("%d> %s", userdata, line);

  return total_size;
}

void add_task (char *url, int num) {
  CURL *ehandle = curl_easy_init ();

  curl_easy_setopt (ehandle, CURLOPT_WRITEFUNCTION,
      (curl_write_callback) write_function);
  curl_easy_setopt (ehandle, CURLOPT_WRITEDATA, num);
  curl_easy_setopt (ehandle, CURLOPT_HEADERFUNCTION,
      (curl_write_callback) header_function);
  curl_easy_setopt (ehandle, CURLOPT_HEADERDATA, num);
  curl_easy_setopt (ehandle, CURLOPT_PRIVATE, num);

  curl_easy_setopt (ehandle, CURLOPT_NOPROGRESS, 1L);
  curl_easy_setopt (ehandle, CURLOPT_NOSIGNAL, 1L);
  curl_easy_setopt (ehandle, CURLOPT_FOLLOWLOCATION, 1L);
  curl_easy_setopt (ehandle, CURLOPT_URL, url);

  tasks_running++;

  curl_multi_add_handle (mhandle, ehandle);
}

void process_messages ()
{
  CURLMsg *msg;
  int nmsgs;

  while ((msg = curl_multi_info_read (mhandle, &nmsgs))) {
    CURL *easy = msg->easy_handle;
    int num;

    if (msg->msg != CURLMSG_DONE || !easy)
      continue;

    curl_easy_getinfo (easy, CURLINFO_PRIVATE, (char **) &num);
    printf ("%d> Task done\n", num);

    curl_multi_remove_handle (mhandle, easy);
    curl_easy_cleanup (easy);

    tasks_running--;
  }
}

int main (int argc, char *argv[]) {
  fd_set rfds, wfds, efds;
  int max_fd;
  int num_queued_tasks;

  printf ("%s\n", curl_version ());

  mhandle = curl_multi_init ();
  curl_multi_setopt (mhandle, CURLMOPT_PIPELINING, 1);

  add_task ("http://ftp.nluug.nl/ftp/graphics/blender/demo/movies/Sintel_4k/tiff16/00000001.tif",
1);
  add_task ("http://ftp.nluug.nl/ftp/graphics/blender/demo/feature_videos/flvplayer.swf",
2);

  while (tasks_running > 0) {
    FD_ZERO (&rfds);
    FD_ZERO (&wfds);
    FD_ZERO (&efds);
    curl_multi_fdset (mhandle, &rfds, &wfds, &efds, &max_fd);
    if (max_fd == -1) {
      /* There is nothing happening: wait 100ms and look again */
      usleep (100000);
    } else if (max_fd > 0) {
      /* There are some active fd's: wait for them (and release the mutex) */
      struct timeval tv;

      tv.tv_sec = 0;
      tv.tv_usec = 100000;
      select (max_fd + 1, &rfds, NULL, NULL, &tv);
    } else {
      /* There are some fd requiring immediate action */
    }

    /* Perform transfers */
    curl_multi_perform (mhandle, &num_queued_tasks);

    /* Keep an eye on possible finished tasks */
    process_messages ();
  }

  curl_multi_cleanup (mhandle);
}

On 31 March 2014 14:20, Xavi Artigas <xartigas_at_fluendo.com> wrote:
>> Why is that strange? We rewrite internals and (try to) maintain API and ABI. The promise to the world is in the backards compatible ABI and API, not that we don't rewrite code.
>
> Sorry for not being clear, I meant that an API break was strange. We
> all love code rewrites!
>
> I hoped that maybe this was a known issue that could be quickly fixed.
> If that is not the case, then yeah, I will try to prepare a simple
> test case and post it here.
> In the mean time, I can give some more details:
>
> - The data being transferred is a series of mp4 files, therefore, the
> nature of the corruption is not easily seen. At some point, one of the
> files cannot be decoded, but I still do not know why.
> - Multi handle setup (I have only one of these):
> context->handle = curl_multi_init ();
> curl_multi_setopt (context->handle, CURLMOPT_PIPELINING, 1);
>
> - Simple handle setup (for each of the downloading tasks):
> task->handle = curl_easy_init ();
> curl_easy_setopt (task->handle, CURLOPT_WRITEFUNCTION,
> (curl_write_callback) _write_function);
> curl_easy_setopt (task->handle, CURLOPT_WRITEDATA, task);
> curl_easy_setopt (task->handle, CURLOPT_HEADERFUNCTION,
> (curl_write_callback) _header_function);
> curl_easy_setopt (task->handle, CURLOPT_HEADERDATA, task);
> curl_easy_setopt (task->handle, CURLOPT_PRIVATE, task);
> curl_easy_setopt (task->handle, CURLOPT_NOPROGRESS, 1L);
> /* We do not want signals, since we are multithreading */
> curl_easy_setopt (task->handle, CURLOPT_NOSIGNAL, 1L);
> /* Allow redirections */
> curl_easy_setopt (task->handle, CURLOPT_FOLLOWLOCATION, 1L);
> curl_easy_setopt (task->handle, CURLOPT_URL, url);
> curl_easy_setopt (task->handle, CURLOPT_RANGE, range);
>
> - And then, every time the current transfer is detected to be above
> 75% completion (and it is the only task running), a new task is added
> with:
> curl_multi_add_handle (context->handle, next_task->handle);
>
> - Every time a task is completed (or cancelled):
> curl_multi_remove_handle (context->handle, task->handle);
> curl_easy_cleanup (task->handle);
>
> I'll try to prepare a simple test.
> Thanks!
-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
Received on 2014-03-31