cURL / Mailing Lists / curl-users / Single Mail

curl-users

[PATCH] Non-blocking stdin on pipes/sockets for curl -T-

From: Eric Wong <normalperson_at_yhbt.net>
Date: Sun, 7 Jun 2009 03:17:00 -0700

This allows curl(1) to be used as a client-side tunnel for
arbitrary stream protocols by abusing chunked transfer encoding
in both the HTTP request and HTTP response. This requires
server support for sending a response while a request is
still being read, of course.

If attempting to read from stdin returns EAGAIN, then we pause
our sender. This leaves curl to attempt to read from the socket
while reading from stdin (and thus sending) is paused. A
1-second alarm is set to the reader to retry stdin.

This change was needed to allow successfully tunneling the git
protocol over HTTP (--no-buffer is needed, as well).

A better solution would be to move to the curl_multi_* interface
allowing us to poll() stdin along with the TCP socket, but
that's probably a bigger project. I realize this implementation
is fairly ugly at the moment, but opens up (AFAIK new)
possibilities for evading firewalls where CONNECT is not
available/viable.

An example for use with git would be creating a simple shell
script to use as GIT_PROXY_COMMAND:
----------------------------- 8< -------------------------------
#!/bin/sh
exec curl --no-buffer -sSfT- http://$1:$2/
----------------------------- 8< -------------------------------
Assuming bash as your shell:

  $ export GIT_PROXY_COMMAND=/path/to/above/script
  $ git clone git://git.bogomips.org:8080/pcu

To test a larger repository git.git, you may also try the above
command with the following repository instead:

  git://git.bogomips.org:8080/mirrors/git

The server software running git.bogomips.org:8080 is a
development version of Unicorn, available at:

  git://git.bogomips.org:8080/unicorn

And the Rack webserver interface that currently allows this
behavior with Unicorn:

  git://git.bogomips.org:8080/mirrors/rack

---
 src/main.c |   76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 74 insertions(+), 2 deletions(-)
diff --git a/src/main.c b/src/main.c
index d00ede8..f2610e8 100644
--- a/src/main.c
+++ b/src/main.c
@@ -158,6 +158,11 @@
 #define O_BINARY 0
 #endif
 
+/* S_ISSOCK is not in POSIX.1-1996. */
+#ifndef S_ISSOCK
+#define S_ISSOCK(st_mode) 0
+#endif
+
 #ifdef MSDOS
 #define USE_WATT32
 #include <dos.h>
@@ -190,6 +195,12 @@ char **__crt0_glob_function (char *arg)
 #define CURL_PROGRESS_STATS 0 /* default progress display */
 #define CURL_PROGRESS_BAR   1
 
+#if defined(HAVE_SIGNAL_H) && defined(HAVE_ALARM) && \
+    defined(HAVE_FCNTL_O_NONBLOCK)
+# include <signal.h>
+# define ALARM_RETRY
+#endif
+
 typedef enum {
   HTTPREQ_UNSPEC,
   HTTPREQ_GET,
@@ -3226,6 +3237,58 @@ static void go_sleep(long ms)
 #endif
 }
 
+static CURL *main_easy;
+
+#if defined(ALARM_RETRY)
+static void unpause_read(int signum)
+{
+  if (main_easy) {
+    curl_easy_pause(main_easy, CURLPAUSE_CONT);
+    alarm(1);
+  }
+}
+#endif
+
+/* maybe we could just use Curl_nonblock() instead ... */
+static void set_nonblocking(struct Configurable *config, int fd)
+{
+#if defined(ALARM_RETRY)
+  int flags;
+  struct stat st_buf;
+
+  if (fstat(fd, &st_buf) >= 0) {
+    warnf(config, "fstat failed on fd=%d: %s\n", fd, strerror(errno));
+  } else if (S_ISFIFO(st_buf.st_mode) || S_ISSOCK(st_buf.st_mode)) {
+    flags = fcntl(fd, F_GETFL, 0);
+    if (flags >= 0)
+      flags = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+    if (flags < 0)
+      warnf(config, "fcntl failed on fd=%: %s\n", fd, strerror(errno));
+  }
+#endif
+}
+
+static void set_unpause_timer(struct Configurable *config, bool enable)
+{
+#if defined(ALARM_RETRY)
+  if (enable) {
+    struct sigaction sa;
+
+    memset(&sa, 0, sizeof(sa));
+    sigemptyset(&sa.sa_mask);
+#ifdef SA_RESTART
+    sa.sa_flags = SA_RESTART;
+#endif
+    sa.sa_handler = unpause_read;
+    main_easy = config->easy;
+    sigaction(SIGALRM, &sa, NULL);
+    alarm(1);
+  } else {
+    signal(SIGALRM, SIG_IGN);
+  }
+#endif
+}
+
 static size_t my_fwrite(void *buffer, size_t sz, size_t nmemb, void *stream)
 {
   size_t rc;
@@ -3320,9 +3383,17 @@ static size_t my_fread(void *buffer, size_t sz, size_t nmemb, void *userp)
   struct InStruct *in=(struct InStruct *)userp;
 
   rc = read(in->fd, buffer, sz*nmemb);
-  if(rc < 0)
+  if(rc < 0) {
+    if(errno == EAGAIN) {
+      errno = 0;
+      set_unpause_timer(in->config, true);
+      return CURL_READFUNC_PAUSE;
+    }
     /* since size_t is unsigned we can't return negative values fine */
-    return 0;
+    rc = 0;
+  }
+  if (main_easy)
+    set_unpause_timer(in->config, false);
   return (size_t)rc;
 }
 
@@ -4491,6 +4562,7 @@ operate(struct Configurable *config, int argc, argv_item_t argv[])
         else if(uploadfile && curlx_strequal(uploadfile, "-")) {
           SET_BINMODE(stdin);
           infd = STDIN_FILENO;
+          set_nonblocking(config, infd);
         }
 
         if(uploadfile && config->resume_from_current)
-- 
Eric Wong
-------------------------------------------------------------------
List admin: http://cool.haxx.se/cgi-bin/mailman/listinfo/curl-users
FAQ:        http://curl.haxx.se/docs/faq.html
Etiquette:  http://curl.haxx.se/mail/etiquette.html
Received on 2009-06-07