curl-library
[PATCH 5/7] pipelining: Add CURLMOPT_PIPELINE_POLICY_FUNCTION
From: Carlo Wood <carlo_at_alinoe.com>
Date: Thu, 6 Nov 2014 03:03:19 +0100
Date: Thu, 6 Nov 2014 03:03:19 +0100
Also adds CURLMOPT_PIPELINE_POLICY_DATA, CURL_SUPPORTS_PIPELINING,
CURL_BLACKLISTED, struct curl_pipeline_policy
and curl_pipeline_policy_callback
This allows the user to tweak the maximum number of connections
per host, the maximum number of requests in a pipeline and
to override blacklisting or implement whitelisting on a per
host basis (since bundles are only per host, not host:port - which
imho is a limitation). Finally they can cause libcurl to start
doing pipelining immediately instead after the first reply
is received.
Usage example:
curl_multi_setopt(multi_handle,
CURLMOPT_PIPELINE_POLICY_FUNCTION,
&pipeline_policy_callback);
void pipeline_policy_callback(
char const* hostname,
int port,
curl_pipeline_policy* policy,
void* userp)
{
if(supports_pipelining(host))
policy->flags |= CURL_SUPPORTS_PIPELINING;
if(port == 80)
policy->flags |= CURL_BLACKLISTED;
if(host == "special.case" && port == 12340) {
policy->flags &= ~CURL_BLACKLISTED;
policy->max_pipeline_length = 100;
}
if((policy->flags & CURL_SUPPORTS_PIPELINING))
policy->max_host_connections = 2;
}
---
include/curl/multi.h | 49 ++++++++++++++++++++++++++++++++++++++++++++++
lib/bundles.c | 3 ++-
lib/bundles.h | 6 ++++--
lib/conncache.c | 19 +++++++++++++++++-
lib/http.c | 11 ++++++-----
lib/multi.c | 8 +++++++-
lib/multihandle.h | 10 +++++++---
lib/url.c | 19 +++++++-----------
packages/OS400/curl.inc.in | 7 +++++++
9 files changed, 107 insertions(+), 25 deletions(-)
diff --git a/include/curl/multi.h b/include/curl/multi.h
index 3c4acb0..e2ac30a 100644
--- a/include/curl/multi.h
+++ b/include/curl/multi.h
@@ -289,6 +289,49 @@ CURL_EXTERN CURLMcode curl_multi_socket_action(CURLM *multi_handle,
CURL_EXTERN CURLMcode curl_multi_socket_all(CURLM *multi_handle,
int *running_handles);
+/*
+ * Name: curl_pipeline_policy
+ *
+ * Desc: Contains the per site pipeline policy variables. The values default
+ * to CURLMOPT_MAX_HOST_CONNECTIONS, CURLMOPT_MAX_PIPELINE_LENGTH,
+ * and whether or not the site appears in CURLMOPT_PIPELINING_SITE_BL
+ * respectively. CURL_SUPPORTS_PIPELINING is not set. However, these
+ * values can be overwritten by the user during the
+ * curl_pipeline_policy_callback set with
+ * CURLMOPT_PIPELINE_POLICY_FUNCTION. If CURL_SUPPORTS_PIPELINING and
+ * CURL_BLACKLISTED are both set then the site is blacklisted from
+ * pipelining.
+ */
+
+#define CURL_SUPPORTS_PIPELINING 1
+#define CURL_BLACKLISTED 2
+
+struct curl_pipeline_policy {
+ size_t max_host_connections; /* the maximum number of simultaneous
+ connections to the site */
+ long max_pipeline_length; /* the maximum amount of requests in
+ a pipelined connection */
+ int flags; /* bit-mask for CURL_SUPPORTS_PIPELINING
+ and CURL_BLACKLISTED */
+};
+
+/* Return TRUE iff bundle supports pipelining and is not blacklisted */
+#define CURL_CAN_PIPELINE(bundle) ((bundle->policy.flags & \
+ (CURL_SUPPORTS_PIPELINING|CURL_BLACKLISTED)) == CURL_SUPPORTS_PIPELINING)
+
+/*
+ * Name: curl_pipeline_policy_callback
+ *
+ * Desc: Called by libcurl whenever the library creates a new bundle for
+ * some hostname:port combination.
+ */
+typedef void (*curl_pipeline_policy_callback)(
+ char const *hostname, /* connected hostname */
+ int port, /* connected port */
+ struct curl_pipeline_policy* policy, /* writable policy */
+ void *userp); /* private callback
+ pointer */
+
#ifndef CURL_ALLOW_OLD_MULTI_SOCKET
/* This macro below was added in 7.16.3 to push users who recompile to use
the new curl_multi_socket_action() instead of the old curl_multi_socket()
@@ -365,6 +408,12 @@ typedef enum {
/* maximum number of open connections in total */
CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13),
+ /* This is the pipeline policy callback function pointer */
+ CINIT(PIPELINE_POLICY_FUNCTION, FUNCTIONPOINT, 14),
+
+ /* This is the argument passed to the pipeline policy callback */
+ CINIT(PIPELINE_POLICY_DATA, OBJECTPOINT, 15),
+
CURLMOPT_LASTENTRY /* the last unused */
} CURLMoption;
diff --git a/lib/bundles.c b/lib/bundles.c
index aadf026..e46b878 100644
--- a/lib/bundles.c
+++ b/lib/bundles.c
@@ -46,6 +46,7 @@ static void conn_llist_dtor(void *user, void *element)
}
CURLcode Curl_bundle_create(struct SessionHandle *data,
+ struct curl_pipeline_policy *policy,
struct connectbundle **cb_ptr)
{
(void)data;
@@ -55,7 +56,7 @@ CURLcode Curl_bundle_create(struct SessionHandle *data,
return CURLE_OUT_OF_MEMORY;
(*cb_ptr)->num_connections = 0;
- (*cb_ptr)->server_supports_pipelining = FALSE;
+ (*cb_ptr)->policy = *policy;
(*cb_ptr)->conn_list = Curl_llist_alloc((curl_llist_dtor) conn_llist_dtor);
if(!(*cb_ptr)->conn_list) {
diff --git a/lib/bundles.h b/lib/bundles.h
index 3816c40..e418c2c 100644
--- a/lib/bundles.h
+++ b/lib/bundles.h
@@ -22,14 +22,16 @@
*
***************************************************************************/
+#include <curl/multi.h>
+
struct connectbundle {
- bool server_supports_pipelining; /* TRUE if server supports pipelining,
- set after first response */
+ struct curl_pipeline_policy policy; /* connect policy */
size_t num_connections; /* Number of connections in the bundle */
struct curl_llist *conn_list; /* The connectdata members of the bundle */
};
CURLcode Curl_bundle_create(struct SessionHandle *data,
+ struct curl_pipeline_policy *policy,
struct connectbundle **cb_ptr);
void Curl_bundle_destroy(struct connectbundle *cb_ptr);
diff --git a/lib/conncache.c b/lib/conncache.c
index d07718e..9d28116 100644
--- a/lib/conncache.c
+++ b/lib/conncache.c
@@ -33,6 +33,7 @@
#include "rawstr.h"
#include "bundles.h"
#include "conncache.h"
+#include "pipeline.h"
#include "curl_memory.h"
/* The last #include file should be: */
@@ -130,7 +131,23 @@ CURLcode Curl_conncache_add_conn(struct conncache *connc,
bundle = Curl_conncache_find_bundle(data->state.conn_cache,
conn->host.name);
if(!bundle) {
- result = Curl_bundle_create(data, &new_bundle);
+ struct Curl_multi *multi = data->multi;
+ struct curl_pipeline_policy policy = { 0, 0, 0 };
+ if(multi) {
+ /* policy default values */
+ policy.max_host_connections = Curl_multi_max_host_connections(multi);
+ policy.max_pipeline_length = Curl_multi_max_pipeline_length(multi);
+ if(Curl_pipeline_site_blacklisted(data, conn)) {
+ policy.flags = CURL_BLACKLISTED;
+ }
+ /* allow user to tweak the policy */
+ if(multi->pipeline_policy_cb)
+ multi->pipeline_policy_cb(conn->host.name,
+ conn->remote_port,
+ &policy,
+ multi->pipeline_policy_userp);
+ }
+ result = Curl_bundle_create(data, &policy, &new_bundle);
if(result)
return result;
diff --git a/lib/http.c b/lib/http.c
index 2487bac..3a54cf9 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -3302,8 +3302,8 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data,
/* Activate pipelining if needed */
cb_ptr = conn->bundle;
if(cb_ptr) {
- if(!Curl_pipeline_site_blacklisted(data, conn))
- cb_ptr->server_supports_pipelining = TRUE;
+ if(!(cb_ptr->policy.flags & CURL_BLACKLISTED))
+ cb_ptr->policy.flags |= CURL_SUPPORTS_PIPELINING;
}
}
@@ -3384,10 +3384,11 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data,
else if(checkprefix("Server:", k->p)) {
char *server_name = Curl_copy_header_value(k->p);
- /* Turn off pipelining if the server version is blacklisted */
- if(conn->bundle && conn->bundle->server_supports_pipelining) {
+ /* Check if the server version is blacklisted when
+ we're doing pipelining */
+ if(conn->bundle && CURL_CAN_PIPELINE(conn->bundle)) {
if(Curl_pipeline_server_blacklisted(data, server_name))
- conn->bundle->server_supports_pipelining = FALSE;
+ conn->bundle->policy.flags |= CURL_BLACKLISTED;
}
Curl_safefree(server_name);
}
diff --git a/lib/multi.c b/lib/multi.c
index 50c4cf8..db1b582 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -1659,7 +1659,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
to remove the bundle: we need to remember that this server is
capable of pipelining. */
bundle = data->easy_conn->bundle;
- keep_bundle = (bundle->server_supports_pipelining &&
+ keep_bundle = (CURL_CAN_PIPELINE(bundle) &&
(data->result == CURLE_OPERATION_TIMEDOUT ||
data->easy_conn->bits.retry));
if(keep_bundle)
@@ -2415,6 +2415,12 @@ CURLMcode curl_multi_setopt(CURLM *multi_handle,
case CURLMOPT_MAX_TOTAL_CONNECTIONS:
multi->max_total_connections = va_arg(param, long);
break;
+ case CURLMOPT_PIPELINE_POLICY_FUNCTION:
+ multi->pipeline_policy_cb = va_arg(param, curl_pipeline_policy_callback);
+ break;
+ case CURLMOPT_PIPELINE_POLICY_DATA:
+ multi->pipeline_policy_userp = va_arg(param, void *);
+ break;
default:
res = CURLM_UNKNOWN_OPTION;
break;
diff --git a/lib/multihandle.h b/lib/multihandle.h
index 1a4b1d9..a673f96 100644
--- a/lib/multihandle.h
+++ b/lib/multihandle.h
@@ -107,14 +107,18 @@ struct Curl_multi {
long maxconnects; /* if >0, a fixed limit of the maximum number of entries
we're allowed to grow the connection cache to */
- long max_host_connections; /* if >0, a fixed limit of the maximum number
+ long max_host_connections; /* if >0, the default limit of the maximum number
of connections per host */
long max_total_connections; /* if >0, a fixed limit of the maximum number
of connections in total */
- long max_pipeline_length; /* if >0, maximum number of requests in a
- pipeline */
+ /* callback function and user data pointer for the pipeline policy API */
+ curl_pipeline_policy_callback pipeline_policy_cb;
+ void *pipeline_policy_userp;
+
+ long max_pipeline_length; /* if >0, the default maximum number of requests
+ in a pipeline */
long content_length_penalty_size; /* a connection with a
content-length bigger than
diff --git a/lib/url.c b/lib/url.c
index 81a27ad..b41f5cb 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -3021,25 +3021,20 @@ ConnectionExists(struct SessionHandle *data,
*force_reuse = FALSE;
- /* We can't pipe if the site is blacklisted */
- if(canPipeline && Curl_pipeline_site_blacklisted(data, needle)) {
- canPipeline = FALSE;
- }
-
/* Look up the bundle with all the connections to this
particular host */
bundle = Curl_conncache_find_bundle(data->state.conn_cache,
needle->host.name);
if(bundle) {
- size_t max_pipe_len = Curl_multi_max_pipeline_length(data->multi);
+ size_t max_pipe_len = bundle->policy.max_pipeline_length;
size_t best_pipe_len = max_pipe_len;
struct curl_llist_element *curr;
infof(data, "Found bundle for host %s: %p\n",
needle->host.name, (void *)bundle);
- /* We can't pipe if we don't know anything about the server */
- if(canPipeline && !bundle->server_supports_pipelining) {
+ /* Check if we are pipelining */
+ if(canPipeline && !CURL_CAN_PIPELINE(bundle)) {
infof(data, "Server doesn't support pipelining\n");
canPipeline = FALSE;
}
@@ -5209,7 +5204,6 @@ static CURLcode create_conn(struct SessionHandle *data,
bool prot_missing = FALSE;
bool no_connections_available = FALSE;
bool force_reuse = FALSE;
- size_t max_host_connections = Curl_multi_max_host_connections(data->multi);
size_t max_total_connections = Curl_multi_max_total_connections(data->multi);
*async = FALSE;
@@ -5555,7 +5549,8 @@ static CURLcode create_conn(struct SessionHandle *data,
infof(data, "Found connection %ld, with requests in the pipe (%zu)\n",
conn_temp->connection_id, pipelen);
- if(conn_temp->bundle->num_connections < max_host_connections &&
+ if(conn_temp->bundle->num_connections <
+ conn_temp->bundle->policy.max_host_connections &&
data->state.conn_cache->num_connections < max_total_connections) {
/* We want a new connection anyway */
reuse = FALSE;
@@ -5594,8 +5589,8 @@ static CURLcode create_conn(struct SessionHandle *data,
bundle = Curl_conncache_find_bundle(data->state.conn_cache,
conn->host.name);
- if(max_host_connections > 0 && bundle &&
- (bundle->num_connections >= max_host_connections)) {
+ if(bundle && bundle->policy.max_host_connections > 0 &&
+ (bundle->num_connections >= bundle->policy.max_host_connections)) {
struct connectdata *conn_candidate;
/* The bundle is full. Let's see if we can kill a connection. */
diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in
index 39adc6a..1d30a99 100644
--- a/packages/OS400/curl.inc.in
+++ b/packages/OS400/curl.inc.in
@@ -1551,6 +1551,10 @@
d c 10012
d CURLMOPT_MAX_TOTAL_CONNECTIONS...
d c 00013
+ d CURLMOPT_PIPELINING_POLICY_FUNCTION...
+ d c 00014
+ d CURLMOPT_PIPELINING_POLICY_DATA...
+ d c 00015
*
* Public API enums for RTSP requests.
*
@@ -1769,6 +1773,9 @@
d curl_socket_callback...
d s * based(######ptr######) procptr
*
+ d curl_pipeline_policy_callback...
+ d s * based(######ptr######) procptr
+ *
d curl_opensocket_callback...
d s * based(######ptr######) procptr
*
--
2.1.1
-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
Received on 2014-11-06