Hi i have one site that use Lib Curl to Spider Orkut.

I use about 45 simultaneos process in spider but it keep open too much
connections with Orkut. With 45 process it keep alive more than 5000
conections, i need to reduce this.

I try to reduce CURLOPT_MAXCONNECTS value but i dont see results.

The PHP cURLClass That i use is this.

<?php
class CURL
{

    function getFile($method,$url,$params,$header,$cookie)
    {

        $headers[] = "Connection: keep-alive";
        $headers[] = "Keep-Alive: 300";

        $ch = curl_init();
        curl_setopt($ch, CURLOPT_HEADER, $header);
        curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
        curl_setopt($ch, CURLOPT_ENCODING, "gzip,deflate");
        curl_setopt($ch, CURLOPT_URL,$url);
        curl_setopt($ch, CURLOPT_FORBID_REUSE,0);
        curl_setopt($ch, CURLOPT_FRESH_CONNECT,0);
        curl_setopt($ch, CURLOPT_MAXCONNECTS, 5);
        curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
        curl_setopt($ch, CURLOPT_COOKIE, $cookie);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, FALSE);
        curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U;
Windows NT 5.1; pt-BR; rv:1.8.1.15) Gecko/20080623 Firefox/2.0.0.15");
        curl_setopt($ch, CURLOPT_RETURNTRANSFER,TRUE);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
        curl_setopt($ch, CURLOPT_TIMEOUT, 150);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT,100);
        if ($method == 'POST') {
            curl_setopt($ch, CURLOPT_CUSTOMREQUEST,'POST');
            curl_setopt($ch, CURLOPT_POST, 1);
            curl_setopt($ch, CURLOPT_POSTFIELDS,$params);
        }

        $result=curl_exec ($ch);
        curl_close ($ch);
        if($result)
        {
            return $result;
        }
        else
        {
            return curl_error($ch);
        }
    }
}

$curl = new CURL;
?>

Thanks
Luciano

_______________________________________________
http://cool.haxx.se/cgi-bin/mailman/listinfo/curl-and-php
Received on 2008-08-28