I use multi thread to list dir on sftp in my test code.
There are 30 dirs and 30 threads.
I also use openssl pthread_mutex to prevent crash.


In test code, I list three times, the first time there are 10 threads list successfully and return 0, the other return error code 2.
the second time there are 20 threads list successfully include the count of first time list success,the third time ,all of the 30 threads are list successfully.


I think it init 10 threads successfully every time and init other 10 threads successfully next time.But why is 10???? where is the limit????






this is my test code:


#define USE_OPENSSL /* or USE_GNUTLS accordingly */


#include <stdio.h>
#include <string.h>
#include <pthread.h>
#include <curl/curl.h>


#define NUMT 30


static pthread_mutex_t *lockarray;


#ifdef USE_OPENSSL
#include <openssl/crypto.h>
static void lock_callback(int mode, int type, char *file, int line)
{
  (void)file;
  (void)line;
  if (mode & CRYPTO_LOCK) {
    pthread_mutex_lock(&(lockarray[type]));
  }
  else {
    pthread_mutex_unlock(&(lockarray[type]));
  }
}


static unsigned long thread_id(void)
{
  unsigned long ret;


  ret=(unsigned long)pthread_self();
  return(ret);
}


static void init_locks(void)
{
  int i;


  lockarray=(pthread_mutex_t *)OPENSSL_malloc(CRYPTO_num_locks() *
                                            sizeof(pthread_mutex_t));
  for (i=0; i<CRYPTO_num_locks(); i++) {
    pthread_mutex_init(&(lockarray[i]),NULL);
  }


  CRYPTO_set_id_callback((unsigned long (*)())thread_id);
  CRYPTO_set_locking_callback((void (*)())lock_callback);
}


static void kill_locks(void)
{
  int i;


  CRYPTO_set_locking_callback(NULL);
  for (i=0; i<CRYPTO_num_locks(); i++)
    pthread_mutex_destroy(&(lockarray[i]));


  OPENSSL_free(lockarray);
}
#endif


#ifdef USE_GNUTLS
#include <gcrypt.h>
#include <errno.h>


GCRY_THREAD_OPTION_PTHREAD_IMPL;


void init_locks(void)
{
  gcry_control(GCRYCTL_SET_THREAD_CBS);
}


#define kill_locks()
#endif


static char *spacechr = "\r\n\t ";


char* pltrim(char *str){
   char *head, *curpos;
   if (str == NULL) return NULL;
   head = curpos = str;
   while (*curpos && strchr(spacechr, *curpos)) curpos++;
   while ((*head = *curpos) != '\0') head++,curpos++;
   return str;
}


char* prtrim(char *str){
   char *head, *curpos;
   if (str == NULL) return NULL;
   head = curpos = str;
   while (*curpos) curpos++;
   while (curpos >= head && strchr(spacechr, *curpos)) curpos--;
   curpos[1] = '\0';
   return head;
}


static long _sftp_list_chunk_(void *buffer,
                              size_t size,
                              size_t nmemb,
                              void *stream) {
  
  size_t bufflen = size*nmemb;
  char buffer_tmp[1024] = { 0x0 };
  memcpy(buffer_tmp, buffer, bufflen);
  prtrim(pltrim(buffer_tmp));


  
  char buf[11][1024];
  int i = 0;
  for (i = 0; i < 11; i++) {
    memset(buf[i], 0x0, 1024);
  }
  
  int buf_count = 0;
  char *last = NULL;
  char *split = strtok_r(buffer_tmp, " ", &last);
  while (NULL != split) {
    prtrim(pltrim(split));
    if (*split != '\0') {
      memcpy(buf[buf_count], split, 1024);
      buf_count++;
    }
    split = strtok_r(NULL, " ", &last);
  }


  return size * nmemb;
}




/* List of URLs to fetch.*/
const char * const urls[]= {
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ccc/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ddd/",
    "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ccc/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ddd/",
    "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ccc/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ddd/",
    "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ccc/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ddd/",
    "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ccc/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ddd/",
    "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ccc/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ddd/",
    "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ccc/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/ddd/",
    "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/aaa/",
  "sftp://test:test123@11.8.13.35/test/tmp/hebh/sftp/bbb/"
};


CURL *curl[NUMT];


static void *pull_one_url(void *curl)
{
  
  curl_easy_setopt(curl, CURLOPT_SSH_AUTH_TYPES, CURLSSH_AUTH_PASSWORD);
  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _sftp_list_chunk_);
  curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
  curl_easy_setopt(curl, CURLOPT_TIMEOUT, 300);
  curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);
  curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
  curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
  int result = curl_easy_perform(curl);
  printf("result = %d\n",result);
  return NULL;
}


int main(int argc, char **argv)
{
  pthread_t tid[NUMT];
  int i;
  int error;
  (void)argc; /* we don't use any arguments in this example */
  (void)argv;


  /* Must initialize libcurl before any threads are started */
  curl_global_init(CURL_GLOBAL_ALL);


  init_locks();
  
  for(i=0; i< NUMT; i++) {
          
          CURL *a;
    a = curl_easy_init();
    curl_easy_setopt(a, CURLOPT_URL, urls[i]);
    curl[i] = a;
  }


  //NUMT
  for(i=0; i< NUMT; i++) {
  
    error = pthread_create(&tid[i],
                           NULL, /* default attributes please */
                           pull_one_url,
                           (void *)curl[i]);
    if(0 != error){
      fprintf(stderr, "Couldn't run thread number %d, errno %d\n", i, error);
    }
    else{
      //fprintf(stderr, "Thread %d, gets %s\n", i, urls[i]);
    }
  }
  
  sleep(2);
  printf("========\n");
  
  for(i=0; i< NUMT; i++) {
  
    error = pthread_create(&tid[i],
                           NULL, /* default attributes please */
                           pull_one_url,
                           (void *)curl[i]);
    if(0 != error){
      fprintf(stderr, "Couldn't run thread number %d, errno %d\n", i, error);
    }
    else{
      //fprintf(stderr, "Thread %d, gets %s\n", i, urls[i]);
    }
  }
  
  sleep(2);
   printf("========\n");
  
  for(i=0; i< NUMT; i++) {
  
    error = pthread_create(&tid[i],
                           NULL, /* default attributes please */
                           pull_one_url,
                           (void *)curl[i]);
    if(0 != error){
      fprintf(stderr, "Couldn't run thread number %d, errno %d\n", i, error);
    }
    else{
      //fprintf(stderr, "Thread %d, gets %s\n", i, urls[i]);
    }
  }


  /* now wait for all threads to terminate */
  for(i=0; i< NUMT; i++) {
    error = pthread_join(tid[i], NULL);
    //fprintf(stderr, "Thread %d terminated\n", i);
  }


  kill_locks();


  return 0;
}

-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
Received on 2014-02-20