I am experiencing sporadic crashes in curl_multi_remove_handle code.
The crashes aren't always reproducible but there is a certain
pattern:

- the immediate reason for the crash is an invalid easy_conn in easy
(easy is obtained by iterating multi.easy.next several times). It
appears as if easy_conn is no longer functional (or wasn't even
allocated?);

- the multi.easy has a flag has a state state = CURLM_STATE_MSGSENT
and result = CURLE_OPERATION_TIMEDOUT:

$21 = {
  next = 0xafd3c87c,
  prev = 0xae5f20e4,
  easy_handle = 0xae6ca724, // looks pretty much intact
  easy_conn = 0xaf957a6c, // !!!___THIS IS NO LONGER A VALID STRUCTURE__!!!
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xae6ca724,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

The crash usually occurs right here:

 643 if(easy->easy_conn &&
 644 (easy->easy_conn->send_pipe->size +
 645 easy->easy_conn->recv_pipe->size > 1) &&
 646 easy->state > CURLM_STATE_WAITDO &&
 647 easy->state < CURLM_STATE_COMPLETED) {

due to the fact that easy_conn->send_pipe is 0x0 or some other kind of
thrash. Just for a test I out the state check prior to
easy->easy_conn->send_pipe->size - this just moved the crash to other
portions of the code.

The crashes started to appear after I fixed (oh well) the timeout code
related to the "select" in my application's curl_multi_perform
sequence. Prior the fix the timeout that I specified to the "select"
was too long and as a result I wasn't getting Easy connection
timeouts. Now I do know almost immediately when a connection timeout
occurs, and most of the time it works fine - just that I started to
get these crashes.

I checked several times whether I am meddling with the easy handles in
my own code. destroying them, but as far as I see I am not touching
any easy handles at all.

I'd like to understand what kind of tests can I do to pinpoint the
problem, which direction to go. As I said, even if timeout occurs the
crash isn't always happening.

Just for the record, here is a dump of all multi.easy handles during a
crash and the related code.

Thanks,
Martin

#0 gethandleathead (data=0xa6ae719c, conn=0xaf957a6c) at url.c:2803
#1 Curl_getoff_all_pipelines (data=0xa6ae719c, conn=0xaf957a6c) at url.c:2817
#2 0x00737f1e in curl_multi_remove_handle (multi_handle=0xa73775f4,
curl_handle=0xa6ae719c) at multi.c:695

2801 static struct SessionHandle* gethandleathead(struct curl_llist
*pipeline)
2802 {
2803 struct curl_llist_element *curr = pipeline->head;
2804 if(curr) {
2805 return (struct SessionHandle *) curr->ptr;
2806 }

2813 void Curl_getoff_all_pipelines(struct SessionHandle *data,
2814 struct connectdata *conn)
2815 {
2816 bool recv_head = (bool)(conn->readchannel_inuse &&
2817 (gethandleathead(conn->recv_pipe) == data));
2818

675 if(easy->easy_conn) {
676
677 /* we must call Curl_done() here (if we still "own it")
so that we don't
678 leave a half-baked one around */
679 if (easy_owns_conn) {
680
681 /* Curl_done() clears the conn->data field to lose the
association
682 between the easy handle and the connection
683
684 Note that this ignores the return code simply because there's
(gdb)
685 nothing really useful to do with it anyway! */
686 (void)Curl_done(&easy->easy_conn, easy->result, premature);
687
688 if(easy->easy_conn)
689 /* the connection is still alive, set back the
association to enable
690 the check below to trigger TRUE */
691 easy->easy_conn->data = easy->easy_handle;
692 }
693 else
694 /* Clear connection pipelines, if Curl_done above was
not called */
(gdb)
695 Curl_getoff_all_pipelines(easy->easy_handle, easy->easy_conn);
696 }

$12 = {
  next = 0xa7380124,
  prev = 0xae321cac,
  easy_handle = 0x0,
  easy_conn = 0x0,
  state = CURLM_STATE_INIT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_NONE,
      easy_handle = 0x0,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$14 = {
  next = 0xa7380acc,
  prev = 0xa73775f8,
  easy_handle = 0xae5de75c,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xae5de75c,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$15 = {
  next = 0xa7381454,
  prev = 0xa7380124,
  easy_handle = 0xafd2ac3c,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xafd2ac3c,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$16 = {
  next = 0xae5f03bc,
  prev = 0xa7380acc,
  easy_handle = 0xae30e75c,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xae30e75c,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$17 = {
  next = 0xae5f0d7c,
  prev = 0xa7381454,
  easy_handle = 0xa7c5e894,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa7c5e894,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$18 = {
  next = 0xae5f172c,
  prev = 0xae5f03bc,
  easy_handle = 0xa7bb3dcc,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa7bb3dcc,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$19 = {
  next = 0xae5f20e4,
  prev = 0xae5f0d7c,
  easy_handle = 0xa7684904,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa7684904,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$20 = {
  next = 0xae5f2a9c,
  prev = 0xae5f172c,
  easy_handle = 0xae6b8efc,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xae6b8efc,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$21 = {
  next = 0xafd3c87c,
  prev = 0xae5f20e4,
  easy_handle = 0xae6ca724,
  easy_conn = 0xaf957a6c,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xae6ca724,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$22 = {
  next = 0xafd3d25c,
  prev = 0xae5f2a9c,
  easy_handle = 0xa6ae719c,
  easy_conn = 0xaf957a6c,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa6ae719c,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$23 = {
  next = 0xafd3dbec,
  prev = 0xafd3c87c,
  easy_handle = 0xa73383dc,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa73383dc,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$24 = {
  next = 0xafd3e5d4,
  prev = 0xafd3d25c,
  easy_handle = 0xa7349c04,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa7349c04,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$25 = {
  next = 0xafd3ef64,
  prev = 0xafd3dbec,
  easy_handle = 0xa7ac00fc,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa7ac00fc,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$27 = {
  next = 0xa77dab3c,
  prev = 0xafd3e5d4,
  easy_handle = 0xa77b78a4,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa77b78a4,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$28 = {
  next = 0xa77db53c,
  prev = 0xafd3ef64,
  easy_handle = 0xa77c90cc,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa77c90cc,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$29 = {
  next = 0xa77dbee4,
  prev = 0xa77dab3c,
  easy_handle = 0xae0d7644,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xae0d7644,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$30 = {
  next = 0xa77dc8ac,
  prev = 0xa77db53c,
  easy_handle = 0xae0e8e6c,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xae0e8e6c,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$31 = {
  next = 0xa77dd26c,
  prev = 0xa77dbee4,
  easy_handle = 0xa777ea94,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa777ea94,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$32 = {
  next = 0xae3204dc,
  prev = 0xa77dc8ac,
  easy_handle = 0xa77902bc,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa77902bc,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$33 = {
  next = 0xae320ea4,
  prev = 0xa77dd26c,
  easy_handle = 0xa76cc97c,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa76cc97c,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$34 = {
  next = 0xa76f8894,
  prev = 0xae3204dc,
  easy_handle = 0xa76de1a4,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa76de1a4,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$35 = {
  next = 0xae321314,
  prev = 0xae320ea4,
  easy_handle = 0xa76ef9cc,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xa76ef9cc,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$36 = {
  next = 0xae321cac,
  prev = 0xa76f8894,
  easy_handle = 0xaf92fe64,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OPERATION_TIMEDOUT,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xaf92fe64,
      data = {
        whatever = 0x1c,
        result = CURLE_OPERATION_TIMEDOUT
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}

$37 = {
  next = 0xa73775f8,
  prev = 0xae321314,
  easy_handle = 0xaf94168c,
  easy_conn = 0x0,
  state = CURLM_STATE_MSGSENT,
  result = CURLE_OK,
  msg = {
    extmsg = {
      msg = CURLMSG_DONE,
      easy_handle = 0xaf94168c,
      data = {
        whatever = 0x0,
        result = CURLE_OK
      }
    }
  },
  sockets = {0, 0, 0, 0, 0},
  numsocks = 0
}
-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
Received on 2011-09-25