curl-users
ftp restart with multiple URLs
From: Ilguiz Latypov <ilatypov_at_infradead.org>
Date: Thu, 19 Sep 2002 00:19:23 -0400 (EDT)
Date: Thu, 19 Sep 2002 00:19:23 -0400 (EDT)
I found that curl-7.10-pre3 would keep restarting FTP download with the
same start offset when more than one URL is specified in the command line.
Apparently the start offset in the above case is the size of the file that
was downloaded on previous invokation. When no other files in command
line are present on disk, the resume_from value from the former file will
propagate to the new file.
Will the following patch fix this problem? I also attached the script I
am using to download RPM updates with curl.
-- Ilguiz ========================================================================= --- ../../curl-7.10-pre3.orig/src/main.c Thu Sep 5 10:41:22 2002 +++ ./main.c Wed Sep 18 22:36:34 2002 @@ -2510,6 +2510,8 @@ config->resume_from = fileinfo.st_size; } /* else let offset remain 0 */ + else + config->resume_from = 0; } if(config->resume_from) { ========================================================================= #! /bin/sh function try_get() { tr_dir="`echo $2 | tr '/' '-'`" dir_list="${DN_DIR}/${tr_dir}.list" { test ! -s "${dir_list}" || \ test $((`date +'%s'` - `date -r ${dir_list} +'%s'`)) -gt 3600 } && { curl -C - -vl "${URL}/${START_DIR}/${2}/" ${USER} > "${dir_list}" \ || { rm -f "${dir_list}" ; return 1 ; } } file_list="" dir_list_filtered="/tmp/try_get_$$" rm -f "${dir_list_filtered}" grep -- "$1" "${dir_list}" | tr -d "\r" > "${dir_list_filtered}" cat "${dir_list_filtered}" while read ; do test ".${file_list}" = "." || file_list="${file_list}," file_list="${file_list}${REPLY}" done < "${dir_list_filtered}" rm -f "${dir_list_filtered}" test ".${file_list}" != "." || return 1 curdir="`pwd`" cd "${DN_DIR}" while ! curl -o "${DN_DIR}/#1" -C - -v \ "${URL}/${START_DIR}/${2}/{${file_list}}" ${USER} ; do echo "Retrying..." done result="$?" cd "${cur_dir}" return ${result} } function usage() { echo "Usage: `basename $0` \"FILEMASK\" [-r(2|1)] [-u(1|0)]" >&2 exit 1 } trap exit 1 2 3 DN_DIR="${HOME}/download" typeset -i redhat=1 URL="ftp://ftp.redhat.com" START_DIR="pub/redhat/linux" # URL="ftp://ftp.muug.mb.ca" # START_DIR="/redhat/linux" # URL="ftp://ftp.crc.ca" # START_DIR="pub/systems/linux/redhat/ftp.redhat.com/linux" # URL="ftp://redhat.eyetap.org" # START_DIR="linux" # URL="ftp://sunsite.ualberta.ca" # START_DIR="pub/unix/Linux/redhat" USER="" RH_RPM_DIR="en/os/i386/RedHat/RPMS" RH_UPDATE_RPM_DIR="en/os/i386" GET_ALL="" RH_VERSIONS="" RH_UPDATES="" FILES="" while test "$#" -gt 0 ; do OPTIND=1 while getopts "aD:L:S:r:u:" option ; do # echo "option $option" case "$option" in "a") GET_ALL="y" ;; "r") case ".${OPTARG}" in ".2") RH_VERSIONS="${RH_VERSIONS} 7.3" ;; ".1") RH_VERSIONS="${RH_VERSIONS} 7.2" ;; *) usage ;; esac ;; "u") if [ ".${OPTARG}" = ".0" -o ".${OPTARG}" = ".1" ] ; then RH_UPDATES="${RH_UPDATES} ${OPTARG}" else usage fi ;; "D") DN_DIR="${OPTARG}" ;; "L") URL="${OPTARG}" ;; "U") USER="--user ${OPTARG}" ;; "S") START_DIR="${OPTARG}" redhat=0 ;; "?") usage ;; # not implemented by shell's getopts ("-" is not allowed in optstring) # "\001") # FILES="${FILES} \"${1}\"" # ;; esac shift $[OPTIND-1] OPTIND=1 done test "$#" -eq 0 || { FILES="${FILES} ${1}" ; shift ; } done test ".${FILES}" = "." && usage test ".${GET_ALL}" = "." && GET_ALL="n" test ".${RH_VERSIONS}" = "." && RH_VERSIONS="7.3 7.2" test ".${RH_UPDATES}" = "." && RH_UPDATES="1 0" for file in ${FILES} ; do rc=1 # echo file=$file if [ $redhat -eq 0 ] ; then try_get "${file}" "" rc="$?" else try_get "${file}" "rawhide/i386/RedHat/RPMS" rc="$?" test "${rc}" -ne 0 && { for ver in ${RH_VERSIONS} ; do # echo $ver for upd in ${RH_UPDATES} ; do # echo upd=$upd case "${upd}" in "1") try_get "${file}" "updates/${ver}/${RH_UPDATE_RPM_DIR}" ;; "0") try_get "${file}" "${ver}/${RH_RPM_DIR}" ;; esac rc="$?" test "${rc}" -ne 0 || break done test "${rc}" -ne 0 || break done } fi done ========================================================================= ------------------------------------------------------- This SF.NET email is sponsored by: AMD - Your access to the experts on Hammer Technology! Open Source & Linux Developers, register now for the AMD Developer Symposium. Code: EX8664 http://www.developwithamd.com/developerlabReceived on 2002-09-19