cURL / Mailing Lists / curl-users / Single Mail

curl-users

ftp restart with multiple URLs

From: Ilguiz Latypov <ilatypov_at_infradead.org>
Date: Thu, 19 Sep 2002 00:19:23 -0400 (EDT)

I found that curl-7.10-pre3 would keep restarting FTP download with the
same start offset when more than one URL is specified in the command line.

Apparently the start offset in the above case is the size of the file that
was downloaded on previous invokation. When no other files in command
line are present on disk, the resume_from value from the former file will
propagate to the new file.

Will the following patch fix this problem? I also attached the script I
am using to download RPM updates with curl.

-- 
Ilguiz
=========================================================================
--- ../../curl-7.10-pre3.orig/src/main.c	Thu Sep  5 10:41:22 2002
+++ ./main.c	Wed Sep 18 22:36:34 2002
@@ -2510,6 +2510,8 @@
             config->resume_from = fileinfo.st_size;
           }
           /* else let offset remain 0 */
+          else
+            config->resume_from = 0;
         }
       
         if(config->resume_from) {
=========================================================================
#! /bin/sh
function try_get() {
	tr_dir="`echo $2 | tr '/' '-'`"
	dir_list="${DN_DIR}/${tr_dir}.list"
	{
	  test ! -s "${dir_list}" || \
	  test $((`date +'%s'` - `date -r ${dir_list} +'%s'`)) -gt 3600 
	} && {
	  curl -C - -vl "${URL}/${START_DIR}/${2}/" ${USER} > "${dir_list}" \
		|| { rm -f "${dir_list}" ; return 1 ; }
	}
	file_list=""
	dir_list_filtered="/tmp/try_get_$$"
	rm -f "${dir_list_filtered}"
	grep -- "$1" "${dir_list}" | tr -d "\r" > "${dir_list_filtered}"
	cat "${dir_list_filtered}"
	while read ; do 
	  test ".${file_list}" = "." || file_list="${file_list},"
	  file_list="${file_list}${REPLY}"
	done < "${dir_list_filtered}"
	rm -f "${dir_list_filtered}"
	test ".${file_list}" != "." || return 1
	curdir="`pwd`"
	cd "${DN_DIR}"
	while ! curl -o "${DN_DIR}/#1" -C - -v \
		"${URL}/${START_DIR}/${2}/{${file_list}}" ${USER} ; do
	  echo "Retrying..."
	done
	result="$?"
	cd "${cur_dir}"
	return ${result}
}
function usage() {
	echo "Usage: `basename $0` \"FILEMASK\" [-r(2|1)] [-u(1|0)]" >&2
	exit 1
}
trap exit 1 2 3
DN_DIR="${HOME}/download"
typeset -i redhat=1
URL="ftp://ftp.redhat.com"
START_DIR="pub/redhat/linux"
# URL="ftp://ftp.muug.mb.ca"
# START_DIR="/redhat/linux"
# URL="ftp://ftp.crc.ca"
# START_DIR="pub/systems/linux/redhat/ftp.redhat.com/linux"
# URL="ftp://redhat.eyetap.org"
# START_DIR="linux"
# URL="ftp://sunsite.ualberta.ca"
# START_DIR="pub/unix/Linux/redhat"
USER=""
RH_RPM_DIR="en/os/i386/RedHat/RPMS"
RH_UPDATE_RPM_DIR="en/os/i386"
GET_ALL=""
RH_VERSIONS=""
RH_UPDATES=""
FILES=""
while test "$#" -gt 0 ; do
  OPTIND=1
  while getopts "aD:L:S:r:u:" option ; do
	# echo "option $option"
	case "$option" in
	  "a")
		GET_ALL="y"
		;;
	  "r")
		case ".${OPTARG}" in
		  ".2")
			RH_VERSIONS="${RH_VERSIONS} 7.3"
			;;
		  ".1")
			RH_VERSIONS="${RH_VERSIONS} 7.2"
			;;
		  *)
			usage
			;;
		esac
		;;
	  "u")
		if [ ".${OPTARG}" = ".0" -o ".${OPTARG}" = ".1" ] ; then
			RH_UPDATES="${RH_UPDATES} ${OPTARG}"
		else
			usage
		fi
		;;
	  "D")
		DN_DIR="${OPTARG}"
		;;
	  "L")
		URL="${OPTARG}"
		;;
	  "U")
		USER="--user ${OPTARG}"
		;;
	  "S")
		START_DIR="${OPTARG}"
		redhat=0
		;;
	  "?")
		usage
		;;
	  # not implemented by shell's getopts ("-" is not allowed in optstring)
	  # "\001")
	  #	FILES="${FILES} \"${1}\""
	  #	;;
	esac
	shift $[OPTIND-1]
	OPTIND=1
  done
  test "$#" -eq 0 || { FILES="${FILES} ${1}" ; shift ; }
done
test ".${FILES}" = "." && usage
test ".${GET_ALL}" = "." && GET_ALL="n"
test ".${RH_VERSIONS}" = "." && RH_VERSIONS="7.3 7.2"
test ".${RH_UPDATES}" = "." && RH_UPDATES="1 0"
for file in ${FILES} ; do
  rc=1
  # echo file=$file
  if [ $redhat -eq 0 ] ; then
    try_get "${file}" ""
    rc="$?"
  else
    try_get "${file}" "rawhide/i386/RedHat/RPMS"
    rc="$?"
    test "${rc}" -ne 0 && {
      for ver in ${RH_VERSIONS} ; do
	# echo $ver
	for upd in ${RH_UPDATES} ; do
	  # echo upd=$upd
	  case "${upd}" in
	    "1")
		try_get "${file}" "updates/${ver}/${RH_UPDATE_RPM_DIR}"
		;;
	    "0")
		try_get "${file}" "${ver}/${RH_RPM_DIR}"
		;;
	  esac
	  rc="$?"
	  test "${rc}" -ne 0 || break
	done
	test "${rc}" -ne 0 || break
      done
    }
  fi
done
=========================================================================
-------------------------------------------------------
This SF.NET email is sponsored by: AMD - Your access to the experts
on Hammer Technology! Open Source & Linux Developers, register now
for the AMD Developer Symposium. Code: EX8664
http://www.developwithamd.com/developerlab
Received on 2002-09-19