Tuesday, February 07, 2006

[UNIX] Download Accelerator using curl (Revised)

This script has been submitted to Sun BigAdmin
#! /bin/bash
#
# Download Accelerator Plus - download URL with multiple threads
#
# Usage:
#     ./dap.sh  [thread=4]
#
# Example:
#     $ url=http://www.swim.ee/videos/breast/Igelstrom-full-50m-race.mpg
#
#     $ time ./dap.sh $url
#     Downloading Igelstrom-full-50m-race.mpg (5640348 bytes) with 4 threads ...
#
#     real    3m50.204s
#     user    0m0.160s
#     sys     0m0.330s
#
#     $ time curl --silent --output `basename $url` $url
#     real    14m40.941s
#     user    0m0.070s
#     sys     0m0.220s
 
 
 
trap 'killall; cleanup; exit 1' 1 2 3 6 9 15
 
 
killall()
{
  for j in $jobs; do
    kill -9 $j > /dev/null 2>&1
  done
}
cleanup()
{
  rm -f ${prefix}*
  rm -f $header
}
 
 
PATH=/usr/bin:/bin:/usr/sbin:$HOME/bin
 
 
if [ $# -lt 1 -o $# -gt 2 ]; then
  echo "Usage: $0  [thread=4]"
  exit 1
fi
 
 
url="$1"
url_base=`basename $url`
thread="${2:-4}"
TMPDIR="${TMPDIR:-/tmp}"
prefix="$TMPDIR/dap.$$"
header="$TMPDIR/header.$$"
curl --head --silent $url > $header
size=`awk '/Content-[Ll]ength:/ {printf("%d",$NF)}' $header`
 
 
#
# HTTP header: Accept-Ranges
#
grep -i "accept-ranges" $header > /dev/null 2>&1
cleanup
if [ $? -ne 0 -o $thread -le 1 ]; then
  echo "Downloading $url_base ($size bytes) with 1 thread ..."
  curl --silent --output $url_base $url
  exit 0
fi
   
 
echo "Downloading $url_base ($size bytes) with $thread threads ..."
count=1
end=-1
step=`expr $size / $thread`
jobs=""
while [ $count -le $thread ]
do
  start=`expr $end + 1`
  if [ $count -eq $thread ]; then
     end=$size
  else
     end=`expr $count \* $step`
     end=`expr $end - 1`
  fi
  pad=`echo $count | awk -v count=$count '{printf("%04d",count)}'`
  curl --silent --range ${start}-${end} --output $prefix.$pad $url &
  jobs="$jobs $!"
  count=`expr $count + 1`
done
 
# wait for all background to finish
wait
cat ${prefix}* > $url_base
cleanup