Wednesday, September 28, 2005

[UNIX] Download Accelerator using curl

#! /bin/sh


PATH=/usr/bin:/bin:/usr/sbin:$HOME/bin


if [ $# -lt 1 -o $# -gt 2 ]; then
 echo "$0  [nthreads=4]"
 exit 1
fi
url=$1
thread=${2:-4}
 

size=`curl -s --head $url | awk '/Content-[Ll]ength:/ {printf("%d",$NF)}'`
count=1
end=-1
step=`expr $size / $thread`
jobs=""
while [ $count -le $thread ]
do
 start=`expr $end + 1`
 if [ $count -eq $thread ]; then
  end=$size
 else
  end=`expr $count \* $step`
  end=`expr $end - 1`
 fi
 pad=`echo $count | awk -v count=$count '{printf("%04d",count)}'`
 curl -s -r ${start}-${end} -o $TMPDIR/tmp$$.$pad $url &
 job=$!
 if [ $count -eq 1 ]; then
  jobs="$job"
 else
  jobs="$jobs $job"
 fi
 count=`expr $count + 1`
done

# wait for all background jobs to finish
while :
do
 running=0
 for j in $jobs
 do
  ps -ef | grep $j | grep -v grep > /dev/null 2>&1
  if [ $? -eq 0 ]; then
   running=`expr $running + 1`
  fi
 done
 if [ $running -eq 0 ]; then
  break
 fi
 sleep 1
done


count=1
while [ $count -le $thread ]
do
 pad=`echo $count | awk -v count=$count '{printf("%04d",count)}'`
 cat $TMPDIR/tmp$$.$pad >> `basename $url`
 rm -f $TMPDIR/tmp$$.$pad
 count=`expr $count + 1`
done

1 Comments:

Blogger Knox Karter said...

Nice article, I am a big time fan of your site, keep up the nice work, and I will be a frequent visitor for a very long time.

event management logo

3:29 PM  

Post a Comment

<< Home