Start work on rsync version of download, as ncbi does also accept rsync, and...

Start work on rsync version of download, as ncbi does also accept rsync, and this will speed up syncing.

getlatest_rsync

+#!/bin/bash
+# This script fetches nr & nt blast databases from ncbi
+# written by Jan van Haarst, PRI
+# needed : wget, curl & perl
+# latest update 28 Februari 2007
+# variables
+# Get location of this script, so we can call the other script
+DIRNAME=`dirname $0`
+SCRIPT_LOCATION=`perl -e 'END {use Cwd qw(realpath);print realpath($ARGV[0]);}' $DIRNAME`
+# rsync --itemize-changes -avPH --stats -z --compress-level=1 ftp.ncbi.nlm.nih.gov::blast/db/n* /home/jvh/data/blast_latest/databases.nobackup/rsync/
+#--exclude=PATTERN
+#--include=PATTERN
+#--log-file=FILE
+
+# Set site and data locations depending on where we are
+if [ $HOSTNAME == "dev1" ] 						# Primary site
+then
+	site_location="ftp://ftp.ncbi.nlm.nih.gov/blast/db/"
+	data_dir="/home/jvh/data/blast_latest/databases.nobackup"
+	extracted_dir=$data_dir"/extracted"
+	# Remove secondary checkfile
+	rm $data_dir"/newest_date.txt"
+	WGET="/home/jvh/bin/wget"
+	CURL="/usr/bin/curl -s"
+	TAR="/bin/tar"
+	PRIMARY="true"
+else										# Secondary sites
+	site_location="http://dev1.ab.wurnet.nl/~jvh/databases.nobackup/"
+	PRIMARY="false"
+	if [ $HOSTNAME == "kwatta" ]
+	then
+		target_dir="/hwraid4/data/blast/databases"
+		data_dir=$target_dir"/updates"
+		extracted_dir=$data_dir"/extracted"
+		WGET="/usr/local/bin/wget"
+		CURL="/opt/sfw/bin/curl"
+		TAR="/usr/local/bin/tar"
+	else
+		target_dir="/state/partition1/blast/db/"
+		data_dir=$target_dir"/updates"
+		extracted_dir=$data_dir"/extracted"
+		WGET="/usr/bin/wget"
+		CURL="/usr/bin/curl"
+		TAR="/bin/tar"
+	fi
+
+	# Only secondary sites need to check the date
+
+	newest_date_local=`cat $data_dir"/newest_date.txt"`
+	if [ $? != "0" ]
+	then
+		echo Problem checking local file !
+		newest_date_local=0
+		#exit
+	fi
+
+	newest_date_remote=`$CURL -s -f $site_location"newest_date.txt"`
+	if [ $? != "0" ]
+	then
+		# Primary site is updating or offline
+		#echo Primary site is updating or offline
+		exit
+	fi
+
+	if [ $newest_date_remote == $newest_date_local ]
+	then
+		# No update needed
+		exit
+	fi
+fi
+
+now=`date '+%Y_%m_%d_%A_%Hh%M'`
+log_dir=$data_dir"/logs"
+logfile=$log_dir"/blastlog_"$now
+lockfile=$data_dir"/update_is_running.lock"
+accept_list="nt*,nr*,taxdb*,*.txt,*.htm*"
+reject_list="*.html*,newest_date.txt"
+wget_options="--proxy=off --mirror --no-parent --no-directories --level=1 --timestamping --passive-ftp --accept "$accept_list" -o "$logfile" --reject "$reject_list
+# make the needed directories
+
+mkdir -p $data_dir
+mkdir -p $log_dir
+mkdir -p $extracted_dir
+
+# change to datadir
+cd $data_dir
+
+# catch sigint and remove lockfile
+
+remove_lock()
+{
+if test -e $lockfile
+    then
+    # remove the update file.
+    rm $lockfile
+    exit
+fi
+}
+
+trap remove_lock SIGINT
+
+# fetch the data
+if test -e $lockfile
+    then
+	# update is running, so print a message and stop.
+	echo "Update is already running."
+	exit
+    else
+	# lockfile does NOT exist, so we create the lockfile, run the update and remove the file.
+	touch $lockfile
+	# get the data
+	$WGET  $wget_options  $site_location
+	# Add return value to log file
+	echo "Return Value="$? >> $logfile
+	# Generate a timestamp, so we know what date the latest file was added
+	perl $SCRIPT_LOCATION"/last_file_time.pl" > $data_dir"/newest_date.txt"
+	# For secondary sites :
+	# Unpack the retrieved set, and remove the original archive
+	if [ $PRIMARY == "false" ]
+	then
+		for NAME in *.tar.gz
+		do
+			$TAR --directory=$extracted_dir -xzf $NAME
+			if [ $? != "0" ]
+			then
+				echo Problem unpacking $NAME
+			else
+				rm $NAME
+			fi
+		done
+		if [ $HOSTNAME == "apbioinf100.wurnet.nl" ]
+		then
+			perl "$DIRNAME"/fastcopy.pl --directory $extracted_dir
+			cluster-fork rsync --archive --include="/*"  -e ssh `hostname`:$extracted_dir $extracted_dir
+		fi
+		if [ $HOSTNAME == "kwatta" ]
+		then
+			# Check if blast is running
+			while [ `/usr/bin/pgrep blastall > /dev/null;echo $?` != "1" ]
+			do
+				echo "Blast is running"
+				# Try again in an hour
+				sleep 3600
+			done
+			# Double check before move
+			/usr/bin/pgrep blastall
+			# If not, move the data to the target location
+			if [ $? == "1" ]
+			then
+				pushd $extracted_dir > /dev/null
+				mv * $target_dir
+				popd > /dev/null
+			else
+				echo "Blast was running"
+			fi
+		fi
+	fi
+	# remove the lockfile.
+	rm $lockfile
+	# done
+	exit
+fi