started work on taxonomy retrieval

get_latest_taxonomy.sh

+#!/bin/bash
+# This script fetches nr & nt blast databases from ncbi
+# written by Jan van Haarst, PRI
+# needed : wget, curl & perl
+# latest update 28 Februari 2007
+# variables
+# Get location of this script, so we can call the other script
+DIRNAME=`dirname $0`
+SCRIPT_LOCATION=`perl -e 'END {use Cwd qw(realpath);print realpath($ARGV[0]);}' $DIRNAME`
+
+# Set site and data locations depending on where we are
+if [ $HOSTNAME == "dev1" ] 						# Primary site
+then
+	site_location="ftp://ftp.ncbi.nlm.nih.gov/blast/db/"
+	data_dir="/home/jvh/data/blast_latest/databases.nobackup"
+	extracted_dir=$data_dir"/extracted"
+	# Remove secondary checkfile
+	rm $data_dir"/newest_date.txt"
+	WGET="/home/jvh/bin/wget"
+	CURL="/usr/bin/curl -s"
+	TAR="/bin/tar"
+	PRIMARY="true"
+else										# Secondary sites
+	site_location="http://dev1.ab.wurnet.nl/~jvh/databases.nobackup/"
+	PRIMARY="false"
+	if [ $HOSTNAME == "kwatta" ]
+	then
+		target_dir="/hwraid4/data/blast/databases"
+		data_dir=$target_dir"/updates"
+		extracted_dir=$data_dir"/extracted"
+		WGET="/usr/local/bin/wget"
+		CURL="/opt/sfw/bin/curl"
+		TAR="/usr/local/bin/tar"
+	else
+		target_dir="/state/partition1/blast/db/"
+		data_dir=$target_dir"/updates"
+		extracted_dir=$data_dir"/extracted"
+		WGET="/usr/bin/wget"
+		CURL="/usr/bin/curl"
+		TAR="/bin/tar"
+	fi
+
+	# Only secondary sites need to check the date
+	newest_date_local=`cat $data_dir"/newest_date.txt"`
+	if [ $? != "0" ]
+	then
+		echo Problem checking local file !
+		#exit
+	fi
+
+	newest_date_remote=`$CURL -s -f $site_location"newest_date.txt"`
+	if [ $? != "0" ]
+	then
+		# Primary site is updating or offline
+		echo Primary site is updating or offline
+		exit
+	fi
+
+	if [ $newest_date_remote == $newest_date_local ]
+	then
+		# No update needed
+		echo No update needed
+		exit
+	fi
+fi
+
+now=`date '+%Y_%m_%d_%A_%Hh%M'`
+log_dir=$data_dir"/logs"
+logfile=$log_dir"/blastlog_"$now
+lockfile=$data_dir"/update_is_running.lock"
+accept_list="nt*,nr*,taxdb*,*.txt,*.htm*"
+reject_list="*.html*,newest_date.txt"
+wget_options="--proxy=off --mirror --no-parent --no-directories --level=1 --timestamping --passive-ftp --accept "$accept_list" -o "$logfile" --reject "$reject_list
+# make the needed directories
+
+mkdir -p $data_dir
+mkdir -p $log_dir
+mkdir -p $extracted_dir
+
+# change to datadir
+cd $data_dir
+
+# catch sigint and remove lockfile
+
+remove_lock()
+{
+if test -e $lockfile
+    then
+    # remove the update file.
+    rm $lockfile
+    exit
+fi
+}
+
+trap remove_lock SIGINT
+
+# fetch the data
+if test -e $lockfile
+    then
+	# update is running, so print a message and stop.
+	echo "Update is already running."
+	exit
+    else
+	# lockfile does NOT exist, so we create the lockfile, run the update and remove the file.
+	touch $lockfile
+	# get the data
+	$WGET  $wget_options  $site_location
+	# Add return value to log file
+	echo "Return Value="$? >> $logfile
+	# Generate a timestamp, so we know what date the latest file was added
+	perl $SCRIPT_LOCATION"/last_file_time.pl" > $data_dir"/newest_date.txt"
+	# For secondary sites :
+	# Unpack the retrieved set, and remove the original archive
+	if [ $PRIMARY == "false" ]
+	then
+		for NAME in *.tar.gz
+		do
+			$TAR --directory=$extracted_dir -xzf $NAME
+			if [ $? != "0" ]
+			then
+				echo Problem unpacking $NAME
+			else
+				rm $NAME
+			fi
+		done
+		if [ $HOSTNAME == "apbioinf100.wurnet.nl" ]
+		then
+			perl "$DIRNAME"/fastcopy.pl --directory $extracted_dir
+			cluster-fork rsync --archive --include="/*"  -e ssh `hostname`:$extracted_dir $extracted_dir
+		fi
+		if [ $HOSTNAME == "kwatta" ]
+		then
+			# Check if blast is running
+			while [ `/usr/bin/pgrep blastall > /dev/null;echo $?` != "1" ]
+			do
+				echo "Blast is running"
+				# Try again in an hour
+				sleep 3600
+			done
+			# Double check before move
+			/usr/bin/pgrep blastall
+			# If not, move the data to the target location
+			if [ $? == "1" ]
+			then
+				pushd $extracted_dir > /dev/null
+				mv * $target_dir
+				popd > /dev/null
+			else
+				echo "Blast was running"
+			fi
+		fi
+	fi
+	# remove the lockfile.
+	rm $lockfile
+	# done
+	exit
+fi
+