PureDarwin/setup/pd_fetch

#!/bin/sh
#
# Copyright (c) 2007-2009 The PureDarwin Project.
# All rights reserved.
#
# @LICENSE_HEADER_START@
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# @LICENSE_HEADER_END@
#

#
# probono <probono@puredarwin.org>
# aladin   <aladin@puredarwin.org>
#

#
# Downloads and check "vital" BinaryRoots and BinaryDrivers from DarwinBuild website
#

#
# Changelog (deprecated)
#
# 20090326 - Please, see `hg log' or `svn log' from now (Mercurial) - aladin
# 20081219 - Added 9G55 and 9G55pd1 (experimental patched packages) - aladin
# 20081210 - Added 9F33pd1 (experimental packages, our patched binaryroots) - aladin
# 20081209 - License header update - aladin
# 20081115 - Minor updates - aladin
# 20080921 - SHA-1 checksum added - aladin
# 20080831 - Minor updates and fixes - aladin
# 20080606 - Cleaned up - aladin
# 20080528 - Now used in order to download binary roots and drivers - aladin
# 20080527 - Fixed download order, curl and minor bugs - aladin
# 20080305 - Recursive {search|fetch} of binary files from DarwinBuild - aladin 
# 20080107 - Initial release - probono
#

################################################################################
# GLOBAL VARIABLES
################################################################################

# Find out where this script lives
DIRNAME=$(cd $(dirname $0) && pwd)

# Reading global variables in pd_config file
source "$DIRNAME"/pd_config

################################################################################
# INITIAL CONDITION
################################################################################

# Autorun
if [ "$1" = "" ]; then
	echo
	echo "Fetch "vital" binary files (roots and drivers)."
	echo
	echo "Legend:"
	echo " . project.root.tar.gz\tThe archive has already been retrieved"
	echo " + project.root.tar.gz\tThe archive comes from a local darwinbuild package"
	echo " * project.root.tar.gz\tThe archive comes from a remote repository"
	echo " ? project.root.tar.gz\tThe archive is missing" 
	#echo " ! project.root.tar.gz\tThe archive is \"corrupted\""

	echo
	echo "Retrieving binaryroots, please wait..."
	echo

	# Clone the PureDarwin binary roots repository (Mercurial)
	# Source: http://code.google.com/p/puredarwin/source/list?repo=roots
	if [ ! -e $BINARYROOTS_DIR ]; then
		hg clone https://roots.puredarwin.googlecode.com/hg/ $TEMPDIR #--debug
	else
		cd $TEMPDIR
			hg pull
			hg update #--debug
		cd - > /dev/null
	fi
	
	# Recursive call of itself
	$0 0 `cat $BINARYROOTS_LIST_FILE $BINARYROOTS_EXTRA_LIST_FILE`
	exit 0
fi

################################################################################
# OTHER VARIABLES
################################################################################

# Recursive level found in arg1 (loop counter)
LEVEL=$1
# Current build version target taken in BINVERSION array from pd_config
BUILD_VERSION=${BINVERSION[$LEVEL]}
# Verbose mode (or not) for curl
CURL_VERBOSE_FLAG="-s"
#CURL_VERBOSE_FLAG="--progress-bar"
#CURL_VERBOSE_FLAG=""

# Eating args $1 
#
# 	Shifting LEVEL
#       $# (arg0..argn) is now the list of missing binary files (from pd_roots file)
#
#	At the beginning, all the files are missing
#	At the end, no file should missing
shift

################################################################################
# RECURSIVE CONDITIONS
################################################################################

# Break condition: no more missing files found in args, success.
if [ $# -eq 0 ]; then
	echo
	echo "Fetching binary files complete ($MISSING_LIST_INIT files)" 
	if [ "$BAD_SHA1_LIST" ]; then
		echo "WARNING! SHA-1 checksum failed on:$BAD_SHA1_LIST"
	fi	
	exit 0
fi

# Break condition: no more level to go in, end of array, some missing files could remain..
if [ $LEVEL -gt $[${#BINVERSION[@]} - 1] ]; then
	echo 
	echo "Fetching binary files uncomplete"
	echo "Missing $[$MISSING_LIST_INIT - $[$MISSING_LIST_INIT - $#]] files: $*"
	if [ "$BAD_SHA1_LIST" ]; then
		echo "WARNING! SHA-1 checksum failed on:$BAD_SHA1_LIST"
	fi	
	exit -1
fi

# Initialization, level 0 is the beginning
if [ $LEVEL -eq 0 ]; then
	# As `cat pd_roots | wc -l` (because all files are missing)
	MISSING_LIST_INIT=$#
	export BAD_SHA1_LIST=""
fi

###############################################################################
# MAIN
###############################################################################

# Matching condition: current build version does belong to the target darwin release
MAJOR_BUILD_VERSION=`echo $BUILD_VERSION | cut -b1-"$(printf "$DARWIN_RELEASE" | wc -c | sed 's/ //g')"`
# Faking major build version for specific build version "pd" (mostly binaryroots for Darwin 9)
if [ "$BUILD_VERSION" = "pd" ]; then
	MAJOR_BUILD_VERSION="$DARWIN_RELEASE"
fi
if [ "$DARWIN_RELEASE" =  "$MAJOR_BUILD_VERSION" ]; then

	echo
	echo "BuildVersion: $BUILD_VERSION ($[$MISSING_LIST_INIT - $#]/$MISSING_LIST_INIT files)"

	for x in $*; do

		BINARYROOT_FOUND=""

		###############################################################
		# LOCAL BINARYROOT
		###############################################################
		if [ "$BINARYROOT_FOUND" = "" ]; then
			# Test missing binary file existence in the local target dir ($BINARYROOTS_DEFAULT_DIR)
			if [ -e "$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz" ]; then
				echo " . ${x}.root.tar.gz"
				BINARYROOT_FOUND="$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz"

				# Verifying SHA1 checksum
				#T_SHA1=`openssl dgst -sha1 $BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz`
				#grep "$T_SHA1" pd_sha1  > /dev/null 2> /dev/null
				#if [ $? -eq 0 ]; then
					# Found, kept and checked!
				#	printf " . ${x}.root.tar.gz"
				#else
					# Bad checksum or not up to date or unknow!
				#	echo " ! ${x}.root.tar.gz";
					# Adding binary to bad SHA-1 binary list
				#	BAD_SHA1_LIST="$BAD_SHA1_LIST $x"
				#fi
			fi
		fi

		###############################################################
		# LOCAL EXTRA BINARYROOT
		###############################################################
		if [ "$BINARYROOT_FOUND" = "" ]; then
			# Test missing binary file existence in the local build directories
			z=0
			while [ $z -lt $[${#BINARYROOTS_DIR[@]}] ]; do
				BINARY_DIR=${BINARYROOTS_DIR[$z]}
				if [ -e "$BINARY_DIR/$BUILD_VERSION/${x}.root.tar.gz" ]; then
					echo " . ${x}.root.tar.gz"
					BINARYROOT_FOUND="$BINARY_DIR/$BUILD_VERSION/${x}.root.tar.gz"
					break # FIXME while stop. / stop at first one found
				fi	
			z=$[$z + 1]
			done # End of WHILE through BINARYROOTS_DIR
		fi

		###############################################################
		# LOCAL DARWINBUILD BINARYROOT
		###############################################################
		if [ "$BINARYROOT_FOUND" = "" ]; then
			# Test missing binary file existence in the darwinbuild build directory
			DARWINBUILD_BUILD_DIR_FOUND=""
			for DARWINBUILD_BUILD_DIR in `ls $DARWINBUILD_DIR`; do
				if [ -e "$DARWINBUILD_DIR/$DARWINBUILD_BUILD_DIR/.build/build" ]; then
					DARWINBUILD_BUILD_DIR_BUILDVERSION=`cat $DARWINBUILD_DIR/$DARWINBUILD_BUILD_DIR/.build/build 2> /dev/null`
					if [ "${BINVERSION[$LEVEL]}"  = "$DARWINBUILD_BUILD_DIR_BUILDVERSION" ]; then
						# Take the first one # FIXME
						DARWINBUILD_BUILD_DIR_FOUND=$DARWINBUILD_DIR/$DARWINBUILD_BUILD_DIR
						break
					fi
				fi
			done
			if [ ! "$DARWINBUILD_BUILD_DIR_FOUND" = "" ]; then
				DARWINBUILD_BINARYROOT_FOUND=""
				if [ -e $DARWINBUILD_BUILD_DIR_FOUND/Roots/.DownloadCache/${x}.root.tar.gz ]; then
					# Took first from ../Roots/.DownloadCache/.. which content is == fetching it remotely in case this step fail to found
					DARWINBUILD_BINARYROOT_FOUND="$DARWINBUILD_BUILD_DIR_FOUND/Roots/.DownloadCache/${x}.root.tar.gz"
				else
					# Then took from ../Packages/..
					if [ -e $DARWINBUILD_BUILD_DIR_FOUND/Packages/${x}.root.tar.gz ]; then
						DARWINBUILD_BINARYROOT_FOUND="$DARWINBUILD_BUILD_DIR_FOUND/Packages/${x}.root.tar.gz"
					fi
				fi
			fi
			if [ ! $DARWINBUILD_BINARYROOT_FOUND = "" ]; then
				echo " + ${x}.root.tar.gz"
				BINARYROOT_FOUND=$DARWINBUILD_BINARYROOT_FOUND
			fi
		fi

		###############################################################
		# REMOTE (DARWINBUILD) BINARYROOT 
		###############################################################
		if [ "$BINARYROOT_FOUND" = "" ]; then
			# Iterative loop in BINARYROOTS_HTTP_PATH
			y=0
			while [ $y -lt $[${#BINARYROOTS_HTTP_PATH[@]}] ]; do
				BINARY_HTTP_PATH=${BINARYROOTS_HTTP_PATH[$y]}	

				# PSEUDO PREFETCH 
				# FIXME add switch or rm to flush the cache or.. test how it behaves before and after an updated page..
				PSEUDOCACHE_FILE="$TEMPDIR/$(basename $BINARY_HTTP_PATH)/cache-$BUILD_VERSION"
				if [ ! -e $PSEUDOCACHE_FILE ]; then
					curl -f $CURL_VERBOSE_FLAG --create-dir --output "$PSEUDOCACHE_FILE" -C -  "$BINARY_HTTP_PATH/$BUILD_VERSION/"
					if [ ! $? -eq 0 ]; then
						touch $PSEUDOCACHE_FILE # create an empty one for next time..
					fi
				fi

				# Test missing binary file existence in the local binary file list cache
				grep "\"${x}.root.tar.gz\"" $PSEUDOCACHE_FILE > /dev/null 2> /dev/null
				if [ $? -eq 0 ]; then
					# Fetching missing or corrupted binary file into $BINARYROOTS_DEFAULT_DIR
					curl -f --create-dir $CURL_VERBOSE_FLAG --output "$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz" -C - "$BINARY_HTTP_PATH/$BUILD_VERSION/${x}.root.tar.gz"
					if [ $? -eq 0 ]; then
						# Found remote archive 

						# Verifying SHA1 checksum
						#T_SHA1=`openssl dgst -sha1 $BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz`
						#grep "$T_SHA1" pd_sha1  > /dev/null 2> /dev/null
						#if [ $? -eq 0 ]; then
							# Found, kept and checked!
							echo " * ${x}.root.tar.gz" # ($y)
		#					# Removing old version since a new one has been fetch
		#					T_LEVEL=$[$LEVEL + 1] 
		#					while [ $T_LEVEL -lt $[${#BINVERSION[@]} - 1] ]; do
		#						####rm -f $BINARYROOTS_DEFAULT_DIR/${BINVERSION[$T_LEVEL]}/${x}.root.tar.gz
		#						T_LEVEL=$[$T_LEVEL + 1] 
		#					done
		#				else
							# Bad checksum!
		#					echo " ! ${x}.root.tar.gz";
							# Adding binary to bad SHA-1 binary list
		#					BAD_SHA1_LIST=$x
						#fi

						BINARYROOT_FOUND="$BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz"
						break # This breaks the WHILE through BINARYROOTS_HTTP_PATH
					fi
				fi
				y=$[$y + 1] # next element in BINARYROOTS_HTTP_PATH
			done # End of WHILE through BINARYROOTS_HTTP_PATH
		fi

		###############################################################
		# MISSING BINARYROOT 
		###############################################################
		if [ "$BINARYROOT_FOUND" = "" ]; then
			echo " ? ${x}.root.tar.gz";
			#####rm -f $BINARYROOTS_DEFAULT_DIR/$BUILD_VERSION/${x}.root.tar.gz
			# Adding missing binary to missing binary list
			MISSING_LIST="${MISSING_LIST} ${x}"
		fi	
			
	done # End of FOR through missing binaryroots list (MISSING_LIST)
else
	#echo "Skipping. Major build version ($MAJOR_BUILD_VERSION (build ver: $BUILD_VERSION)) does not match Major Darwin release ($DARWIN_RELEASE)."
	# Adding initial missing binary to missing binary list since no current BuildVersion directory has been found
	MISSING_LIST="$*"
fi

################################################################################
# RECURSIVE CALL
################################################################################

# Recursive call + propagating MISSING_LIST_INIT in env
MISSING_LIST_INIT="$MISSING_LIST_INIT" $0 $[$LEVEL + 1] $MISSING_LIST

exit 0