Mercurial > hg > aboriginal
changeset 1253:d4b251e78fc0
Break out download_functions.sh into their own file.
author | Rob Landley <rob@landley.net> |
---|---|
date | Sat, 25 Sep 2010 14:10:23 -0500 |
parents | ba951c11fb48 |
children | 9df779cde3e9 |
files | sources/download_functions.sh sources/functions.sh sources/include.sh |
diffstat | 3 files changed, 244 insertions(+), 243 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sources/download_functions.sh Sat Sep 25 14:10:23 2010 -0500 @@ -0,0 +1,233 @@ +#!/bin/echo "This file is sourced, not run" + +# Give filename.tar.ext minus the version number. + +noversion() +{ + echo "$1" | sed -e 's/-*\(\([0-9\.]\)*\([_-]rc\)*\(-pre\)*\([0-9][a-zA-Z]\)*\)*\(\.tar\..z2*\)$/'"$2"'\6/' +} + +# Apply any patches to this package +patch_package() +{ + ls "$PATCHDIR/${PACKAGE}"-* 2> /dev/null | sort | while read i + do + if [ -f "$i" ] + then + echo "Applying $i" + (cd "${SRCTREE}/${PACKAGE}" && + patch -p1 -i "$i" && + sha1file "$i" >> "$SHA1FILE") || + ([ -z "$ALLOW_PATCH_FAILURE" ] && dienow) + fi + done +} + +# Extract tarball named in $1 and apply all relevant patches into +# "$BUILD/packages/$1". Record sha1sum of tarball and patch files in +# sha1-for-source.txt. Re-extract if tarball or patches change. + +extract_package() +{ + mkdir -p "$SRCTREE" || dienow + + # Figure out whether we're using an unstable package. + + PACKAGE="$1" + is_in_list "$PACKAGE" $USE_UNSTABLE && PACKAGE=alt-"$PACKAGE" + + # Announce to the world that we're cracking open a new package + + echo "=== $PACKAGE ($ARCH_NAME $STAGE_NAME)" + set_titlebar "$ARCH_NAME $STAGE_NAME $PACKAGE" + + # Find tarball, and determine type + + FILENAME="$(ls -tc "$SRCDIR/${PACKAGE}-"*.tar* 2>/dev/null | head -n 1)" + DECOMPRESS="" + [ "$FILENAME" != "${FILENAME/%\.tar\.bz2/}" ] && DECOMPRESS="j" + [ "$FILENAME" != "${FILENAME/%\.tar\.gz/}" ] && DECOMPRESS="z" + + # If the source tarball doesn't exist, but the extracted directory is there, + # assume everything's ok. + + SHA1FILE="$SRCTREE/$PACKAGE/sha1-for-source.txt" + if [ -z "$FILENAME" ] + then + [ ! -e "$SRCTREE/$PACKAGE" ] && dienow "No tarball for $PACKAGE" + + # If the sha1sum file isn't there, re-patch the package. + [ ! -e "$SHA1FILE" ] && patch_package + return 0 + fi + + # Check the sha1 list from the previous extract. If the source is already + # up to date (including patches), keep it. + + SHA1TAR="$(sha1file "$FILENAME")" + SHALIST=$(cat "$SHA1FILE" 2> /dev/null) + if [ ! -z "$SHALIST" ] + then + for i in "$SHA1TAR" $(sha1file "$PATCHDIR/$PACKAGE"-* 2>/dev/null) + do + # Is this sha1 in the file? + if [ -z "$(echo "$SHALIST" | sed -n "s/$i/$i/p" )" ] + then + SHALIST=missing + break + fi + # Remove it + SHALIST="$(echo "$SHALIST" | sed "s/$i//" )" + done + # If we matched all the sha1sums, nothing more to do. + [ -z "$SHALIST" ] && return 0 + fi + + # Re-extract the package, deleting the old one (if any).. + + echo -n "Extracting '$PACKAGE'" + ( + UNIQUE=$(readlink /proc/self) + trap 'rm -rf "$BUILD/temp-'$UNIQUE'"' EXIT + rm -rf "$SRCTREE/$PACKAGE" 2>/dev/null + mkdir -p "$BUILD"/{temp-$UNIQUE,packages} || dienow + + { tar -xv${DECOMPRESS} -f "$FILENAME" -C "$BUILD/temp-$UNIQUE" || dienow + } | dotprogress + + mv "$BUILD/temp-$UNIQUE/"* "$SRCTREE/$PACKAGE" && + echo "$SHA1TAR" > "$SHA1FILE" + ) + + [ $? -ne 0 ] && dienow + + patch_package +} + +# Confirm that a file has the appropriate checksum (or exists but SHA1 is blank) +# Delete invalid file. + +confirm_checksum() +{ + SUM="$(sha1file "$SRCDIR/$FILENAME" 2>/dev/null)" + if [ x"$SUM" == x"$SHA1" ] || [ -z "$SHA1" ] && [ -f "$SRCDIR/$FILENAME" ] + then + if [ -z "$SHA1" ] + then + echo "No SHA1 for $FILENAME ($SUM)" + else + echo "Confirmed $FILENAME" + fi + + # Preemptively extract source packages? + + [ -z "$EXTRACT_ALL" ] && return 0 + extract_package "$BASENAME" + return $? + fi + + # If there's a corrupted file, delete it. In theory it would be nice + # to resume downloads, but wget creates "*.1" files instead. + + rm "$SRCDIR/$FILENAME" 2> /dev/null + + return 1 +} + +# Attempt to obtain file from a specific location + +download_from() +{ + # Return success if we already have a valid copy of the file + + confirm_checksum && return 0 + + # If we have another source, try to download file from there. + + [ -z "$1" ] && return 1 + wget -t 2 -T 20 -O "$SRCDIR/$FILENAME" "$1" || + (rm "$SRCDIR/$FILENAME"; return 2) + touch -c "$SRCDIR/$FILENAME" + + confirm_checksum +} + +# Confirm a file matches sha1sum, else try to download it from mirror list. + +download() +{ + FILENAME=`echo "$URL" | sed 's .*/ '` + [ -z "$RENAME" ] || FILENAME="$(echo "$FILENAME" | sed -r "$RENAME")" + ALTFILENAME=alt-"$(noversion "$FILENAME" -0)" + + echo -ne "checking $FILENAME\r" + + # Update timestamps on both stable and unstable tarballs (if any) + # so cleanup_oldfiles doesn't delete stable when we're building unstable + # or vice versa + + touch -c "$SRCDIR"/{"$FILENAME","$ALTFILENAME"} 2>/dev/null + + # Give package name, minus file's version number and archive extension. + BASENAME="$(noversion "$FILENAME" | sed 's/\.tar\..z2*$//')" + + # If unstable version selected, try from listed location, and fall back + # to PREFERRED_MIRROR. Do not try normal mirror locations for unstable. + + if is_in_list "$BASENAME" $USE_UNSTABLE + then + # If extracted source directory exists, don't download alt-tarball. + [ -e "$SRCTREE/alt-$BASENAME" ] && return 0 + + # Download new one as alt-packagename.tar.ext + FILENAME="$ALTFILENAME" + SHA1= + + ([ ! -z "$PREFERRED_MIRROR" ] && + download_from "$PREFERRED_MIRROR/$ALTFILENAME") || + download_from "$UNSTABLE" + return $? + fi + + # If environment variable specifies a preferred mirror, try that first. + + if [ ! -z "$PREFERRED_MIRROR" ] + then + download_from "$PREFERRED_MIRROR/$FILENAME" && return 0 + fi + + # Try original location, then mirrors. + # Note: the URLs in mirror list cannot contain whitespace. + + download_from "$URL" && return 0 + for i in $MIRROR_LIST + do + download_from "$i/$FILENAME" && return 0 + done + + # Return failure. + + echo "Could not download $FILENAME" + echo -en "\e[0m" + return 1 +} + +# Clean obsolete files out of the source directory + +START_TIME=`date +%s` + +cleanup_oldfiles() +{ + # wait for asynchronous downloads to complete + + wait + + for i in "${SRCDIR}"/* + do + if [ -f "$i" ] && [ "$(date +%s -r "$i")" -lt "${START_TIME}" ] + then + echo Removing old file "$i" + rm -rf "$i" + fi + done +}
--- a/sources/functions.sh Sat Sep 25 13:40:04 2010 -0500 +++ b/sources/functions.sh Sat Sep 25 14:10:23 2010 -0500 @@ -2,8 +2,6 @@ # Lots of reusable functions. This file is sourced, not run. -source sources/utility_functions.sh - # Output path to cross compiler. cc_path() @@ -128,7 +126,7 @@ done } -# Strip the version number off a tarball +# Delete a working copy of source code once the build's done. cleanup() { @@ -162,245 +160,6 @@ WORKDIR_LIST= } -# Give filename.tar.ext minus the version number. - -noversion() -{ - echo "$1" | sed -e 's/-*\(\([0-9\.]\)*\([_-]rc\)*\(-pre\)*\([0-9][a-zA-Z]\)*\)*\(\.tar\..z2*\)$/'"$2"'\6/' -} - -# Given a filename.tar.ext, return the version number. - -getversion() -{ - echo "$1" | sed -e 's/.*-\(\([0-9\.]\)*\([_-]rc\)*\(-pre\)*\([0-9][a-zA-Z]\)*\)*\(\.tar\..z2*\)$/'"$2"'\1/' -} - -# Apply any patches to this package -patch_package() -{ - ls "$PATCHDIR/${PACKAGE}"-* 2> /dev/null | sort | while read i - do - if [ -f "$i" ] - then - echo "Applying $i" - (cd "${SRCTREE}/${PACKAGE}" && - patch -p1 -i "$i" && - sha1file "$i" >> "$SHA1FILE") || - ([ -z "$ALLOW_PATCH_FAILURE" ] && dienow) - fi - done -} - -# Extract tarball named in $1 and apply all relevant patches into -# "$BUILD/packages/$1". Record sha1sum of tarball and patch files in -# sha1-for-source.txt. Re-extract if tarball or patches change. - -extract_package() -{ - mkdir -p "$SRCTREE" || dienow - - # Figure out whether we're using an unstable package. - - PACKAGE="$1" - is_in_list "$PACKAGE" $USE_UNSTABLE && PACKAGE=alt-"$PACKAGE" - - # Announce to the world that we're cracking open a new package - - echo "=== $PACKAGE ($ARCH_NAME $STAGE_NAME)" - set_titlebar "$ARCH_NAME $STAGE_NAME $PACKAGE" - - # Find tarball, and determine type - - FILENAME="$(ls -tc "$SRCDIR/${PACKAGE}-"*.tar* 2>/dev/null | head -n 1)" - DECOMPRESS="" - [ "$FILENAME" != "${FILENAME/%\.tar\.bz2/}" ] && DECOMPRESS="j" - [ "$FILENAME" != "${FILENAME/%\.tar\.gz/}" ] && DECOMPRESS="z" - - # If the source tarball doesn't exist, but the extracted directory is there, - # assume everything's ok. - - SHA1FILE="$SRCTREE/$PACKAGE/sha1-for-source.txt" - if [ -z "$FILENAME" ] - then - [ ! -e "$SRCTREE/$PACKAGE" ] && dienow "No tarball for $PACKAGE" - - # If the sha1sum file isn't there, re-patch the package. - [ ! -e "$SHA1FILE" ] && patch_package - return 0 - fi - - # Check the sha1 list from the previous extract. If the source is already - # up to date (including patches), keep it. - - SHA1TAR="$(sha1file "$FILENAME")" - SHALIST=$(cat "$SHA1FILE" 2> /dev/null) - if [ ! -z "$SHALIST" ] - then - for i in "$SHA1TAR" $(sha1file "$PATCHDIR/$PACKAGE"-* 2>/dev/null) - do - # Is this sha1 in the file? - if [ -z "$(echo "$SHALIST" | sed -n "s/$i/$i/p" )" ] - then - SHALIST=missing - break - fi - # Remove it - SHALIST="$(echo "$SHALIST" | sed "s/$i//" )" - done - # If we matched all the sha1sums, nothing more to do. - [ -z "$SHALIST" ] && return 0 - fi - - # Re-extract the package, deleting the old one (if any).. - - echo -n "Extracting '$PACKAGE'" - ( - UNIQUE=$(readlink /proc/self) - trap 'rm -rf "$BUILD/temp-'$UNIQUE'"' EXIT - rm -rf "$SRCTREE/$PACKAGE" 2>/dev/null - mkdir -p "$BUILD"/{temp-$UNIQUE,packages} || dienow - - { tar -xv${DECOMPRESS} -f "$FILENAME" -C "$BUILD/temp-$UNIQUE" || dienow - } | dotprogress - - mv "$BUILD/temp-$UNIQUE/"* "$SRCTREE/$PACKAGE" && - echo "$SHA1TAR" > "$SHA1FILE" - ) - - [ $? -ne 0 ] && dienow - - patch_package -} - -# Confirm that a file has the appropriate checksum (or exists but SHA1 is blank) -# Delete invalid file. - -confirm_checksum() -{ - SUM="$(sha1file "$SRCDIR/$FILENAME" 2>/dev/null)" - if [ x"$SUM" == x"$SHA1" ] || [ -z "$SHA1" ] && [ -f "$SRCDIR/$FILENAME" ] - then - if [ -z "$SHA1" ] - then - echo "No SHA1 for $FILENAME ($SUM)" - else - echo "Confirmed $FILENAME" - fi - - # Preemptively extract source packages? - - [ -z "$EXTRACT_ALL" ] && return 0 - extract_package "$BASENAME" - return $? - fi - - # If there's a corrupted file, delete it. In theory it would be nice - # to resume downloads, but wget creates "*.1" files instead. - - rm "$SRCDIR/$FILENAME" 2> /dev/null - - return 1 -} - -# Attempt to obtain file from a specific location - -download_from() -{ - # Return success if we already have a valid copy of the file - - confirm_checksum && return 0 - - # If we have another source, try to download file from there. - - [ -z "$1" ] && return 1 - wget -t 2 -T 20 -O "$SRCDIR/$FILENAME" "$1" || - (rm "$SRCDIR/$FILENAME"; return 2) - touch -c "$SRCDIR/$FILENAME" - - confirm_checksum -} - -# Confirm a file matches sha1sum, else try to download it from mirror list. - -download() -{ - FILENAME=`echo "$URL" | sed 's .*/ '` - [ -z "$RENAME" ] || FILENAME="$(echo "$FILENAME" | sed -r "$RENAME")" - ALTFILENAME=alt-"$(noversion "$FILENAME" -0)" - - echo -ne "checking $FILENAME\r" - - # Update timestamps on both stable and unstable tarballs (if any) - # so cleanup_oldfiles doesn't delete stable when we're building unstable - # or vice versa - - touch -c "$SRCDIR"/{"$FILENAME","$ALTFILENAME"} 2>/dev/null - - # Give package name, minus file's version number and archive extension. - BASENAME="$(noversion "$FILENAME" | sed 's/\.tar\..z2*$//')" - - # If unstable version selected, try from listed location, and fall back - # to PREFERRED_MIRROR. Do not try normal mirror locations for unstable. - - if is_in_list "$BASENAME" $USE_UNSTABLE - then - # If extracted source directory exists, don't download alt-tarball. - [ -e "$SRCTREE/alt-$BASENAME" ] && return 0 - - # Download new one as alt-packagename.tar.ext - FILENAME="$ALTFILENAME" - SHA1= - - ([ ! -z "$PREFERRED_MIRROR" ] && - download_from "$PREFERRED_MIRROR/$ALTFILENAME") || - download_from "$UNSTABLE" - return $? - fi - - # If environment variable specifies a preferred mirror, try that first. - - if [ ! -z "$PREFERRED_MIRROR" ] - then - download_from "$PREFERRED_MIRROR/$FILENAME" && return 0 - fi - - # Try original location, then mirrors. - # Note: the URLs in mirror list cannot contain whitespace. - - download_from "$URL" && return 0 - for i in $MIRROR_LIST - do - download_from "$i/$FILENAME" && return 0 - done - - # Return failure. - - echo "Could not download $FILENAME" - echo -en "\e[0m" - return 1 -} - -# Clean obsolete files out of the source directory - -START_TIME=`date +%s` - -cleanup_oldfiles() -{ - # wait for asynchronous downloads to complete - - wait - - for i in "${SRCDIR}"/* - do - if [ -f "$i" ] && [ "$(date +%s -r "$i")" -lt "${START_TIME}" ] - then - echo Removing old file "$i" - rm -rf "$i" - fi - done -} - # Create a working directory under TMPDIR, deleting existing contents (if any), # and tracking created directories so cleanup can delete them automatically. @@ -459,6 +218,13 @@ fi } +# Given a filename.tar.ext, return the version number. + +getversion() +{ + echo "$1" | sed -e 's/.*-\(\([0-9\.]\)*\([_-]rc\)*\(-pre\)*\([0-9][a-zA-Z]\)*\)*\(\.tar\..z2*\)$/'"$2"'\1/' +} + # Figure out what version of a package we last built get_download_version()
--- a/sources/include.sh Sat Sep 25 13:40:04 2010 -0500 +++ b/sources/include.sh Sat Sep 25 14:10:23 2010 -0500 @@ -11,11 +11,13 @@ # Set up all the environment variables and functions for a build stage. # This file is sourced, not run. -# Include config and sources/functions.sh +# Include config and source shell function files. [ -e config ] && source config +source sources/utility_functions.sh source sources/functions.sh +source sources/download_functions.sh # Avoid trouble from unexpected environment settings