Mercurial > hg > aboriginal
changeset 222:8b0d2080707c
Be smarter about extracting tarballs: extract one copy of each tarball
into build/sources and then make a symlink tree (cp -sR) in each temp
directory to build with. Makes ./forkbomb.sh --fork much faster. :)
author | Rob Landley <rob@landley.net> |
---|---|
date | Wed, 19 Sep 2007 18:52:56 -0500 |
parents | a0a17e78c07d |
children | f6973e090c4f |
files | cross-compiler.sh download.sh forkbomb.sh host-tools.sh include.sh sources/configs/armv4l |
diffstat | 6 files changed, 107 insertions(+), 60 deletions(-) [+] |
line wrap: on
line diff
--- a/cross-compiler.sh Tue Sep 18 15:10:29 2007 -0500 +++ b/cross-compiler.sh Wed Sep 19 18:52:56 2007 -0500 @@ -25,14 +25,14 @@ make install && cd .. && mkdir -p "${CROSS}/include" && -cp binutils-*/include/libiberty.h "${CROSS}/include" && -$CLEANUP binutils-* build-binutils +cp binutils/include/libiberty.h "${CROSS}/include" && +$CLEANUP binutils build-binutils [ $? -ne 0 ] && dienow # Build and install gcc -setupfor gcc-core build-gcc gcc- +setupfor gcc-core build-gcc AR_FOR_TARGET="${ARCH}-ar" "${CURSRC}/configure" $GCC_FLAGS \ --prefix="${CROSS}" --host=${CROSS_HOST} --target=${CROSS_TARGET} \ --enable-languages=c --disable-threads --disable-multilib \ @@ -74,7 +74,7 @@ # Install Linux kernel headers (for use by uClibc). make -j $CPUS headers_install ARCH="${KARCH}" INSTALL_HDR_PATH="${CROSS}" && cd .. && -$CLEANUP linux-* +$CLEANUP linux [ $? -ne 0 ] && dienow @@ -96,7 +96,7 @@ $CC -Os -s -I include utils/readelf.c -o "${CROSS}/bin/${ARCH}-readelf" && $CC -Os -s -I ldso/include utils/ldd.c -o "${CROSS}/bin/${ARCH}-ldd" && cd .. && -$CLEANUP uClibc* +$CLEANUP uClibc [ $? -ne 0 ] && dienow
--- a/download.sh Tue Sep 18 15:10:29 2007 -0500 +++ b/download.sh Wed Sep 19 18:52:56 2007 -0500 @@ -3,6 +3,8 @@ NO_ARCH=none source include.sh +[ x"$1" == x--extract ] && EXTRACT_ALL=yes + # Dark blue echo -e "\e[34m" @@ -15,14 +17,22 @@ echo "=== Download source code." && +# Note, a blank SHA1 value means accept anything, and the download script +# prints out the sha1 of such files after downloading it, so to update to +# a new version of a file, set SHA1= and updat the URL, run ./download.sh, +# then cut and paste the sha1 from the output and run it again to confirm. + # Required for cross compile toolchain -URL=http://www.kernel.org/pub/linux/kernel/v2.6/testing/linux-2.6.23-rc5.tar.bz2 \ -SHA1=958b5f726dc1ea64d75a9a0913febf2bec700e77 \ +URL=http://www.kernel.org/pub/linux/kernel/v2.6/testing/linux-2.6.23-rc6.tar.bz2 \ +SHA1=c343c10a7eab0eda40938569f72fc97c73e13fb6 \ download && URL=http://www.uclibc.org/downloads/uClibc-0.9.29.tar.bz2 \ SHA1=1c5a36dc2cfa58b41db413190e45675c44ca4691 \ download && +#URL=http://uclibc.org/downloads/snapshots/uClibc-20070918.tar.bz2 \ +#SHA1= \ +#download && URL=ftp://ftp.gnu.org/gnu/binutils/binutils-2.17.tar.bz2 \ SHA1=a557686eef68362ea31a3aa41ce274e3eeae1ef0 \
--- a/forkbomb.sh Tue Sep 18 15:10:29 2007 -0500 +++ b/forkbomb.sh Wed Sep 19 18:52:56 2007 -0500 @@ -16,7 +16,9 @@ then if [ $# -ne 0 ] then - (nice -n 20 ./download.sh && nice -n 20 ./host-tools.sh) || exit 1 + (nice -n 20 ./download.sh && + nice -n 20 ./download.sh --extract && + nice -n 20 ./host-tools.sh) || exit 1 fi for i in `cd sources/configs; ls` do
--- a/host-tools.sh Tue Sep 18 15:10:29 2007 -0500 +++ b/host-tools.sh Wed Sep 19 18:52:56 2007 -0500 @@ -19,7 +19,9 @@ make defconfig && make && make instlist && - make install_flat PREFIX="${HOSTTOOLS}" + make install_flat PREFIX="${HOSTTOOLS}" && + cd .. && + $CLEANUP toybox [ $? -ne 0 ] && dienow fi @@ -44,7 +46,7 @@ make -j $CPUS ARCH=um && cp linux "${HOSTTOOLS}" && cd .. && - rm -rf linux-* + $CLEANUP linux [ $? -ne 0 ] && dienow fi
--- a/include.sh Tue Sep 18 15:10:29 2007 -0500 +++ b/include.sh Wed Sep 19 18:52:56 2007 -0500 @@ -1,7 +1,62 @@ #!/bin/bash +function noversion() +{ + echo "$1" | sed -r -e 's/-*([0-9\.]|[_-]rc|-pre|[0-9][a-zA-Z])*(\.tar\..z2*)$/\2/' +} + function extract() { + SRCTREE="${BUILD}/sources" + BASENAME=`noversion "$1"` + BASENAME="${BASENAME/%\.tar\.*/}" + + # Sanity check: don't ever "rm -rf /". Just don't. + + if [ -z "$BASENAME" ] || [ -z "$SRCTREE" ] + then + dienow + fi + + # If it's already extracted, do nothing. + if [ -f "${SRCTREE}/${BASENAME}/sha1-for-source.txt" ] + then + SHA2="$(cat "${SRCTREE}/${BASENAME}/sha1-for-source.txt" 2>/dev/null)" + if [ -z "$2" ] || [ "$2" == "$SHA2" ] + then + return 0 + fi + fi + + rm -rf "${BUILD}/temp" "${SRCTREE}/${BASENAME}" 2>/dev/null + mkdir -p "${BUILD}"/{temp,sources} || dienow + + # Is it a bzip2 or gzip tarball? + DECOMPRESS="" + [ "$1" != "${1/%\.tar\.bz2/}" ] && DECOMPRESS="j" + [ "$1" != "${1/%\.tar\.gz/}" ] && DECOMPRESS="z" + + echo -n "Extracting '${BASENAME}'" && + cd "${WORK}" && + { tar xv${DECOMPRESS}fC "${SRCDIR}/$1" "${BUILD}/temp" || dienow + } | dotprogress + + mv "${BUILD}/temp/"* "${SRCTREE}/${BASENAME}" && + rmdir "${BUILD}/temp" && + echo "$2" > "${SRCTREE}/${BASENAME}/sha1-for-source.txt" + + [ $? -ne 0 ] && dienow + + # Apply any patches to this package + + ls "${SOURCES}/patches/$BASENAME"* 2> /dev/null | sort | while read i + do + if [ -f "$i" ] + then + echo "Applying $i" + (cd "${SRCTREE}/${BASENAME}" && patch -p1 -i "$i") || dienow + fi + done } function download() @@ -9,31 +64,28 @@ FILENAME=`echo "$URL" | sed 's .*/ '` BASENAME=`echo "$FILENAME" | sed -r -e 's/-*([0-9\.]|[_-]rc|-pre|[0-9][a-zA-Z])*(\.tar\..z2*)$/\2/'` - if [ ! -z "$LINKDIR" ] - then - rm -f "$LINKDIR/$BASENAME" 2> /dev/null - ln -s "$FROMSRC/$FILENAME" "$LINKDIR/$BASENAME" || dienow - fi - # The extra "" is so we test the sha1sum after the last download. - for i in "$URL" http://www.landley.net/code/firmware/mirror/"$FILENAME" \ - http://engineering.timesys.com/~landley/mirror/"$FILENAME" "" + for i in "$URL" http://www.landley.net/code/firmware/mirror/"$FILENAME" "" do # Return success if we have a valid copy of the file # Test first (so we don't re-download a file we've already got). SUM=`cat "$SRCDIR/$FILENAME" | sha1sum | awk '{print $1}'` - if [ -z "$SHA1" ] && [ -f "$SRCDIR/$FILENAME" ] + if [ x"$SUM" == x"$SHA1" ] || [ -z "$SHA1" ] && [ -f "$SRCDIR/$FILENAME" ] then touch "$SRCDIR/$FILENAME" - echo "No SHA1 for $FILENAME ($SUM)" - return 0 - elif [ x"$SUM" == x"$SHA1" ] - then - touch "$SRCDIR/$FILENAME" - echo "Confirmed $FILENAME" + if [ -z "$SHA1" ] + then + echo "No SHA1 for $FILENAME ($SUM)" + else + echo "Confirmed $FILENAME" + fi + if [ ! -z "$EXTRACT_ALL" ] + then + extract "$FILENAME" "$SUM" + fi return 0 fi @@ -94,29 +146,30 @@ echo } -# Extract package $1, use work directory $2 (or $1 if no $2), use source -# directory $3 (or $1 if no $3) +# Extract package $1, use out-of-tree build directory $2 (or $1 if no $2) +# Use symlink directory $3 (or $1 if no $3) function setupfor() { - # Is it a bzip2 or gzip tarball? + # Make sure the + cd "${SRCDIR}" && + extract "${1}-"*.tar* "" - FILE="$1".tar.bz2 - DECOMPRESS="j" + # Set CURSRC - if [ ! -f "${LINKDIR}/${FILE}" ] - then - FILE="$1".tar.gz - DECOMPRESS="z" - fi + export CURSRC="$1" + [ ! -z "$3" ] && CURSRC="$3" + CURSRC="${WORK}/${CURSRC}" - # Announce package, with easy-to-grep-for "===" marker. Extract it. + # Announce package, with easy-to-grep-for "===" marker. echo "=== Building $1 ($ARCH_NAME)" - echo -n "Extracting '${FILE}'" + echo "Snapshot '$1'..." cd "${WORK}" && - { tar xv${DECOMPRESS}f "${LINKDIR}/${FILE}" || dienow - } | dotprogress + mkdir -p "${CURSRC}" && + cp -sfR "${SRCTREE}/$1/"* "${CURSRC}" + + [ $? -ne 0 ] && dienow # Do we have a separate working directory? @@ -127,24 +180,6 @@ mkdir -p "$2" && cd "$2" || dienow fi - - # Set CURSRC - - export CURSRC="$1" - [ ! -z "$3" ] && CURSRC="$3" - export CURSRC=`echo "${WORK}/${CURSRC}"*` - [ ! -d "${CURSRC}" ] && dienow - - # Apply any patches to this package - - ls "${SOURCES}/patches/$1"* 2> /dev/null | sort | while read i - do - if [ -f "$i" ] - then - echo "Applying $i" - (cd "${CURSRC}" && patch -p1 -i "$i") || dienow - fi - done } # Setup @@ -162,12 +197,11 @@ export SOURCES="${TOP}/sources" export SRCDIR="${SOURCES}/packages" export FROMSRC=../packages -export LINKDIR="${SOURCES}/build-links" export BUILD="${TOP}/build" export HOSTTOOLS="${BUILD}/host" export WORK="${BUILD}/host-temp" export PATH="${HOSTTOOLS}:$PATH" -mkdir -p "${SRCDIR}" "${LINKDIR}" +mkdir -p "${SRCDIR}" # For bash: check the $PATH for new executables added after startup. set +h
--- a/sources/configs/armv4l Tue Sep 18 15:10:29 2007 -0500 +++ b/sources/configs/armv4l Wed Sep 19 18:52:56 2007 -0500 @@ -10,7 +10,6 @@ # of the installed base). # For details, see http://www.arm.com/products/CPUs/architecture.html -CLEANUP=echo KARCH=arm KERNEL_PATH=arch/${KARCH}/boot/zImage GCC_FLAGS="--with-float=soft"