changeset 222:8b0d2080707c

Be smarter about extracting tarballs: extract one copy of each tarball into build/sources and then make a symlink tree (cp -sR) in each temp directory to build with. Makes ./forkbomb.sh --fork much faster. :)
author Rob Landley <rob@landley.net>
date Wed, 19 Sep 2007 18:52:56 -0500
parents a0a17e78c07d
children f6973e090c4f
files cross-compiler.sh download.sh forkbomb.sh host-tools.sh include.sh sources/configs/armv4l
diffstat 6 files changed, 107 insertions(+), 60 deletions(-) [+]
line wrap: on
line diff
--- a/cross-compiler.sh	Tue Sep 18 15:10:29 2007 -0500
+++ b/cross-compiler.sh	Wed Sep 19 18:52:56 2007 -0500
@@ -25,14 +25,14 @@
 make install &&
 cd .. &&
 mkdir -p "${CROSS}/include" &&
-cp binutils-*/include/libiberty.h "${CROSS}/include" &&
-$CLEANUP binutils-* build-binutils
+cp binutils/include/libiberty.h "${CROSS}/include" &&
+$CLEANUP binutils build-binutils
 
 [ $? -ne 0 ] && dienow
 
 # Build and install gcc
 
-setupfor gcc-core build-gcc gcc-
+setupfor gcc-core build-gcc
 AR_FOR_TARGET="${ARCH}-ar" "${CURSRC}/configure" $GCC_FLAGS \
 	--prefix="${CROSS}" --host=${CROSS_HOST} --target=${CROSS_TARGET} \
 	--enable-languages=c --disable-threads --disable-multilib \
@@ -74,7 +74,7 @@
 # Install Linux kernel headers (for use by uClibc).
 make -j $CPUS headers_install ARCH="${KARCH}" INSTALL_HDR_PATH="${CROSS}" &&
 cd .. &&
-$CLEANUP linux-*
+$CLEANUP linux
 
 [ $? -ne 0 ] && dienow
 
@@ -96,7 +96,7 @@
 $CC -Os -s -I include utils/readelf.c -o "${CROSS}/bin/${ARCH}-readelf" &&
 $CC -Os -s -I ldso/include utils/ldd.c -o "${CROSS}/bin/${ARCH}-ldd" &&
 cd .. &&
-$CLEANUP uClibc*
+$CLEANUP uClibc
 
 [ $? -ne 0 ] && dienow
 
--- a/download.sh	Tue Sep 18 15:10:29 2007 -0500
+++ b/download.sh	Wed Sep 19 18:52:56 2007 -0500
@@ -3,6 +3,8 @@
 NO_ARCH=none
 source include.sh
 
+[ x"$1" == x--extract ] && EXTRACT_ALL=yes
+
 # Dark blue
 echo -e "\e[34m"
 
@@ -15,14 +17,22 @@
 
 echo "=== Download source code." &&
 
+# Note, a blank SHA1 value means accept anything, and the download script
+# prints out the sha1 of such files after downloading it, so to update to
+# a new version of a file, set SHA1= and updat the URL, run ./download.sh,
+# then cut and paste the sha1 from the output and run it again to confirm.
+
 # Required for cross compile toolchain
-URL=http://www.kernel.org/pub/linux/kernel/v2.6/testing/linux-2.6.23-rc5.tar.bz2 \
-SHA1=958b5f726dc1ea64d75a9a0913febf2bec700e77 \
+URL=http://www.kernel.org/pub/linux/kernel/v2.6/testing/linux-2.6.23-rc6.tar.bz2 \
+SHA1=c343c10a7eab0eda40938569f72fc97c73e13fb6 \
 download &&
 
 URL=http://www.uclibc.org/downloads/uClibc-0.9.29.tar.bz2 \
 SHA1=1c5a36dc2cfa58b41db413190e45675c44ca4691 \
 download &&
+#URL=http://uclibc.org/downloads/snapshots/uClibc-20070918.tar.bz2 \
+#SHA1= \
+#download &&
 
 URL=ftp://ftp.gnu.org/gnu/binutils/binutils-2.17.tar.bz2 \
 SHA1=a557686eef68362ea31a3aa41ce274e3eeae1ef0 \
--- a/forkbomb.sh	Tue Sep 18 15:10:29 2007 -0500
+++ b/forkbomb.sh	Wed Sep 19 18:52:56 2007 -0500
@@ -16,7 +16,9 @@
 then
   if [ $# -ne 0 ]
   then
-    (nice -n 20 ./download.sh && nice -n 20 ./host-tools.sh) || exit 1
+    (nice -n 20 ./download.sh &&
+     nice -n 20 ./download.sh --extract &&
+     nice -n 20 ./host-tools.sh) || exit 1
   fi
   for i in `cd sources/configs; ls`
   do
--- a/host-tools.sh	Tue Sep 18 15:10:29 2007 -0500
+++ b/host-tools.sh	Wed Sep 19 18:52:56 2007 -0500
@@ -19,7 +19,9 @@
   make defconfig &&
   make &&
   make instlist &&
-  make install_flat PREFIX="${HOSTTOOLS}"
+  make install_flat PREFIX="${HOSTTOOLS}" &&
+  cd .. &&
+  $CLEANUP toybox
 
   [ $? -ne 0 ] && dienow
 fi
@@ -44,7 +46,7 @@
   make -j $CPUS ARCH=um &&
   cp linux "${HOSTTOOLS}" &&
   cd .. &&
-  rm -rf linux-*
+  $CLEANUP linux
 
   [ $? -ne 0 ] && dienow
 fi
--- a/include.sh	Tue Sep 18 15:10:29 2007 -0500
+++ b/include.sh	Wed Sep 19 18:52:56 2007 -0500
@@ -1,7 +1,62 @@
 #!/bin/bash
 
+function noversion()
+{
+  echo "$1" | sed -r -e 's/-*([0-9\.]|[_-]rc|-pre|[0-9][a-zA-Z])*(\.tar\..z2*)$/\2/'
+}
+
 function extract()
 {
+  SRCTREE="${BUILD}/sources"
+  BASENAME=`noversion "$1"`
+  BASENAME="${BASENAME/%\.tar\.*/}"
+
+  # Sanity check: don't ever "rm -rf /".  Just don't.
+
+  if [ -z "$BASENAME" ] || [ -z "$SRCTREE" ]
+  then
+    dienow
+  fi
+
+  # If it's already extracted, do nothing.
+  if [ -f "${SRCTREE}/${BASENAME}/sha1-for-source.txt" ]
+  then
+    SHA2="$(cat "${SRCTREE}/${BASENAME}/sha1-for-source.txt" 2>/dev/null)"
+    if [ -z "$2" ] || [ "$2" == "$SHA2" ]
+    then
+      return 0
+    fi
+  fi
+
+  rm -rf "${BUILD}/temp" "${SRCTREE}/${BASENAME}" 2>/dev/null
+  mkdir -p "${BUILD}"/{temp,sources} || dienow
+
+  # Is it a bzip2 or gzip tarball?
+  DECOMPRESS=""
+  [ "$1" != "${1/%\.tar\.bz2/}" ] && DECOMPRESS="j"
+  [ "$1" != "${1/%\.tar\.gz/}" ] && DECOMPRESS="z"
+
+  echo -n "Extracting '${BASENAME}'" &&
+  cd "${WORK}" &&
+  { tar xv${DECOMPRESS}fC "${SRCDIR}/$1" "${BUILD}/temp" || dienow
+  } | dotprogress
+
+  mv "${BUILD}/temp/"* "${SRCTREE}/${BASENAME}" &&
+  rmdir "${BUILD}/temp" &&
+  echo "$2" > "${SRCTREE}/${BASENAME}/sha1-for-source.txt"
+
+  [ $? -ne 0 ] && dienow
+
+  # Apply any patches to this package
+
+  ls "${SOURCES}/patches/$BASENAME"* 2> /dev/null | sort | while read i
+  do
+    if [ -f "$i" ]
+    then
+      echo "Applying $i"
+      (cd "${SRCTREE}/${BASENAME}" && patch -p1 -i "$i") || dienow
+    fi
+  done
 }
 
 function download()
@@ -9,31 +64,28 @@
   FILENAME=`echo "$URL" | sed 's .*/  '`
   BASENAME=`echo "$FILENAME" | sed -r -e 's/-*([0-9\.]|[_-]rc|-pre|[0-9][a-zA-Z])*(\.tar\..z2*)$/\2/'`
 
-  if [ ! -z "$LINKDIR" ]
-  then
-    rm -f "$LINKDIR/$BASENAME" 2> /dev/null
-    ln -s "$FROMSRC/$FILENAME" "$LINKDIR/$BASENAME" || dienow
-  fi
-
   # The extra "" is so we test the sha1sum after the last download.
 
-  for i in "$URL" http://www.landley.net/code/firmware/mirror/"$FILENAME" \
-           http://engineering.timesys.com/~landley/mirror/"$FILENAME" ""
+  for i in "$URL" http://www.landley.net/code/firmware/mirror/"$FILENAME" ""
   do
     # Return success if we have a valid copy of the file
 
     # Test first (so we don't re-download a file we've already got).
 
     SUM=`cat "$SRCDIR/$FILENAME" | sha1sum | awk '{print $1}'`
-    if [ -z "$SHA1" ] && [ -f "$SRCDIR/$FILENAME" ]
+    if [ x"$SUM" == x"$SHA1" ] || [ -z "$SHA1" ] && [ -f "$SRCDIR/$FILENAME" ]
     then
       touch "$SRCDIR/$FILENAME"
-      echo "No SHA1 for $FILENAME ($SUM)"
-      return 0
-    elif [ x"$SUM" == x"$SHA1" ]
-    then
-      touch "$SRCDIR/$FILENAME"
-      echo "Confirmed $FILENAME"
+      if [ -z "$SHA1" ]
+      then
+        echo "No SHA1 for $FILENAME ($SUM)"
+      else
+        echo "Confirmed $FILENAME"
+      fi
+      if [ ! -z "$EXTRACT_ALL" ]
+      then
+        extract "$FILENAME" "$SUM"
+      fi
       return 0
     fi
 
@@ -94,29 +146,30 @@
   echo
 }
 
-# Extract package $1, use work directory $2 (or $1 if no $2), use source
-# directory $3 (or $1 if no $3)
+# Extract package $1, use out-of-tree build directory $2 (or $1 if no $2)
+# Use symlink directory $3 (or $1 if no $3)
 
 function setupfor()
 {
-  # Is it a bzip2 or gzip tarball?
+  # Make sure the 
+  cd "${SRCDIR}" &&
+  extract "${1}-"*.tar* ""
 
-  FILE="$1".tar.bz2
-  DECOMPRESS="j"
+  # Set CURSRC
 
-  if [ ! -f "${LINKDIR}/${FILE}" ]
-  then
-    FILE="$1".tar.gz
-    DECOMPRESS="z"
-  fi
+  export CURSRC="$1"
+  [ ! -z "$3" ] && CURSRC="$3"
+  CURSRC="${WORK}/${CURSRC}"
 
-  # Announce package, with easy-to-grep-for "===" marker.  Extract it.
+  # Announce package, with easy-to-grep-for "===" marker.
 
   echo "=== Building $1 ($ARCH_NAME)"
-  echo -n "Extracting '${FILE}'"
+  echo "Snapshot '$1'..."
   cd "${WORK}" &&
-  { tar xv${DECOMPRESS}f "${LINKDIR}/${FILE}" || dienow
-  } | dotprogress
+  mkdir -p "${CURSRC}" &&
+  cp -sfR "${SRCTREE}/$1/"* "${CURSRC}"
+
+  [ $? -ne 0 ] && dienow
 
   # Do we have a separate working directory?
 
@@ -127,24 +180,6 @@
     mkdir -p "$2" &&
     cd "$2" || dienow
   fi
-
-  # Set CURSRC
-
-  export CURSRC="$1"
-  [ ! -z "$3" ] && CURSRC="$3"
-  export CURSRC=`echo "${WORK}/${CURSRC}"*`
-  [ ! -d "${CURSRC}" ] && dienow
-
-  # Apply any patches to this package
-
-  ls "${SOURCES}/patches/$1"* 2> /dev/null | sort | while read i
-  do
-    if [ -f "$i" ]
-    then
-      echo "Applying $i"
-      (cd "${CURSRC}" && patch -p1 -i "$i") || dienow
-    fi
-  done
 }
 
 # Setup
@@ -162,12 +197,11 @@
 export SOURCES="${TOP}/sources"
 export SRCDIR="${SOURCES}/packages"
 export FROMSRC=../packages
-export LINKDIR="${SOURCES}/build-links"
 export BUILD="${TOP}/build"
 export HOSTTOOLS="${BUILD}/host"
 export WORK="${BUILD}/host-temp"
 export PATH="${HOSTTOOLS}:$PATH"
-mkdir -p "${SRCDIR}" "${LINKDIR}"
+mkdir -p "${SRCDIR}"
 
 # For bash: check the $PATH for new executables added after startup.
 set +h
--- a/sources/configs/armv4l	Tue Sep 18 15:10:29 2007 -0500
+++ b/sources/configs/armv4l	Wed Sep 19 18:52:56 2007 -0500
@@ -10,7 +10,6 @@
 # of the installed base).
 
 # For details, see http://www.arm.com/products/CPUs/architecture.html
-CLEANUP=echo
 KARCH=arm
 KERNEL_PATH=arch/${KARCH}/boot/zImage
 GCC_FLAGS="--with-float=soft"