changeset 815:8129df56091b

Extended setupfor/cleanup to create binary package tarballs if the configure option BINARY_PACKAGE_TARBALLS is set. These tarballs extract into the current directory and add all the changed files installed by this package build, so you can pick and choose when assembling your own filesystem.
author Rob Landley <rob@landley.net>
date Wed, 26 Aug 2009 02:37:57 -0500
parents 5dd054d1c789
children 0f9c421d8504
files config cross-compiler.sh host-tools.sh root-filesystem.sh sources/functions.sh system-image.sh
diffstat 6 files changed, 119 insertions(+), 77 deletions(-) [+]
line wrap: on
line diff
--- a/config	Mon Aug 24 00:29:35 2009 -0500
+++ b/config	Wed Aug 26 02:37:57 2009 -0500
@@ -123,3 +123,8 @@
 # Don't update the title bar in the display
 
 # export NO_TITLE_BAR=1
+
+# Create a tarball with the files installed by each individual package at
+# each stage of the build.
+
+# export BINARY_PACKAGE_TARBALLS=1
--- a/cross-compiler.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/cross-compiler.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -22,7 +22,7 @@
 
 # Build and install binutils
 
-setupfor binutils build-binutils &&
+setupfor binutils build-binutils
 AR=ar AS=as LD=ld NM=nm OBJDUMP=objdump OBJCOPY=objcopy \
 	"${CURSRC}/configure" --prefix="${STAGE_DIR}" --host=${CROSS_HOST} \
 	--target=${CROSS_TARGET} --with-lib-path=lib --disable-nls \
@@ -31,11 +31,10 @@
 make -j $CPUS configure-host &&
 make -j $CPUS CFLAGS="-O2 $STATIC_FLAGS" &&
 make -j $CPUS install &&
-cd .. &&
 mkdir -p "${STAGE_DIR}/include" &&
-cp binutils/include/libiberty.h "${STAGE_DIR}/include"
+cp "$CURSRC/include/libiberty.h" "${STAGE_DIR}/include"
 
-cleanup binutils build-binutils
+cleanup build-binutils
 
 # Build and install gcc
 
@@ -54,9 +53,9 @@
 
 make -j $CPUS all-gcc LDFLAGS="$STATIC_FLAGS" &&
 make -j $CPUS install-gcc &&
-cd ..
 
-cleanup gcc-core build-gcc
+# We're done with that source and could theoretically cleanup gcc-core and
+# build-gcc here, but we still need the timestamps if we do binary packaging.
 
 echo Fixup toolchain... &&
 
@@ -82,9 +81,13 @@
 rm "${ARCH}-c++" &&
 ln -s "${ARCH}-g++" "${ARCH}-rawc++" &&
 ln -s "${ARCH}-gcc" "${ARCH}-g++" &&
-ln -s "${ARCH}-gcc" "${ARCH}-c++"
+ln -s "${ARCH}-gcc" "${ARCH}-c++" &&
 
-cleanup "${STAGE_DIR}"/{lib/gcc,{libexec/gcc,gcc/lib}/install-tools}
+# When tarring up individual binary packages, we want this one to be called
+# "gcc" and that's not what we fed to setupfor, so rename it.
+
+mv "$WORK"/gcc-core "$WORK"/gcc
+PACKAGE=gcc cleanup build-gcc "${STAGE_DIR}"/{lib/gcc,{libexec/gcc,gcc/lib}/install-tools}
 
 # Install kernel headers.
 
@@ -92,10 +95,9 @@
 # Install Linux kernel headers (for use by uClibc).
 make -j $CPUS headers_install ARCH="${KARCH}" INSTALL_HDR_PATH="${STAGE_DIR}" &&
 # This makes some very old package builds happy.
-ln -s ../sys/user.h "${STAGE_DIR}/include/asm/page.h" &&
-cd ..
+ln -s ../sys/user.h "${STAGE_DIR}/include/asm/page.h"
 
-cleanup linux
+cleanup
 
 # Build and install uClibc
 
@@ -108,9 +110,8 @@
 do
   cp utils/"$i".host "$STAGE_DIR/bin/$ARCH-$i" || dienow
 done
-cd ..
 
-cleanup uClibc
+cleanup
 
 cat > "${STAGE_DIR}"/README << EOF &&
 Cross compiler for $ARCH
--- a/host-tools.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/host-tools.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -31,6 +31,9 @@
 
 STAGE_DIR="${HOSTTOOLS}"
 
+# Blank $WORK but accept $STAGE_DIR if it exists.  Re-running this script
+# should be a NOP.
+
 blank_tempdir "${WORK}"
 mkdir -p "${STAGE_DIR}" || dienow
 
@@ -111,9 +114,8 @@
       [ ! -f "${STAGE_DIR}/$i" ] &&
         (ln -sf busybox "${STAGE_DIR}/$i" || dienow)
     done
-    cd ..
 
-    cleanup busybox
+    cleanup
   fi
 
   # Build toybox
@@ -131,9 +133,8 @@
     else
       make install_flat PREFIX="${STAGE_DIR}" || dienow
     fi
-    cd ..
 
-    cleanup toybox
+    cleanup
   fi
 
   # Create symlinks to the host toolchain.  We need a usable existing host
@@ -186,10 +187,9 @@
   setupfor distcc &&
   ./configure --with-included-popt --disable-Werror &&
   make -j "$CPUS" &&
-  cp distcc distccd "${STAGE_DIR}" &&
-  cd ..
+  cp distcc distccd "${STAGE_DIR}"
 
-  cleanup distcc
+  cleanup
 fi
 
 # Build genext2fs.  We use it to build the ext2 image to boot qemu with
@@ -200,10 +200,9 @@
   setupfor genext2fs &&
   ./configure &&
   make -j $CPUS &&
-  cp genext2fs "${STAGE_DIR}" &&
-  cd ..
+  cp genext2fs "${STAGE_DIR}"
 
-  cleanup genext2fs
+  cleanup
 fi
 
 # Build e2fsprogs.
@@ -226,10 +225,9 @@
   ./configure --disable-tls --enable-htree &&
   make -j "$CPUS" &&
   cp misc/{mke2fs,tune2fs} resize/resize2fs "${STAGE_DIR}" &&
-  cp e2fsck/e2fsck "$STAGE_DIR"/fsck.ext2 &&
-  cd ..
+  cp e2fsck/e2fsck "$STAGE_DIR"/fsck.ext2
 
-  cleanup e2fsprogs
+  cleanup
 fi
 
 # Squashfs is an alternate packaging option.
@@ -239,10 +237,9 @@
   setupfor squashfs &&
   cd squashfs-tools &&
   make -j $CPUS &&
-  cp mksquashfs unsquashfs "${STAGE_DIR}" &&
-  cd ..
+  cp mksquashfs unsquashfs "${STAGE_DIR}"
 
-  cleanup squashfs
+  cleanup
 fi
 
 # Here's some stuff that isn't used to build a cross compiler or system
@@ -272,10 +269,9 @@
     make -j $CPUS &&
     # Copy the executable files and ROM files
     cp $(find -type f -perm +111 -name "qemu*") "$STAGE_DIR" &&
-    cp -r pc-bios "$STAGE_DIR" &&
-    cd ..
+    cp -r pc-bios "$STAGE_DIR"
 
-    cleanup qemu
+    cleanup
   else
     # Symlink qemu out of the host, if found.  Since run-from-build.sh uses
     # $PATH=.../build/host if it exists, add the various qemu instances to that.
--- a/root-filesystem.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/root-filesystem.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -28,8 +28,8 @@
 echo -e "$NATIVE_COLOR"
 echo "=== Building $STAGE_NAME"
 
+blank_tempdir "$STAGE_DIR"
 blank_tempdir "$WORK"
-blank_tempdir "$STAGE_DIR"
 
 # Determine which directory layout we're using
 
@@ -53,10 +53,9 @@
 # Install Linux kernel headers (for use by uClibc).
 make headers_install -j "$CPUS" ARCH="${KARCH}" INSTALL_HDR_PATH="$ROOT_TOPDIR" &&
 # This makes some very old package builds happy.
-ln -s ../sys/user.h "$ROOT_TOPDIR/include/asm/page.h" &&
-cd ..
+ln -s ../sys/user.h "$ROOT_TOPDIR/include/asm/page.h"
 
-cleanup linux
+cleanup
 
 # Build and install uClibc.  (We could just copy the one from the compiler
 # toolchain, but this is cleaner.)
@@ -85,9 +84,7 @@
   done
 fi
 
-cd ..
-
-cleanup uClibc
+cleanup
 
 if [ "$NATIVE_TOOLCHAIN" == "none" ]
 then
@@ -117,11 +114,10 @@
 make -j $CPUS configure-host &&
 make -j $CPUS CFLAGS="-O2 $STATIC_FLAGS" &&
 make -j $CPUS install &&
-cd .. &&
 mkdir -p "$ROOT_TOPDIR/include" &&
-cp binutils/include/libiberty.h "$ROOT_TOPDIR/include"
+cp "$CURSRC/include/libiberty.h" "$ROOT_TOPDIR/include"
 
-cleanup binutils build-binutils
+cleanup build-binutils
 
 # Build and install native gcc, with c++ support this time.
 
@@ -166,9 +162,9 @@
 cd "$CROSS_TARGET"/libstdc++-v3/libsupc++ &&
 make -j $CPUS &&
 mv .libs/libsupc++.a "$ROOT_TOPDIR"/lib &&
-cd ../../../..
 
-cleanup gcc-core build-gcc
+# We're done with that source and could theoretically cleanup gcc-core and
+# build-gcc here, but we still need the timestamps if we do binary packaging.
 
 # Move the gcc internal libraries and headers somewhere sane
 
@@ -190,9 +186,14 @@
 mv "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}g++" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}rawg++" &&
 ln "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}gcc" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}g++" &&
 rm "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}c++" &&
-ln -s "${PROGRAM_PREFIX}g++" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}c++"
+ln -s "${PROGRAM_PREFIX}g++" "$ROOT_TOPDIR/bin/${PROGRAM_PREFIX}c++" &&
 
-cleanup "$ROOT_TOPDIR"/{lib/gcc,gcc/lib/install-tools,bin/${ARCH}-unknown-*}
+# When tarring up individual binary packages, we want this one to be called
+# "gcc" and that's not what we fed to setupfor, so rename it.
+
+mv "$WORK"/{gcc-core,gcc}
+PACKAGE=gcc cleanup build-gcc \
+  "$ROOT_TOPDIR"/{lib/gcc,gcc/lib/install-tools,bin/${ARCH}-unknown-*}
 
 # Tell future packages to link against the libraries in the new root filesystem,
 # rather than the ones in the cross compiler directory.
@@ -214,10 +215,9 @@
 mv "$ROOT_TOPDIR"/c++/lib/* "$ROOT_TOPDIR"/lib &&
 rm -rf "$ROOT_TOPDIR"/c++/{lib,bin} &&
 ln -s libuClibc++.so "$ROOT_TOPDIR"/lib/libstdc++.so &&
-ln -s libuClibc++.a "$ROOT_TOPDIR"/lib/libstdc++.a &&
-cd ..
+ln -s libuClibc++.a "$ROOT_TOPDIR"/lib/libstdc++.a
 
-cleanup uClibc++
+cleanup
 
 fi # End of NATIVE_TOOLCHAIN build
 
@@ -240,15 +240,13 @@
   cp toybox "$ROOT_TOPDIR/bin" &&
   ln -s toybox "$ROOT_TOPDIR/bin/patch" &&
   ln -s toybox "$ROOT_TOPDIR/bin/oneit" &&
-  ln -s toybox "$ROOT_TOPDIR/bin/netcat" &&
-  cd ..
+  ln -s toybox "$ROOT_TOPDIR/bin/netcat"
 else
   CFLAGS="$CFLAGS $STATIC_FLAGS" \
-    make install_flat PREFIX="$ROOT_TOPDIR"/bin CROSS="${ARCH}-" &&
-  cd ..
+    make install_flat PREFIX="$ROOT_TOPDIR"/bin CROSS="${ARCH}-"
 fi
 
-cleanup toybox
+cleanup
 
 # Build and install busybox
 
@@ -258,18 +256,15 @@
 LDFLAGS="$LDFLAGS $STATIC_FLAGS" \
   make -j $CPUS CROSS_COMPILE="${ARCH}-" $VERBOSITY &&
 make busybox.links &&
-cp busybox "$ROOT_TOPDIR/bin"
-
-[ $? -ne 0 ] && dienow
+cp busybox "$ROOT_TOPDIR/bin" || dienow
 
 for i in $(sed 's@.*/@@' busybox.links)
 do
   # Allowed to fail.
-  ln -s busybox "$ROOT_TOPDIR/bin/$i" 2>/dev/null
+  ln -s busybox "$ROOT_TOPDIR/bin/$i" 2>/dev/null || true
 done
-cd ..
 
-cleanup busybox
+cleanup
 
 # Build and install make
 
@@ -277,10 +272,9 @@
 LDFLAGS="$STATIC_FLAGS $LDFLAGS" CC="${ARCH}-cc" ./configure \
   --prefix="$ROOT_TOPDIR" --build="${CROSS_HOST}" --host="${CROSS_TARGET}" &&
 make -j $CPUS &&
-make -j $CPUS install &&
-cd ..
+make -j $CPUS install
 
-cleanup make
+cleanup
 
 # Build and install bash.  (Yes, this is an old version.  It's intentional.)
 
@@ -302,10 +296,9 @@
 make &&
 make install &&
 # Make bash the default shell.
-ln -sf bash "$ROOT_TOPDIR/bin/sh" &&
-cd ..
+ln -sf bash "$ROOT_TOPDIR/bin/sh"
 
-cleanup bash
+cleanup
 
 setupfor distcc
 rsync_cv_HAVE_C99_VSNPRINTF=yes \
@@ -320,9 +313,8 @@
 do
   ln -s ../bin/distcc "$ROOT_TOPDIR/distcc/$i" || dienow
 done
-cd ..
 
-cleanup distcc
+cleanup
 
 # Put statically and dynamically linked hello world programs on there for
 # test purposes.
--- a/sources/functions.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/sources/functions.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -97,22 +97,58 @@
   dienow
 }
 
+# Find all files in $STAGE_DIR newer than $CURSRC.
+
+function recent_binary_files()
+{
+  PREVIOUS=
+  (cd "$STAGE_DIR" || dienow
+   # Note $WORK/$PACKAGE != $CURSRC here for renamed packages like gcc-core.
+   find . -newer "$WORK/$PACKAGE/FWL-TIMESTAMP" -depth \
+     | sed -e 's/^.//' -e 's/^.//' -e '/^$/d'
+  ) | while read i
+  do
+    TEMP="${PREVIOUS##"$i"/}"
+
+    if [ $[${#PREVIOUS}-${#TEMP}] -ne $[${#i}+1] ]
+    then
+      # Because the expanded $i might have \ chars in it, that's why.
+      echo -n "$i"
+      echo -ne '\0'
+    fi
+    PREVIOUS="$i"
+  done
+}
+
 # Strip the version number off a tarball
 
 function cleanup()
 {
+  # If package build exited with an error, do not continue.
 
   [ $? -ne 0 ] && dienow
 
+  if [ ! -z "$BINARY_PACKAGE_TARBALLS" ]
+  then
+    TARNAME="$PACKAGE-$STAGE_NAME-${ARCH_NAME}".tar.bz2
+    echo -n Creating "$TARNAME"
+    { recent_binary_files | xargs -0 tar -cjvf \
+        "$BUILD/${TARNAME}".tar.bz2 -C "$STAGE_DIR" || dienow
+    } | dotprogress
+  fi
+
   if [ ! -z "$NO_CLEANUP" ]
   then
-    echo "skip cleanup $@"
+    echo "skip cleanup $PACKAGE $@"
     return
   fi
 
-  for i in "$@"
+  # Loop deleting directories
+
+  cd "$WORK" || dienow
+  for i in "$PACKAGE" "$@"
   do
-    unstable "$i" && i="$PACKAGE"
+    [ -z "$i" ] && continue
     echo "cleanup $i"
     rm -rf "$i" || dienow
  done
@@ -198,7 +234,6 @@
   [ "$FILENAME" != "${FILENAME/%\.tar\.bz2/}" ] && DECOMPRESS="j"
   [ "$FILENAME" != "${FILENAME/%\.tar\.gz/}" ] && DECOMPRESS="z"
 
-  cd "${WORK}" &&
   { tar -xv${DECOMPRESS} -f "${SRCDIR}/${FILENAME}" -C "${BUILD}/temp" || dienow
   } | dotprogress
 
@@ -420,6 +455,21 @@
   # Change window title bar to package now
   [ -z "$NO_TITLE_BAR" ] &&
     echo -en "\033]2;$ARCH_NAME $STAGE_NAME $PACKAGE\007"
+
+  # Ugly bug workaround: timestamp granularity in a lot of filesystems is only
+  # 1 second, so find -newer misses things installed in the same second, so we
+  # make sure it's a new second before we start actually doing anything.
+
+  if [ ! -z "$BINARY_PACKAGE_TARBALLS" ]
+  then
+    touch "${CURSRC}/FWL-TIMESTAMP" || dienow
+    TIME=$(date +%s)
+    while true
+    do
+      [ $TIME != "$(date +%s)" ] && break
+      sleep .1
+    done
+  fi
 }
 
 # Figure out what version of a package we last built
--- a/system-image.sh	Mon Aug 24 00:29:35 2009 -0500
+++ b/system-image.sh	Wed Aug 26 02:37:57 2009 -0500
@@ -17,8 +17,6 @@
   exit 1
 fi
 
-blank_tempdir "$WORK"
-
 # This little song and dance makes us run in our own session, to prevent the
 # "kill 0" below from taking down the shell that called us when it cleans up
 # our background tasks.  (We run the kernel build and root filesystem image
@@ -43,6 +41,7 @@
 echo "=== Packaging system image from root-filesystem"
 
 blank_tempdir "$STAGE_DIR"
+blank_tempdir "$WORK"
 
 
 [ -z "$SYSIMAGE_TYPE" ] && SYSIMAGE_TYPE=squashfs
@@ -166,10 +165,9 @@
 # Install kernel
 
 [ -d "${TOOLS}/src" ] && cp .config "${TOOLS}"/src/config-linux
-cp "${KERNEL_PATH}" "${STAGE_DIR}/zImage-${ARCH}" &&
-cd ..
+cp "${KERNEL_PATH}" "${STAGE_DIR}/zImage-${ARCH}"
 
-cleanup linux
+cleanup
 
 # Provide qemu's common command line options between architectures.  The lack
 # of ending quotes on -append is intentional, callers append more kernel