Mercurial > hg > aboriginal
changeset 352:1782b77fae15
Add command logging. Set RECORD_COMMANDS=1 to log every command line run
by the build into log files named "build/cmdlines.$STAGE.$PACKAGE".
author | Rob Landley <rob@landley.net> |
---|---|
date | Wed, 02 Jul 2008 22:37:41 -0500 |
parents | 7f529baf0b57 |
children | e89a2760e6cb |
files | build.sh forkbomb.sh host-tools.sh include.sh sources/toys/wrappy.c |
diffstat | 5 files changed, 297 insertions(+), 50 deletions(-) [+] |
line wrap: on
line diff
--- a/build.sh Tue Jul 01 14:42:39 2008 -0500 +++ b/build.sh Wed Jul 02 22:37:41 2008 -0500 @@ -17,8 +17,6 @@ # so we can ditch the old $PATH afterwards. time ./host-tools.sh || exit 1 -PATH=`pwd`/build/host -[ -f "$PATH"/toybox ] || exit 1 # Run the steps in order for each architecture listed on the command line for i in "$@"
--- a/forkbomb.sh Tue Jul 01 14:42:39 2008 -0500 +++ b/forkbomb.sh Wed Jul 02 22:37:41 2008 -0500 @@ -5,6 +5,8 @@ # With --nofork, it build them sequentially # With --watch, it displays output from an existing parallel build +# Build and package one architecture. + function buildarch() { nice -n 20 ./cross-compiler.sh $1 && @@ -16,22 +18,69 @@ then if [ $# -ne 0 ] then - (nice -n 20 ./download.sh && + + # The first thing we need to do is download the source, build the host + # tools, and extract the source packages. This is only done once (not + # repeated for each architecure), so we do it up front here. Doing this + # multiple times in parallel would collide, which is the main reason + # you can't just run build.sh several times in parallel. + + (touch .kthxbye && + nice -n 20 ./download.sh && # host-tools populates one directory with every command the build needs, # so we can ditch the old $PATH afterwards. - nice -n 20 ./host-tools.sh && - PATH=`pwd`/build/host && - nice -n 20 ./download.sh --extract || touch .kthxbye) 2>&1 | tee out.txt + nice -n 20 ./host-tools.sh || exit 1 + + # Ordinarily the build extracts packages the first time it needs them, + # but letting multiple builds do that in parallel can collide, so + # extract them all now up front. (Adjust the path manually here so we're + # using the busybox tools rather than the host distro's to do the + # extract, just to be consistent.) + if [ -f `pwd`/build/host/busybox ] + then + PATH=`pwd`/build/host + fi + nice -n 20 ./download.sh --extract && + rm .kthxbye) 2>&1 | tee out.txt + + # Exiting from a sub-shell doesn't exit the parent shell, and because + # we piped the output of the subshell to tee, we can't get the exit code + # of the subshell. So we use a sentinel file: if it wasn't deleted, the + # build went "boing" and should stop now. rm .kthxbye 2>/dev/null && exit 1 + + # If we're using host tools and logging the commands run, save the logs + # into a subdirectory. + if [ ! -z "$RECORD_COMMANDS" ] + then + mkdir -p build/cmdlines-host && + mv build/cmdlines.* build/cmdlines-host || exit 1 + fi fi + + # Loop through each architecture and call "buildarch" as appropriate. + for i in `cd sources/configs; ls` do + # Build sequentially. + if [ "$1" == "--nofork" ] then buildarch $i 2>&1 | tee out-$i.txt || exit 1 + if [ ! -z "$RECORD_COMMANDS" ] + then + mkdir -p build/cmdlines-$i && + mv build/cmdlines.* build/cmdlines-$i || exit 1 + fi + + # Build in parallel + elif [ "$1" == "--fork" ] then (buildarch $i > out-$i.txt 2>&1 &)& + + # Didn't understand command line arguments, dump help. + else echo "Usage: forkbomb.sh [--fork] [--nofork] [--watch] [--stat]" echo -e "\t--nofork Build all targets one after another." @@ -43,17 +92,21 @@ done fi +# Show which builds did or didn't work. + if [ "$1" == "--stat" ] then echo "Success:" - grep -l qemu-image- out-*.txt + grep -l system-image- out-*.txt echo "Failed:" - (ls -1 out-*.txt; grep -l qemu-image- out-*.txt) | sort | uniq -u + (ls -1 out-*.txt; grep -l system-image- out-*.txt) | sort | uniq -u exit 0 fi +# Show progress indicators for parallel build. + if [ "$1" != "--nofork" ] then watch -n 3 'X=; for i in out-*.txt; do /bin/echo -e "$X$i"; X="\n"; tail -n 1 $i; done'
--- a/host-tools.sh Tue Jul 01 14:42:39 2008 -0500 +++ b/host-tools.sh Wed Jul 02 22:37:41 2008 -0500 @@ -14,52 +14,113 @@ mkdir -p "${HOSTTOOLS}" || dienow -# Here are the utilities the build needs that this script doesn't -# build, but which me must instead use from the host system. +# If we want to record the host command lines, so we know exactly what commands +# the build uses. + +if [ ! -z "$RECORD_COMMANDS" ] +then + echo setup wrapdir + + # Build the wrapper and install it into build/wrapdir/wrappy + rm -rf "$BUILD/wrapdir" + mkdir "$BUILD/wrapdir" && + $CC -Os "$SOURCES/toys/wrappy.c" -o "$BUILD/wrapdir/wrappy" || dienow + + # Loop through each $PATH element and create a symlink to the wrapper with + # that name. + + for i in $(echo $PATH | sed 's/:/ /g') + do + for j in $(ls $i) + do + ln -s wrappy "$BUILD/wrapdir/$j" + done + done + + # Adjust things to use wrapper directory -# The first seven are from packages already in mini-native. -# The last six need to be added to toybox. (The build breaks if we use -# the busybox-1.2.2 versions.) + export WRAPPY_REALPATH="$PATH" + PATH="$BUILD/wrapdir" + +# If we're not recording the host command lines, then populate a directory +# with host versions of all the command line utilities we're going to install +# into mini-native. When we're done, PATH can be set to include just this +# directory and nothing else. + +# This serves three purposes: +# +# 1) Enumerate exactly what we need to build the system, so we can make sure +# mini-native has everything it needs to rebuild us. If anything is missing +# from this list, the resulting mini-native probably won't have it either, +# so it's nice to know as early as possible that we actually needed it. +# +# 2) Quick smoke test that the versions of the tools we're using can compile +# everything from source correctly, and thus mini-native should be able to +# rebuild from source using those same tools. +# +# 3) Reduce variation from distro to distro. The build always uses the +# same command line utilities no matter where we're running, because we +# provide our own. + +else -for i in ar as nm cc gcc make ld bzip2 find install od sort diff wget -do - [ ! -f "${HOSTTOOLS}/$i" ] && (ln -s `which $i` "${HOSTTOOLS}/$i" || dienow) -done + # Start by creating symlinks to the host toolchain, since we need to use + # that to build anything else. We build a cross compiler, and a native + # compiler for the target, but we don't build a host toolchain. We use the + # one that's already there. + + for i in ar as nm cc gcc make ld + do + [ ! -f "${HOSTTOOLS}/$i" ] && (ln -s `which $i` "${HOSTTOOLS}/$i" || dienow) + done + + # These commands need to be added to toybox. The build breaks if we use + # the busybox-1.2.2 versions, where available. I'm working to remove this + # hunk... + + for i in bzip2 find install od diff wget + do + [ ! -f "${HOSTTOOLS}/$i" ] && (ln -s `which $i` "${HOSTTOOLS}/$i" || dienow) + done + + # Build toybox + + if [ ! -f "${HOSTTOOLS}/toybox" ] + then + setupfor toybox && + make defconfig && + make install_flat PREFIX="${HOSTTOOLS}" && + cd .. -# Build toybox -if [ ! -f "${HOSTTOOLS}/toybox" ] -then - setupfor toybox && - make defconfig && - make install_flat PREFIX="${HOSTTOOLS}" && - cd .. + cleanup toybox + fi + + # Build busybox + + # Yes this is an old version of busybox. (It's the last version I released + # as busybox maintainer.) We're gradually replacing busybox with toybox, one + # command at a time. - cleanup toybox + if [ ! -f "${HOSTTOOLS}/busybox" ] + then + setupfor busybox && + cp "${SOURCES}/config-busybox" .config && + yes "" | make oldconfig && + make && + cp busybox "${HOSTTOOLS}" + + [ $? -ne 0 ] && dienow + + for i in $(sed 's@.*/@@' busybox.links) + do + ln -s busybox "${HOSTTOOLS}"/$i || dienow + done + cd .. + + cleanup busybox + fi fi -# Yes this is an old version of busybox. (It's the last version I released -# as busybox maintainer.) We're gradually replacing busybox with toybox, one -# command at a time. - -# Build busybox -if [ ! -f "${HOSTTOOLS}/busybox" ] -then - setupfor busybox && - cp "${SOURCES}/config-busybox" .config && - yes "" | make oldconfig && - make && - cp busybox "${HOSTTOOLS}" - - [ $? -ne 0 ] && dienow - - for i in $(sed 's@.*/@@' busybox.links) - do - ln -s busybox "${HOSTTOOLS}"/$i || dienow - done - cd .. - - cleanup busybox -fi # This is optionally used by mini-native to accelerate native builds when # running under qemu. It's not used to build mini-native, or to build @@ -113,4 +174,13 @@ # #cleanup squashfs +if [ ! -z "$RECORD_COMMANDS" ] +then + # Add the host tools we just built to wrapdir + for j in $(ls "$HOSTTOOLS") + do + ln -s wrappy "$BUILD/wrapdir/$j" + done +fi + echo -e "\e[32mHost tools build complete.\e[0m"
--- a/include.sh Tue Jul 01 14:42:39 2008 -0500 +++ b/include.sh Wed Jul 02 22:37:41 2008 -0500 @@ -8,6 +8,11 @@ # export BUILD_SHORT=1 +# If this is set, the build records the command lines run by each build into +# log files in the build directory, ala "build/cmdlines.$PACKAGENAME" + +# export RECORD_COMMANDS=1 + # What host compiler should we use? [ -z "$CC" ] && CC=gcc @@ -38,7 +43,27 @@ export FROMSRC=../packages export BUILD="${TOP}/build" export HOSTTOOLS="${BUILD}/host" -[ "$PATH" != "$HOSTTOOLS" ] && export PATH="${HOSTTOOLS}:$PATH" + +# Adjust $PATH + +if [ "$PATH" != "$HOSTTOOLS" ] +then + if [ -f "$HOSTTOOLS/busybox" ] + then + PATH="$HOSTTOOLS" + else + PATH="${HOSTTOOLS}:$PATH" + fi +fi + +STAGE_NAME=`echo $0 | sed 's@.*/\(.*\)\.sh@\1@'` +export WRAPPY_LOGPATH="$BUILD/cmdlines.${STAGE_NAME}.setupfor" +if [ -f "$BUILD/wrapdir/wrappy" ] +then + export WRAPPY_REALPATH="$PATH" + PATH="$BUILD/wrapdir" +fi + mkdir -p "${SRCDIR}" # Tell bash not to cache the $PATH because we modify it. Without this, bash @@ -190,7 +215,7 @@ # The extra "" is so we test the sha1sum after the last download. for i in "$URL" http://impactlinux.com/firmware/mirror/"$FILENAME" \ - http://127.0.0.1/code/firmware/mirror/"$FILENAME" "" + http://landley.net/code/firmware/mirror/"$FILENAME" "" do # Return success if we have a valid copy of the file @@ -275,6 +300,8 @@ function setupfor() { + export WRAPPY_LOGPATH="$BUILD/cmdlines.${STAGE_NAME}.setupfor" + # Make sure the source is already extracted and up-to-date. cd "${SRCDIR}" && extract "${1}-"*.tar* || exit 1 @@ -308,4 +335,5 @@ mkdir -p "$2" && cd "$2" || dienow fi + export WRAPPY_LOGPATH="$BUILD/cmdlines.${STAGE_NAME}.$1" }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sources/toys/wrappy.c Wed Jul 02 22:37:41 2008 -0500 @@ -0,0 +1,98 @@ +// A little wrapper to figure out what commands you're actually using. + +// To use it, you'll need to make a directory of symlinks and set up two +// environment variables. Something like: +// +// export WRAPPY_LOGPATH=/path/to/wrappy.log +// export WRAPPY_REALPATH="$PATH" +// +// WRAPPYDIR=/path/to/wrappy +// cp wrappy $WRAPPYDIR +// for i in `echo $PATH | sed 's/:/ /g'` +// do +// for j in `ls $i` +// do +// ln -s wrappy $WRAPPYDIR/$j +// done +// done +// +// PATH="$WRAPPYDIR" make thingy + +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +char blah[65536]; + +int main(int argc, char *argv[], char *env[]) +{ + char *logpath, *realpath, *p, *p2, *p3; + + int i, fd; + + // If these environment variables didn't propogate down to our build, how + // did $PATH make it there? (Faily noisily if they are missing...) + + logpath = getenv("WRAPPY_LOGPATH"); + realpath = getenv("WRAPPY_REALPATH"); + if (!logpath || !realpath) { + fprintf(stderr, "No WRAPPY_%s\n", logpath ? "REALPATH" : "LOGPATH"); + exit(1); + } + + // Figure out name of command being run. + + p2 = strrchr(*argv, '/'); + if (!p2) p2=*argv; + else p2++; + + // Write command line to a buffer. (We need the whole command line in one + //buffer so we can do one write.) + + p=blah + sprintf(blah, "%s ",p2); + for (i=1; i<argc && (p-blah)<sizeof(blah); i++) { + *(p++)='"'; + for (p3 = argv[i]; *p3 && (p-blah)<sizeof(blah); p3++) { + char *s = "\n\\\"", *ss = strchr(s, *p3); + + if (ss) { + *(p++)='\\'; + *(p++)="n\\\""[ss-s]; + } else *(p++) = *p3; + } + *(p++)='"'; + *(p++)=' '; + } + p[-1]='\n'; + + // Log the command line, using O_APPEND and an atomic write so this entry + // always goes at the end no matter what other processes are writing + // into the file. + + fd=open(logpath, O_WRONLY|O_CREAT|O_APPEND, 0777); + write(fd, blah, p-blah); + close(fd); + + // Touch the file that got used. + + // sprintf(blah, ROOTPATH "/used/%s", p2); + // close(open(blah, O_WRONLY|O_CREAT, 0777)); + + // Hand off control to the real executable + + for (p = p3 = realpath; *p3; p3++) { + if (*p3==':' || !*p3) { + char snapshot = *p3; + *p3 = 0; + snprintf(blah, sizeof(blah)-1, "%s/%s", p, p2); + *p3 = snapshot; + execve(blah, argv, env); + p = p3+1; + } + } + + // Should never happen, means environment setup is wrong. + fprintf(stderr, "Didn't find %s\n", logpath); + exit(1); +}