changeset 352:1782b77fae15

Add command logging. Set RECORD_COMMANDS=1 to log every command line run by the build into log files named "build/cmdlines.$STAGE.$PACKAGE".
author Rob Landley <rob@landley.net>
date Wed, 02 Jul 2008 22:37:41 -0500
parents 7f529baf0b57
children e89a2760e6cb
files build.sh forkbomb.sh host-tools.sh include.sh sources/toys/wrappy.c
diffstat 5 files changed, 297 insertions(+), 50 deletions(-) [+]
line wrap: on
line diff
--- a/build.sh	Tue Jul 01 14:42:39 2008 -0500
+++ b/build.sh	Wed Jul 02 22:37:41 2008 -0500
@@ -17,8 +17,6 @@
 # so we can ditch the old $PATH afterwards.
 
 time ./host-tools.sh || exit 1
-PATH=`pwd`/build/host
-[ -f "$PATH"/toybox ] || exit 1
 
 # Run the steps in order for each architecture listed on the command line
 for i in "$@"
--- a/forkbomb.sh	Tue Jul 01 14:42:39 2008 -0500
+++ b/forkbomb.sh	Wed Jul 02 22:37:41 2008 -0500
@@ -5,6 +5,8 @@
 #  With --nofork, it build them sequentially
 #  With --watch, it displays output from an existing parallel build
 
+# Build and package one architecture.
+
 function buildarch()
 {
   nice -n 20 ./cross-compiler.sh $1 &&
@@ -16,22 +18,69 @@
 then
   if [ $# -ne 0 ]
   then
-    (nice -n 20 ./download.sh &&
+
+    # The first thing we need to do is download the source, build the host
+    # tools, and extract the source packages.  This is only done once (not
+    # repeated for each architecure), so we do it up front here.  Doing this
+    # multiple times in parallel would collide, which is the main reason
+    # you can't just run build.sh several times in parallel.
+
+    (touch .kthxbye &&
+     nice -n 20 ./download.sh &&
      # host-tools populates one directory with every command the build needs,
      # so we can ditch the old $PATH afterwards.
-     nice -n 20 ./host-tools.sh &&
-     PATH=`pwd`/build/host &&
-     nice -n 20 ./download.sh --extract || touch .kthxbye) 2>&1 | tee out.txt
+     nice -n 20 ./host-tools.sh || exit 1
+
+     # Ordinarily the build extracts packages the first time it needs them,
+     # but letting multiple builds do that in parallel can collide, so
+     # extract them all now up front.  (Adjust the path manually here so we're
+     # using the busybox tools rather than the host distro's to do the
+     # extract, just to be consistent.)
+     if [ -f `pwd`/build/host/busybox ]
+     then
+       PATH=`pwd`/build/host
+     fi
+     nice -n 20 ./download.sh --extract &&
+     rm .kthxbye) 2>&1 | tee out.txt
+
+     # Exiting from a sub-shell doesn't exit the parent shell, and because
+     # we piped the output of the subshell to tee, we can't get the exit code
+     # of the subshell.  So we use a sentinel file: if it wasn't deleted, the
+     # build went "boing" and should stop now.
      rm .kthxbye 2>/dev/null && exit 1
+
+     # If we're using host tools and logging the commands run, save the logs
+     # into a subdirectory.
+     if [ ! -z "$RECORD_COMMANDS" ]
+     then
+       mkdir -p build/cmdlines-host &&
+       mv build/cmdlines.* build/cmdlines-host || exit 1
+     fi
   fi
+
+  # Loop through each architecture and call "buildarch" as appropriate.
+
   for i in `cd sources/configs; ls`
   do
+    # Build sequentially.
+
     if [ "$1" == "--nofork" ]
     then
       buildarch $i 2>&1 | tee out-$i.txt || exit 1
+      if [ ! -z "$RECORD_COMMANDS" ]
+      then
+        mkdir -p build/cmdlines-$i &&
+        mv build/cmdlines.* build/cmdlines-$i || exit 1
+      fi
+
+    # Build in parallel
+
     elif [ "$1" == "--fork" ]
     then
       (buildarch $i > out-$i.txt 2>&1 &)&
+
+    # Didn't understand command line arguments, dump help.
+
     else
       echo "Usage: forkbomb.sh [--fork] [--nofork] [--watch] [--stat]"
       echo -e "\t--nofork  Build all targets one after another."
@@ -43,17 +92,21 @@
   done
 fi
 
+# Show which builds did or didn't work.
+
 if [ "$1" == "--stat" ]
 then
   echo "Success:"
-  grep -l qemu-image- out-*.txt
+  grep -l system-image- out-*.txt
 
   echo "Failed:"
-  (ls -1 out-*.txt; grep -l qemu-image- out-*.txt) | sort | uniq -u
+  (ls -1 out-*.txt; grep -l system-image- out-*.txt) | sort | uniq -u
 
   exit 0
 fi
 
+# Show progress indicators for parallel build.
+
 if [ "$1" != "--nofork" ]
 then
   watch -n 3 'X=; for i in out-*.txt; do /bin/echo -e "$X$i"; X="\n"; tail -n 1 $i; done'
--- a/host-tools.sh	Tue Jul 01 14:42:39 2008 -0500
+++ b/host-tools.sh	Wed Jul 02 22:37:41 2008 -0500
@@ -14,52 +14,113 @@
 
 mkdir -p "${HOSTTOOLS}" || dienow
 
-# Here are the utilities the build needs that this script doesn't
-# build, but which me must instead use from the host system.
+# If we want to record the host command lines, so we know exactly what commands
+# the build uses.
+
+if [ ! -z "$RECORD_COMMANDS" ]
+then
+  echo setup wrapdir
+
+  # Build the wrapper and install it into build/wrapdir/wrappy
+  rm -rf "$BUILD/wrapdir"
+  mkdir "$BUILD/wrapdir" &&
+  $CC -Os "$SOURCES/toys/wrappy.c" -o "$BUILD/wrapdir/wrappy"  || dienow
+
+  # Loop through each $PATH element and create a symlink to the wrapper with
+  # that name.
+
+  for i in $(echo $PATH | sed 's/:/ /g')
+  do
+    for j in $(ls $i)
+    do
+      ln -s wrappy "$BUILD/wrapdir/$j"
+    done
+  done
+
+  # Adjust things to use wrapper directory
 
-# The first seven are from packages already in mini-native.
-# The last six need to be added to toybox.  (The build breaks if we use
-# the busybox-1.2.2 versions.)
+  export WRAPPY_REALPATH="$PATH"
+  PATH="$BUILD/wrapdir"
+
+# If we're not recording the host command lines, then populate a directory
+# with host versions of all the command line utilities we're going to install
+# into mini-native.  When we're done, PATH can be set to include just this
+# directory and nothing else.
+
+# This serves three purposes:
+#
+# 1) Enumerate exactly what we need to build the system, so we can make sure
+#    mini-native has everything it needs to rebuild us.  If anything is missing
+#    from this list, the resulting mini-native probably won't have it either,
+#    so it's nice to know as early as possible that we actually needed it.
+#
+# 2) Quick smoke test that the versions of the tools we're using can compile
+#    everything from source correctly, and thus mini-native should be able to
+#    rebuild from source using those same tools.
+#
+# 3) Reduce variation from distro to distro.  The build always uses the
+#    same command line utilities no matter where we're running, because we
+#    provide our own.
+
+else
 
-for i in ar as nm cc gcc make ld   bzip2 find install od sort diff wget
-do
-  [ ! -f "${HOSTTOOLS}/$i" ] && (ln -s `which $i` "${HOSTTOOLS}/$i" || dienow)
-done
+  # Start by creating symlinks to the host toolchain, since we need to use
+  # that to build anything else.  We build a cross compiler, and a native
+  # compiler for the target, but we don't build a host toolchain.  We use the
+  # one that's already there.
+
+  for i in ar as nm cc gcc make ld
+  do
+    [ ! -f "${HOSTTOOLS}/$i" ] && (ln -s `which $i` "${HOSTTOOLS}/$i" || dienow)
+  done
+
+  # These commands need to be added to toybox.  The build breaks if we use
+  # the busybox-1.2.2 versions, where available.  I'm working to remove this
+  # hunk...
+
+  for i in bzip2 find install od diff wget
+  do
+    [ ! -f "${HOSTTOOLS}/$i" ] && (ln -s `which $i` "${HOSTTOOLS}/$i" || dienow)
+  done
+
+  # Build toybox
+
+  if [ ! -f "${HOSTTOOLS}/toybox" ]
+  then
+    setupfor toybox &&
+    make defconfig &&
+    make install_flat PREFIX="${HOSTTOOLS}" &&
+    cd ..
 
-# Build toybox
-if [ ! -f "${HOSTTOOLS}/toybox" ]
-then
-  setupfor toybox &&
-  make defconfig &&
-  make install_flat PREFIX="${HOSTTOOLS}" &&
-  cd ..
+    cleanup toybox
+  fi
+
+  # Build busybox
+
+  # Yes this is an old version of busybox.  (It's the last version I released
+  # as busybox maintainer.)  We're gradually replacing busybox with toybox, one
+  # command at a time.
 
-  cleanup toybox
+  if [ ! -f "${HOSTTOOLS}/busybox" ]
+  then
+    setupfor busybox &&
+    cp "${SOURCES}/config-busybox" .config &&
+    yes "" | make oldconfig &&
+    make &&
+    cp busybox "${HOSTTOOLS}"
+
+    [ $? -ne 0 ] && dienow
+
+    for i in $(sed 's@.*/@@' busybox.links)
+    do
+      ln -s busybox "${HOSTTOOLS}"/$i || dienow
+    done
+    cd ..
+
+    cleanup busybox
+  fi
 fi
 
-# Yes this is an old version of busybox.  (It's the last version I released
-# as busybox maintainer.)  We're gradually replacing busybox with toybox, one
-# command at a time.
-
-# Build busybox
-if [ ! -f "${HOSTTOOLS}/busybox" ]
-then
-  setupfor busybox &&
-  cp "${SOURCES}/config-busybox" .config &&
-  yes "" | make oldconfig &&
-  make &&
-  cp busybox "${HOSTTOOLS}"
-
-  [ $? -ne 0 ] && dienow
-
-  for i in $(sed 's@.*/@@' busybox.links)
-  do
-    ln -s busybox "${HOSTTOOLS}"/$i || dienow
-  done
-  cd ..
-
-  cleanup busybox
-fi
 
 # This is optionally used by mini-native to accelerate native builds when
 # running under qemu.  It's not used to build mini-native, or to build
@@ -113,4 +174,13 @@
 #
 #cleanup squashfs
 
+if [ ! -z "$RECORD_COMMANDS" ]
+then 
+  # Add the host tools we just built to wrapdir
+  for j in $(ls "$HOSTTOOLS")
+  do
+    ln -s wrappy "$BUILD/wrapdir/$j"
+  done
+fi
+
 echo -e "\e[32mHost tools build complete.\e[0m"
--- a/include.sh	Tue Jul 01 14:42:39 2008 -0500
+++ b/include.sh	Wed Jul 02 22:37:41 2008 -0500
@@ -8,6 +8,11 @@
 
 # export BUILD_SHORT=1
 
+# If this is set, the build records the command lines run by each build into
+# log files in the build directory, ala "build/cmdlines.$PACKAGENAME"
+
+# export RECORD_COMMANDS=1
+
 # What host compiler should we use?
 
 [ -z "$CC" ] && CC=gcc
@@ -38,7 +43,27 @@
 export FROMSRC=../packages
 export BUILD="${TOP}/build"
 export HOSTTOOLS="${BUILD}/host"
-[ "$PATH" != "$HOSTTOOLS" ] && export PATH="${HOSTTOOLS}:$PATH"
+
+# Adjust $PATH
+
+if [ "$PATH" != "$HOSTTOOLS" ]
+then
+  if [ -f "$HOSTTOOLS/busybox" ]
+  then
+    PATH="$HOSTTOOLS"
+  else
+    PATH="${HOSTTOOLS}:$PATH"
+  fi
+fi
+
+STAGE_NAME=`echo $0 | sed 's@.*/\(.*\)\.sh@\1@'`
+export WRAPPY_LOGPATH="$BUILD/cmdlines.${STAGE_NAME}.setupfor"
+if [ -f "$BUILD/wrapdir/wrappy" ]
+then
+  export WRAPPY_REALPATH="$PATH"
+  PATH="$BUILD/wrapdir"
+fi
+
 mkdir -p "${SRCDIR}"
 
 # Tell bash not to cache the $PATH because we modify it.  Without this, bash
@@ -190,7 +215,7 @@
   # The extra "" is so we test the sha1sum after the last download.
 
   for i in "$URL" http://impactlinux.com/firmware/mirror/"$FILENAME" \
-    http://127.0.0.1/code/firmware/mirror/"$FILENAME" ""
+    http://landley.net/code/firmware/mirror/"$FILENAME" ""
   do
     # Return success if we have a valid copy of the file
 
@@ -275,6 +300,8 @@
 
 function setupfor()
 {
+  export WRAPPY_LOGPATH="$BUILD/cmdlines.${STAGE_NAME}.setupfor"
+
   # Make sure the source is already extracted and up-to-date.
   cd "${SRCDIR}" &&
   extract "${1}-"*.tar* || exit 1
@@ -308,4 +335,5 @@
     mkdir -p "$2" &&
     cd "$2" || dienow
   fi
+  export WRAPPY_LOGPATH="$BUILD/cmdlines.${STAGE_NAME}.$1"
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sources/toys/wrappy.c	Wed Jul 02 22:37:41 2008 -0500
@@ -0,0 +1,98 @@
+// A little wrapper to figure out what commands you're actually using.
+
+// To use it, you'll need to make a directory of symlinks and set up two
+// environment variables.  Something like:
+//
+// export WRAPPY_LOGPATH=/path/to/wrappy.log
+// export WRAPPY_REALPATH="$PATH"
+//
+// WRAPPYDIR=/path/to/wrappy
+// cp wrappy $WRAPPYDIR
+// for i in `echo $PATH | sed 's/:/ /g'`
+// do
+//   for j in `ls $i`
+//   do
+//     ln -s wrappy $WRAPPYDIR/$j
+//   done
+// done
+//
+// PATH="$WRAPPYDIR" make thingy
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+char blah[65536];
+
+int main(int argc, char *argv[], char *env[])
+{
+  char *logpath, *realpath, *p, *p2, *p3;
+
+  int i, fd;
+
+  // If these environment variables didn't propogate down to our build, how
+  // did $PATH make it there?  (Faily noisily if they are missing...)
+
+  logpath = getenv("WRAPPY_LOGPATH");
+  realpath = getenv("WRAPPY_REALPATH");
+  if (!logpath || !realpath) {
+    fprintf(stderr, "No WRAPPY_%s\n", logpath ? "REALPATH" : "LOGPATH");
+    exit(1);
+  }
+
+  // Figure out name of command being run.
+
+  p2 = strrchr(*argv, '/');
+  if (!p2) p2=*argv;
+  else p2++;
+
+  // Write command line to a buffer.  (We need the whole command line in one
+  //buffer so we can do one write.)
+
+  p=blah + sprintf(blah, "%s ",p2);
+  for (i=1; i<argc && (p-blah)<sizeof(blah); i++) {
+    *(p++)='"';
+    for (p3 = argv[i]; *p3 && (p-blah)<sizeof(blah); p3++) {
+      char *s = "\n\\\"", *ss = strchr(s, *p3);
+
+      if (ss) {
+        *(p++)='\\';
+        *(p++)="n\\\""[ss-s];
+      } else *(p++) = *p3;
+    }
+    *(p++)='"';
+    *(p++)=' ';
+  }
+  p[-1]='\n';
+
+  // Log the command line, using O_APPEND and an atomic write so this entry
+  // always goes at the end no matter what other processes are writing
+  // into the file.
+
+  fd=open(logpath, O_WRONLY|O_CREAT|O_APPEND, 0777);
+  write(fd, blah, p-blah);
+  close(fd);
+
+  // Touch the file that got used.
+
+  // sprintf(blah, ROOTPATH "/used/%s", p2);
+  // close(open(blah, O_WRONLY|O_CREAT, 0777));
+
+  // Hand off control to the real executable
+
+  for (p = p3 = realpath; *p3; p3++) {
+    if (*p3==':' || !*p3) {
+      char snapshot = *p3;
+      *p3 = 0;
+      snprintf(blah, sizeof(blah)-1, "%s/%s", p, p2);
+      *p3 = snapshot;
+      execve(blah, argv, env);
+      p = p3+1;
+    }
+  }
+
+  // Should never happen, means environment setup is wrong.
+  fprintf(stderr, "Didn't find %s\n", logpath);
+  exit(1);
+}