Mercurial > hg > aboriginal
changeset 1157:300e6d919d86
Move "sources/more/" to just "more/", add a README explaining why those scripts are there, and adjust calls to them.
author | Rob Landley <rob@landley.net> |
---|---|
date | Mon, 05 Jul 2010 15:37:04 -0500 |
parents | 8ab750338f73 |
children | df7814a794a1 |
files | cross-compiler.sh more/README more/bisectinate.sh more/build-control-images.sh more/buildall.sh more/cronjob.sh more/cross-smoke-test.sh more/for-each-target.sh more/migrate-kernel.sh more/record-commands.sh more/report-recorded-commands.sh more/smoketest-all.sh more/smoketest.sh more/test.sh more/timeout.sh simple-cross-compiler.sh smoketest.sh sources/more/bisectinate.sh sources/more/build-control-images.sh sources/more/buildall.sh sources/more/cronjob.sh sources/more/cross-smoke-test.sh sources/more/for-each-target.sh sources/more/migrate-kernel.sh sources/more/record-commands.sh sources/more/report-recorded-commands.sh sources/more/smoketest-all.sh sources/more/test.sh sources/more/timeout.sh |
diffstat | 29 files changed, 553 insertions(+), 550 deletions(-) [+] |
line wrap: on
line diff
--- a/cross-compiler.sh Sun Jul 04 20:36:50 2010 -0500 +++ b/cross-compiler.sh Mon Jul 05 15:37:04 2010 -0500 @@ -23,5 +23,5 @@ if [ ! -z "$CROSS_SMOKE_TEST" ] then - sources/more/cross-smoke-test.sh "$ARCH" || exit 1 + more/cross-smoke-test.sh "$ARCH" || exit 1 fi
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/README Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,3 @@ +The scripts in the top level directory are the build stages, called in order +by the build.sh wrapper. This directory contains additional commands the +user may want to run directly, but which aren't build stages.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/bisectinate.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,110 @@ +#!/bin/bash + +# Find the first breakage (since the last known good version) via git bisect. + +if [ $# -ne 4 ] +then + echo usage: bisectinate PACKAGE repodir@branch start arch >&2 + exit 1 +fi + +# Parse command line options + +PKG="$1" +REPO="${2/@*/}" +BRANCH="${2/*@/}" +[ "$BRANCH" == "$2" ] && BRANCH=master +START="$3" +ARCH="$4" + +TOP="$(pwd)" +[ -z "$SRCDIR" ] && SRCDIR="$TOP/packages" +[ -z "$BUILD" ] && BUILD="$TOP/build" + +if [ ! -d "$REPO/.git" ] +then + echo "No git repo at $REPO" + exit 1 +fi + +# For kernel and busybox bisects, only redo part of the build + +if [ "$PKG" == linux ] && [ -e "$BUILD/root-filesystem-$ARCH".tar.bz2 ] +then + ZAPJUST=system-image +elif [ "$PKG" == busybox ] && + [ -e "$BUILD/simple-cross-compiler-$ARCH.tar.bz2" ] +then + ZAPJUST=root-filesystem +else + ZAPJUST= +fi + +# Start bisecting repository + +mkdir -p "$BUILD"/logs +cd "$REPO" && +git clean -fdx && git checkout -f && +git bisect reset && +git bisect start && +git bisect good "$START" || exit 1 +RESULT="$(git bisect bad "$BRANCH")" +cd "$TOP" + +set -o pipefail + +# Loop through bisection results + +while true +do + echo "$RESULT" + + # Are we done? + + [ ! "$(echo "$RESULT" | head -n 1 | grep "^Bisecting:")" ] && exit + + cd "$REPO" + git show > "$BUILD/logs/test-${ARCH}.txt" + # The "cat" bypasses git's stupid overengineered built-in call to less. + git log HEAD -1 | cat + echo "Testing..." + git archive --prefix="$PKG/" HEAD | bzip2 \ + > "$SRCDIR/alt-$PKG-0.tar.bz2" || exit 1 + cd "$TOP" + + # Perform actual build + + RESULT=bad + + [ ! -z "$ZAPJUST" ] && + rm -rf "$BUILD/${ZAPJUST}-$ARCH"{,.tar.bz2} || + rm -rf "$BUILD"/*-"$ARCH"{,.tar.bz2} + EXTRACT_ALL=yes USE_UNSTABLE="$PKG" ./build.sh "$ARCH" \ + | tee -a "$BUILD"/logs/test-"$ARCH".txt + if [ -e "$BUILD"/system-image-"$ARCH".tar.bz2 ] + then + if [ -z "$LONG" ] + then + RESULT=good + else + rm -rf "$BUILD"/cron-temp/"$ARCH"-dropbearmulti + more/native-static-build.sh "$ARCH" 2>&1 \ + | tee -a "$BUILD"/logs/test-"$ARCH".txt + + [ -e "$BUILD"/cron-temp/"$ARCH"-dropbearmulti ] && RESULT=good + fi + fi + + # If it built, try the native compile + + if [ "$RESULT" == "bad" ] + then + mv "$BUILD"/logs/{test,testfail}-"$ARCH".txt + else + rm "$BUILD"/logs/test-"$ARCH".txt + fi + + cd "$REPO" + RESULT="$(git bisect $RESULT)" + cd "$TOP" +done
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/build-control-images.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,15 @@ +#!/bin/bash + +# Iterate through sources/native-builds and run each script, writing output +# to build/control-images/$SCRIPTNAME.hdc + +. sources/include.sh || exit 1 + +mkdir -p build/control-images || dienow +for i in sources/native-builds/*.sh +do + SCRIPTNAME=$(echo $i | sed 's@.*/\(.*\)\.sh@\1@') + # Forking doesn't work yet due to extract collisions with the same package + #maybe_fork "$i build/control-images/$SCRIPTNAME.hdc | maybe_quiet" + $i build/control-images/$SCRIPTNAME.hdc | maybe_quiet +done
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/buildall.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,62 @@ +#!/bin/bash + +# Build every target architecture, creating build-$ARCH.txt log files. +# If $FORK is set, build them in parallel. + +. sources/functions.sh || exit 1 + +[ -z "$STATIC_CC_HOST" ] && export STATIC_CC_HOST=i686 + +trap "killtree $$" EXIT + +# Build the host architecture. This has to be built first so the other +# architectures can canadian cross static compilers to run on the host using +# this toolchain to link against a host version of uClibc. + +# This also performs the download.sh and host-tools.sh steps, which don't +# parallelize well if many build.sh instances try to call them at once. + +# If this fails, don't bother trying to build the other targets. + +blank_tempdir build +mkdir -p build/logs && +(EXTRACT_ALL=1 ./download.sh 2>&1 && + ./host-tools.sh 2>&1 && + ./simple-cross-compiler.sh 2>&1 "$STATIC_CC_HOST" || + dienow) | tee build/logs/build-host-cc.txt | maybe_quiet + +cp packages/MANIFEST build || dienow + +# Build all non-hw targets, possibly in parallel + +more/for-each-target.sh \ + './build.sh $TARGET 2>&1 | tee build/logs/build-${TARGET}.txt' + +# Run smoketest.sh for each non-hw target. + +more/for-each-target.sh \ + 'more/smoketest.sh $TARGET 2>&1 | tee build/logs/smoketest-$TARGET.txt' + +more/build-control-images.sh + +# Build all control images + +mkdir -p build/control-images || dienow +for i in sources/native-builds/*.sh +do + X=$(echo $i | sed 's@.*/\(.*\)\.sh@\1@') + # Don't use maybe_fork here, the extract stages conflict. + $i build/control-images/${X}.hdc | maybe_quiet +done + +wait + +# Build static-tools (dropbear and strace) for each target + +mkdir -p build/native-static && +more/for-each-target.sh \ + 'more/timeout.sh 60 "(cd build/system-image-$TARGET && ln -s ../native-static upload && ./native-build.sh ../control-images/static-tools.hdc) | tee build/logs/native-$TARGET.txt"' + +# Create a file containing simple pass/fail results for all architectures. + +more/smoketest-all.sh --logs | tee build/logs/status.txt
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/cronjob.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,103 @@ +#!/bin/bash + +# Build stable versions of all packages with current scripts. + +# This gets run in the aboriginal top directory. + +pull_repo() +{ + # Clone working copy + + rm -rf "packages/alt-$1-0.tar.bz2" build/packages/alt-$1 + mkdir -p build/packages/alt-$1 + pushd build/packages/alt-$1 && + ln -s ../../../repos/$1/.git .git && + git checkout -f master && + git pull + popd +} + +# Expire snapshots directory + +SNAPSHOTS="$(find snapshots -mindepth 1 -maxdepth 1 -type d)" +COUNT=$(( $(echo "$SNAPSHOTS" | wc -l) - 30 )) +if [ "$COUNT" -gt 0 ] +then + # Delete appropriate number of oldest entries, then dead symlinks. + rm -rf $( echo "$SNAPSHOTS" | sort | head -n $COUNT ) + rm -rf $(find -L snapshots -type l) +fi + +echo === Begin cron job + +# Start a new snapshot + +export SNAPSHOT_DATE=$(date +"%Y-%m-%d") +mkdir -p snapshots/$SNAPSHOT_DATE/base && +rm snapshots/latest && +ln -sf $SNAPSHOT_DATE snapshots/latest || exit 1 + +# build base repo + +export FORK=1 +export CROSS_HOST_ARCH=i686 +hg pull -u + +build_snapshot() +{ + if [ -z "$USE_UNSTABLE" ] + then + SNAPNAME=base + else + pull_repo $USE_UNSTABLE + SNAPNAME=$USE_UNSTABLE + fi + + echo === Building snapshot $SNAPNAME + + [ "$USE_UNSTABLE" == linux ] && + more/for-each-arch.sh 'more/migrate-kernel.sh $TARGET' + + # Update manifest + + ./download.sh + + # If it's unchanged, just hardlink the previous binaries instead of rebuilding + + if cmp -s snapshots/latest/$SNAPNAME/MANIFEST packages/MANIFEST + then + cp -rl snapshots/latest/$SNAPNAME/* snapshots/$SNAPSHOT_DATE/$SNAPNAME + return + fi + + # Build it + + nice -n 20 more/buildall.sh + rm build/simple-cross-compiler-*.tar.bz2 + mv build/*.tar.bz2 build/logs build/MANIFEST snapshots/$SNAPSHOT_DATE/$SNAPNAME +} + +build_snapshot base + +echo === Building QEMU + +# build qemu-git + +QPATH="" +CPUS=$(echo /sys/devices/system/cpu/cpu[0-9]* | wc -w) +pull_repo qemu +pushd build/packages/alt-qemu +./configure --disable-werror && +nice -n 20 make -j $CPUS 2>&1 | tee build/logs/alt-qemu.txt && +QPATH="$(for i in *-softmmu;do echo -n $(pwd)/$i:; done)" +popd + +# test all with qemu-git + +[ -z "$QPATH" ] || + PATH="$QPATH:$PATH" more/for-each-target.sh \ + 'more/smoketest.sh $TARGET | tee snapshots/$SNAPSHOT_DATE/base/logs/newqemu-smoketest-$TARGET.txt' + +#USE_UNSTABLE=linux build_snapshot +#USE_UNSTABLE=uClibc build_snapshot +#USE_UNSTABLE=busybox build_snapshot
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/cross-smoke-test.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,27 @@ +#!/bin/bash + +source sources/include.sh && read_arch_dir "$1" || exit 1 + +# Build statically linked hello world, if necessary + +if [ ! -e "$WORK/hello" ] +then + "${ARCH}-gcc" -Os -static "${SOURCES}/toys/hello.c" -o "$WORK"/hello + + if [ $? -ne 0 ] + then + echo "Compiler doesn't seem to work" >&2 + dienow + fi +fi + +# Attempt to run statically linked hello world + +RESULT="$(PATH="$OLDPATH" qemu-"$QEMU_TEST" "$WORK/hello")" +if [ "$RESULT" == "Hello world!" ] +then + echo "Cross toolchain seems to work." +else + echo "Can't run hello world" >&2 + dienow +fi
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/for-each-target.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,18 @@ +#!/bin/bash + +# Iterate through every target architecture, running rest of command line +# on each $TARGET. + +# If $FORK is set, run them in parallel with filtered output. + +. sources/functions.sh || exit 1 + +[ -z "${ARCHES}" ] && + ARCHES="$(cd sources/targets/; ls | grep -v '^hw-')" + +for TARGET in $ARCHES +do + maybe_fork "$* | maybe_quiet" +done + +wait
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/migrate-kernel.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,47 @@ +#!/bin/bash + +# Calculate new config for UNSTABLE kernel based on stable kernel config. +# I.E. calculate miniconfig-alt-linux based on miniconfig-linux for a target. + +# Expand miniconfig with the old kernel, copy .config to new kernel, run +# make oldconfig, compress to miniconfig, copy to sources/targets/$TARGET + +. sources/include.sh + +read_arch_dir "$1" +rmdir "$STAGE_DIR" + +[ -z "$BOOT_KARCH" ] && BOOT_KARCH="$KARCH" + +# Expand config against current kernel + +USE_UNSTABLE= + +setupfor linux + +cp "$(getconfig linux)" mini.conf || dienow +[ "$SYSIMAGE_TYPE" == "initramfs" ] && + (echo "CONFIG_BLK_DEV_INITRD=y" >> mini.conf || dienow) +make ARCH="$BOOT_KARCH" KCONFIG_ALLCONFIG=mini.conf $LINUX_FLAGS \ + allnoconfig > /dev/null && +cp .config "$WORK" + +cleanup + +USE_UNSTABLE=linux + +setupfor linux + +cp "$WORK/.config" . && +yes "" | make ARCH="$BOOT_KARCH" oldconfig && +mv .config walrus && +ARCH="${BOOT_KARCH}" "$SOURCES/toys/miniconfig.sh" walrus || dienow + +CFG="$CONFIG_DIR/$ARCH_NAME/miniconfig-alt-linux" +if [ -e "$CFG" ] && ! cmp mini.config "$CFG" +then + mv "$CFG" "${CFG}.bak" || dienow +fi +mv mini.config "$CFG" + +cleanup
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/record-commands.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,23 @@ +#!/bin/bash + +source sources/include.sh || exit 1 + +# Build a wrapper that records each command line the build runs out of the +# host's $PATH, so we know exactly what commands the build uses. + +# (Note: this misses things called via absolute paths, such as the #!/bin/bash +# at the start of shell scripts.) + +echo "=== Setting up command recording wrapper" + +[ -f "$WRAPDIR/wrappy" ] && PATH="$OLDPATH" +[ -f "$HOSTTOOLS/busybox" ] && PATH="$HOSTTOOLS" +blank_tempdir "$WRAPDIR" +blank_tempdir "$BUILD/logs" + +# Populate a directory of symlinks with every command in the $PATH. + +path_search "$PATH" "*" 'ln -s wrappy "$WRAPDIR/$FILE"' | dotprogress + +# Build the wrapper +$CC -Os "$SOURCES/toys/wrappy.c" -o "$WRAPDIR/wrappy" || dienow
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/report-recorded-commands.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,37 @@ +#!/bin/bash + +# List the commands used to build each architecture. + +# If given an argument it's the architecture to compare others against, +# which shows just the extra commands used by those other architectures. + +# Mines the output created by build.sh after record-commands.sh. + +COMPARE="$1" + +# Output the list of commands used in a command log. + +function mine_commands() +{ + awk '{print $1}' build/logs/cmdlines.$1.* | sort -u +} + +# Iterate through architectures + +for i in `ls -1 build/logs/cmdlines.* | sed 's@.*/cmdlines\.\([^.]*\).*@\1@' | sort -u` +do + [ "$COMPARE" == "$i" ] && continue + + # Start of new group, announce build stage we're looking at. + echo + echo -n Checking $i: + + if [ -z "$COMPARE" ] + then + # Show all commands in first architecture. + echo $(mine_commands $i) + else + # Show commands that differ from first architecture (if any). + echo $(sort <(mine_commands $COMPARE) <(mine_commands $i) | uniq -u) + fi +done
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/smoketest-all.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,45 @@ +#!/bin/bash + +# Run smoketest script on every build/system-image-* architecture. + +# If $FORK is set, run them in parallel. + +. sources/functions.sh || exit 1 + +if [ "$1" == "--logs" ] +then + for i in build/logs/smoketest-*.txt + do + NAME="$(echo $i | sed 's/.*smoketest-\(.*\)\.txt/\1/')" + echo -n "Testing $NAME:" + RESULT="$(grep 'Hello world!' "$i")" + [ -z "$RESULT" ] && echo "FAIL" || echo "PASS" + done + + exit +fi + +function dotest() +{ + [ -z "$FORK" ] && echo -n "Testing $1:" + [ ! -z "$VERBOSE" ] && VERBOSITY="tee >(cat >&2) |" + RESULT="$(more/smoketest.sh "$1" 2>&1 | eval "$VERBOSITY grep 'Hello world!'")" + [ -z "$RESULT" ] && RESULT="FAIL" || RESULT="PASS" + [ -z "$FORK" ] && echo "$RESULT" || echo "Testing $1:$RESULT" + rm -f build/system-image-"$1"/hdb.img 2>/dev/null +} + +# Test all non-hw targets to see whether or not they can compile and run +# the included "hello world" program. + +for i in $(ls -d sources/targets/* | sed 's@.*/@@' | grep -v "^hw-") +do + if [ -e "build/system-image-$i" ] + then + maybe_fork "dotest $i" + else + echo "Testing $i:NONE" + fi +done + +wait
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/smoketest.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,25 @@ +#!/bin/bash + +. sources/functions.sh || exit 1 + +# This script compiles stuff under the final system, using distcc to call out +# to the cross compiler. It calls run-from-build with a here document. + +# Note that the first line of the script is a few spaces followed by a comment +# character. This gives some harmless data for the linux boot process (serial +# initialization) to consume and discard before it gets to the command prompt. +# (The comment character is just so you can see how much got eaten.) + +# If you cat your own script into emulator-build.sh, you probably also need +# to start with a line of spaces like that. Just FYI. + +more/timeout.sh 60 cd build/system-image-$1 "&&" ./run-emulator.sh << 'EOF' + # +# Show free space +df +# Smoke test for the compiler +gcc -s /usr/src/thread-hello2.c -lpthread -o /tmp/hello && +/tmp/hello +sync +exit +EOF
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/test.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,16 @@ +#!/bin/bash + +# Run a command with sources/include.sh and an architecture loaded + +if [ $# -eq 0 ] +then + echo "Usage: test.sh ARCH COMMAND..." >&2 + echo "You generally want to set STAGE_NAME= too." + exit 1 +fi + +. sources/include.sh || exit 1 + +read_arch_dir "$1" +shift +eval "$@"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/more/timeout.sh Mon Jul 05 15:37:04 2010 -0500 @@ -0,0 +1,20 @@ +#!/bin/bash + +# Run a command line with a hang timeout, which kills the child process if it +# doesn't produce a new line of output for $1 seconds. + +# This script has to be a separate process (rather than just a shell function) +# so killing it doesn't kill the parent process. + +source sources/functions.sh + +if [ $# -lt 1 ] +then + echo "Usage: timeout.sh SECONDS COMMANDS..." >&2 + exit 1 +fi + +trap "killtree $$" EXIT +TIMEOUT="$1" +shift +( eval "$@" ) | tee >(while read -t "$TIMEOUT" -n 32 i; do true; done; sleep 1; kill -TERM $$ 2>/dev/null )
--- a/simple-cross-compiler.sh Sun Jul 04 20:36:50 2010 -0500 +++ b/simple-cross-compiler.sh Mon Jul 05 15:37:04 2010 -0500 @@ -74,7 +74,7 @@ if [ ! -z "$CROSS_SMOKE_TEST" ] then - sources/more/cross-smoke-test.sh "$ARCH" || exit 1 + more/cross-smoke-test.sh "$ARCH" || exit 1 fi # Tar it up
--- a/smoketest.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,25 +0,0 @@ -#!/bin/bash - -. sources/functions.sh || exit 1 - -# This script compiles stuff under the final system, using distcc to call out -# to the cross compiler. It calls run-from-build with a here document. - -# Note that the first line of the script is a few spaces followed by a comment -# character. This gives some harmless data for the linux boot process (serial -# initialization) to consume and discard before it gets to the command prompt. -# (The comment character is just so you can see how much got eaten.) - -# If you cat your own script into emulator-build.sh, you probably also need -# to start with a line of spaces like that. Just FYI. - -sources/more/timeout.sh 60 cd build/system-image-$1 "&&" ./run-emulator.sh << 'EOF' - # -# Show free space -df -# Smoke test for the compiler -gcc -s /usr/src/thread-hello2.c -lpthread -o /tmp/hello && -/tmp/hello -sync -exit -EOF
--- a/sources/more/bisectinate.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,110 +0,0 @@ -#!/bin/bash - -# Find the first breakage (since the last known good version) via git bisect. - -if [ $# -ne 4 ] -then - echo usage: bisectinate PACKAGE repodir@branch start arch >&2 - exit 1 -fi - -# Parse command line options - -PKG="$1" -REPO="${2/@*/}" -BRANCH="${2/*@/}" -[ "$BRANCH" == "$2" ] && BRANCH=master -START="$3" -ARCH="$4" - -TOP="$(pwd)" -[ -z "$SRCDIR" ] && SRCDIR="$TOP/packages" -[ -z "$BUILD" ] && BUILD="$TOP/build" - -if [ ! -d "$REPO/.git" ] -then - echo "No git repo at $REPO" - exit 1 -fi - -# For kernel and busybox bisects, only redo part of the build - -if [ "$PKG" == linux ] && [ -e "$BUILD/root-filesystem-$ARCH".tar.bz2 ] -then - ZAPJUST=system-image -elif [ "$PKG" == busybox ] && - [ -e "$BUILD/simple-cross-compiler-$ARCH.tar.bz2" ] -then - ZAPJUST=root-filesystem -else - ZAPJUST= -fi - -# Start bisecting repository - -mkdir -p "$BUILD"/logs -cd "$REPO" && -git clean -fdx && git checkout -f && -git bisect reset && -git bisect start && -git bisect good "$START" || exit 1 -RESULT="$(git bisect bad "$BRANCH")" -cd "$TOP" - -set -o pipefail - -# Loop through bisection results - -while true -do - echo "$RESULT" - - # Are we done? - - [ ! "$(echo "$RESULT" | head -n 1 | grep "^Bisecting:")" ] && exit - - cd "$REPO" - git show > "$BUILD/logs/test-${ARCH}.txt" - # The "cat" bypasses git's stupid overengineered built-in call to less. - git log HEAD -1 | cat - echo "Testing..." - git archive --prefix="$PKG/" HEAD | bzip2 \ - > "$SRCDIR/alt-$PKG-0.tar.bz2" || exit 1 - cd "$TOP" - - # Perform actual build - - RESULT=bad - - [ ! -z "$ZAPJUST" ] && - rm -rf "$BUILD/${ZAPJUST}-$ARCH"{,.tar.bz2} || - rm -rf "$BUILD"/*-"$ARCH"{,.tar.bz2} - EXTRACT_ALL=yes USE_UNSTABLE="$PKG" ./build.sh "$ARCH" \ - | tee -a "$BUILD"/logs/test-"$ARCH".txt - if [ -e "$BUILD"/system-image-"$ARCH".tar.bz2 ] - then - if [ -z "$LONG" ] - then - RESULT=good - else - rm -rf "$BUILD"/cron-temp/"$ARCH"-dropbearmulti - sources/more/native-static-build.sh "$ARCH" 2>&1 \ - | tee -a "$BUILD"/logs/test-"$ARCH".txt - - [ -e "$BUILD"/cron-temp/"$ARCH"-dropbearmulti ] && RESULT=good - fi - fi - - # If it built, try the native compile - - if [ "$RESULT" == "bad" ] - then - mv "$BUILD"/logs/{test,testfail}-"$ARCH".txt - else - rm "$BUILD"/logs/test-"$ARCH".txt - fi - - cd "$REPO" - RESULT="$(git bisect $RESULT)" - cd "$TOP" -done
--- a/sources/more/build-control-images.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -#!/bin/bash - -# Iterate through sources/native-builds and run each script, writing output -# to build/control-images/$SCRIPTNAME.hdc - -. sources/include.sh || exit 1 - -mkdir -p build/control-images || dienow -for i in sources/native-builds/*.sh -do - SCRIPTNAME=$(echo $i | sed 's@.*/\(.*\)\.sh@\1@') - # Forking doesn't work yet due to extract collisions with the same package - #maybe_fork "$i build/control-images/$SCRIPTNAME.hdc | maybe_quiet" - $i build/control-images/$SCRIPTNAME.hdc | maybe_quiet -done
--- a/sources/more/buildall.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,62 +0,0 @@ -#!/bin/bash - -# Build every target architecture, creating build-$ARCH.txt log files. -# If $FORK is set, build them in parallel. - -. sources/functions.sh || exit 1 - -[ -z "$STATIC_CC_HOST" ] && export STATIC_CC_HOST=i686 - -trap "killtree $$" EXIT - -# Build the host architecture. This has to be built first so the other -# architectures can canadian cross static compilers to run on the host using -# this toolchain to link against a host version of uClibc. - -# This also performs the download.sh and host-tools.sh steps, which don't -# parallelize well if many build.sh instances try to call them at once. - -# If this fails, don't bother trying to build the other targets. - -blank_tempdir build -mkdir -p build/logs && -(EXTRACT_ALL=1 ./download.sh 2>&1 && - ./host-tools.sh 2>&1 && - ./simple-cross-compiler.sh 2>&1 "$STATIC_CC_HOST" || - dienow) | tee build/logs/build-host-cc.txt | maybe_quiet - -cp packages/MANIFEST build || dienow - -# Build all non-hw targets, possibly in parallel - -sources/more/for-each-target.sh \ - './build.sh $TARGET 2>&1 | tee build/logs/build-${TARGET}.txt' - -# Run smoketest.sh for each non-hw target. - -sources/more/for-each-target.sh \ - './smoketest.sh $TARGET 2>&1 | tee build/logs/smoketest-$TARGET.txt' - -sources/more/build-control-images.sh - -# Build all control images - -mkdir -p build/control-images || dienow -for i in sources/native-builds/*.sh -do - X=$(echo $i | sed 's@.*/\(.*\)\.sh@\1@') - # Don't use maybe_fork here, the extract stages conflict. - $i build/control-images/${X}.hdc | maybe_quiet -done - -wait - -# Build static-tools (dropbear and strace) for each target - -mkdir -p build/native-static && -sources/more/for-each-target.sh \ - 'sources/more/timeout.sh 60 "(cd build/system-image-$TARGET && ln -s ../native-static upload && ./native-build.sh ../control-images/static-tools.hdc) | tee build/logs/native-$TARGET.txt"' - -# Create a file containing simple pass/fail results for all architectures. - -sources/more/smoketest-all.sh --logs | tee build/logs/status.txt
--- a/sources/more/cronjob.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,103 +0,0 @@ -#!/bin/bash - -# Build stable versions of all packages with current scripts. - -# This gets run in the aboriginal top directory. - -pull_repo() -{ - # Clone working copy - - rm -rf "packages/alt-$1-0.tar.bz2" build/packages/alt-$1 - mkdir -p build/packages/alt-$1 - pushd build/packages/alt-$1 && - ln -s ../../../repos/$1/.git .git && - git checkout -f master && - git pull - popd -} - -# Expire snapshots directory - -SNAPSHOTS="$(find snapshots -mindepth 1 -maxdepth 1 -type d)" -COUNT=$(( $(echo "$SNAPSHOTS" | wc -l) - 30 )) -if [ "$COUNT" -gt 0 ] -then - # Delete appropriate number of oldest entries, then dead symlinks. - rm -rf $( echo "$SNAPSHOTS" | sort | head -n $COUNT ) - rm -rf $(find -L snapshots -type l) -fi - -echo === Begin cron job - -# Start a new snapshot - -export SNAPSHOT_DATE=$(date +"%Y-%m-%d") -mkdir -p snapshots/$SNAPSHOT_DATE/base && -rm snapshots/latest && -ln -sf $SNAPSHOT_DATE snapshots/latest || exit 1 - -# build base repo - -export FORK=1 -export CROSS_HOST_ARCH=i686 -hg pull -u - -build_snapshot() -{ - if [ -z "$USE_UNSTABLE" ] - then - SNAPNAME=base - else - pull_repo $USE_UNSTABLE - SNAPNAME=$USE_UNSTABLE - fi - - echo === Building snapshot $SNAPNAME - - [ "$USE_UNSTABLE" == linux ] && - sources/more/for-each-arch.sh 'sources/more/migrate-kernel.sh $TARGET' - - # Update manifest - - ./download.sh - - # If it's unchanged, just hardlink the previous binaries instead of rebuilding - - if cmp -s snapshots/latest/$SNAPNAME/MANIFEST packages/MANIFEST - then - cp -rl snapshots/latest/$SNAPNAME/* snapshots/$SNAPSHOT_DATE/$SNAPNAME - return - fi - - # Build it - - nice -n 20 sources/more/buildall.sh - rm build/simple-cross-compiler-*.tar.bz2 - mv build/*.tar.bz2 build/logs build/MANIFEST snapshots/$SNAPSHOT_DATE/$SNAPNAME -} - -build_snapshot base - -echo === Building QEMU - -# build qemu-git - -QPATH="" -CPUS=$(echo /sys/devices/system/cpu/cpu[0-9]* | wc -w) -pull_repo qemu -pushd build/packages/alt-qemu -./configure --disable-werror && -nice -n 20 make -j $CPUS 2>&1 | tee build/logs/alt-qemu.txt && -QPATH="$(for i in *-softmmu;do echo -n $(pwd)/$i:; done)" -popd - -# test all with qemu-git - -[ -z "$QPATH" ] || - PATH="$QPATH:$PATH" sources/more/for-each-target.sh \ - './smoketest.sh $TARGET | tee snapshots/$SNAPSHOT_DATE/base/logs/newqemu-smoketest-$TARGET.txt' - -#USE_UNSTABLE=linux build_snapshot -#USE_UNSTABLE=uClibc build_snapshot -#USE_UNSTABLE=busybox build_snapshot
--- a/sources/more/cross-smoke-test.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -#!/bin/bash - -source sources/include.sh && read_arch_dir "$1" || exit 1 - -# Build statically linked hello world, if necessary - -if [ ! -e "$WORK/hello" ] -then - "${ARCH}-gcc" -Os -static "${SOURCES}/toys/hello.c" -o "$WORK"/hello - - if [ $? -ne 0 ] - then - echo "Compiler doesn't seem to work" >&2 - dienow - fi -fi - -# Attempt to run statically linked hello world - -RESULT="$(PATH="$OLDPATH" qemu-"$QEMU_TEST" "$WORK/hello")" -if [ "$RESULT" == "Hello world!" ] -then - echo "Cross toolchain seems to work." -else - echo "Can't run hello world" >&2 - dienow -fi
--- a/sources/more/for-each-target.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,18 +0,0 @@ -#!/bin/bash - -# Iterate through every target architecture, running rest of command line -# on each $TARGET. - -# If $FORK is set, run them in parallel with filtered output. - -. sources/functions.sh || exit 1 - -[ -z "${ARCHES}" ] && - ARCHES="$(cd sources/targets/; ls | grep -v '^hw-')" - -for TARGET in $ARCHES -do - maybe_fork "$* | maybe_quiet" -done - -wait
--- a/sources/more/migrate-kernel.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,47 +0,0 @@ -#!/bin/bash - -# Calculate new config for UNSTABLE kernel based on stable kernel config. -# I.E. calculate miniconfig-alt-linux based on miniconfig-linux for a target. - -# Expand miniconfig with the old kernel, copy .config to new kernel, run -# make oldconfig, compress to miniconfig, copy to sources/targets/$TARGET - -. sources/include.sh - -read_arch_dir "$1" -rmdir "$STAGE_DIR" - -[ -z "$BOOT_KARCH" ] && BOOT_KARCH="$KARCH" - -# Expand config against current kernel - -USE_UNSTABLE= - -setupfor linux - -cp "$(getconfig linux)" mini.conf || dienow -[ "$SYSIMAGE_TYPE" == "initramfs" ] && - (echo "CONFIG_BLK_DEV_INITRD=y" >> mini.conf || dienow) -make ARCH="$BOOT_KARCH" KCONFIG_ALLCONFIG=mini.conf $LINUX_FLAGS \ - allnoconfig > /dev/null && -cp .config "$WORK" - -cleanup - -USE_UNSTABLE=linux - -setupfor linux - -cp "$WORK/.config" . && -yes "" | make ARCH="$BOOT_KARCH" oldconfig && -mv .config walrus && -ARCH="${BOOT_KARCH}" "$SOURCES/toys/miniconfig.sh" walrus || dienow - -CFG="$CONFIG_DIR/$ARCH_NAME/miniconfig-alt-linux" -if [ -e "$CFG" ] && ! cmp mini.config "$CFG" -then - mv "$CFG" "${CFG}.bak" || dienow -fi -mv mini.config "$CFG" - -cleanup
--- a/sources/more/record-commands.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,23 +0,0 @@ -#!/bin/bash - -source sources/include.sh || exit 1 - -# Build a wrapper that records each command line the build runs out of the -# host's $PATH, so we know exactly what commands the build uses. - -# (Note: this misses things called via absolute paths, such as the #!/bin/bash -# at the start of shell scripts.) - -echo "=== Setting up command recording wrapper" - -[ -f "$WRAPDIR/wrappy" ] && PATH="$OLDPATH" -[ -f "$HOSTTOOLS/busybox" ] && PATH="$HOSTTOOLS" -blank_tempdir "$WRAPDIR" -blank_tempdir "$BUILD/logs" - -# Populate a directory of symlinks with every command in the $PATH. - -path_search "$PATH" "*" 'ln -s wrappy "$WRAPDIR/$FILE"' | dotprogress - -# Build the wrapper -$CC -Os "$SOURCES/toys/wrappy.c" -o "$WRAPDIR/wrappy" || dienow
--- a/sources/more/report-recorded-commands.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,37 +0,0 @@ -#!/bin/bash - -# List the commands used to build each architecture. - -# If given an argument it's the architecture to compare others against, -# which shows just the extra commands used by those other architectures. - -# Mines the output created by build.sh after record-commands.sh. - -COMPARE="$1" - -# Output the list of commands used in a command log. - -function mine_commands() -{ - awk '{print $1}' build/logs/cmdlines.$1.* | sort -u -} - -# Iterate through architectures - -for i in `ls -1 build/logs/cmdlines.* | sed 's@.*/cmdlines\.\([^.]*\).*@\1@' | sort -u` -do - [ "$COMPARE" == "$i" ] && continue - - # Start of new group, announce build stage we're looking at. - echo - echo -n Checking $i: - - if [ -z "$COMPARE" ] - then - # Show all commands in first architecture. - echo $(mine_commands $i) - else - # Show commands that differ from first architecture (if any). - echo $(sort <(mine_commands $COMPARE) <(mine_commands $i) | uniq -u) - fi -done
--- a/sources/more/smoketest-all.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,45 +0,0 @@ -#!/bin/bash - -# Run smoketest script on every build/system-image-* architecture. - -# If $FORK is set, run them in parallel. - -. sources/functions.sh || exit 1 - -if [ "$1" == "--logs" ] -then - for i in build/logs/smoketest-*.txt - do - NAME="$(echo $i | sed 's/.*smoketest-\(.*\)\.txt/\1/')" - echo -n "Testing $NAME:" - RESULT="$(grep 'Hello world!' "$i")" - [ -z "$RESULT" ] && echo "FAIL" || echo "PASS" - done - - exit -fi - -function dotest() -{ - [ -z "$FORK" ] && echo -n "Testing $1:" - [ ! -z "$VERBOSE" ] && VERBOSITY="tee >(cat >&2) |" - RESULT="$(./smoketest.sh "$1" 2>&1 | eval "$VERBOSITY grep 'Hello world!'")" - [ -z "$RESULT" ] && RESULT="FAIL" || RESULT="PASS" - [ -z "$FORK" ] && echo "$RESULT" || echo "Testing $1:$RESULT" - rm -f build/system-image-"$1"/hdb.img 2>/dev/null -} - -# Test all non-hw targets to see whether or not they can compile and run -# the included "hello world" program. - -for i in $(ls -d sources/targets/* | sed 's@.*/@@' | grep -v "^hw-") -do - if [ -e "build/system-image-$i" ] - then - maybe_fork "dotest $i" - else - echo "Testing $i:NONE" - fi -done - -wait
--- a/sources/more/test.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,16 +0,0 @@ -#!/bin/bash - -# Run a command with sources/include.sh and an architecture loaded - -if [ $# -eq 0 ] -then - echo "Usage: test.sh ARCH COMMAND..." >&2 - echo "You generally want to set STAGE_NAME= too." - exit 1 -fi - -. sources/include.sh || exit 1 - -read_arch_dir "$1" -shift -eval "$@"
--- a/sources/more/timeout.sh Sun Jul 04 20:36:50 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,20 +0,0 @@ -#!/bin/bash - -# Run a command line with a hang timeout, which kills the child process if it -# doesn't produce a new line of output for $1 seconds. - -# This script has to be a separate process (rather than just a shell function) -# so killing it doesn't kill the parent process. - -source sources/functions.sh - -if [ $# -lt 1 ] -then - echo "Usage: timeout.sh SECONDS COMMANDS..." >&2 - exit 1 -fi - -trap "killtree $$" EXIT -TIMEOUT="$1" -shift -( eval "$@" ) | tee >(while read -t "$TIMEOUT" -n 32 i; do true; done; sleep 1; kill -TERM $$ 2>/dev/null )