changeset 972:cfa6262528f3

Genericize native build. Break out dropbear and strace packages from download.sh (and clean up what remains, mirror list is now in include.sh). New script setup-native.build.sh downloads, extracts, and creates a system image with "here document" build script to build dropbear and strace. New native-build.sh script works on arbitrary hdc image, providing FTP upload to a directory. The setup/fork logic to build all architectures is now in buildall.sh for this, just like everything else. Update buildall.sh to take optional list of architectures to build on command line. Factor out unique-port.sh (in embeddable manner) so distcc (in run-emulator.sh) and ftp server (in native-build.sh) can both use it. Upgrade to new busybox version while we're there. HGi removed sources/more/setup-native-static-build.sh
author Rob Landley <rob@landley.net>
date Fri, 05 Feb 2010 10:33:06 -0600
parents 9488f5ad5029
children cfeaae1c55ed
files download.sh sources/include.sh sources/more/buildall.sh sources/more/native-build.sh sources/more/native-static-build.sh sources/more/setup-native-build.sh sources/more/setup-native-static-build.sh sources/toys/run-emulator.sh sources/toys/unique-port.sh sources/trimconfig-busybox system-image.sh
diffstat 11 files changed, 195 insertions(+), 180 deletions(-) [+]
line wrap: on
line diff
--- a/download.sh	Fri Feb 05 07:11:11 2010 -0600
+++ b/download.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -11,10 +11,6 @@
 
 echo "=== Download source code."
 
-# List of fallback mirrors for these files
-
-MIRROR_LIST="http://impactlinux.com/firmware/mirror http://landley.net/code/firmware/mirror http://127.0.0.1/code/firmware/mirror"
-
 # Note: set SHA1= blank to skip checksum validation.
 
 # A blank SHA1 value means accept anything, and the download script
@@ -60,8 +56,8 @@
 # BusyBox.  Adding a native toolchain requires binutils and gcc (above) plus
 # make and bash.
 
-URL=http://www.busybox.net/downloads/busybox-1.15.2.tar.bz2 \
-SHA1=2f396a4cb35db438a9b4af43df6224f343b8a7ae \
+URL=http://www.busybox.net/downloads/busybox-1.16.0.tar.bz2 \
+SHA1=727f6280729cd9e819ae2bb0065b9cd12a27efb1 \
 UNSTABLE=http://busybox.net/downloads/busybox-snapshot.tar.bz2 \
 download || dienow
 
@@ -92,14 +88,6 @@
 SHA1=30663e8ff94f13c0553fbfb928adba91814e1b3a \
 download || dienow
 
-URL=http://downloads.sf.net/sourceforge/strace/strace-4.5.19.tar.bz2 \
-SHA1=5554c2fd8ffae5c1e2b289b2024aa85a0889c989 \
-download || dienow
-
-URL=http://matt.ucc.asn.au/dropbear/releases/dropbear-0.52.tar.bz2 \
-SHA1=8c1745a9b64ffae79f28e25c6fe9a8b96cac86d8 \
-download || dienow
-
 # The following packages are built and run on the host only.  (host-tools.sh
 # also builds host versions of many packages in the native root filesystem,
 # but the following packages are not cross compiled for the target, and thus
@@ -118,14 +106,9 @@
 RENAME="s/(squashfs)(.*)/\1-\2/" \
 download || dienow
 
-# Todo:
-
-# ftp://ftp.denx.de/pub/u-boot/u-boot-1.2.0.tar.bz2
-
 echo === Got all source.
 
-rm -f "$SRCDIR"/MANIFEST
-
+rm -f "$SRCDIR"/MANIFEST  # So cleanup_oldfiles doesn't warn about it.
 cleanup_oldfiles
 
 # Create a MANIFEST file listing package versions.
@@ -135,8 +118,4 @@
 # are intentionally excluded from the new path setup by host-tools.sh, so
 # just in case we've already run that use $OLDPATH for this.
 
-blank_tempdir "$WORK"
 PATH="$OLDPATH" do_readme > "$SRCDIR"/MANIFEST || dienow
-
-# Set color back to normal.
-echo -e "\e[0m"
--- a/sources/include.sh	Fri Feb 05 07:11:11 2010 -0600
+++ b/sources/include.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -9,6 +9,10 @@
 
 source sources/functions.sh
 
+# List of fallback mirrors to download package source from
+
+MIRROR_LIST="http://impactlinux.com/firmware/mirror http://landley.net/code/firmware/mirror http://127.0.0.1/code/firmware/mirror"
+
 # Where are our working directories?
 
 TOP=`pwd`
--- a/sources/more/buildall.sh	Fri Feb 05 07:11:11 2010 -0600
+++ b/sources/more/buildall.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -8,11 +8,16 @@
 [ -z "$STATIC_CC_HOST" ] && export STATIC_CC_HOST=i686
 export FAIL_QUIET=1
 
-[ -z "${ARCHES}" ] &&
-  ARCHES="$(cd sources/targets/; ls | grep -v '^hw-')"
+if [ -z "$*" ]
+then
+  [ -z "${ARCHES}" ] &&
+    ARCHES="$(cd sources/targets/; ls | grep -v '^hw-')"
 
-[ -z "$HWARCHES" ] &&
-  HWARCHES="$(cd sources/targets; ls | grep '^hw-')"
+  [ -z "$HWARCHES" ] &&
+    HWARCHES="$(cd sources/targets; ls | grep '^hw-')"
+else
+  ARCHES="$*"
+fi
 
 [ ! -z "$FORK" ] && QUIET=1
 
@@ -61,9 +66,14 @@
 
 wait
 
-# This one has to do its own forking to avoid redundantly creating hdc.sqf
+# Build dropbear and strace
 
-sources/more/native-static-build.sh
+sources/more/setup-native-build.sh build/host-temp/hdc.sqf &&
+mkdir -p build/native-static &&
+for i in ${ARCHES}
+do
+  maybe_fork "sources/more/native-build.sh $i build/host-temp/hdc.sqf build/native-static | tee build/logs/native-$i.txt | maybe_quiet"
+done
 
 # Create a file containing simple pass/fail results for all architectures.
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sources/more/native-build.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# setup hdc.sqf
+# find build -name "hdb.img" | xargs rm
+
+# Launch a system image under the emulator under the control of a filesystem
+# image, with an FTP server to upload results to.
+
+. sources/functions.sh || exit 1
+
+# Parse arguments
+
+if [ $# -ne 3 ]
+then
+  echo "usage: $0 ARCH HDCFILE OUTPUTDIR" >&2
+  exit 1
+fi
+
+ARCH="$1"
+if [ ! -f "$2" ]
+then
+  echo "Filesystem image $2 missing" >&2
+  exit 1
+fi
+HDCFILE="$(readlink -f $2)"
+mkdir -p "$3" || dienow
+STAGE_DIR="$(readlink -f $3)"
+
+# Fire off the ftp daemon, making sure it's killed when this script exits
+
+. sources/toys/unique-port.sh || exit 1
+PORT=$(unique_port)
+build/host/netcat -s 127.0.0.1 -p $PORT -L build/host/ftpd -w "$STAGE_DIR" &
+trap "kill $(jobs -p)" EXIT
+disown $(jobs -p)
+
+# Run emulator as a child process, feeding in -hdc and some extra environment
+# variables so it auto-launches the build process.
+
+echo === Begin native build for $ARCH
+
+rm -f sources/system-image-"$ARCH"/hdb.img
+HDC="$HDCFILE" KERNEL_EXTRA="OUTPORT=$PORT ARCH=$ARCH" \
+  sources/timeout.sh 60 ./run-from-build.sh "$ARCH"
+
+echo === End native build for $ARCH
--- a/sources/more/native-static-build.sh	Fri Feb 05 07:11:11 2010 -0600
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,53 +0,0 @@
-#!/bin/bash
-
-# Launch a system image under the emulator and natively build static versions
-# of a few packages.
-
-# Takes the name of the architecture to build for as its first argument,
-# with no arguments builds all architectures in build/system-image-*
-# (If $FORK is set, run them in parallel.)
-
-. sources/functions.sh || exit 1
-
-STAGE_DIR="$(pwd)/build/cron-temp"
-
-# Create an hdc image with the source code and build script
-
-sources/more/setup-native-static-build.sh "$STAGE_DIR"
-
-# Fire off the ftp daemon, making sure it's killed when this script exits
-
-build/host/netcat -s 127.0.0.1 -p 9876 -L build/host/ftpd -w "$STAGE_DIR" &
-trap "kill $(jobs -p)" EXIT
-disown $(jobs -p)
-
-find build -name "hdb.img" | xargs rm
-
-# Run emulator as a child process, feeding in -hdc and some extra environment
-# variables so it auto-launches the build process.
-
-function do_arch()
-{
-  set_titlebar "Native build for $1"
-  HDC="$STAGE_DIR/hdc.sqf" KERNEL_EXTRA="OUTPORT=9876 ARCH=$1" \
-  sources/timeout.sh 60 ./run-from-build.sh $1
-}
-
-# If we have a command line argument, build just that arch, otherwise build
-# all arches that managed to create a system image.
-
-if [ ! -z "$1" ]
-then
-  do_arch "$1"
-else
-  mkdir -p build/logs
-
-  for i in $(ls build/system-image-*.tar.bz2 | sed 's@build/system-image-\(.*\)\.tar\.bz2@\1@' | grep -v system-image-hw-)
-  do
-    maybe_fork "do_arch $i | tee build/logs/native-static-$i.txt | maybe_quiet"
-  done
-
-  wait
-fi
-
-echo End of native build
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sources/more/setup-native-build.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Download all the source tarballs we haven't got up-to-date copies of.
+
+# The tarballs are downloaded into the "packages" directory, which is
+# created as needed.
+
+source sources/include.sh || exit 1
+
+if [ $# -ne 1 ]
+then
+  echo "usage: $0 FILENAME" >&2
+  exit 1
+fi
+
+if [ -e "$1" ]
+then
+  echo "$1" exists
+  exit 0
+fi
+
+SRCDIR="$SRCDIR/native"
+mkdir -p "$SRCDIR" || dienow
+
+echo "=== Download source code."
+
+# Note: set SHA1= blank to skip checksum validation.
+
+URL=http://downloads.sf.net/sourceforge/strace/strace-4.5.19.tar.bz2 \
+SHA1=5554c2fd8ffae5c1e2b289b2024aa85a0889c989 \
+download || dienow
+
+URL=http://matt.ucc.asn.au/dropbear/releases/dropbear-0.52.tar.bz2 \
+SHA1=8c1745a9b64ffae79f28e25c6fe9a8b96cac86d8 \
+download || dienow
+
+echo === Got all source.
+
+cleanup_oldfiles
+
+# Set up working directories
+
+WORK="$WORK"/sub
+blank_tempdir "$WORK"
+
+# Extract source code into new image directory
+
+setupfor dropbear
+setupfor strace
+
+cat > "$WORK"/init << 'EOF' || dienow
+#!/bin/bash
+
+echo Started second stage init
+
+cd /home &&
+mkdir output &&
+
+echo === Native build static dropbear
+
+cp -sfR /mnt/dropbear dropbear &&
+cd dropbear &&
+LDFLAGS="--static" ./configure --disable-zlib &&
+make -j $CPUS PROGRAMS="dropbear dbclient dropbearkey dropbearconvert scp" MULTI=1 SCPPROGRESS=1 &&
+cp dropbearmulti /home/output &&
+cd .. &&
+rm -rf dropbear || exit 1
+
+echo === Native build static strace
+
+cp -sfR /mnt/strace strace &&
+cd strace &&
+CFLAGS="--static" ./configure &&
+make -j $CPUS &&
+cp strace /home/output &&
+cd .. &&
+rm -rf strace || dienow
+
+echo === Upload
+
+cd /home/output
+for i in *
+do
+  ftpput 10.0.2.2 -P $OUTPORT $ARCH-$i $i
+done
+
+sync
+
+EOF
+
+chmod +x "$WORK"/init || dienow
+
+cd "$TOP"
+
+mksquashfs "$WORK" "$1" -noappend -all-root
--- a/sources/more/setup-native-static-build.sh	Fri Feb 05 07:11:11 2010 -0600
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,73 +0,0 @@
-#!/bin/bash
-
-# Create hdc image to build dropbear and strace statically.
-
-. sources/include.sh
-
-if [ -z "$1" ]
-then
-  print "Need directory name" >&2
-  exit 1
-fi
-
-if [ -e "$1/hdc.sqf" ]
-then
-  echo "$1/hdc.sqf" exists
-  exit 0
-fi
-
-# Set up working directories
-
-WORK="$1"
-blank_tempdir "$WORK"
-WORK="$WORK"/sub
-mkdir -p "$WORK" || dienow
-
-# Extract source code into new image directory
-
-setupfor dropbear
-setupfor strace
-
-cat > "$WORK"/init << 'EOF' || dienow
-#!/bin/bash
-
-echo Started second stage init
-
-cd /home &&
-mkdir output &&
-
-echo === Native build static dropbear
-
-cp -sfR /mnt/dropbear dropbear &&
-cd dropbear &&
-LDFLAGS="--static" ./configure --disable-zlib &&
-make -j $CPUS PROGRAMS="dropbear dbclient dropbearkey dropbearconvert scp" MULTI=1 SCPPROGRESS=1 &&
-cp dropbearmulti /home/output &&
-cd .. &&
-rm -rf dropbear || exit 1
-
-echo === Native build static strace
-
-cp -sfR /mnt/strace strace &&
-cd strace &&
-CFLAGS="--static" ./configure &&
-make -j $CPUS &&
-cp strace /home/output &&
-cd .. &&
-rm -rf strace || dienow
-
-echo === Upload
-
-cd /home/output
-for i in *
-do
-  ftpput 10.0.2.2 -P $OUTPORT $ARCH-$i $i
-done
-
-sync
-
-EOF
-
-chmod +x "$WORK"/init || dienow
-
-mksquashfs "$WORK" "$WORK"/../hdc.sqf -noappend -all-root
--- a/sources/toys/run-emulator.sh	Fri Feb 05 07:11:11 2010 -0600
+++ b/sources/toys/run-emulator.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -1,4 +1,6 @@
-#!/bin/bash
+# The ARCH gets substituted in here by system-image.sh
+
+ARCH=
 
 # The following environment variables affect the behavior of this script:
 
@@ -39,28 +41,7 @@
 
 [ ! -z "$HDC" ] && [ -e "$HDC" ] && WITH_HDC="-hdc $HDC"
 
-# Start of distcc setup
-
-unique_port()
-{
-  # Try to find a unique port number for each running instance of the program.
-
-  # To reduce the chance of the port already being in use by another program,
-  # we use a range either before or after that used by normal programs, but
-  # beyond that allocated to most persistent demons.  There's a small chance
-  # even these ports are already in use, but this at least prevents
-  # simultaneous run-emulator instances for different targets from
-  # trivially interfering with each other.
-
-  START=8192
-  RANGE=$[$(awk '{print $1}' /proc/sys/net/ipv4/ip_local_port_range)-$START]
-  if [ $RANGE -lt 1 ]
-  then
-    START=$[$(awk '{print $2}' /proc/sys/net/ipv4/ip_local_port_range)]
-    RANGE=$[65535-$START]
-  fi
-  echo $[($$%$RANGE)+$START]
-}
+# Setup distcc
 
 # The ARCH gets substituted in here by system-image.sh
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sources/toys/unique-port.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+unique_port()
+{
+  # Try to find a unique port number for each running instance of the program.
+
+  # To reduce the chance of the port already being in use by another program,
+  # we use a range either before or after that used by normal programs, but
+  # beyond that allocated to most persistent demons.  There's a small chance
+  # even these ports are already in use, but this at least prevents
+  # simultaneous run-emulator instances for different targets from
+  # trivially interfering with each other.
+
+  START=8192
+  RANGE=$[$(awk '{print $1}' /proc/sys/net/ipv4/ip_local_port_range)-$START]
+  if [ $RANGE -lt 1 ]
+  then
+    START=$[$(awk '{print $2}' /proc/sys/net/ipv4/ip_local_port_range)]
+    RANGE=$[65535-$START]
+  fi
+  echo $[($$%$RANGE)+$START]
+}
--- a/sources/trimconfig-busybox	Fri Feb 05 07:11:11 2010 -0600
+++ b/sources/trimconfig-busybox	Fri Feb 05 10:33:06 2010 -0600
@@ -3,6 +3,9 @@
 # These are provided by other packages, because the busybox ones don't
 # work yet.
 
+# No ext2 support (tune2fs -j)
+CONFIG_MKFS_EXT2=n
+
 # binutils (no write support)
 CONFIG_AR=n
 # toybox
--- a/system-image.sh	Fri Feb 05 07:11:11 2010 -0600
+++ b/system-image.sh	Fri Feb 05 10:33:06 2010 -0600
@@ -180,7 +180,8 @@
 QEMU_MEMORY=256 HDB=hdb.img HDBMEGS=2048 ./run-emulator.sh
 EOF
 chmod +x "$STAGE_DIR/dev-environment.sh" &&
-sed -e 's/^ARCH=.*/ARCH='"$ARCH"/  "$SOURCES/toys/run-emulator.sh" > \
+cp "$SOURCES/toys/unique-port.sh" "$STAGE_DIR/run-emulator.sh" &&
+sed -e 's/^ARCH=.*/ARCH='"$ARCH"/  "$SOURCES/toys/run-emulator.sh" >> \
   "$STAGE_DIR/run-emulator.sh" &&
 chmod +x "$STAGE_DIR/run-emulator.sh" &&
 emulator_command "$IMAGE" zImage-$ARCH >> "$STAGE_DIR/run-emulator.sh"