From 95b79373bac8a8d17f2b6bc72842680fe3cbb5c3 Mon Sep 17 00:00:00 2001 From: Rob Landley Date: Sun, 15 May 2022 19:39:27 -0500 Subject: [PATCH] Clean up build plumbing. --- configure | 2 +- scripts/change.sh | 10 +-- scripts/make.sh | 153 +++++++++++++++++------------------------ scripts/portability.sh | 3 + 4 files changed, 72 insertions(+), 96 deletions(-) diff --git a/configure b/configure index d511d948..6dc2a49b 100755 --- a/configure +++ b/configure @@ -12,7 +12,7 @@ then fi # Warn about stuff, disable stupid warnings, be 8-bit clean for utf8. -CFLAGS="$CFLAGS -Wall -Wundef -Werror=implicit-function-declaration -Wno-char-subscripts -Wno-pointer-sign -Wno-string-plus-int -funsigned-char" +CFLAGS+=" -Wall -Wundef -Werror=implicit-function-declaration -Wno-char-subscripts -Wno-pointer-sign -Wno-string-plus-int -funsigned-char" # Set default values if variable not already set : ${CC:=cc} ${HOSTCC:=cc} ${GENDIR:=generated} ${KCONFIG_CONFIG:=.config} diff --git a/scripts/change.sh b/scripts/change.sh index 74889aa7..dda5ee14 100755 --- a/scripts/change.sh +++ b/scripts/change.sh @@ -9,15 +9,11 @@ ${HOSTCC:-cc} -I . scripts/install.c -o "$UNSTRIPPED"/instlist && export PREFIX=${PREFIX:-change/} && mkdir -p "$PREFIX" || exit 1 -# Build all the commands standalone except: - -# sh - shell builtins like "cd" and "exit" need the multiplexer -# help - needs to know what other commands are enabled (use command --help) - -for i in $("$UNSTRIPPED"/instlist | egrep -vw "sh|help") +# Build all the commands standalone +for i in $("$UNSTRIPPED"/instlist) do echo -n " $i" && - scripts/single.sh $i > /dev/null 2>$PREFIX/${i}.bad && + scripts/single.sh $i &>$PREFIX/${i}.bad && rm $PREFIX/${i}.bad || echo -n '*' done echo diff --git a/scripts/make.sh b/scripts/make.sh index b7bb6930..cd0715f2 100755 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -1,62 +1,49 @@ #!/bin/bash # Grab default values for $CFLAGS and such. - set -o pipefail source scripts/portability.sh - -# Default to running one more parallel cc instance than we have processors -: ${CPUS:=$(($(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null)+1))} +mkdir -p "$UNSTRIPPED" # Respond to V= by echoing command lines as well as running them -DOTPROG= +unset DOTPROG do_loudly() { - [ ! -z "$V" ] && echo "$@" || echo -n "$DOTPROG" + [ -n "$V" ] && echo "$@" || echo -n "$DOTPROG" "$@" } -# Is anything under directory $2 newer than file $1 +# Is anything under directory $2 newer than generated/$1 (or does it not exist)? isnewer() { - CHECK="$1" - shift - [ ! -z "$(find "$@" -newer "$CHECK" 2>/dev/null || echo yes)" ] + find "${@:2}" -newer "$GENDIR/$1" &>/dev/null && return 1 + [ "$((DIDNEWER++))" -eq 0 ] && echo -n "$GENDIR/{" || echo -n , + echo -n $1 + return 0 } -echo "Generate headers from toys/*/*.c..." - -mkdir -p "$UNSTRIPPED" - -if isnewer "$GENDIR"/Config.in toys || isnewer "$GENDIR"/Config.in Config.in -then - echo "Extract configuration information from toys/*.c files..." - scripts/genconfig.sh -fi - -# Create a list of all the commands toybox can provide. Note that the first -# entry is out of order on purpose (the toybox multiplexer command must be the -# first element of the array). The rest must be sorted in alphabetical order -# for fast binary search. - -if isnewer "$GENDIR"/newtoys.h toys -then - echo -n "$GENDIR/newtoys.h " - - echo "USE_TOYBOX(NEWTOY(toybox, NULL, TOYFLAG_STAYROOT|TOYFLAG_NOHELP))" \ - > "$GENDIR"/newtoys.h - $SED -n -e 's/^USE_[A-Z0-9_]*(/&/p' toys/*/*.c \ - | $SED 's/\(.*TOY(\)\([^,]*\),\(.*\)/\2 \1\2,\3/' | sort -s -k 1,1 \ - | $SED 's/[^ ]* //' >> "$GENDIR"/newtoys.h - [ $? -ne 0 ] && exit 1 -fi +# Build a tool that runs on the host +hostcomp() +{ + if [ ! -f "$UNSTRIPPED"/$1 ] || [ "$UNSTRIPPED"/$1 -ot scripts/$1.c ] + then + do_loudly $HOSTCC scripts/$1.c -o "$UNSTRIPPED"/$1 || exit 1 + fi +} -[ ! -z "$V" ] && echo "Which C files to build..." +# Used once for dependency checking, then again to write build.sh +genbuildsh() +{ + LLINK="$(echo $LDOPTIMIZE $LDFLAGS $(cat "$GENDIR"/optlibs.dat))" + echo -e "#!/bin/sh\n\nPATH='$PATH'\nBUILD='$BUILD'\nLINK='$LLINK'\n" + echo -e "\$BUILD lib/*.c $TOYFILES \$LINK -o $OUTNAME" +} # Extract a list of toys/*/*.c files to compile from the data in $KCONFIG_CONFIG # (First command names, then filenames with relevant {NEW,OLD}TOY() macro.) -[ -d ".git" ] && [ ! -z "$(which git 2>/dev/null)" ] && +[ -n "$V" ] && echo -e "\nWhich C files to build..." +[ -d ".git" ] && [ -n "$(which git 2>/dev/null)" ] && GITHASH="-DTOYBOX_VERSION=\"$(git describe --tags --abbrev=12 2>/dev/null)\"" TOYFILES="$($SED -n 's/^CONFIG_\([^=]*\)=.*/\1/p' "$KCONFIG_CONFIG" | xargs | tr ' [A-Z]' '|[a-z]')" TOYFILES="main.c $(egrep -l "TOY[(]($TOYFILES)[ ,]" toys/*/*.c | xargs)" @@ -67,17 +54,9 @@ then echo -e "\n\033[1;31mwarning: using unfinished code from toys/pending\033[0m" fi -genbuildsh() -{ - # Write a canned build line for use on crippled build machines. - - LLINK="$(echo $LDOPTIMIZE $LDFLAGS $(cat "$GENDIR"/optlibs.dat))" - echo -e "#!/bin/sh\n\nPATH='$PATH'\nBUILD='$BUILD'\nLINK='$LLINK'\n" - echo -e "\$BUILD lib/*.c $TOYFILES \$LINK -o $OUTNAME" -} - +# Probe library list if our compiler/linker options changed if ! cmp -s <(genbuildsh 2>/dev/null | head -n 5) \ - <(head -n 5 "$GENDIR"/build.sh 2>/dev/null | $SED '5s/ -o .*//') + <(head -n 5 "$GENDIR"/build.sh 2>/dev/null) then echo -n "Library probe" @@ -88,22 +67,38 @@ then > "$GENDIR"/optlibs.dat for i in util crypt m resolv selinux smack attr crypto z log iconv tls ssl do - echo "int main(int argc, char *argv[]) {return 0;}" | \ - ${CROSS_COMPILE}${CC} $CFLAGS $LDFLAGS -xc - -o "$GENDIR"/libprobe -l$i > /dev/null 2>/dev/null && - echo -l$i >> "$GENDIR"/optlibs.dat + ${CROSS_COMPILE}${CC} $CFLAGS $LDFLAGS -xc - -o "$UNSTRIPPED"/libprobe -l$i\ + &>/dev/null <<<"int main(int argc, char *argv[]) {return 0;}" && + echo -l$i >> "$GENDIR"/optlibs.dat echo -n . done - rm -f "$GENDIR"/libprobe + rm -f "$UNSTRIPPED"/libprobe echo fi +# Write a canned build line for use on crippled build machines. genbuildsh > "$GENDIR"/build.sh && chmod +x "$GENDIR"/build.sh || exit 1 -#TODO: "make $SED && make" doesn't regenerate config.h because diff .config -if true #isnewer "$GENDIR"/config.h "$KCONFIG_CONFIG" +if isnewer Config.in toys || isnewer Config.in Config.in +then + scripts/genconfig.sh +fi + +# Create a list of all the commands toybox can provide. +if isnewer newtoys.h toys then - echo "Make $GENDIR/config.h from $KCONFIG_CONFIG." + # The multiplexer is the first element in the array + echo "USE_TOYBOX(NEWTOY(toybox, 0, TOYFLAG_STAYROOT|TOYFLAG_NOHELP))" \ + > "$GENDIR"/newtoys.h + # Sort rest by name for binary search (copy name to front, sort, remove copy) + $SED -n 's/^\(USE_[^(]*(.*TOY(\)\([^,]*\)\(,.*\)/\2 \1\2\3/p' toys/*/*.c \ + | sort -s -k 1,1 | $SED 's/[^ ]* //' >> "$GENDIR"/newtoys.h + [ $? -ne 0 ] && exit 1 +fi +#TODO: "make $SED && make" doesn't regenerate config.h because diff .config +if true #isnewer config.h "$KCONFIG_CONFIG" +then # This long and roundabout sed invocation is to make old versions of sed # happy. New ones have '\n' so can replace one line with two without all # the branches and tedious mucking about with hyperspace. @@ -130,20 +125,14 @@ then $KCONFIG_CONFIG > "$GENDIR"/config.h || exit 1 fi -if [ ! -f "$GENDIR"/mkflags ] || [ "$GENDIR"/mkflags -ot scripts/mkflags.c ] -then - do_loudly $HOSTCC scripts/mkflags.c -o "$UNSTRIPPED"/mkflags || exit 1 -fi - # Process config.h and newtoys.h to generate FLAG_x macros. Note we must # always #define the relevant macro, even when it's disabled, because we # allow multiple NEWTOY() in the same C file. (When disabled the FLAG is 0, # so flags&0 becomes a constant 0 allowing dead code elimination.) -if isnewer "$GENDIR"/flags.h toys "$KCONFIG_CONFIG" +hostcomp mkflags +if isnewer flags.h toys "$KCONFIG_CONFIG" then - echo -n "$GENDIR/flags.h " - # Parse files through C preprocessor twice, once to get flags for current # .config and once to get flags for allyesconfig for I in A B @@ -192,13 +181,12 @@ function getglobals() DATA="$($SED -n -e '/^GLOBALS(/,/^)/b got;b;:got' \ -e 's/^GLOBALS(/_data {/' \ -e 's/^)/};/' -e 'p' $i)" - [ ! -z "$DATA" ] && echo -e "// $i\n\nstruct $NAME$DATA\n" + [ -n "$DATA" ] && echo -e "// $i\n\nstruct $NAME$DATA\n" done } -if isnewer "$GENDIR"/globals.h toys +if isnewer globals.h toys then - echo -n "$GENDIR/globals.h " GLOBSTRUCT="$(getglobals)" ( echo "$GLOBSTRUCT" @@ -210,33 +198,24 @@ then ) > "$GENDIR"/globals.h fi -if [ ! -f "$UNSTRIPPED"/mktags ] || [ "$UNSTRIPPED"/mktags -ot scripts/mktags.c ] -then - do_loudly $HOSTCC scripts/mktags.c -o "$UNSTRIPPED"/mktags || exit 1 -fi - -if isnewer "$GENDIR"/tags.h toys +hostcomp mktags +if isnewer tags.h toys then - echo -n "$GENDIR/tags.h " - $SED -n '/TAGGED_ARRAY(/,/^)/{s/.*TAGGED_ARRAY[(]\([^,]*\),/\1/;p}' \ toys/*/*.c lib/*.c | "$UNSTRIPPED"/mktags > "$GENDIR"/tags.h fi -if [ ! -f "$UNSTRIPPED"/config2help ] || [ "$UNSTRIPPED"/config2help -ot scripts/config2help.c ] -then - do_loudly $HOSTCC scripts/config2help.c -o "$UNSTRIPPED"/config2help || exit 1 -fi -if isnewer "$GENDIR"/help.h "$GENDIR"/Config.in +hostcomp config2help +if isnewer help.h "$GENDIR"/Config.in then - echo "$GENDIR/help.h" "$UNSTRIPPED"/config2help Config.in $KCONFIG_CONFIG > "$GENDIR"/help.h || exit 1 fi +[ -z "$DIDNEWER" ] || echo } -[ ! -z "$NOBUILD" ] && exit 0 +[ -n "$NOBUILD" ] && exit 0 echo -n "Compile $OUTNAME" -[ ! -z "$V" ] && echo +[ -n "$V" ] && echo DOTPROG=. # This is a parallel version of: do_loudly $BUILD $FILES $LLINK || exit 1 @@ -244,7 +223,7 @@ DOTPROG=. # Any headers newer than the oldest generated/obj file? X="$(ls -1t "$GENDIR"/obj/* 2>/dev/null | tail -n 1)" # TODO: redo this -if [ ! -e "$X" ] || [ ! -z "$(find toys -name "*.h" -newer "$X")" ] +if [ ! -e "$X" ] || [ -n "$(find toys -name "*.h" -newer "$X")" ] then rm -rf "$GENDIR"/obj && mkdir -p "$GENDIR"/obj || exit 1 else @@ -266,9 +245,7 @@ do OUT="$GENDIR/obj/${X%%.c}.o" LNKFILES="$LNKFILES $OUT" - # Library files don't need to be rebuilt if older than .config. - # ($TOYFILES contents can depend on CONFIG symbols, lib/*.c never should.) - + # Library files don't get rebuilt if older than .config, commands do. [ "$OUT" -nt "$i" ] && [ -z "$CLICK" -o "$OUT" -nt "$KCONFIG_CONFIG" ] && continue @@ -278,8 +255,8 @@ do [ $((++COUNT)) -ge $CPUS ] && { wait $DASHN; DONE=$?; : $((--COUNT)); } [ $DONE -ne 0 ] && break done -# wait for all background jobs, detecting errors +# wait for all background jobs, detecting errors while [ $((COUNT--)) -gt 0 ] do wait $DASHN; @@ -289,7 +266,7 @@ done UNSTRIPPED="$UNSTRIPPED/${OUTNAME/*\//}" do_loudly $BUILD $LNKFILES $LLINK -o "$UNSTRIPPED" || exit 1 -if [ ! -z "$NOSTRIP" ] || +if [ -n "$NOSTRIP" ] || ! do_loudly ${CROSS_COMPILE}${STRIP} "$UNSTRIPPED" -o "$OUTNAME" then [ -z "$NOSTRIP" ] && echo "strip failed, using unstripped" diff --git a/scripts/portability.sh b/scripts/portability.sh index 3af2be5f..e59c76c5 100644 --- a/scripts/portability.sh +++ b/scripts/portability.sh @@ -34,6 +34,9 @@ if [ ! -z "$ASAN" ]; then unset ASAN fi +# Probe number of available processors, and add one. +: ${CPUS:=$(($(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null)+1))} + # Centos 7 bug workaround, EOL June 30 2024. TODO DASHN=-n; wait -n 2>/dev/null; [ $? -eq 2 ] && unset DASHN -- 2.39.2