changeset 83:13a980b307f8

Largeish rewrite of make.sh, breaking things out into functions. This has the advantage of letting subsets of this functionality easily be called by new wrapper scripts (instead of turning make.sh into a nest of --options), but doesn't leave us with 8 gazillion tiny little scripts that share no common code. (Probably buggy, this is just the first good stopping point.)
author Rob Landley <rob@landley.net>
date Wed, 07 Nov 2007 01:43:39 -0600
parents a80ed9124e61
children 598061b944e5
files make/functions.sh make/make.sh
diffstat 2 files changed, 288 insertions(+), 72 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/make/functions.sh	Wed Nov 07 01:43:39 2007 -0600
@@ -0,0 +1,254 @@
+#!/bin/bash
+
+# This file defines functions used by make.sh
+
+# Safety check: Only define these functions if WEBDIR and LNXDIR defined.
+
+if [ -z "$WEBDIR" ] || [ -z "$LNXDIR" ]
+then
+  echo set WEBDIR and LNXDIR
+  exit 1
+fi
+
+# Update Documentation directory
+
+function update_documentation_dir()
+{
+  rm -rf $WEBDIR/Documentation/
+  cp -a $LNXDIR/hg/Documentation $WEBDIR/Documentation
+  # Make the change directory local to this function
+  (
+    cd $WEBDIR
+    for i in make/patches/*.patch
+    do
+      echo $i
+      patch -p1 -i $i || exit 1
+    done
+    make/docdiridx.py
+  )
+}
+
+# Clone or update mercurial Linux kernel repository from kernel.org
+
+function get_hg()
+{
+  # Make the chdir local to this function
+  (
+    if [ ! -d "$LNXDIR" ]
+    then
+      echo "Clone Linux kernel repository from kernel.org"
+
+      mkdir -p "$LNXDIR"/hg &&
+      cd "$LNXDIR"/hg &&
+      hg clone http://kernel.org/hg/linux-2.6 hg
+    else
+      # Update kernel mercurial repository.
+      cd "$LNXDIR"/hg &&
+      echo "Update Linux kernel from kernel.org" &&
+      hg pull -u
+    fi
+    [ $? -ne 0 ] && exit 1
+
+    echo "Linux version: $(hg tags | head -n 2 | tail -n 1 | awk '{print $1}')"
+  )
+}
+
+function make_htmldocs()
+{
+  # Make the chdir local
+  (
+    cd "$LNXDIR"/temp &&
+    make -j 2 htmldocs
+  )
+}
+
+function make_htmlnochunks()
+{
+  # Make the chdir local
+  (
+    cd "$LNXDIR"/temp &&
+    # Probably redundant, but just in case...
+    make -j 2 xmldocs &&
+    cd Documentation/DocBook &&
+    for i in *.xml
+    do
+      echo "Converting $i"
+      xmlto xhtml-nochunks $i -o xhtml-nochunks
+    done
+  )
+}
+
+function install_htmldocs()
+{
+  (
+    rm -f $WEBDIR/htmldocs
+    cd "$LNXDIR"/Documentation/DocBook &&
+    # Install nochunks versions
+    mv xhtml-nochunks "$WEBDIR"/htmldocs &&
+
+    head -n 2 index.html > "$WEBDIR"/htmldocs/index.html &&
+    echo "<ul>" >> "$WEBDIR"/htmldocs/index.html
+
+    [ $? -ne 0 ] && exit 1
+
+    # Install broken-up versions and index.html
+    for i in $(ls -p | sed -n 's@/$@@p')
+    do
+      mv $i "$WEBDIR"/htmldocs &&
+      echo "<li>(<a href=\"$i\">chunks</a>) (<a href=\"${i}.html\">nochunks</a>) " \
+        >> "$WEBDIR"/htmldocs/index.html &&
+      sed -nr 's@.*<title>(.*)</title>.*@\1@p' ${i}.xml | head -n 1 >> \
+        "$WEBDIR"/htmldocs/index.html &&
+      echo "</li>" >> "$WEBDIR"/htmldocs/index.html
+
+      [ $? -ne 0 ] && exit 1
+    done
+
+    cd "$WEBDIR"/htmldocs &&
+    echo "</ul>" >> index.html &&
+    # Work around a bug in xmlto: it puts an incorrect code page into the
+    # converted html documents, which produces line noise in the output.
+    sed -i -e 's/.*ISO-8859-1.*//' $(find . -name "*.html")
+  )
+}
+
+function update_master()
+{
+  "$WEBDIR"/make/indexsections.py $WEBDIR/master.idx > $WEBDIR/index.html
+}
+
+function htmlize_menuconfig()
+{
+  (
+    cd "$LNXDIR"/hg &&
+    LNXTAG="$(hg tags | head -n 2 | tail -n 1 | awk '{print $1}')"
+
+    rm -rf "$WEBDIR"/menuconfig/
+    mkdir -p "$WEBDIR"/menuconfig &&
+    echo "<html><title>Menuconfig docs for Linux $LNXTAG</title><body><ul>" > \
+      "$WEBDIR"/menuconfig/index.html &&
+    for i in $(find arch -maxdepth 2 -name Kconfig | sort)
+    do
+      echo Converting "$i" &&
+      ARCH=$(echo $i | sed -r 's@.*/(.*)/.*@\1@') &&
+      echo "<li><a href=${ARCH}.html>${ARCH}</a></li>" >> \
+        "$WEBDIR"/menuconfig/index.html &&
+      "$WEBDIR"/make/menuconfig2html.py $i "$WEBDIR"/menuconfig "$LNXTAG" > \
+        "$WEBDIR"/menuconfig/$ARCH.html
+    done
+  ) &&
+  echo "</ul></body></html>" >> "$WEBDIR"/menuconfig/index.html
+}
+
+function mirror_files()
+{
+  (
+    cd "$WEBDIR" &&
+    make/mirror.py
+    [ $? -ne 0 ] && exit 1
+
+    cat mirror/ols2002.pdf.gz | gunzip > mirror/ols2002.pdf &&
+    if [ -z "$(find mirror/ols2001 -name "*.pdf")" ]
+    then
+      mkdir -p mirror/ols2001 &&
+      cd mirror/ols2001 &&
+      wget -r -l 1 -nd http://lwn.net/2001/features/OLS/pdf/pdf/ &&
+      cd ../..
+
+      [ $? -ne 0 ] && exit 1
+    fi
+
+    if [ -z "$(find ols/2007 -name "*.pdf")" ]
+    then
+      echo "Split ols files"
+      mkdir -p ols/{1999,2000,2001,2002,2003,2004,2005,2006,2007} &&
+      cp mirror/ols2001/*.pdf ols/2001 &&
+      make/splitols.py mirror ols
+
+      [ $? -ne 0 ] && exit 1
+    fi
+  )
+}
+
+function find_readmes()
+{
+  (
+    rm -rf "$WEBDIR"/readme
+    mkdir "$WEBDIR"/readme &&
+    cd "$LNXDIR"/hg &&
+    make help > "$WEBDIR"/makehelp.txt &&
+    for i in $(find . -name "*[Rr][Ee][Aa][Dd][Mm][Ee]*" | grep -v "Documentation/")
+    do
+      OUT=$(echo $i | sed -e 's@\./@@' -e 's@/@-@g')
+      cp $i "$WEBDIR"/readme/"$(echo $i | sed -e 's@\./@@' -e 's@/@-@g')"
+    done
+  )
+}
+
+function findrfc()
+{
+  (
+    cd "$LNXDIR" || exit 1
+    echo -n "Search for RFC references"
+    x=0
+    # Iterate through *.c and *.h in the Linux source code.
+    for i in `find "$LNXDIR"/hg -name "*.[hc]"`
+    do
+      # Progress indicator
+      x=$[$x+1]
+      if [ $x -gt 250 ]
+      then
+        echo -n . >&2
+        x=0
+      fi
+      # Grep each source file for mention of an RFC, filtering out the more
+      # obvious false positives.
+      for j in $(grep -io "rfc *[1-9][0-9]*" $i | grep -v "  " | grep -v "@rfc822\.org" | sed -r 's/[Rr][Ff][Cc] *([0-9]*)/rfc\1/g')
+      do
+        echo $i:$j
+      done
+    done
+  ) > "$WEBDIR">temp-linuxrfc.txt
+
+  # Process the temporary directory into an html file.
+
+  echo Converting to html...
+  (
+    cd "$WEBDIR" || exit 1
+    echo "<html><head><title>RFCs used by the Linux kernel</title></head>"
+    echo "<ul>"
+    for i in $(sed 's/.*://' temp-linuxrfc.txt | sort -u | sort -t c -k 2,2 -n)
+    do
+      echo '<li><h2><a href="http://tools.ietf.org/html/'$i'">'$i'</a></h2><p>'
+      for j in $(sed -rn 's@\'"$LNXDIR"/hg'(.*):'$i'$@\1@p' temp-linuxrfc.txt)
+      do
+        echo '<a href="http://kernel.org/hg/linux-2.6/file/tip/'$j'">'$j'</a>'
+      done
+      echo "</p></li>"
+    done
+    echo "</ul></html>"
+  ) > "$WEBDIR"/rfc-linux.html
+
+  # Remove the temporary file
+  rm temp-linuxrfc.txt
+}
+
+# Set up for out of tree build of Linux.
+
+function setuplinuxtemp()
+{
+  if [ ! -d "$WEBDIR"/Documentation ]
+  then
+    "$WEBDIR"/make/do-Documentation.sh || exit 1
+  fi
+
+  rm -rf "$LNXDIR"/temp &&
+  mkdir "$LNXDIR"/temp &&
+  ( cd "$LNXDIR"/hg &&
+    make allnoconfig O=$LNXDIR/temp &&
+    # Adjust to use the patched Documentation directory
+    ln -s "$WEBDIR"/Documentation "$LNXDIR"/temp &&
+    # We'll need this later.
+    make -j 2 xmldocs
+  )
+}
--- a/make/make.sh	Wed Oct 31 16:40:10 2007 -0500
+++ b/make/make.sh	Wed Nov 07 01:43:39 2007 -0600
@@ -5,95 +5,57 @@
 [ -z "$WEBDIR" ] && export WEBDIR=~/www/kdocs
 [ -z "$LNXDIR" ] && export LNXDIR=~/linux
 
+source "$WEBDIR"/make/functions.sh
+
 # Update mercurial repository
 if [ "$1" != "--nonet" ]
 then
-  "$WEBDIR"/make/do-gethg.sh || exit 1
+  echo "Update Linux source from mercurial on kernel.org"
+  get_hg || exit 1
 fi
 
-echo -n "Grab version number..."
-export LNXTAG="$(hg tags | head -n 2 | tail -n 1 | awk '{print $1}')"
-echo $LNXTAG
+echo "Install updated (and patched) Documentation directory."
+update_documentation_dir || exit 1
+
+echo "Set up Linux build directory (uses patched Documentation dir)"
+setuplinuxtemp || exit 1
+
+if [ "$1" == "--long" ]
+then
+  # I have a dual processor laptop, so overlap work here.  Have make htmldocs
+  # produce split-up versions in the background (which is very slow)
+  # while other work is done in the foreground.
+  make_htmldocs > /dev/null &
+
+  # This is also very slow.
+
+  # Search for references to RFC documents
+  "$WEBDIR"/make/do-rfc.sh &
+fi
 
 echo "Update master index."
-"$WEBDIR"/make/do-master.sh || exit 1
+update_master || exit 1
 
 echo "Mirror resources"
-"$WEBDIR"/make/do-mirror.sh || exit 1
+mirror_files || exit 1
 
 echo Copy readme files from Linux kernel
-"$WEBDIR"/make/do-readme.sh || exit 1
+find_readmes || exit 1
 
 echo Convert kconfig to html for all architectures
-"$WEBDIR"/make/do-menuconfig.sh || exit 1
+htmlize_menuconfig || exit 1
 
-echo Install updated Documentation
+echo Build nochunks htmldocs.
+make_htmlnochunks || exit 1
 
-rm -rf $WEBDIR/Documentation/
-cp -a $LNXDIR/hg/Documentation $WEBDIR/Documentation
-cd $WEBDIR
-for i in make/patches/*.patch
-do
-  echo $i
-  patch -p1 -i $i || exit 1
-done
-make/docdiridx.py
+# Wait for background task(s) to finish.  If there aren't any, this continues
+# immediately.
+"$WEBDIR"/do-wait.sh
 
-[ "$1" == "--short" ] && exit 1
-
-echo Build htmldocs and xhtml-nochunks
-
-rm -rf $LNXDIR/temp
-cd $LNXDIR/hg
-mkdir $LNXDIR/temp
-make allnoconfig O=$LNXDIR/temp
-
-# Adjust to use the patched Documentation directory
-ln -s $WEBDIR/Documentation $LNXDIR/temp
-
-# I have a dual processor laptop, so overlap work here.  Generate the xml
-# first, then have make htmldocs produce split-up versions in the background
-# while a script produces nochunks versions in the foreground.
-cd $LNXDIR/temp
-make -j 2 xmldocs
-[ "$1" != "--skimp" ] && make -j 2 htmldocs &
-cd Documentation/DocBook
-for i in *.xml
-do
-  echo "Converting $i"
-  xmlto xhtml-nochunks $i -o xhtml-nochunks 
-done
-# Wait for background task to finish before retrieving result
-[ "$1" != "--skimp" ] && while [ $(jobs | wc -l) -ne 0 ]
-do
-  sleep 1
-  # Without this next line, bash never notices a change in the number of jobs.
-  # Bug noticed in Ubuntu 7.04
-  jobs > /dev/null
-done
-mv xhtml-nochunks/*.html .
-rmdir xhtml-nochunks
-
-echo Install htmldocs and update index
-
-rm -f $WEBDIR/htmldocs
-ln -s Documentation/DocBook $WEBDIR/htmldocs
-cd $WEBDIR/htmldocs
-# Work around a bug in xmlto: it puts an incorrect code page into the converted
-# html documents, which produces line noise in the output.
-find . -name "*.html" | xargs sed -i -e 's/.*ISO-8859-1.*//'
-head -n 2 index.html > index2.html
-echo "<ul>" >> index2.html
-for i in $(ls -p | sed -n 's@/$@@p')
-do
-  echo "<li>(<a href=\"$i\">chunks</a>) (<a href=\"${i}.html\">nochunks</a>) " >> index2.html
-  sed -nr 's@.*<title>(.*)</title>.*@\1@p' ${i}.xml | head -n 1 >> index2.html
-  echo "</li>" >> index2.html
-done
-echo "</ul>" >> index2.html
-mv index2.html index.html
+# Install the htmldocs
+install_htmldocs || exit 1
 
 # Search for references to RFC documents
-"$WEBDIR"/make/findrfc.sh > "$WEBDIR"/rfc-linux.html || exit 1
+"$WEBDIR"/make/do-rfc.sh > "$WEBDIR"/rfc-linux.html || exit 1
 
 echo "Done with kdocs"