# HG changeset patch # User Rob Landley # Date 1191008834 18000 # Node ID 93c68f362860800837b412e27b3c6d6a93206a3d # Parent 74555f989db5e43aa24873d09dedf36d1d869d2d New script to download OLS files (and others), and split up volumes into individual papers, plus teach make.sh to run it. diff -r 74555f989db5 -r 93c68f362860 make/make.sh --- a/make/make.sh Fri Sep 28 14:44:05 2007 -0500 +++ b/make/make.sh Fri Sep 28 14:47:14 2007 -0500 @@ -9,6 +9,32 @@ $WEBDIR/make/indexsections.py $WEBDIR/master.idx > $WEBDIR/index.html || exit 1 +echo "Mirror resources" + +cd "$WEBDIR" +make/mirror.py +cat mirror/ols2002.pdf.gz | gunzip > mirror/ols2002.pdf +if [ -z "$(find mirror/ols2001 -name "*.pdf")" ] +then + mkdir -p mirror/ols2001 && + cd mirror/ols2001 && + wget -r -l 1 -nd http://lwn.net/2001/features/OLS/pdf/pdf/ && + cd ../.. + + [ $? -ne 0 ] && exit 1 +fi + +if [ -z "$(find ols/2007 -name "*.pdf")" ] +then + echo "Split ols files" + mkdir -p ols/{1999,2000,2001,2002,2003,2004,2005,2006,2007} && + cp mirror/ols2001/*.pdf ols/2001 && + make/splitols.py mirror ols + [ $? -ne 0 ] && exit 1 +fi + +exit 1 + echo "Update Linux kernel from kernel.org" if [ ! -d "$LNXDIR" ] diff -r 74555f989db5 -r 93c68f362860 make/mirror.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/make/mirror.py Fri Sep 28 14:47:14 2007 -0500 @@ -0,0 +1,36 @@ +#!/usr/bin/python + +import os, sys, popen2 + +# Download stuff to populate mirror/ + +try: os.mkdir("mirror") +except: pass +os.chdir("mirror") + +# Fetch mirrored files if we haven't already got them + +for i in (["b6398cf2d5b3cc6e2bfe572adb856dd8dddb707a", "http://www.moses.uklinux.net/patches/lki-single.html"], + ["08727c048f443926adadb9bf86248e71eab3cbf9", "http://www.moses.uklinux.net/patches/lki.pdf"], + ["de58dfc537e24d5e0b7cbf50666e50efafe996e7", "http://www.linux.org.uk/~ajh/ols2002_proceedings.pdf.gz","ols2002.pdf.gz"], + ["1e7c6f9fd7722805bcba78ea4afa57ceea87e16c", "http://web.archive.org/web/20031002044046/http://archive.linuxsymposium.org/ols2003/Proceedings/LinuxSymposium2003.pdf", "ols2003.pdf"], + ["710e98915c0b6474cabf4dd27b3d80483cb14aac", "http://web.archive.org/web/20040910220019/http://www.finux.org/proceedings/LinuxSymposium2004_V1.pdf", "ols2004v1.pdf"], + ["0e5ab31d392a7c03213c23c3ca84c8be37a0727d", "http://web.archive.org/web/20040910220019/http://www.finux.org/proceedings/LinuxSymposium2004_V2.pdf", "ols2004v2.pdf"], + ["2abfcb9265191e64e9c22d3f6f9dc9404b27419e", "http://www.linuxsymposium.org/2005/linuxsymposium_procv1.pdf", "ols2005v1.pdf"], + ["437a4c35c72667f9d35012546e0d5a4720693f3f", "http://www.linuxsymposium.org/2005/linuxsymposium_procv2.pdf", "ols2005v2.pdf"], + ["355ac65470c38333b26d55b7c8493d93c419cd2e", "http://www.linuxsymposium.org/2006/linuxsymposium_procv1.pdf","ols2006v1.pdf"], + ["ab250f4549b13b47425e0400ea7762f21eb01cde", "http://www.linuxsymposium.org/2006/linuxsymposium_procv2.pdf","ols2006v2.pdf"], + ["765e049036ca947af05fb5b4f29b9daf4b363558", "https://ols2006.108.redhat.com/2007/Reprints/OLS2007-Proceedings-V1.pdf", "ols2007v1.pdf"], + ["60fdaa6a74dec29a0538325b742bee4097247c6d", "https://ols2006.108.redhat.com/2007/Reprints/OLS2007-Proceedings-V2.pdf", "ols2007v2.pdf"] + ): + file=i[1].split('/')[-1] + if len(i)>2: file=i[2] + print "check %s" % file + if i[0]: + sha1=popen2.Popen3('sha1sum %s' % file).fromchild.read().split() + if sha1 and i[0] == sha1[0]: + continue + print "fetch %s" % i[1] + if os.system('wget -O "%s" "%s"' % (file, i[1])): + sys.stderr.write("wget failed\n") + sys.exit(1) diff -r 74555f989db5 -r 93c68f362860 make/splitols.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/make/splitols.py Fri Sep 28 14:47:14 2007 -0500 @@ -0,0 +1,31 @@ +#!/usr/bin/python + +# Break up the OLS volumes into individual papers in the ols/$YEAR directories. + +# usage: splitols.py mirrordir olsdir + +import os, sys + +inpath=sys.argv[1] +outpath=sys.argv[2] + +def splitvolume(infile, outpath, credits, numbers): + for i in xrange(len(numbers)-1): + next = int(numbers[i+1])-1 + command="pdftk %s/%s.pdf cat %s-%s 1-1 %s-%s output %s/%s-pages-%s-%s.pdf" % (inpath,infile,numbers[i],next,credits,credits,outpath,infile,numbers[i],next) + print command + os.system(command) + +volumes = ( ("ols2002", "2002", 7, (8, 31, 40, 50, 55, 65, 73, 93, 107, 117, 130, 146, 176, 183, 191, 197, 213, 223, 242, 250, 260, 265, 274, 289, 301, 310, 317, 330, 338, 368, 376, 383, 390, 407, 414, 425, 439, 451, 459, 468, 479, 496, 521, 530, 545, 556, 565, 573, 594, 604, 618, 632)), + ("ols2003", "2003", 7, (8, 23, 34, 44, 50, 63, 76, 89, 98, 109, 117, 128, 134, 150, 169, 185, 201, 213, 219, 233, 242, 249, 258, 260, 275, 285, 297, 304, 315, 321, 325, 340, 351, 367, 373, 380, 387, 398, 410, 424, 434, 447, 457, 470, 480, 494, 507, 524, 531)), + ("ols2004v1", "2004", 7, (9, 23, 33, 41, 51, 63, 79, 89, 103, 113, 121, 133, 149, 163, 167, 187, 203, 207, 215, 227, 239, 255, 269, 287, 295)), + ("ols2004v2", "2004", 7, (9, 15, 25, 31, 43, 61, 71, 75, 81, 85, 95, 105, 117, 139, 163, 175, 181, 195, 209, 225, 239, 251, 261, 271, 287, 297, 307)), + ("ols2005v1", "2005", 7, (9, 21, 37, 49, 59, 77, 105, 117, 135, 141, 155, 177, 189, 203, 209, 227, 241, 247, 253, 257, 277, 279, 289, 297, 305, 313, 321, 335, 347)), + + ("ols2005v2", "2005", 7, (9, 19, 33, 43, 57, 73, 87, 99, 119, 141, 149, 159, 183, 201, 213, 217, 227, 229, 243, 259, 273, 279, 291, 299, 305, 321)), + ("ols2006v1", "2006", 7, (9, 19, 27, 39, 57, 71, 87, 101, 113, 127, 147, 153, 177, 193, 209, 225, 235, 247, 261, 269, 289, 303, 313, 333, 347, 369, 385, 395, 409, 421, 427, 441, 451)), + ("ols2006v2", "2006", 7, (9, 25, 35, 45, 59, 73, 79, 83, 91, 99, 109, 125, 131, 147, 151, 159, 173, 181, 193, 209, 223, 239, 249, 263, 275, 295, 303, 313, 321, 337, 349, 357, 371, 387, 401, 411, 417, 423, 437)), + ("ols2007v1", "2007", 7, (9, 21, 29, 41, 53, 65, 75, 81, 87, 93, 103, 113, 125, 131, 141, 153, 167, 179, 189, 201, 205, 215, 225, 231, 238, 245, 251, 263, 277, 285, 293, 303, 315)), + ("ols2007v2", "2007", 7, (9, 21, 35, 45, 59, 65, 71, 79, 87, 97, 107, 119, 127, 135, 145, 151, 161, 173, 179, 187, 201, 209, 223, 229, 235, 245, 253, 261, 273, 285, 297, 305, 311, 319))) + +for i in volumes: splitvolume(i[0],"%s/%s" % (outpath,i[1]),i[2],i[3])