changeset 51:93c68f362860

New script to download OLS files (and others), and split up volumes into individual papers, plus teach make.sh to run it.
author Rob Landley <rob@landley.net>
date Fri, 28 Sep 2007 14:47:14 -0500
parents 74555f989db5
children ecfa70db0bb3
files make/make.sh make/mirror.py make/splitols.py
diffstat 3 files changed, 93 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/make/make.sh	Fri Sep 28 14:44:05 2007 -0500
+++ b/make/make.sh	Fri Sep 28 14:47:14 2007 -0500
@@ -9,6 +9,32 @@
 
 $WEBDIR/make/indexsections.py $WEBDIR/master.idx > $WEBDIR/index.html || exit 1
 
+echo "Mirror resources"
+
+cd "$WEBDIR"
+make/mirror.py
+cat mirror/ols2002.pdf.gz | gunzip > mirror/ols2002.pdf
+if [ -z "$(find mirror/ols2001 -name "*.pdf")" ]
+then
+  mkdir -p mirror/ols2001 &&
+  cd mirror/ols2001 &&
+  wget -r -l 1 -nd http://lwn.net/2001/features/OLS/pdf/pdf/ &&
+  cd ../..
+
+  [ $? -ne 0 ] && exit 1
+fi
+
+if [ -z "$(find ols/2007 -name "*.pdf")" ]
+then
+  echo "Split ols files"
+  mkdir -p ols/{1999,2000,2001,2002,2003,2004,2005,2006,2007} &&
+  cp mirror/ols2001/*.pdf ols/2001 &&
+  make/splitols.py mirror ols
+  [ $? -ne 0 ] && exit 1
+fi
+
+exit 1
+
 echo "Update Linux kernel from kernel.org"
 
 if [ ! -d "$LNXDIR" ]
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/make/mirror.py	Fri Sep 28 14:47:14 2007 -0500
@@ -0,0 +1,36 @@
+#!/usr/bin/python
+
+import os, sys, popen2
+
+# Download stuff to populate mirror/
+
+try: os.mkdir("mirror")
+except: pass
+os.chdir("mirror")
+
+# Fetch mirrored files if we haven't already got them
+
+for i in (["b6398cf2d5b3cc6e2bfe572adb856dd8dddb707a", "http://www.moses.uklinux.net/patches/lki-single.html"],
+          ["08727c048f443926adadb9bf86248e71eab3cbf9", "http://www.moses.uklinux.net/patches/lki.pdf"],
+          ["de58dfc537e24d5e0b7cbf50666e50efafe996e7", "http://www.linux.org.uk/~ajh/ols2002_proceedings.pdf.gz","ols2002.pdf.gz"],
+          ["1e7c6f9fd7722805bcba78ea4afa57ceea87e16c", "http://web.archive.org/web/20031002044046/http://archive.linuxsymposium.org/ols2003/Proceedings/LinuxSymposium2003.pdf", "ols2003.pdf"],
+          ["710e98915c0b6474cabf4dd27b3d80483cb14aac", "http://web.archive.org/web/20040910220019/http://www.finux.org/proceedings/LinuxSymposium2004_V1.pdf", "ols2004v1.pdf"],
+          ["0e5ab31d392a7c03213c23c3ca84c8be37a0727d", "http://web.archive.org/web/20040910220019/http://www.finux.org/proceedings/LinuxSymposium2004_V2.pdf", "ols2004v2.pdf"],
+          ["2abfcb9265191e64e9c22d3f6f9dc9404b27419e", "http://www.linuxsymposium.org/2005/linuxsymposium_procv1.pdf", "ols2005v1.pdf"],
+          ["437a4c35c72667f9d35012546e0d5a4720693f3f", "http://www.linuxsymposium.org/2005/linuxsymposium_procv2.pdf", "ols2005v2.pdf"],
+          ["355ac65470c38333b26d55b7c8493d93c419cd2e", "http://www.linuxsymposium.org/2006/linuxsymposium_procv1.pdf","ols2006v1.pdf"],
+          ["ab250f4549b13b47425e0400ea7762f21eb01cde", "http://www.linuxsymposium.org/2006/linuxsymposium_procv2.pdf","ols2006v2.pdf"],
+          ["765e049036ca947af05fb5b4f29b9daf4b363558", "https://ols2006.108.redhat.com/2007/Reprints/OLS2007-Proceedings-V1.pdf", "ols2007v1.pdf"],
+          ["60fdaa6a74dec29a0538325b742bee4097247c6d", "https://ols2006.108.redhat.com/2007/Reprints/OLS2007-Proceedings-V2.pdf", "ols2007v2.pdf"]
+         ):
+  file=i[1].split('/')[-1]
+  if len(i)>2: file=i[2]
+  print "check %s" % file
+  if i[0]:
+    sha1=popen2.Popen3('sha1sum %s' % file).fromchild.read().split()
+    if sha1 and i[0] == sha1[0]:
+      continue
+  print "fetch %s" % i[1]
+  if os.system('wget -O "%s" "%s"' % (file, i[1])):
+    sys.stderr.write("wget failed\n")
+    sys.exit(1)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/make/splitols.py	Fri Sep 28 14:47:14 2007 -0500
@@ -0,0 +1,31 @@
+#!/usr/bin/python
+
+# Break up the OLS volumes into individual papers in the ols/$YEAR directories.
+
+# usage: splitols.py mirrordir olsdir
+
+import os, sys
+
+inpath=sys.argv[1]
+outpath=sys.argv[2]
+
+def splitvolume(infile, outpath, credits, numbers):
+  for i in xrange(len(numbers)-1):
+    next = int(numbers[i+1])-1
+    command="pdftk %s/%s.pdf cat %s-%s 1-1 %s-%s output %s/%s-pages-%s-%s.pdf" % (inpath,infile,numbers[i],next,credits,credits,outpath,infile,numbers[i],next)
+    print command
+    os.system(command)
+
+volumes = ( ("ols2002", "2002", 7, (8, 31, 40, 50, 55, 65, 73, 93, 107, 117, 130, 146, 176, 183, 191, 197, 213, 223, 242, 250, 260, 265, 274, 289, 301, 310, 317, 330, 338, 368, 376, 383, 390, 407, 414, 425, 439, 451, 459, 468, 479, 496, 521, 530, 545, 556, 565, 573, 594, 604, 618, 632)),
+            ("ols2003", "2003", 7, (8, 23, 34, 44, 50, 63, 76, 89, 98, 109, 117, 128, 134, 150, 169, 185, 201, 213, 219, 233, 242, 249, 258, 260, 275, 285, 297, 304, 315, 321, 325, 340, 351, 367, 373, 380, 387, 398, 410, 424, 434, 447, 457, 470, 480, 494, 507, 524, 531)),
+            ("ols2004v1", "2004", 7, (9, 23, 33, 41, 51, 63, 79, 89, 103, 113, 121, 133, 149, 163, 167, 187, 203, 207, 215, 227, 239, 255, 269, 287, 295)),
+            ("ols2004v2", "2004", 7, (9, 15, 25, 31, 43, 61, 71, 75, 81, 85, 95, 105, 117, 139, 163, 175, 181, 195, 209, 225, 239, 251, 261, 271, 287, 297, 307)),
+            ("ols2005v1", "2005", 7, (9, 21, 37, 49, 59, 77, 105, 117, 135, 141, 155, 177, 189, 203, 209, 227, 241, 247, 253, 257, 277, 279, 289, 297, 305, 313, 321, 335, 347)),
+
+            ("ols2005v2", "2005", 7, (9, 19, 33, 43, 57, 73, 87, 99, 119, 141, 149, 159, 183, 201, 213, 217, 227, 229, 243, 259, 273, 279, 291, 299, 305, 321)),
+            ("ols2006v1", "2006", 7, (9, 19, 27, 39, 57, 71, 87, 101, 113, 127, 147, 153, 177, 193, 209, 225, 235, 247, 261, 269, 289, 303, 313, 333, 347, 369, 385, 395, 409, 421, 427, 441, 451)),
+            ("ols2006v2", "2006", 7, (9, 25, 35, 45, 59, 73, 79, 83, 91, 99, 109, 125, 131, 147, 151, 159, 173, 181, 193, 209, 223, 239, 249, 263, 275, 295, 303, 313, 321, 337, 349, 357, 371, 387, 401, 411, 417, 423, 437)),
+            ("ols2007v1", "2007", 7, (9, 21, 29, 41, 53, 65, 75, 81, 87, 93, 103, 113, 125, 131, 141, 153, 167, 179, 189, 201, 205, 215, 225, 231, 238, 245, 251, 263, 277, 285, 293, 303, 315)),
+            ("ols2007v2", "2007", 7, (9, 21, 35, 45, 59, 65, 71, 79, 87, 97, 107, 119, 127, 135, 145, 151, 161, 173, 179, 187, 201, 209, 223, 229, 235, 245, 253, 261, 273, 285, 297, 305, 311, 319)))
+
+for i in volumes: splitvolume(i[0],"%s/%s" % (outpath,i[1]),i[2],i[3])