Added -x option to check external links. Slooooow!

diff --git a/Tools/webchecker/webchecker.py b/Tools/webchecker/webchecker.py
index 2ec9b6121eb1d043b6dc41c7fd7a1a991234b867..d6c81cce64af9afebc1911c6eaf5d7f9106d07f0 100755 (executable)
--- a/Tools/webchecker/webchecker.py
+++ b/Tools/webchecker/webchecker.py
@@ -70,9 +70,11 @@ default locations of Netscape and the NCSA HTTP daemon.
 links in <FORM> or <IMG> or whatever other tags might contain
 hyperlinks.  It does honor the <BASE> tag.
 
-- It could be argued that it should also check external links for
-validity.  This is true, but is is more error-prone.  I think I will
-make this an option in the future.
+- Checking external links is not done by default; use -x to enable
+this feature.  This is done because checking external links usually
+takes a lot of time.  When enabled, this check is executed during the
+report generation phase (so -x is ignored when -q is specified).  Even
+when -x is enabled, only ``http:'' URLs are checked.
 
 
 Usage: webchecker.py [option] ... [rooturl] ...
@@ -85,6 +87,7 @@ Options:
 -q        -- quiet operation (also suppresses external links report)
 -r number -- number of links processed per round (default %(ROUNDSIZE)d)
 -v        -- verbose operation; repeating -v will increase verbosity
+-x        -- check external links (during report phase)
 
 Arguments:
 
@@ -131,9 +134,10 @@ def main():
     global verbose, maxpage, roundsize
     dumpfile = DUMPFILE
     restart = 0
+    checkext = 0
 
     try:
-       opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:qr:v')
+       opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:qr:vx')
     except getopt.error, msg:
        sys.stdout = sys.stderr
        print msg
@@ -151,6 +155,8 @@ def main():
            roundsize = string.atoi(a)
        if o == '-v':
            verbose = verbose + 1
+       if o == '-x':
+           checkext = 1
 
     if verbose:
        print AGENTNAME, "version", __version__
@@ -180,8 +186,12 @@ def main():
        c.run()
     except KeyboardInterrupt:
        if verbose > 0:
-           print "[interrupted]"
-    c.report()
+           print "[run interrupted]"
+    try:
+       c.report(checkext)
+    except KeyboardInterrupt:
+       if verbose > 0:
+           print "[report interrupted]"
     if not needsave:
        if verbose > 0:
            print
@@ -266,7 +276,7 @@ class Checker:
                self.done[url] = self.todo[url]
                del self.todo[url]
 
-    def report(self):
+    def report(self, checkext=0):
        print
        if not self.todo: print "Final",
        else: print "Interim",
@@ -274,22 +284,34 @@ class Checker:
            len(self.todo), len(self.done),
            len(self.ext), len(self.bad))
        if verbose > 0:
-           self.report_extrefs()
+           self.report_extrefs(checkext)
        # Report errors last because the output may get truncated
        self.report_errors()
 
-    def report_extrefs(self):
+    def report_extrefs(self, checkext=0):
        if not self.ext:
            print
            print "No external URLs"
            return
        print
-       print "External URLs:"
+       if checkext:
+           print "External URLs (checking validity):"
+       else:
+           print "External URLs (not checked):"
        print
        urls = self.ext.keys()
        urls.sort()
        for url in urls:
            show("HREF ", url, " from", self.ext[url])
+           if not checkext:
+               continue
+           if verbose > 2: print "Checking", url, "..."
+           try:
+               f = self.urlopener.open(url)
+               f.close()
+               if verbose > 3: print "OK"
+           except IOError, msg:
+               print "Error:", msg
 
     def report_errors(self):
        if not self.bad: