-t root -- specify root dir which should be treated as internal (can repeat)
-v -- verbose operation; repeating -v will increase verbosity
-x -- don't check external links (these are often slow to check)
+-a -- don't check name anchors
Arguments:
ROUNDSIZE = 50 # Number of links processed per round
DUMPFILE = "@webchecker.pickle" # Pickled checkpoint
AGENTNAME = "webchecker" # Agent name for robots.txt parser
+NONAMES = 0 # Force name anchor checking
# Global variables
try:
- # Begin SLB 2/24/99: Added -t option here.
- opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:t:vx')
- # End SLB 2/24/99
+ # Begin SLB 2/24/99: Added -t option here.
+ opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:t:vxa')
+ # End SLB 2/24/99
except getopt.error, msg:
sys.stdout = sys.stderr
# Begin SLB 2/24/99: Added extra_roots variable to
# collect extra roots.
extra_roots = []
+ nonames = NONAMES
# End SLB 2/24/99
for o, a in opts:
if o == '-r':
roundsize = string.atoi(a)
- # Begin SLB 2/24/99: Added processing for
- # -t option.
- if o == '-t':
- extra_roots.append(a)
- # End SLB 2/24/99
-
+ # Begin SLB 2/24/99: Added processing for
+ # -t option.
+ if o == '-t':
+ extra_roots.append(a)
+ if o == '-a':
+ nonames = not nonames
+ # End SLB 2/24/99
+
if o == '-v':
verbose = verbose + 1
if o == '-x':
c = Checker()
c.setflags(checkext=checkext, verbose=verbose,
- maxpage=maxpage, roundsize=roundsize)
+ maxpage=maxpage, roundsize=roundsize,
+ nonames=nonames
+ )
if not restart and not args:
args.append(DEFROOT)
# links are not to be checked. So -t values are ignored unless
# -x was specified.
if not checkext:
- for root in extra_roots:
- # Make sure it's terminated by a slash,
- # so that addroot doesn't discard the last
- # directory component.
- if root[-1] != "/":
- root = root + "/"
- c.addroot(root)
+ for root in extra_roots:
+ # Make sure it's terminated by a slash,
+ # so that addroot doesn't discard the last
+ # directory component.
+ if root[-1] != "/":
+ root = root + "/"
+ c.addroot(root, add_to_do = 0)
# End SLB 2/24/99
try:
verbose = VERBOSE
maxpage = MAXPAGE
roundsize = ROUNDSIZE
+ nonames = NONAMES
validflags = tuple(dir())
self.done = {}
self.bad = {}
- # Begin SLB 2/24/99: Added a name table, so that the
- # name URLs can be checked. Also serves as an implicit
- # cache for which URLs are done.
- self.name_table = {}
- # End SLB 2/24/99
+ # Begin SLB 2/24/99: Added a name table, so that the
+ # name URLs can be checked. Also serves as an implicit
+ # cache for which URLs are done.
+ self.name_table = {}
+ # End SLB 2/24/99
self.round = 0
# The following are not pickled:
self.errors = {}
self.urlopener = MyURLopener()
self.changed = 0
-
+
def note(self, level, format, *args):
if self.verbose > level:
if args:
format = format%args
self.message(format)
-
+
def message(self, format, *args):
if args:
format = format%args
for url in self.bad.keys():
self.markerror(url)
- def addroot(self, root):
+ def addroot(self, root, add_to_do = 1):
if root not in self.roots:
troot = root
scheme, netloc, path, params, query, fragment = \
params, query, fragment))
self.roots.append(troot)
self.addrobot(root)
- # Begin SLB 2/24/99: Modified this call to respect
- # the fact that the "done" and "todo" dictionaries
- # are now (URL, fragment) pairs
- self.newlink((root, ""), ("<root>", root))
- # End SLB 2/24/99
+ # Begin SLB 2/24/99: Modified this call to respect
+ # the fact that the "done" and "todo" dictionaries
+ # are now (URL, fragment) pairs
+ if add_to_do:
+ self.newlink((root, ""), ("<root>", root))
+ # End SLB 2/24/99
def addrobot(self, root):
root = urlparse.urljoin(root, "/")
self.message("%d Errors in %s", len(triples), source)
else:
self.message("Error in %s", source)
- # Begin SLB 2/24/99: Modified this loop to
- # call self.format_url() instead of referring
- # to the URL directly, since the URLs in these
- # triples is now a (URL, fragment) pair. The value
- # of the "source" variable comes from the list of
- # origins, and is a URL, not a pair.
- for url, rawlink, msg in triples:
+ # Begin SLB 2/24/99: Modified this loop to
+ # call self.format_url() instead of referring
+ # to the URL directly, since the URLs in these
+ # triples is now a (URL, fragment) pair. The value
+ # of the "source" variable comes from the list of
+ # origins, and is a URL, not a pair.
+ for url, rawlink, msg in triples:
if rawlink != self.format_url(url): s = " (%s)" % rawlink
else: s = ""
self.message(" HREF %s%s\n msg %s",
- self.format_url(url), s, msg)
- # End SLB 2/24/99.
+ self.format_url(url), s, msg)
+ # End SLB 2/24/99.
def dopage(self, url_pair):
-
- # Begin SLB 2/24/99: Substantial modification to reflect the change
- # to dictionary entries being (URL, fragment) pairs. All
- # printing of URLs uses format_url(); argument changed to
- # url_pair for clarity.
+
+ # Begin SLB 2/24/99: Substantial modification to reflect the change
+ # to dictionary entries being (URL, fragment) pairs. All
+ # printing of URLs uses format_url(); argument changed to
+ # url_pair for clarity.
if self.verbose > 1:
if self.verbose > 2:
self.show("Check ", self.format_url(url_pair),
- " from", self.todo[url_pair])
+ " from", self.todo[url_pair])
else:
self.message("Check %s", self.format_url(url_pair))
+ url, local_fragment = url_pair
+ if local_fragment and self.nonames:
+ self.markdone(url_pair)
+ return
page = self.getpage(url_pair)
if page:
- url, local_fragment = url_pair
- # Store the page which corresponds to this URL.
- self.name_table[url] = page
- # If there is a fragment in this url_pair, and it's not
- # in the list of names for the page, call setbad(), since
- # it's a missing anchor.
- if local_fragment and local_fragment not in page.getnames():
- self.setbad(url_pair, ("Missing name anchor `%s'" % local_fragment))
- for info in page.getlinkinfos():
- # getlinkinfos() now returns the fragment as well,
- # and we store that fragment here in the "todo" dictionary.
+ # Store the page which corresponds to this URL.
+ self.name_table[url] = page
+ # If there is a fragment in this url_pair, and it's not
+ # in the list of names for the page, call setbad(), since
+ # it's a missing anchor.
+ if local_fragment and local_fragment not in page.getnames():
+ self.setbad(url_pair, ("Missing name anchor `%s'" % local_fragment))
+ for info in page.getlinkinfos():
+ # getlinkinfos() now returns the fragment as well,
+ # and we store that fragment here in the "todo" dictionary.
link, rawlink, fragment = info
- # However, we don't want the fragment as the origin, since
- # the origin is logically a page.
+ # However, we don't want the fragment as the origin, since
+ # the origin is logically a page.
origin = url, rawlink
self.newlink((link, fragment), origin)
- else:
- # If no page has been created yet, we want to
- # record that fact.
- self.name_table[url_pair[0]] = None
+ else:
+ # If no page has been created yet, we want to
+ # record that fact.
+ self.name_table[url_pair[0]] = None
self.markdone(url_pair)
- # End SLB 2/24/99
+ # End SLB 2/24/99
def newlink(self, url, origin):
if self.done.has_key(url):
self.newtodolink(url, origin)
def newdonelink(self, url, origin):
- self.done[url].append(origin)
-
- # Begin SLB 2/24/99: changed reference to URL
- # to call self.format_url(), since the URL here
- # is now a (URL, fragment) pair.
+
+ if origin not in self.done[url]:
+ self.done[url].append(origin)
+
+ # Begin SLB 2/24/99: changed reference to URL
+ # to call self.format_url(), since the URL here
+ # is now a (URL, fragment) pair.
self.note(3, " Done link %s", self.format_url(url))
- # End SLB 2/24/99
+
+ # SLB 11/11/99: Make sure that if it's bad, that
+ # the origin gets added.
+
+ if self.bad.has_key(url):
+ source, rawlink = origin
+ triple = url, rawlink, self.bad[url]
+ self.seterror(source, triple)
+
+ # End SLB 2/24/99
def newtodolink(self, url, origin):
-
- # Begin SLB 2/24/99: changed reference to URL
- # to call self.format_url(), since the URL here
- # is now a (URL, fragment) pair.
+
+ # Begin SLB 2/24/99: changed reference to URL
+ # to call self.format_url(), since the URL here
+ # is now a (URL, fragment) pair.
if self.todo.has_key(url):
- self.todo[url].append(origin)
+ if origin not in self.todo[url]:
+ self.todo[url].append(origin)
self.note(3, " Seen todo link %s", self.format_url(url))
else:
self.todo[url] = [origin]
self.note(3, " New todo link %s", self.format_url(url))
- # End SLB 2/24/99
+ # End SLB 2/24/99
# Begin SLB 2/24/99: Added method format_url().
- def format_url(self, url):
- link, fragment = url
- if fragment: return link + "#" + fragment
- else: return link
+ def format_url(self, url):
+ link, fragment = url
+ if fragment: return link + "#" + fragment
+ else: return link
# End SLB 2/24/99
def markdone(self, url):
if url[:len(root)] == root:
return self.isallowed(root, url)
return 0
-
+
def isallowed(self, root, url):
root = urlparse.urljoin(root, "/")
return self.robots[root].can_fetch(AGENTNAME, url)
def getpage(self, url_pair):
- # Begin SLB 2/24/99: changed incoming argument name to
- # url_pair, since it's now a (URL, fragment) pair.
- # The page may have been cached in the name_table variable.
- url, fragment = url_pair
- if self.name_table.has_key(url):
- return self.name_table[url]
- # End SLB 2/24/99
-
+ # Begin SLB 2/24/99: changed incoming argument name to
+ # url_pair, since it's now a (URL, fragment) pair.
+ # The page may have been cached in the name_table variable.
+ url, fragment = url_pair
+ if self.name_table.has_key(url):
+ return self.name_table[url]
+ # End SLB 2/24/99
+
if url[:7] == 'mailto:' or url[:5] == 'news:':
self.note(1, " Not checking mailto/news URL")
return None
isint = self.inroots(url)
-
- # Begin SLB 2/24/99: Changed calls to openpage and readhtml
- # in order to ensure that openpage gets the URL pair to
- # print out its error message and record the error pair
- # correctly.
+
+ # Begin SLB 2/24/99: Changed calls to openpage and readhtml
+ # in order to ensure that openpage gets the URL pair to
+ # print out its error message and record the error pair
+ # correctly.
if not isint:
if not self.checkext:
self.note(1, " Not checking ext link")
self.safeclose(f)
return None
text, nurl = self.readhtml(url_pair)
- # End SLB 2/24/99
+ # End SLB 2/24/99
if nurl != url:
self.note(1, " Redirected to %s", nurl)
# to take (URL, fragment) pairs as arguments, so that openpage()
# receives the appropriate tuple to record error messages.
def readhtml(self, url_pair):
- url, fragment = url_pair
+ url, fragment = url_pair
text = None
f, url = self.openhtml(url_pair)
if f:
return text, url
def openhtml(self, url_pair):
- url, fragment = url_pair
+ url, fragment = url_pair
f = self.openpage(url_pair)
if f:
url = f.geturl()
return f, url
def openpage(self, url_pair):
- url, fragment = url_pair
+ url, fragment = url_pair
try:
return self.urlopener.open(url)
except IOError, msg:
self.bad[url] = msg
self.changed = 1
self.markerror(url)
-
+
def markerror(self, url):
try:
origins = self.todo[url]
def seterror(self, url, triple):
try:
-
- # Begin SLB 2/24/99: Because of the way the
- # URLs are now processed, I need to check to make
- # sure the URL hasn't been entered in the error list.
- # The first element of the triple here is a (URL, fragment)
- # pair, but the URL key is not, since it's from the
- # list of origins.
- if triple not in self.errors[url]:
- self.errors[url].append(triple)
- # End SLB 2/24/99
-
+
+ # Begin SLB 2/24/99: Because of the way the
+ # URLs are now processed, I need to check to make
+ # sure the URL hasn't been entered in the error list.
+ # The first element of the triple here is a (URL, fragment)
+ # pair, but the URL key is not, since it's from the
+ # list of origins.
+ if triple not in self.errors[url]:
+ self.errors[url].append(triple)
+ # End SLB 2/24/99
+
except KeyError:
self.errors[url] = [triple]
self.verbose = verbose
self.maxpage = maxpage
self.checker = checker
-
- # Begin SLB 2/24/99: Moved the parsing of the page into
- # the __init__() routine in order to initialize the list of
- # names the file contains. Stored the parser in an instance
- # variable. Passed the URL to MyHTMLParser().
- size = len(self.text)
+
+ # Begin SLB 2/24/99: Moved the parsing of the page into
+ # the __init__() routine in order to initialize the list of
+ # names the file contains. Stored the parser in an instance
+ # variable. Passed the URL to MyHTMLParser().
+ size = len(self.text)
if size > self.maxpage:
self.note(0, "Skip huge file %s (%.0f Kbytes)", self.url, (size*0.001))
self.parser = None
- return
+ return
self.checker.note(2, " Parsing %s (%d bytes)", self.url, size)
self.parser = MyHTMLParser(url, verbose=self.verbose,
- checker=self.checker)
+ checker=self.checker)
self.parser.feed(self.text)
self.parser.close()
- # End SLB 2/24/99
+ # End SLB 2/24/99
def note(self, level, msg, *args):
if self.checker:
# Begin SLB 2/24/99: Added method to retrieve names.
def getnames(self):
- return self.parser.names
+ return self.parser.names
# End SLB 2/24/99
def getlinkinfos(self):
- # Begin SLB 2/24/99: Moved file reading to __init__() routine.
- # Stored parser in local variable to indicate success of
- # parsing.
-
- # If no parser was stored, fail.
- if not self.parser: return []
-
+ # Begin SLB 2/24/99: Moved file reading to __init__() routine.
+ # Stored parser in local variable to indicate success of
+ # parsing.
+
+ # If no parser was stored, fail.
+ if not self.parser: return []
+
rawlinks = self.parser.getlinks()
base = urlparse.urljoin(self.url, self.parser.getbase() or "")
infos = []
for rawlink in rawlinks:
t = urlparse.urlparse(rawlink)
- # DON'T DISCARD THE FRAGMENT! Instead, include
- # it in the tuples which are returned. See Checker.dopage().
- fragment = t[-1]
+ # DON'T DISCARD THE FRAGMENT! Instead, include
+ # it in the tuples which are returned. See Checker.dopage().
+ fragment = t[-1]
t = t[:-1] + ('',)
rawlink = urlparse.urlunparse(t)
link = urlparse.urljoin(base, rawlink)
- infos.append((link, rawlink, fragment))
+ infos.append((link, rawlink, fragment))
# End SLB 2/24/99
-
+
return infos
def open_file(self, url):
path = urllib.url2pathname(urllib.unquote(url))
- if path[-1] != os.sep:
- url = url + '/'
if os.path.isdir(path):
+ if path[-1] != os.sep:
+ url = url + '/'
indexpath = os.path.join(path, "index.html")
if os.path.exists(indexpath):
return self.open_file(url + "index.html")
s.write('<A HREF="%s">%s</A>\n' % (q, q))
s.seek(0)
return s
- return urllib.FancyURLopener.open_file(self, path)
+ return urllib.FancyURLopener.open_file(self, url)
class MyHTMLParser(sgmllib.SGMLParser):
self.checker = checker
self.base = None
self.links = {}
-
- # Begin SLB 2/24/99: Added names instance variable.
- # Modified arglist to take the URL as an argument.
- self.names = []
- self.url = url
- # End SLB 2/24/99
-
+
+ # Begin SLB 2/24/99: Added names instance variable.
+ # Modified arglist to take the URL as an argument.
+ self.names = []
+ self.url = url
+ # End SLB 2/24/99
+
sgmllib.SGMLParser.__init__(self)
def start_a(self, attributes):
self.link_attr(attributes, 'href')
-
- # Begin SLB 2/24/99: We must rescue the NAME
- # attributes from the anchor, in order to
- # cache the internal anchors which are made
- # available in the page.
- for name, value in attributes:
- if name == "name":
- if value in self.names:
- self.checker.message("WARNING: duplicate name %s in %s",
- value, self.url)
- else: self.names.append(value)
- break
- # End SLB 2/24/99
+
+ # Begin SLB 2/24/99: We must rescue the NAME
+ # attributes from the anchor, in order to
+ # cache the internal anchors which are made
+ # available in the page.
+ for name, value in attributes:
+ if name == "name":
+ if value in self.names:
+ self.checker.message("WARNING: duplicate name %s in %s",
+ value, self.url)
+ else: self.names.append(value)
+ break
+ # End SLB 2/24/99
def end_a(self): pass
projects / python / commitdiff