# XXX TO DO: make the distribution functions below into methods.
def makeseed(a=None):
- """Turn a hashable value into three seed values for whrandom.seed().
-
- None or no argument returns (0, 0, 0), to seed from current time.
-
- """
- if a is None:
- return (0, 0, 0)
- a = hash(a)
- a, x = divmod(a, 256)
- a, y = divmod(a, 256)
- a, z = divmod(a, 256)
- x = (x + a) % 256 or 1
- y = (y + a) % 256 or 1
- z = (z + a) % 256 or 1
- return (x, y, z)
+ """Turn a hashable value into three seed values for whrandom.seed().
+
+ None or no argument returns (0, 0, 0), to seed from current time.
+
+ """
+ if a is None:
+ return (0, 0, 0)
+ a = hash(a)
+ a, x = divmod(a, 256)
+ a, y = divmod(a, 256)
+ a, z = divmod(a, 256)
+ x = (x + a) % 256 or 1
+ y = (y + a) % 256 or 1
+ z = (z + a) % 256 or 1
+ return (x, y, z)
def seed(a=None):
- """Seed the default generator from any hashable value.
+ """Seed the default generator from any hashable value.
- None or no argument seeds from current time.
+ None or no argument seeds from current time.
- """
- x, y, z = makeseed(a)
- whrandom.seed(x, y, z)
+ """
+ x, y, z = makeseed(a)
+ whrandom.seed(x, y, z)
class generator(whrandom.whrandom):
- """Random generator class."""
+ """Random generator class."""
- def __init__(self, a=None):
- """Constructor. Seed from current time or hashable value."""
- self.seed(a)
+ def __init__(self, a=None):
+ """Constructor. Seed from current time or hashable value."""
+ self.seed(a)
- def seed(self, a=None):
- """Seed the generator from current time or hashable value."""
- x, y, z = makeseed(a)
- whrandom.whrandom.seed(self, x, y, z)
+ def seed(self, a=None):
+ """Seed the generator from current time or hashable value."""
+ x, y, z = makeseed(a)
+ whrandom.whrandom.seed(self, x, y, z)
def new_generator(a=None):
- """Return a new random generator instance."""
- return generator(a)
+ """Return a new random generator instance."""
+ return generator(a)
# Housekeeping function to verify that magic constants have been
# computed correctly
def verify(name, expected):
- computed = eval(name)
- if abs(computed - expected) > 1e-7:
- raise ValueError, \
- 'computed value for %s deviates too much (computed %g, expected %g)' % \
- (name, computed, expected)
+ computed = eval(name)
+ if abs(computed - expected) > 1e-7:
+ raise ValueError, \
+'computed value for %s deviates too much (computed %g, expected %g)' % \
+(name, computed, expected)
# -------------------- normal distribution --------------------
NV_MAGICCONST = 4*exp(-0.5)/sqrt(2.0)
verify('NV_MAGICCONST', 1.71552776992141)
def normalvariate(mu, sigma):
- # mu = mean, sigma = standard deviation
-
- # Uses Kinderman and Monahan method. Reference: Kinderman,
- # A.J. and Monahan, J.F., "Computer generation of random
- # variables using the ratio of uniform deviates", ACM Trans
- # Math Software, 3, (1977), pp257-260.
-
- while 1:
- u1 = random()
- u2 = random()
- z = NV_MAGICCONST*(u1-0.5)/u2
- zz = z*z/4.0
- if zz <= -log(u2):
- break
- return mu+z*sigma
+ # mu = mean, sigma = standard deviation
+
+ # Uses Kinderman and Monahan method. Reference: Kinderman,
+ # A.J. and Monahan, J.F., "Computer generation of random
+ # variables using the ratio of uniform deviates", ACM Trans
+ # Math Software, 3, (1977), pp257-260.
+
+ while 1:
+ u1 = random()
+ u2 = random()
+ z = NV_MAGICCONST*(u1-0.5)/u2
+ zz = z*z/4.0
+ if zz <= -log(u2):
+ break
+ return mu+z*sigma
# -------------------- lognormal distribution --------------------
def lognormvariate(mu, sigma):
- return exp(normalvariate(mu, sigma))
+ return exp(normalvariate(mu, sigma))
# -------------------- circular uniform --------------------
def cunifvariate(mean, arc):
- # mean: mean angle (in radians between 0 and pi)
- # arc: range of distribution (in radians between 0 and pi)
+ # mean: mean angle (in radians between 0 and pi)
+ # arc: range of distribution (in radians between 0 and pi)
- return (mean + arc * (random() - 0.5)) % pi
+ return (mean + arc * (random() - 0.5)) % pi
# -------------------- exponential distribution --------------------
def expovariate(lambd):
- # lambd: rate lambd = 1/mean
- # ('lambda' is a Python reserved word)
+ # lambd: rate lambd = 1/mean
+ # ('lambda' is a Python reserved word)
- u = random()
- while u <= 1e-7:
- u = random()
- return -log(u)/lambd
+ u = random()
+ while u <= 1e-7:
+ u = random()
+ return -log(u)/lambd
# -------------------- von Mises distribution --------------------
verify('TWOPI', 6.28318530718)
def vonmisesvariate(mu, kappa):
- # mu: mean angle (in radians between 0 and 2*pi)
- # kappa: concentration parameter kappa (>= 0)
- # if kappa = 0 generate uniform random angle
+ # mu: mean angle (in radians between 0 and 2*pi)
+ # kappa: concentration parameter kappa (>= 0)
+ # if kappa = 0 generate uniform random angle
- # Based upon an algorithm published in: Fisher, N.I.,
- # "Statistical Analysis of Circular Data", Cambridge
- # University Press, 1993.
+ # Based upon an algorithm published in: Fisher, N.I.,
+ # "Statistical Analysis of Circular Data", Cambridge
+ # University Press, 1993.
- # Thanks to Magnus Kessler for a correction to the
- # implementation of step 4.
+ # Thanks to Magnus Kessler for a correction to the
+ # implementation of step 4.
- if kappa <= 1e-6:
- return TWOPI * random()
+ if kappa <= 1e-6:
+ return TWOPI * random()
- a = 1.0 + sqrt(1.0 + 4.0 * kappa * kappa)
- b = (a - sqrt(2.0 * a))/(2.0 * kappa)
- r = (1.0 + b * b)/(2.0 * b)
+ a = 1.0 + sqrt(1.0 + 4.0 * kappa * kappa)
+ b = (a - sqrt(2.0 * a))/(2.0 * kappa)
+ r = (1.0 + b * b)/(2.0 * b)
- while 1:
- u1 = random()
+ while 1:
+ u1 = random()
- z = cos(pi * u1)
- f = (1.0 + r * z)/(r + z)
- c = kappa * (r - f)
+ z = cos(pi * u1)
+ f = (1.0 + r * z)/(r + z)
+ c = kappa * (r - f)
- u2 = random()
+ u2 = random()
- if not (u2 >= c * (2.0 - c) and u2 > c * exp(1.0 - c)):
- break
+ if not (u2 >= c * (2.0 - c) and u2 > c * exp(1.0 - c)):
+ break
- u3 = random()
- if u3 > 0.5:
- theta = (mu % TWOPI) + acos(f)
- else:
- theta = (mu % TWOPI) - acos(f)
+ u3 = random()
+ if u3 > 0.5:
+ theta = (mu % TWOPI) + acos(f)
+ else:
+ theta = (mu % TWOPI) - acos(f)
- return theta
+ return theta
# -------------------- gamma distribution --------------------
verify('LOG4', 1.38629436111989)
def gammavariate(alpha, beta):
- # beta times standard gamma
- ainv = sqrt(2.0 * alpha - 1.0)
- return beta * stdgamma(alpha, ainv, alpha - LOG4, alpha + ainv)
+ # beta times standard gamma
+ ainv = sqrt(2.0 * alpha - 1.0)
+ return beta * stdgamma(alpha, ainv, alpha - LOG4, alpha + ainv)
SG_MAGICCONST = 1.0 + log(4.5)
verify('SG_MAGICCONST', 2.50407739677627)
def stdgamma(alpha, ainv, bbb, ccc):
- # ainv = sqrt(2 * alpha - 1)
- # bbb = alpha - log(4)
- # ccc = alpha + ainv
-
- if alpha <= 0.0:
- raise ValueError, 'stdgamma: alpha must be > 0.0'
-
- if alpha > 1.0:
-
- # Uses R.C.H. Cheng, "The generation of Gamma
- # variables with non-integral shape parameters",
- # Applied Statistics, (1977), 26, No. 1, p71-74
-
- while 1:
- u1 = random()
- u2 = random()
- v = log(u1/(1.0-u1))/ainv
- x = alpha*exp(v)
- z = u1*u1*u2
- r = bbb+ccc*v-x
- if r + SG_MAGICCONST - 4.5*z >= 0.0 or r >= log(z):
- return x
-
- elif alpha == 1.0:
- # expovariate(1)
- u = random()
- while u <= 1e-7:
- u = random()
- return -log(u)
-
- else: # alpha is between 0 and 1 (exclusive)
-
- # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle
-
- while 1:
- u = random()
- b = (e + alpha)/e
- p = b*u
- if p <= 1.0:
- x = pow(p, 1.0/alpha)
- else:
- # p > 1
- x = -log((b-p)/alpha)
- u1 = random()
- if not (((p <= 1.0) and (u1 > exp(-x))) or
- ((p > 1) and (u1 > pow(x, alpha - 1.0)))):
- break
- return x
+ # ainv = sqrt(2 * alpha - 1)
+ # bbb = alpha - log(4)
+ # ccc = alpha + ainv
+
+ if alpha <= 0.0:
+ raise ValueError, 'stdgamma: alpha must be > 0.0'
+
+ if alpha > 1.0:
+
+ # Uses R.C.H. Cheng, "The generation of Gamma
+ # variables with non-integral shape parameters",
+ # Applied Statistics, (1977), 26, No. 1, p71-74
+
+ while 1:
+ u1 = random()
+ u2 = random()
+ v = log(u1/(1.0-u1))/ainv
+ x = alpha*exp(v)
+ z = u1*u1*u2
+ r = bbb+ccc*v-x
+ if r + SG_MAGICCONST - 4.5*z >= 0.0 or r >= log(z):
+ return x
+
+ elif alpha == 1.0:
+ # expovariate(1)
+ u = random()
+ while u <= 1e-7:
+ u = random()
+ return -log(u)
+
+ else: # alpha is between 0 and 1 (exclusive)
+
+ # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle
+
+ while 1:
+ u = random()
+ b = (e + alpha)/e
+ p = b*u
+ if p <= 1.0:
+ x = pow(p, 1.0/alpha)
+ else:
+ # p > 1
+ x = -log((b-p)/alpha)
+ u1 = random()
+ if not (((p <= 1.0) and (u1 > exp(-x))) or
+ ((p > 1) and (u1 > pow(x, alpha - 1.0)))):
+ break
+ return x
# -------------------- Gauss (faster alternative) --------------------
gauss_next = None
def gauss(mu, sigma):
- # When x and y are two variables from [0, 1), uniformly
- # distributed, then
- #
- # cos(2*pi*x)*sqrt(-2*log(1-y))
- # sin(2*pi*x)*sqrt(-2*log(1-y))
- #
- # are two *independent* variables with normal distribution
- # (mu = 0, sigma = 1).
- # (Lambert Meertens)
- # (corrected version; bug discovered by Mike Miller, fixed by LM)
-
- # Multithreading note: When two threads call this function
- # simultaneously, it is possible that they will receive the
- # same return value. The window is very small though. To
- # avoid this, you have to use a lock around all calls. (I
- # didn't want to slow this down in the serial case by using a
- # lock here.)
-
- global gauss_next
-
- z = gauss_next
- gauss_next = None
- if z is None:
- x2pi = random() * TWOPI
- g2rad = sqrt(-2.0 * log(1.0 - random()))
- z = cos(x2pi) * g2rad
- gauss_next = sin(x2pi) * g2rad
-
- return mu + z*sigma
+ # When x and y are two variables from [0, 1), uniformly
+ # distributed, then
+ #
+ # cos(2*pi*x)*sqrt(-2*log(1-y))
+ # sin(2*pi*x)*sqrt(-2*log(1-y))
+ #
+ # are two *independent* variables with normal distribution
+ # (mu = 0, sigma = 1).
+ # (Lambert Meertens)
+ # (corrected version; bug discovered by Mike Miller, fixed by LM)
+
+ # Multithreading note: When two threads call this function
+ # simultaneously, it is possible that they will receive the
+ # same return value. The window is very small though. To
+ # avoid this, you have to use a lock around all calls. (I
+ # didn't want to slow this down in the serial case by using a
+ # lock here.)
+
+ global gauss_next
+
+ z = gauss_next
+ gauss_next = None
+ if z is None:
+ x2pi = random() * TWOPI
+ g2rad = sqrt(-2.0 * log(1.0 - random()))
+ z = cos(x2pi) * g2rad
+ gauss_next = sin(x2pi) * g2rad
+
+ return mu + z*sigma
# -------------------- beta --------------------
def betavariate(alpha, beta):
- # Discrete Event Simulation in C, pp 87-88.
+ # Discrete Event Simulation in C, pp 87-88.
- y = expovariate(alpha)
- z = expovariate(1.0/beta)
- return z/(y+z)
+ y = expovariate(alpha)
+ z = expovariate(1.0/beta)
+ return z/(y+z)
# -------------------- Pareto --------------------
def paretovariate(alpha):
- # Jain, pg. 495
+ # Jain, pg. 495
- u = random()
- return 1.0 / pow(u, 1.0/alpha)
+ u = random()
+ return 1.0 / pow(u, 1.0/alpha)
# -------------------- Weibull --------------------
def weibullvariate(alpha, beta):
- # Jain, pg. 499; bug fix courtesy Bill Arms
+ # Jain, pg. 499; bug fix courtesy Bill Arms
- u = random()
- return alpha * pow(-log(u), 1.0/beta)
+ u = random()
+ return alpha * pow(-log(u), 1.0/beta)
# -------------------- shuffle --------------------
# Not quite a random distribution, but a standard algorithm.
"""
for i in xrange(len(x)-1, 0, -1):
- # pick an element in x[:i+1] with which to exchange x[i]
+ # pick an element in x[:i+1] with which to exchange x[i]
j = int(random() * (i+1))
x[i], x[j] = x[j], x[i]
# -------------------- test program --------------------
def test(N = 200):
- print 'TWOPI =', TWOPI
- print 'LOG4 =', LOG4
- print 'NV_MAGICCONST =', NV_MAGICCONST
- print 'SG_MAGICCONST =', SG_MAGICCONST
- test_generator(N, 'random()')
- test_generator(N, 'normalvariate(0.0, 1.0)')
- test_generator(N, 'lognormvariate(0.0, 1.0)')
- test_generator(N, 'cunifvariate(0.0, 1.0)')
- test_generator(N, 'expovariate(1.0)')
- test_generator(N, 'vonmisesvariate(0.0, 1.0)')
- test_generator(N, 'gammavariate(0.5, 1.0)')
- test_generator(N, 'gammavariate(0.9, 1.0)')
- test_generator(N, 'gammavariate(1.0, 1.0)')
- test_generator(N, 'gammavariate(2.0, 1.0)')
- test_generator(N, 'gammavariate(20.0, 1.0)')
- test_generator(N, 'gammavariate(200.0, 1.0)')
- test_generator(N, 'gauss(0.0, 1.0)')
- test_generator(N, 'betavariate(3.0, 3.0)')
- test_generator(N, 'paretovariate(1.0)')
- test_generator(N, 'weibullvariate(1.0, 1.0)')
+ print 'TWOPI =', TWOPI
+ print 'LOG4 =', LOG4
+ print 'NV_MAGICCONST =', NV_MAGICCONST
+ print 'SG_MAGICCONST =', SG_MAGICCONST
+ test_generator(N, 'random()')
+ test_generator(N, 'normalvariate(0.0, 1.0)')
+ test_generator(N, 'lognormvariate(0.0, 1.0)')
+ test_generator(N, 'cunifvariate(0.0, 1.0)')
+ test_generator(N, 'expovariate(1.0)')
+ test_generator(N, 'vonmisesvariate(0.0, 1.0)')
+ test_generator(N, 'gammavariate(0.5, 1.0)')
+ test_generator(N, 'gammavariate(0.9, 1.0)')
+ test_generator(N, 'gammavariate(1.0, 1.0)')
+ test_generator(N, 'gammavariate(2.0, 1.0)')
+ test_generator(N, 'gammavariate(20.0, 1.0)')
+ test_generator(N, 'gammavariate(200.0, 1.0)')
+ test_generator(N, 'gauss(0.0, 1.0)')
+ test_generator(N, 'betavariate(3.0, 3.0)')
+ test_generator(N, 'paretovariate(1.0)')
+ test_generator(N, 'weibullvariate(1.0, 1.0)')
def test_generator(n, funccall):
- import time
- print n, 'times', funccall
- code = compile(funccall, funccall, 'eval')
- sum = 0.0
- sqsum = 0.0
- smallest = 1e10
- largest = -1e10
- t0 = time.time()
- for i in range(n):
- x = eval(code)
- sum = sum + x
- sqsum = sqsum + x*x
- smallest = min(x, smallest)
- largest = max(x, largest)
- t1 = time.time()
- print round(t1-t0, 3), 'sec,',
- avg = sum/n
- stddev = sqrt(sqsum/n - avg*avg)
- print 'avg %g, stddev %g, min %g, max %g' % \
- (avg, stddev, smallest, largest)
+ import time
+ print n, 'times', funccall
+ code = compile(funccall, funccall, 'eval')
+ sum = 0.0
+ sqsum = 0.0
+ smallest = 1e10
+ largest = -1e10
+ t0 = time.time()
+ for i in range(n):
+ x = eval(code)
+ sum = sum + x
+ sqsum = sqsum + x*x
+ smallest = min(x, smallest)
+ largest = max(x, largest)
+ t1 = time.time()
+ print round(t1-t0, 3), 'sec,',
+ avg = sum/n
+ stddev = sqrt(sqsum/n - avg*avg)
+ print 'avg %g, stddev %g, min %g, max %g' % \
+ (avg, stddev, smallest, largest)
if __name__ == '__main__':
- test()
+ test()
import regex
-from regex_syntax import * # RE_*
+from regex_syntax import * # RE_*
# Default translation table
mastertable = {
# their special meaning regardless of the surrounding context.
# 1 means that special characters may act as normal characters in some
# contexts. Specifically, this applies to:
-# ^ - only special at the beginning, or after ( or |
-# $ - only special at the end, or before ) or |
-# *, +, ? - only special when not after the beginning, (, or |
+# ^ - only special at the beginning, or after ( or |
+# $ - only special at the end, or before ) or |
+# *, +, ? - only special when not after the beginning, (, or |
RE_CONTEXT_INDEP_OPS = 32
# ANSI sequences (\n etc) and \xhh
import warnings
warnings.warn("the regsub module is deprecated; please use re.sub()",
- DeprecationWarning)
+ DeprecationWarning)
# Ignore further deprecation warnings about this module
warnings.filterwarnings("ignore", "", DeprecationWarning, __name__)
# compiled pattern.
def sub(pat, repl, str):
- prog = compile(pat)
- if prog.search(str) >= 0:
- regs = prog.regs
- a, b = regs[0]
- str = str[:a] + expand(repl, regs, str) + str[b:]
- return str
+ prog = compile(pat)
+ if prog.search(str) >= 0:
+ regs = prog.regs
+ a, b = regs[0]
+ str = str[:a] + expand(repl, regs, str) + str[b:]
+ return str
# Replace all (non-overlapping) occurrences of pattern pat in string
# a previous match, so e.g. gsub('', '-', 'abc') returns '-a-b-c-'.
def gsub(pat, repl, str):
- prog = compile(pat)
- new = ''
- start = 0
- first = 1
- while prog.search(str, start) >= 0:
- regs = prog.regs
- a, b = regs[0]
- if a == b == start and not first:
- if start >= len(str) or prog.search(str, start+1) < 0:
- break
- regs = prog.regs
- a, b = regs[0]
- new = new + str[start:a] + expand(repl, regs, str)
- start = b
- first = 0
- new = new + str[start:]
- return new
+ prog = compile(pat)
+ new = ''
+ start = 0
+ first = 1
+ while prog.search(str, start) >= 0:
+ regs = prog.regs
+ a, b = regs[0]
+ if a == b == start and not first:
+ if start >= len(str) or prog.search(str, start+1) < 0:
+ break
+ regs = prog.regs
+ a, b = regs[0]
+ new = new + str[start:a] + expand(repl, regs, str)
+ start = b
+ first = 0
+ new = new + str[start:]
+ return new
# Split string str in fields separated by delimiters matching pattern
# The optional 3rd argument sets the number of splits that are performed.
def split(str, pat, maxsplit = 0):
- return intsplit(str, pat, maxsplit, 0)
+ return intsplit(str, pat, maxsplit, 0)
# Split string str in fields separated by delimiters matching pattern
# pat. Only non-empty matches for the pattern are considered, so e.g.
def splitx(str, pat, maxsplit = 0):
- return intsplit(str, pat, maxsplit, 1)
-
+ return intsplit(str, pat, maxsplit, 1)
+
# Internal function used to implement split() and splitx().
def intsplit(str, pat, maxsplit, retain):
- prog = compile(pat)
- res = []
- start = next = 0
- splitcount = 0
- while prog.search(str, next) >= 0:
- regs = prog.regs
- a, b = regs[0]
- if a == b:
- next = next + 1
- if next >= len(str):
- break
- else:
- res.append(str[start:a])
- if retain:
- res.append(str[a:b])
- start = next = b
- splitcount = splitcount + 1
- if (maxsplit and (splitcount >= maxsplit)):
- break
- res.append(str[start:])
- return res
+ prog = compile(pat)
+ res = []
+ start = next = 0
+ splitcount = 0
+ while prog.search(str, next) >= 0:
+ regs = prog.regs
+ a, b = regs[0]
+ if a == b:
+ next = next + 1
+ if next >= len(str):
+ break
+ else:
+ res.append(str[start:a])
+ if retain:
+ res.append(str[a:b])
+ start = next = b
+ splitcount = splitcount + 1
+ if (maxsplit and (splitcount >= maxsplit)):
+ break
+ res.append(str[start:])
+ return res
# Capitalize words split using a pattern
def capwords(str, pat='[^a-zA-Z0-9_]+'):
- import string
- words = splitx(str, pat)
- for i in range(0, len(words), 2):
- words[i] = string.capitalize(words[i])
- return string.joinfields(words, "")
+ import string
+ words = splitx(str, pat)
+ for i in range(0, len(words), 2):
+ words[i] = string.capitalize(words[i])
+ return string.joinfields(words, "")
# Internal subroutines:
cache = {}
def compile(pat):
- if type(pat) != type(''):
- return pat # Assume it is a compiled regex
- key = (pat, regex.get_syntax())
- if cache.has_key(key):
- prog = cache[key] # Get it from the cache
- else:
- prog = cache[key] = regex.compile(pat)
- return prog
+ if type(pat) != type(''):
+ return pat # Assume it is a compiled regex
+ key = (pat, regex.get_syntax())
+ if cache.has_key(key):
+ prog = cache[key] # Get it from the cache
+ else:
+ prog = cache[key] = regex.compile(pat)
+ return prog
def clear_cache():
- global cache
- cache = {}
+ global cache
+ cache = {}
# Expand \digit in the replacement.
# the \ and the following character are both copied).
def expand(repl, regs, str):
- if '\\' not in repl:
- return repl
- new = ''
- i = 0
- ord0 = ord('0')
- while i < len(repl):
- c = repl[i]; i = i+1
- if c != '\\' or i >= len(repl):
- new = new + c
- else:
- c = repl[i]; i = i+1
- if '0' <= c <= '9':
- a, b = regs[ord(c)-ord0]
- new = new + str[a:b]
- elif c == '\\':
- new = new + c
- else:
- new = new + '\\' + c
- return new
+ if '\\' not in repl:
+ return repl
+ new = ''
+ i = 0
+ ord0 = ord('0')
+ while i < len(repl):
+ c = repl[i]; i = i+1
+ if c != '\\' or i >= len(repl):
+ new = new + c
+ else:
+ c = repl[i]; i = i+1
+ if '0' <= c <= '9':
+ a, b = regs[ord(c)-ord0]
+ new = new + str[a:b]
+ elif c == '\\':
+ new = new + c
+ else:
+ new = new + '\\' + c
+ return new
# Test program, reads sequences "pat repl str" from stdin.
# Optional argument specifies pattern used to split lines.
def test():
- import sys
- if sys.argv[1:]:
- delpat = sys.argv[1]
- else:
- delpat = '[ \t\n]+'
- while 1:
- if sys.stdin.isatty(): sys.stderr.write('--> ')
- line = sys.stdin.readline()
- if not line: break
- if line[-1] == '\n': line = line[:-1]
- fields = split(line, delpat)
- if len(fields) != 3:
- print 'Sorry, not three fields'
- print 'split:', `fields`
- continue
- [pat, repl, str] = split(line, delpat)
- print 'sub :', `sub(pat, repl, str)`
- print 'gsub:', `gsub(pat, repl, str)`
+ import sys
+ if sys.argv[1:]:
+ delpat = sys.argv[1]
+ else:
+ delpat = '[ \t\n]+'
+ while 1:
+ if sys.stdin.isatty(): sys.stderr.write('--> ')
+ line = sys.stdin.readline()
+ if not line: break
+ if line[-1] == '\n': line = line[:-1]
+ fields = split(line, delpat)
+ if len(fields) != 3:
+ print 'Sorry, not three fields'
+ print 'split:', `fields`
+ continue
+ [pat, repl, str] = split(line, delpat)
+ print 'sub :', `sub(pat, repl, str)`
+ print 'gsub:', `gsub(pat, repl, str)`
import string
class Repr:
- def __init__(self):
- self.maxlevel = 6
- self.maxtuple = 6
- self.maxlist = 6
- self.maxdict = 4
- self.maxstring = 30
- self.maxlong = 40
- self.maxother = 20
- def repr(self, x):
- return self.repr1(x, self.maxlevel)
- def repr1(self, x, level):
- typename = `type(x)`[7:-2] # "<type '......'>"
- if ' ' in typename:
- parts = string.split(typename)
- typename = string.joinfields(parts, '_')
- if hasattr(self, 'repr_' + typename):
- return getattr(self, 'repr_' + typename)(x, level)
- else:
- s = `x`
- if len(s) > self.maxother:
- i = max(0, (self.maxother-3)/2)
- j = max(0, self.maxother-3-i)
- s = s[:i] + '...' + s[len(s)-j:]
- return s
- def repr_tuple(self, x, level):
- n = len(x)
- if n == 0: return '()'
- if level <= 0: return '(...)'
- s = ''
- for i in range(min(n, self.maxtuple)):
- if s: s = s + ', '
- s = s + self.repr1(x[i], level-1)
- if n > self.maxtuple: s = s + ', ...'
- elif n == 1: s = s + ','
- return '(' + s + ')'
- def repr_list(self, x, level):
- n = len(x)
- if n == 0: return '[]'
- if level <= 0: return '[...]'
- s = ''
- for i in range(min(n, self.maxlist)):
- if s: s = s + ', '
- s = s + self.repr1(x[i], level-1)
- if n > self.maxlist: s = s + ', ...'
- return '[' + s + ']'
- def repr_dictionary(self, x, level):
- n = len(x)
- if n == 0: return '{}'
- if level <= 0: return '{...}'
- s = ''
- keys = x.keys()
- keys.sort()
- for i in range(min(n, self.maxdict)):
- if s: s = s + ', '
- key = keys[i]
- s = s + self.repr1(key, level-1)
- s = s + ': ' + self.repr1(x[key], level-1)
- if n > self.maxdict: s = s + ', ...'
- return '{' + s + '}'
- def repr_string(self, x, level):
- s = `x[:self.maxstring]`
- if len(s) > self.maxstring:
- i = max(0, (self.maxstring-3)/2)
- j = max(0, self.maxstring-3-i)
- s = `x[:i] + x[len(x)-j:]`
- s = s[:i] + '...' + s[len(s)-j:]
- return s
- def repr_long_int(self, x, level):
- s = `x` # XXX Hope this isn't too slow...
- if len(s) > self.maxlong:
- i = max(0, (self.maxlong-3)/2)
- j = max(0, self.maxlong-3-i)
- s = s[:i] + '...' + s[len(s)-j:]
- return s
- def repr_instance(self, x, level):
- try:
- s = `x`
- # Bugs in x.__repr__() can cause arbitrary
- # exceptions -- then make up something
- except:
- return '<' + x.__class__.__name__ + ' instance at ' + \
- hex(id(x))[2:] + '>'
- if len(s) > self.maxstring:
- i = max(0, (self.maxstring-3)/2)
- j = max(0, self.maxstring-3-i)
- s = s[:i] + '...' + s[len(s)-j:]
- return s
+ def __init__(self):
+ self.maxlevel = 6
+ self.maxtuple = 6
+ self.maxlist = 6
+ self.maxdict = 4
+ self.maxstring = 30
+ self.maxlong = 40
+ self.maxother = 20
+ def repr(self, x):
+ return self.repr1(x, self.maxlevel)
+ def repr1(self, x, level):
+ typename = `type(x)`[7:-2] # "<type '......'>"
+ if ' ' in typename:
+ parts = string.split(typename)
+ typename = string.joinfields(parts, '_')
+ if hasattr(self, 'repr_' + typename):
+ return getattr(self, 'repr_' + typename)(x, level)
+ else:
+ s = `x`
+ if len(s) > self.maxother:
+ i = max(0, (self.maxother-3)/2)
+ j = max(0, self.maxother-3-i)
+ s = s[:i] + '...' + s[len(s)-j:]
+ return s
+ def repr_tuple(self, x, level):
+ n = len(x)
+ if n == 0: return '()'
+ if level <= 0: return '(...)'
+ s = ''
+ for i in range(min(n, self.maxtuple)):
+ if s: s = s + ', '
+ s = s + self.repr1(x[i], level-1)
+ if n > self.maxtuple: s = s + ', ...'
+ elif n == 1: s = s + ','
+ return '(' + s + ')'
+ def repr_list(self, x, level):
+ n = len(x)
+ if n == 0: return '[]'
+ if level <= 0: return '[...]'
+ s = ''
+ for i in range(min(n, self.maxlist)):
+ if s: s = s + ', '
+ s = s + self.repr1(x[i], level-1)
+ if n > self.maxlist: s = s + ', ...'
+ return '[' + s + ']'
+ def repr_dictionary(self, x, level):
+ n = len(x)
+ if n == 0: return '{}'
+ if level <= 0: return '{...}'
+ s = ''
+ keys = x.keys()
+ keys.sort()
+ for i in range(min(n, self.maxdict)):
+ if s: s = s + ', '
+ key = keys[i]
+ s = s + self.repr1(key, level-1)
+ s = s + ': ' + self.repr1(x[key], level-1)
+ if n > self.maxdict: s = s + ', ...'
+ return '{' + s + '}'
+ def repr_string(self, x, level):
+ s = `x[:self.maxstring]`
+ if len(s) > self.maxstring:
+ i = max(0, (self.maxstring-3)/2)
+ j = max(0, self.maxstring-3-i)
+ s = `x[:i] + x[len(x)-j:]`
+ s = s[:i] + '...' + s[len(s)-j:]
+ return s
+ def repr_long_int(self, x, level):
+ s = `x` # XXX Hope this isn't too slow...
+ if len(s) > self.maxlong:
+ i = max(0, (self.maxlong-3)/2)
+ j = max(0, self.maxlong-3-i)
+ s = s[:i] + '...' + s[len(s)-j:]
+ return s
+ def repr_instance(self, x, level):
+ try:
+ s = `x`
+ # Bugs in x.__repr__() can cause arbitrary
+ # exceptions -- then make up something
+ except:
+ return '<' + x.__class__.__name__ + ' instance at ' + \
+ hex(id(x))[2:] + '>'
+ if len(s) > self.maxstring:
+ i = max(0, (self.maxstring-3)/2)
+ j = max(0, self.maxstring-3-i)
+ s = s[:i] + '...' + s[len(s)-j:]
+ return s
aRepr = Repr()
repr = aRepr.repr
def r_unload(self, m):
return self.importer.unload(m)
-
+
# The s_* methods are similar but also swap std{in,out,err}
def make_delegate_files(self):
self.restricted_stdin = s.stdin
self.restricted_stdout = s.stdout
self.restricted_stderr = s.stderr
-
+
def save_files(self):
self.save_stdin = sys.stdin
sys.stdin = self.save_stdin
sys.stdout = self.save_stdout
sys.stderr = self.save_stderr
-
+
def s_apply(self, func, args=(), kw=None):
self.save_files()
try:
r = apply(func, args)
finally:
self.restore_files()
-
+
def s_exec(self, *args):
self.s_apply(self.r_exec, args)
-
+
def s_eval(self, *args):
self.s_apply(self.r_eval, args)
-
+
def s_execfile(self, *args):
self.s_apply(self.r_execfile, args)
-
+
def s_import(self, *args):
self.s_apply(self.r_import, args)
-
+
def s_reload(self, *args):
self.s_apply(self.r_reload, args)
-
+
def s_unload(self, *args):
self.s_apply(self.r_unload, args)
-
+
# Restricted open(...)
-
+
def r_open(self, file, mode='r', buf=-1):
if mode not in ('r', 'rb'):
raise IOError, "can't open files for writing in restricted mode"
that initial \code{tell} when passing in an unseekable object such as
a a file object created from a socket object. If it is 1 on entry --
which it is by default -- the tell() method of the open file object is
-called once; if this raises an exception, seekable is reset to 0. For
+called once; if this raises an exception, seekable is reset to 0. For
other nonzero values of seekable, this test is not made.
To get the text of a particular header there are several methods:
class Message:
"""Represents a single RFC-822-compliant message."""
-
+
def __init__(self, fp, seekable = 1):
"""Initialize the class instance and read the headers."""
if seekable == 1:
self.startofbody = self.fp.tell()
except IOError:
self.seekable = 0
-
+
def rewindbody(self):
"""Rewind the file to the start of the body (if seekable)."""
if not self.seekable:
raise IOError, "unseekable file"
self.fp.seek(self.startofbody)
-
+
def readheaders(self):
"""Read header lines.
-
+
Read header lines up to the entirely blank line that
terminates them. The (normally blank) line that ends the
headers is skipped, but not included in the returned list.
If a non-header line ends the headers, (which is an error),
an attempt is made to backspace over it; it is never
included in the returned list.
-
+
The variable self.status is set to the empty string if all
went well, otherwise it is an error message.
The variable self.headers is a completely uninterpreted list
return line[:i].lower()
else:
return None
-
+
def islast(self, line):
"""Determine whether a line is a legal end of RFC-822 headers.
-
+
You may override this method if your application wants
to bend the rules, e.g. to strip trailing whitespace,
or to recognize MH template separators ('--------').
For convenience (e.g. for code reading from sockets) a
- line consisting of \r\n also matches.
+ line consisting of \r\n also matches.
"""
return line in _blanklines
comments or free-text data.
"""
return None
-
+
def getallmatchingheaders(self, name):
"""Find all header lines matching a given header name.
-
+
Look through the list of headers and find all lines
matching a given header name (and their continuation
lines). A list of the lines is returned, without
if hit:
list.append(line)
return list
-
+
def getfirstmatchingheader(self, name):
"""Get the first header line matching name.
-
+
This is similar to getallmatchingheaders, but it returns
only the first matching header (and its continuation
lines).
if hit:
list.append(line)
return list
-
+
def getrawheader(self, name):
"""A higher-level interface to getfirstmatchingheader().
-
+
Return a string containing the literal text of the
header but with the keyword stripped. All leading,
trailing and embedded whitespace is kept in the
string, however.
Return None if the header does not occur.
"""
-
+
list = self.getfirstmatchingheader(name)
if not list:
return None
list[0] = list[0][len(name) + 1:]
return ''.join(list)
-
+
def getheader(self, name, default=None):
"""Get the header value for a name.
-
+
This is the normal interface: it returns a stripped
version of the header value for a given header name,
or None if it doesn't exist. This uses the dictionary
if have_header:
result.append(current)
return result
-
+
def getaddr(self, name):
"""Get a single address from a header, as a tuple.
-
+
An example return value:
('Guido van Rossum', 'guido@cwi.nl')
"""
return alist[0]
else:
return (None, None)
-
+
def getaddrlist(self, name):
"""Get a list of addresses from a header.
alladdrs = ''.join(raw)
a = AddrlistClass(alladdrs)
return a.getaddrlist()
-
+
def getdate(self, name):
"""Retrieve a date field from a header.
-
+
Retrieves a date field from the named header, returning
a tuple compatible with time.mktime().
"""
except KeyError:
return None
return parsedate(data)
-
+
def getdate_tz(self, name):
"""Retrieve a date field from a header as a 10-tuple.
-
+
The first 9 elements make up a tuple compatible with
time.mktime(), and the 10th is the offset of the poster's
time zone from GMT/UTC.
except KeyError:
return None
return parsedate_tz(data)
-
-
+
+
# Access as a dictionary (only finds *last* header of each type):
-
+
def __len__(self):
"""Get the number of headers in a message."""
return len(self.dict)
-
+
def __getitem__(self, name):
"""Get a specific header, as from a dictionary."""
return self.dict[name.lower()]
def __setitem__(self, name, value):
"""Set the value of a header.
- Note: This is not a perfect inversion of __getitem__, because
+ Note: This is not a perfect inversion of __getitem__, because
any changed headers get stuck at the end of the raw-headers list
rather than where the altered header was.
"""
lines = text.split("\n")
for line in lines:
self.headers.append(line + "\n")
-
+
def __delitem__(self, name):
"""Delete all occurrences of a specific header, if it is present."""
name = name.lower()
def has_key(self, name):
"""Determine whether a message contains the named header."""
return self.dict.has_key(name.lower())
-
+
def keys(self):
"""Get all of a message's header field names."""
return self.dict.keys()
-
+
def values(self):
"""Get all of a message's header field values."""
return self.dict.values()
-
+
def items(self):
"""Get all of a message's headers.
-
+
Returns a list of name, value tuples.
"""
return self.dict.items()
class AddrlistClass:
"""Address parser class by Ben Escoto.
-
+
To understand what this class does, it helps to have a copy of
RFC-822 in front of you.
Note: this class interface is deprecated and may be removed in the future.
Use rfc822.AddressList instead.
"""
-
+
def __init__(self, field):
"""Initialize a new instance.
-
+
`field' is an unparsed address header field, containing
one or more addresses.
"""
self.atomends = self.specials + self.LWS + self.CR
self.field = field
self.commentlist = []
-
+
def gotonext(self):
"""Parse up to the start of the next address."""
while self.pos < len(self.field):
elif self.field[self.pos] == '(':
self.commentlist.append(self.getcomment())
else: break
-
+
def getaddrlist(self):
"""Parse all addresses.
-
+
Returns a list containing all of the addresses.
"""
ad = self.getaddress()
if ad:
return ad + self.getaddrlist()
else: return []
-
+
def getaddress(self):
"""Parse the next address."""
self.commentlist = []
self.gotonext()
-
+
oldpos = self.pos
oldcl = self.commentlist
plist = self.getphraselist()
-
+
self.gotonext()
returnlist = []
-
+
if self.pos >= len(self.field):
# Bad email address technically, no domain.
if plist:
returnlist = [(' '.join(self.commentlist), plist[0])]
-
+
elif self.field[self.pos] in '.@':
# email address is just an addrspec
# this isn't very efficient since we start over
self.commentlist = oldcl
addrspec = self.getaddrspec()
returnlist = [(' '.join(self.commentlist), addrspec)]
-
+
elif self.field[self.pos] == ':':
# address is a group
returnlist = []
-
+
fieldlen = len(self.field)
self.pos = self.pos + 1
while self.pos < len(self.field):
self.pos = self.pos + 1
break
returnlist = returnlist + self.getaddress()
-
+
elif self.field[self.pos] == '<':
# Address is a phrase then a route addr
routeaddr = self.getrouteaddr()
-
+
if self.commentlist:
returnlist = [(' '.join(plist) + ' (' + \
' '.join(self.commentlist) + ')', routeaddr)]
else: returnlist = [(' '.join(plist), routeaddr)]
-
+
else:
if plist:
returnlist = [(' '.join(self.commentlist), plist[0])]
elif self.field[self.pos] in self.specials:
self.pos = self.pos + 1
-
+
self.gotonext()
if self.pos < len(self.field) and self.field[self.pos] == ',':
self.pos = self.pos + 1
return returnlist
-
+
def getrouteaddr(self):
"""Parse a route address (Return-path value).
-
+
This method just skips all the route stuff and returns the addrspec.
"""
if self.field[self.pos] != '<':
return
-
+
expectroute = 0
self.pos = self.pos + 1
self.gotonext()
self.pos = self.pos + 1
break
self.gotonext()
-
+
return adlist
-
+
def getaddrspec(self):
"""Parse an RFC-822 addr-spec."""
aslist = []
-
+
self.gotonext()
while self.pos < len(self.field):
if self.field[self.pos] == '.':
break
else: aslist.append(self.getatom())
self.gotonext()
-
+
if self.pos >= len(self.field) or self.field[self.pos] != '@':
return ''.join(aslist)
-
+
aslist.append('@')
self.pos = self.pos + 1
self.gotonext()
return ''.join(aslist) + self.getdomain()
-
+
def getdomain(self):
"""Get the complete domain name from an address."""
sdlist = []
break
else: sdlist.append(self.getatom())
return ''.join(sdlist)
-
+
def getdelimited(self, beginchar, endchars, allowcomments = 1):
"""Parse a header fragment delimited by special characters.
-
+
`beginchar' is the start character for the fragment.
If self is not looking at an instance of `beginchar' then
getdelimited returns the empty string.
-
+
`endchars' is a sequence of allowable end-delimiting characters.
Parsing stops when one of these is encountered.
-
+
If `allowcomments' is non-zero, embedded RFC-822 comments
are allowed within the parsed fragment.
"""
if self.field[self.pos] != beginchar:
return ''
-
+
slist = ['']
quote = 0
self.pos = self.pos + 1
else:
slist.append(self.field[self.pos])
self.pos = self.pos + 1
-
+
return ''.join(slist)
-
+
def getquote(self):
"""Get a quote-delimited fragment from self's field."""
return self.getdelimited('"', '"\r', 0)
-
+
def getcomment(self):
"""Get a parenthesis-delimited fragment from self's field."""
return self.getdelimited('(', ')\r', 1)
-
+
def getdomainliteral(self):
"""Parse an RFC-822 domain-literal."""
return '[%s]' % self.getdelimited('[', ']\r', 0)
-
+
def getatom(self):
"""Parse an RFC-822 atom."""
atomlist = ['']
-
+
while self.pos < len(self.field):
if self.field[self.pos] in self.atomends:
break
else: atomlist.append(self.field[self.pos])
self.pos = self.pos + 1
-
+
return ''.join(atomlist)
-
+
def getphraselist(self):
"""Parse a sequence of RFC-822 phrases.
-
+
A phrase is a sequence of words, which are in turn either
RFC-822 atoms or quoted-strings. Phrases are canonicalized
by squeezing all runs of continuous whitespace into one space.
"""
plist = []
-
+
while self.pos < len(self.field):
if self.field[self.pos] in self.LWS:
self.pos = self.pos + 1
elif self.field[self.pos] in self.atomends:
break
else: plist.append(self.getatom())
-
+
return plist
class AddressList(AddrlistClass):
# zones. RFC1123 recommends that numeric timezone indicators be used
# instead of timezone names.
-_timezones = {'UT':0, 'UTC':0, 'GMT':0, 'Z':0,
+_timezones = {'UT':0, 'UTC':0, 'GMT':0, 'Z':0,
'AST': -400, 'ADT': -300, # Atlantic (used in Canada)
'EST': -500, 'EDT': -400, # Eastern
'CST': -600, 'CDT': -500, # Central
'MST': -700, 'MDT': -600, # Mountain
'PST': -800, 'PDT': -700 # Pacific
- }
+ }
def parsedate_tz(data):
"""Convert a date string to a time tuple.
-
+
Accounts for military timezones.
"""
data = data.split()
if _timezones.has_key(tz):
tzoffset = _timezones[tz]
else:
- try:
+ try:
tzoffset = int(tz)
- except ValueError:
+ except ValueError:
pass
# Convert a timezone offset into seconds ; -0500 -> -18000
if tzoffset:
t = parsedate_tz(data)
if type(t) == type( () ):
return t[:9]
- else: return t
+ else: return t
def mktime_tz(data):
projects / python / commitdiff