diff --git a/README.md b/README.md new file mode 100644 index 0000000..0ba683b --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# zbase62 +A fork of [simplegeo/zbase62](https://github.com/simplegeo/zbase62). + +Changes: + - Remove dependency on [pyutil](https://github.com/simplegeo/pyutil) so that file [zbase62.py](/zbase62/zbase62.py) is self-contained thus can be used in a drop-in way. + - Make code work on Python 3 too. + +Python: 2.4+ and 3.0+ diff --git a/ez_setup.py b/ez_setup.py index 6d76468..804d30f 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -13,6 +13,9 @@ This file can also be run as a script to install or upgrade setuptools. """ + +from __future__ import print_function + import sys DEFAULT_VERSION = "0.6c11" DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3] @@ -70,9 +73,9 @@ def _validate_md5(egg_name, data): if egg_name in md5_data: digest = md5(data).hexdigest() if digest != md5_data[egg_name]: - print >>sys.stderr, ( + print( "md5 validation of %s failed! (Possible download problem?)" - % egg_name + % egg_name, file=sys.stderr ) sys.exit(2) return data @@ -103,14 +106,13 @@ def do_download(): return do_download() try: pkg_resources.require("setuptools>="+version); return - except pkg_resources.VersionConflict, e: + except pkg_resources.VersionConflict as e: if was_imported: - print >>sys.stderr, ( + print( "The required version of setuptools (>=%s) is not available, and\n" "can't be installed while this script is running. Please install\n" " a more recent version first, using 'easy_install -U setuptools'." - "\n\n(Currently using %r)" - ) % (version, e.args[0]) + "\n\n(Currently using %r)" % (version, e.args[0]), file=sys.stderr) sys.exit(2) else: del pkg_resources, sys.modules['pkg_resources'] # reload ok @@ -165,41 +167,6 @@ def download_setuptools( if dst: dst.close() return os.path.realpath(saveto) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def main(argv, version=DEFAULT_VERSION): """Install or upgrade setuptools and EasyInstall""" try: @@ -216,9 +183,10 @@ def main(argv, version=DEFAULT_VERSION): os.unlink(egg) else: if setuptools.__version__ == '0.0.1': - print >>sys.stderr, ( + print( "You have an obsolete version of setuptools installed. Please\n" - "remove it from your system entirely before rerunning this script." + "remove it from your system entirely before rerunning this script.", + file=sys.stderr ) sys.exit(2) @@ -238,8 +206,8 @@ def main(argv, version=DEFAULT_VERSION): from setuptools.command.easy_install import main main(argv) else: - print "Setuptools version",version,"or greater has been installed." - print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)' + print("Setuptools version",version,"or greater has been installed.") + print('(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)') def update_md5(filenames): """Update our built-in md5 registry""" @@ -262,7 +230,7 @@ def update_md5(filenames): match = re.search("\nmd5_data = {\n([^}]+)}", src) if not match: - print >>sys.stderr, "Internal error!" + print("Internal error!", file=sys.stderr) sys.exit(2) src = src[:match.start(1)] + repl + src[match.end(1):] @@ -276,9 +244,3 @@ def update_md5(filenames): update_md5(sys.argv[2:]) else: main(sys.argv[1:]) - - - - - - diff --git a/setup.py b/setup.py index 722fbb5..040297e 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,8 @@ # # See README.txt for licensing information. +from __future__ import print_function + import glob, os, re, sys eggs = glob.glob('darcsver-*.egg') @@ -51,7 +53,7 @@ if mo: verstr = mo.group(1) else: - print "unable to find version in %s" % (VERSIONFILE,) + print("unable to find version in %s" % (VERSIONFILE,)) raise RuntimeError("if %s.py exists, it must be well-formed" % (VERSIONFILE,)) setup_requires = [] @@ -66,7 +68,7 @@ # "sdist" or "bdist_egg"), unless there is a zbase62.egg-info/SOURCE.txt file # present which contains a complete list of files that should be included. # http://pypi.python.org/pypi/setuptools_darcs -setup_requires.append('setuptools_darcs >= 1.1.0') +#setup_requires.append('setuptools_darcs >= 1.1.0') data_fnames=[ 'COPYING.GPL', 'COPYING.TGPPL.html', 'COPYING.SPL.txt', 'README.txt' ] diff --git a/zbase62/test/test_base62.py b/zbase62/test/test_base62.py index 6f859c1..d261508 100644 --- a/zbase62/test/test_base62.py +++ b/zbase62/test/test_base62.py @@ -5,16 +5,25 @@ # Permission is hereby granted, free of charge, to any person obtaining a copy # of this work to deal in this work without restriction (including the rights # to use, modify, distribute, sublicense, and/or sell copies). - +import os, sys import random, unittest -# http://zooko.com/repos/pyutil -from pyutil import mathutil, randutil +IS_PY2 = sys.version_info[0] == 2 +if not IS_PY2: + unicode = str + +def div_ceil(n, d): + """ + The smallest integer k such that k*d >= n. + + Copied from https://pypi.org/project/pyutil/ + """ + return int((n//d) + (n%d != 0)) from zbase62 import zbase62 -def insecurerandstr(n): - return ''.join(map(chr, map(random.randrange, [0]*n, [256]*n))) +def random_bytes(n): + return os.urandom(n) class T(unittest.TestCase): def _test_num_octets_that_encode_to_this_many_chars(self, chars, octets): @@ -23,55 +32,65 @@ def _test_num_octets_that_encode_to_this_many_chars(self, chars, octets): def _test_ende(self, bs): alphas=zbase62.b2a(bs) bs2=zbase62.a2b(alphas) - assert bs2 == bs, "bs2: %s:%s, bs: %s:%s, alphas: %s:%s" % (len(bs2), `bs2`, len(bs), `bs`, len(alphas), `alphas`) + assert bs2 == bs, "bs2: %s:%s, bs: %s:%s, alphas: %s:%s" % (len(bs2), repr(bs2), len(bs), repr(bs), len(alphas), repr(alphas)) def test_num_octets_that_encode_to_this_many_chars(self): - return self._test_num_octets_that_encode_to_this_many_chars(2, 1) - return self._test_num_octets_that_encode_to_this_many_chars(3, 2) - return self._test_num_octets_that_encode_to_this_many_chars(5, 3) - return self._test_num_octets_that_encode_to_this_many_chars(6, 4) + self._test_num_octets_that_encode_to_this_many_chars(2, 1) + self._test_num_octets_that_encode_to_this_many_chars(3, 2) + self._test_num_octets_that_encode_to_this_many_chars(5, 3) + self._test_num_octets_that_encode_to_this_many_chars(6, 4) + + def test_empty(self): + self._test_ende(b'') def test_ende_0x00(self): - return self._test_ende('\x00') + self._test_ende(b'\x00') def test_ende_0x01(self): - return self._test_ende('\x01') + self._test_ende(b'\x01') def test_ende_0x0100(self): - return self._test_ende('\x01\x00') + self._test_ende(b'\x01\x00') def test_ende_0x000000(self): - return self._test_ende('\x00\x00\x00') + self._test_ende(b'\x00\x00\x00') def test_ende_0x010000(self): - return self._test_ende('\x01\x00\x00') + self._test_ende(b'\x01\x00\x00') def test_ende_randstr(self): - return self._test_ende(insecurerandstr(2**4)) + self._test_ende(random_bytes(2 ** 4)) def test_ende_longrandstr(self): - return self._test_ende(insecurerandstr(random.randrange(0, 2**10))) + self._test_ende(random_bytes(random.randrange(0, 2 ** 10))) def test_odd_sizes(self): for j in range(2**6): lib = random.randrange(1, 2**8) - numos = mathutil.div_ceil(lib, 8) - bs = insecurerandstr(numos) + numos = div_ceil(lib, 8) + bs = random_bytes(numos) # zero-out unused least-sig bits if lib%8: - b=ord(bs[-1]) + b=ord(bs[-1]) if IS_PY2 else bs[-1] b = b >> (8 - (lib%8)) b = b << (8 - (lib%8)) - bs = bs[:-1] + chr(b) - asl = zbase62.b2a_l(bs, lib) + bs = bs[:-1] + (chr(b) if IS_PY2 else bytes([b])) + asl = zbase62.b2a(bs) assert len(asl) == zbase62.num_chars_that_this_many_octets_encode_to(numos) # the size of the base-62 encoding must be just right bs2l = zbase62.a2b_l(asl, lib) assert len(bs2l) == numos # the size of the result must be just right assert bs == bs2l -def suite(): - suite = unittest.makeSuite(T, 'test') - return suite + def test_invalid(self): + # doesn't fail + zbase62.a2b('~!~') + + def test_types(self): + assert type(zbase62.a2b(u'x')) == bytes + assert type(zbase62.a2b(b'x')) == bytes + + assert type(zbase62.b2a(u'x')) == unicode + assert type(zbase62.b2a(b'x')) == unicode if __name__ == "__main__": unittest.main() diff --git a/zbase62/zbase62.py b/zbase62/zbase62.py index b47dd2e..1558c54 100644 --- a/zbase62/zbase62.py +++ b/zbase62/zbase62.py @@ -8,16 +8,61 @@ # from the Python Standard Library import string - -from pyutil.assertutil import _assert, precondition, postcondition -from pyutil.mathutil import div_ceil, log_ceil, log_floor +import sys + +#/ Copied from |pyutil|: +## https://github.com/simplegeo/pyutil/blob/bd40e624771c84859045911082480e74ff01fcd4/pyutil/mathutil.py#L44 +## +## |zbase62| (https://github.com/simplegeo/zbase62) +## and |pyutil| (https://github.com/simplegeo/pyutil) have same license options. +## +## ---BEG +def log_ceil(n, b): + """ + The smallest integer k such that b^k >= n. + log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. + the number of bits needed to store any of 128 possible values is 7. + """ + p = 1 + k = 0 + while p < n: + p *= b + k += 1 + return k + +def log_floor(n, b): + """ + The largest integer k such that b^k <= n. + """ + p = 1 + k = 0 + while p <= n: + p *= b + k += 1 + return k - 1 +## ---END + +IS_PY2 = sys.version_info[0] == 2 + +if sys.version_info[:2] <= (3, 0): + maketrans = string.maketrans +else: + maketrans = bytes.maketrans + +if IS_PY2: + translate = string.translate +else: + translate = bytes.translate chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - +if not IS_PY2: + chars = chars.encode('ascii') vals = ''.join([chr(i) for i in range(62)]) -c2vtranstable = string.maketrans(chars, vals) -v2ctranstable = string.maketrans(vals, chars) -identitytranstable = string.maketrans(chars, chars) +if not IS_PY2: + vals = vals.encode('latin') +c2vtranstable = maketrans(chars, vals) +v2ctranstable = maketrans(vals, chars) +identitytranstable = maketrans(chars, chars) def b2a(os): """ @@ -25,36 +70,13 @@ def b2a(os): @return the contents of os in base-62 encoded form """ - cs = b2a_l(os, len(os)*8) - assert num_octets_that_encode_to_this_many_chars(len(cs)) == len(os), "%s != %s, numchars: %s" % (num_octets_that_encode_to_this_many_chars(len(cs)), len(os), len(cs)) - return cs - -def b2a_l(os, lengthinbits): - """ - @param os the data to be encoded (a string) - @param lengthinbits the number of bits of data in os to be encoded - - b2a_l() will generate a base-62 encoded string big enough to encode - lengthinbits bits. So for example if os is 3 bytes long and lengthinbits is - 17, then b2a_l() will generate a 3-character- long base-62 encoded string - (since 3 chars is sufficient to encode more than 2^17 values). If os is 3 - bytes long and lengthinbits is 18 (or None), then b2a_l() will generate a - 4-character string (since 4 chars are required to hold 2^18 values). Note - that if os is 3 bytes long and lengthinbits is 17, the least significant 7 - bits of os are ignored. - - Warning: if you generate a base-62 encoded string with b2a_l(), and then someone else tries to - decode it by calling a2b() instead of a2b_l(), then they will (potentially) get a different - string than the one you encoded! So use b2a_l() only when you are sure that the encoding and - decoding sides know exactly which lengthinbits to use. If you do not have a way for the - encoder and the decoder to agree upon the lengthinbits, then it is best to use b2a() and - a2b(). The only drawback to using b2a() over b2a_l() is that when you have a number of - bits to encode that is not a multiple of 8, b2a() can sometimes generate a base-62 encoded - string that is one or two characters longer than necessary. - - @return the contents of os in base-62 encoded form - """ - os = [ord(o) for o in reversed(os)] # treat os as big-endian -- and we want to process the least-significant o first + original_len = len(os) + if not isinstance(os, bytes): + os = os.encode('utf-8') + os = reversed(os) # treat os as big-endian -- and we want to process the least-significant o first + + if IS_PY2: + os = [ord(o) for o in os] value = 0 numvalues = 1 # the number of possible values that value could be @@ -68,8 +90,24 @@ def b2a_l(os, lengthinbits): chars.append(value % 62) value //= 62 numvalues //= 62 + + schars = ''.join([chr(c) for c in reversed(chars)]) + + if IS_PY2: + bchars = schars + else: + bchars = bytes(schars, 'latin') + + bchars = translate(bchars, v2ctranstable) # make it big-endian + assert type(bchars) == bytes - return string.translate(''.join([chr(c) for c in reversed(chars)]), v2ctranstable) # make it big-endian + bchars = bchars.decode('utf-8') + + assert num_octets_that_encode_to_this_many_chars(len(bchars)) == original_len, "%s != %s, numchars: %s" % ( + num_octets_that_encode_to_this_many_chars(len(bchars)), original_len(os), len(bchars) + ) + + return bchars def num_octets_that_encode_to_this_many_chars(numcs): return log_floor(62**numcs, 256) @@ -98,7 +136,13 @@ def a2b_l(cs, lengthinbits): @return the data encoded in cs """ - cs = [ord(c) for c in reversed(string.translate(cs, c2vtranstable))] # treat cs as big-endian -- and we want to process the least-significant c first + if not isinstance(cs, bytes): + cs = cs.encode('utf-8') + + cs = reversed(translate(cs, c2vtranstable)) # treat cs as big-endian -- and we want to process the least-significant c first + + if IS_PY2: + cs = [ord(c) for c in cs] value = 0 numvalues = 1 # the number of possible values that value could be @@ -108,10 +152,16 @@ def a2b_l(cs, lengthinbits): numvalues *= 62 numvalues = 2**lengthinbits - bytes = [] + byte_list = [] while numvalues > 1: - bytes.append(value % 256) + byte_list.append(value % 256) value //= 256 numvalues //= 256 - return ''.join([chr(b) for b in reversed(bytes)]) # make it big-endian + # make it big-endian + byte_list = reversed(byte_list) + + if IS_PY2: + return b''.join([chr(b) for b in byte_list]) + else: + return bytes(byte_list)