Support Everything (James Saryerwinnie)

There comes a time in the life of a library where it must leave your machine and find a home in a far away computer. That computer might even run windows. This talk will show how you can write a library that supports Linux/Mac/Windows and runs on python 2 and 3.

Transcript

1. SUPPORT EVERYTHING Supporting python 2/3, Mac/Linux/Windows Saturday, August 10, 13
2. SUPPORT EVERYTHING Supporting python 2/3, Mac/Linux/Windows ALMOST Saturday, August 10, 13
3. ✓ ✗ ✗ ✗ ✗ ✗ ✗ ✗ ✗ Saturday, August 10, 13
4. Works for me ✓ ✗ ✗ ✗ ✗ ✗ ✗ ✗ ✗ Saturday, August 10, 13
5. A lot of times ✓ ✓ ✓ ✓ ✗ ? ? ✗ ✗ Saturday, August 10, 13
6. requests ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ Saturday, August 10, 13
7. jinja2 ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ Saturday, August 10, 13
8. simplejson ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ Saturday, August 10, 13
9. django ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ Saturday, August 10, 13
10. Find what varies and encapsulate it Saturday, August 10, 13
11. PLATFORMS Saturday, August 10, 13
12. Interface Differences Saturday, August 10, 13
13. class mmap.mmap(fileno, length[, tagname[, access[, offset]]]) (Windows version) class mmap. mmap(fileno, length[, flags[, prot[, access[, offset]]]]) (Unix version) Saturday, August 10, 13
14. O_APPEND = 8 O_ASYNC = 64 O_CREAT = 512 O_DIRECTORY = 1048576 O_DSYNC = 4194304 O_EXCL = 2048 O_EXLOCK = 32 O_NDELAY = 4 O_NOCTTY = 131072 O_NOFOLLOW = 256 O_NONBLOCK = 4 O_RDONLY = 0 O_RDWR = 2 O_SHLOCK = 16 O_SYNC = 128 O_TRUNC = 1024 O_WRONLY = 1 O_APPEND = 8 O_BINARY = 32768 O_CREAT = 256 O_EXCL = 1024 O_NOINHERIT = 128 O_RANDOM = 16 O_RDONLY = 0 O_RDWR = 2 O_SEQUENTIAL = 32 O_SHORT_LIVED = 4096 O_TEMPORARY = 64 O_TEXT = 16384 O_TRUNC = 512 O_WRONLY = 1 import os Saturday, August 10, 13
15. epoll kqueue ? import select Saturday, August 10, 13
16. If preexec_fn is set to a callable object, this object will be called in the child process just before the child is executed. (Unix only) class subprocess.Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False,shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0) Saturday, August 10, 13
17. import curses import curses Traceback (most recent call last): File "<stdin>", line 1, in <module> File "C:\...\__init__.py", line 13, in <module> from _curses import * ImportError: No module named '_curses' Saturday, August 10, 13
18. Semantic Differences Saturday, August 10, 13
19. import os f = open('foo', 'w') os.remove('foo') Saturday, August 10, 13
20. Saturday, August 10, 13
21. from tempfile import NamedTemporaryFile temp = NamedTemporaryFile(mode='w') temp.write('foobar') temp.flush() print(open(temp.name, 'r').read()) Traceback (most recent call last): File "<stdin>", line 1, in <module> IOError: [Errno 13] Permission denied: 'c:\\users\\a\\local\\temp\\2\\thny' Saturday, August 10, 13
22. with open('tempfile', 'w') as f: f.write(contents) f.flush(contents) os.fsync(f.fileno()) os.rename(f.name, 'originalfile') Traceback (most recent call last): File "<stdin>", line 1, in <module> FileExistsError: [WinError 183] Cannot create a file when that file already exists: 'originalfile' Saturday, August 10, 13
23. PYTHON 2/3 Saturday, August 10, 13
24. Module renames - queue vs. Queue API - iter([]).__next__ vs. iter([]).next Syntax - u'' Saturday, August 10, 13
25. str vs. bytes Saturday, August 10, 13
26. Dealing with it Use a single codebase Saturday, August 10, 13
27. Use six Saturday, August 10, 13
28. Use six Dependency Vendor DIY Saturday, August 10, 13
29. compat.py Saturday, August 10, 13
30. import six if six.PY3: from urllib.parse import quote from urllib.parse import unquote else: from urllib import quote from urllib import unquote botocore/compat.py Saturday, August 10, 13
31. import sys PY2 = sys.version_info[0] == 2 _identity = lambda x: x if not PY2: implements_iterator = _identity get_next = lambda x: x.__next__ else: def implements_iterator(cls): cls.next = cls.__next__ del cls.__next__ return cls get_next = lambda x: x.next jinja2/_compat.py Saturday, August 10, 13
32. class Popen(object): def __init__(self, # ...): # ... if mswindows: if preexec_fn is not None: raise ValueError( "preexec_fn is not supported " "on Windows platforms") # ... else: # POSIX if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: close_fds = True subprocess.py Saturday, August 10, 13
33. class Popen(object): def __init__(self, # ...): # ... 60 lines later if mswindows: if p2cwrite != -1: p2cwrite = msvcrt.open_osfhandle( p2cwrite.Detach(), 0) if c2pread != -1: c2pread = msvcrt.open_osfhandle( c2pread.Detach(), 0) if errread != -1: errread = msvcrt.open_osfhandle( errread.Detach(), 0) subprocess.py Saturday, August 10, 13
34. MERCURIAL . ├── posix.py ├── util.py └── windows.py Saturday, August 10, 13
35. MERCURIAL . ├── posix.py ├── util.py └── windows.py Saturday, August 10, 13
36. MERCURIAL if os.name == 'nt': import windows as platform else: import posix as platform util.py Saturday, August 10, 13
37. if 'posix' in _names: name = 'posix' linesep = '\n' from posix import * # ... elif 'nt' in _names: name = 'nt' linesep = '\r\n' from nt import * try: from nt import _exit except ImportError: pass import ntpath as path # ... os.py stdlib Saturday, August 10, 13
38. TORNADO ├── platform │ ├── __init__.py │ ├── auto.py │ ├── interface.py │ ├── posix.py │ └── windows.py Saturday, August 10, 13
39. if os.name == 'nt': from tornado.platform.common import Waker from tornado.platform.windows import set_close_exec else: from tornado.platform.posix import set_close_exec, Waker platform/auto.py Tornado Saturday, August 10, 13
40. Factory Saturday, August 10, 13
41. __getitem__ __setitem__ __delitem__ SemiDBM __call__ Renamer iter_keys Loader MMapLoader SimpleFileLoader PosixRenamer WindowsRenamer Saturday, August 10, 13
42. def open(filename, flag='r', mode=0o666, verify_checksums=False): if sys.platform.startswith('win'): renamer = _WindowsRenamer() else: renamer = _Renamer() if sys.platform.startswith('java'): from semidbm.loaders.simpleload import SimpleFileLoader data_loader = SimpleFileLoader() else: from semidbm.loaders.mmapload import MMapLoader data_loader = MMapLoader() kwargs = {'renamer': renamer, 'verify_checksums': verify_checksums, 'data_loader': data_loader} # ... return _SemiDBM(filename, **kwargs) Saturday, August 10, 13
43. def open(filename, flag='r', mode=0o666, verify_checksums=False): if sys.platform.startswith('win'): renamer = _WindowsRenamer() else: renamer = _Renamer() if sys.platform.startswith('java'): from semidbm.loaders.simpleload import SimpleFileLoader data_loader = SimpleFileLoader() else: from semidbm.loaders.mmapload import MMapLoader data_loader = MMapLoader() kwargs = {'renamer': renamer, 'verify_checksums': verify_checksums, 'data_loader': data_loader} # ... return _SemiDBM(filename, **kwargs) Saturday, August 10, 13
44. def open(filename, flag='r', mode=0o666, verify_checksums=False): if sys.platform.startswith('win'): renamer = _WindowsRenamer() else: renamer = _Renamer() if sys.platform.startswith('java'): from semidbm.loaders.simpleload import SimpleFileLoader data_loader = SimpleFileLoader() else: from semidbm.loaders.mmapload import MMapLoader data_loader = MMapLoader() kwargs = {'renamer': renamer, 'verify_checksums': verify_checksums, 'data_loader': data_loader} # ... return _SemiDBM(filename, **kwargs) Saturday, August 10, 13
45. def open(filename, flag='r', mode=0o666, verify_checksums=False): if sys.platform.startswith('win'): renamer = _WindowsRenamer() else: renamer = _Renamer() if sys.platform.startswith('java'): from semidbm.loaders.simpleload import SimpleFileLoader data_loader = SimpleFileLoader() else: from semidbm.loaders.mmapload import MMapLoader data_loader = MMapLoader() kwargs = {'renamer': renamer, 'verify_checksums': verify_checksums, 'data_loader': data_loader} # ... return _SemiDBM(filename, **kwargs) Saturday, August 10, 13
46. Testing Saturday, August 10, 13
47. Testing Yes Please Saturday, August 10, 13
48. Photos: http://www.flickr.com/photos/korou/320051042/ http://www.flickr.com/photos/crystallineradical/3049570916/ Saturday, August 10, 13
49. More Info http://nedbatchelder.com/text/unipain.html http://lucumr.pocoo.org/2013/5/21/porting-to-python-3-redux/ http://python3porting.com/ http://python3porting.com/cextensions.html Additional info: http://docs.python.org/3/using/windows.html http://docs.python.org/2/using/windows.html https://github.com/mitsuhiko/jinja2/blob/master/jinja2/_compat.py https://github.com/simplejson/simplejson/blob/master/simplejson/compat.py Saturday, August 10, 13
50. Thanks! @jsaryer http://github.com/jamesls Saturday, August 10, 13