-
-
Save agumonkey/a5a89bc11d52ee308ae7 to your computer and use it in GitHub Desktop.
readable / neigh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<html class="js no-touch geolocation fontface generatedcontent svg formvalidation placeholder boxsizing no-retina" lang="en" dir="ltr" style=""><!--<![endif]--><head> | |
<meta charset="utf-8"> | |
<meta http-equiv="X-UA-Compatible" content="IE=edge"> | |
<link rel="prefetch" href="http://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"> | |
<meta name="application-name" content="Python.org"> | |
<meta name="msapplication-tooltip" content="The official home of the Python Programming Language"> | |
<meta name="apple-mobile-web-app-title" content="Python.org"> | |
<meta name="apple-mobile-web-app-capable" content="yes"> | |
<meta name="apple-mobile-web-app-status-bar-style" content="black"> | |
<meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
<meta name="HandheldFriendly" content="True"> | |
<meta name="format-detection" content="telephone=no"> | |
<meta http-equiv="cleartype" content="on"> | |
<meta http-equiv="imagetoolbar" content="false"> | |
<script id="twitter-wjs" src="https://platform.twitter.com/widgets.js"></script><script type="text/javascript" async="" src="https://ssl.google-analytics.com/ga.js"></script><script src="/static/js/libs/modernizr.js"></script> | |
<link href="/static/stylesheets/style.css" rel="stylesheet" type="text/css" title="default"> | |
<link href="/static/stylesheets/mq.css" rel="stylesheet" type="text/css" media="not print, braille, embossed, speech, tty"> | |
<!--[if (lte IE 8)&(!IEMobile)]> | |
<link href="/static/stylesheets/no-mq.css" rel="stylesheet" type="text/css" media="screen" /> | |
<![endif]--> | |
<link rel="icon" type="image/x-icon" href="/static/favicon.ico"> | |
<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/static/apple-touch-icon-144x144-precomposed.png"> | |
<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/static/apple-touch-icon-114x114-precomposed.png"> | |
<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/static/apple-touch-icon-72x72-precomposed.png"> | |
<link rel="apple-touch-icon-precomposed" href="/static/apple-touch-icon-precomposed.png"> | |
<link rel="apple-touch-icon" href="/static/apple-touch-icon-precomposed.png"> | |
<meta name="msapplication-TileImage" content="/static/metro-icon-144x144-precomposed.png"><!-- white shape --> | |
<meta name="msapplication-TileColor" content="#3673a5"><!-- python blue --> | |
<meta name="msapplication-navbutton-color" content="#3673a5"> | |
<meta property="og:site_name" content="Python.org"> | |
<meta property="og:type" content="website"> | |
<title>Python Patterns - An Optimization Anecdote | Python.org</title> | |
<meta property="og:title" content="Welcome to Python.org"> | |
<meta name="description" content="The official home of the Python Programming Language"> | |
<meta name="og:description" content="The official home of the Python Programming Language"> | |
<meta name="keywords" content="Python programming language object oriented web free open source software license documentation download community"> | |
<meta property="og:tag" content="Python programming language object oriented web free open source software license documentation download community"> | |
<meta property="og:published_time" content=""> | |
<meta property="og:modified_time" content=""> | |
<meta property="og:author" content=""> | |
<meta property="og:section" content=""> | |
<meta property="og:url" content=""> | |
<meta property="og:image" content=""> | |
<meta property="og:video" content=""> | |
<link rel="author" href="/static/humans.txt"> | |
<script type="application/ld+json"> | |
{ | |
"@context": "http://schema.org", | |
"@type": "WebSite", | |
"url": "https://www.python.org/", | |
"potentialAction": { | |
"@type": "SearchAction", | |
"target": "https://www.python.org/search/?q={search_term_string}", | |
"query-input": "required name=search_term_string" | |
} | |
} | |
</script> | |
<script type="text/javascript"> | |
var _gaq = _gaq || []; | |
_gaq.push(['_setAccount', 'UA-39055973-1']); | |
_gaq.push(['_trackPageview']); | |
(function() { | |
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; | |
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; | |
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); | |
})(); | |
</script> | |
</head> | |
<body class="python pages default-page" data-twttr-rendered="true"> | |
<div id="touchnav-wrapper"> | |
<div id="nojs" class="do-not-print"> | |
<p><strong>Notice:</strong> While Javascript is not essential for this website, your interaction with the content will be limited. Please turn Javascript on for the full experience. </p> | |
</div> | |
<!--[if lt IE 8]> | |
<div id="oldie-warning" class="do-not-print"> | |
<p><strong>Notice:</strong> Your browser is <em>ancient</em> and <a href="http://www.ie6countdown.com/">Microsoft agrees</a>. <a href="http://browsehappy.com/">Upgrade to a different browser</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to experience a better web.</p> | |
</div> | |
<![endif]--> | |
<!-- Sister Site Links --> | |
<div id="top" class="top-bar do-not-print"> | |
<nav class="meta-navigation container" role="navigation"> | |
<div class="skip-link screen-reader-text"> | |
<a href="#content" title="Skip to content">Skip to content</a> | |
</div> | |
<a id="close-python-network" class="jump-link" href="#python-network" aria-hidden="true"> | |
<span aria-hidden="true" class="icon-arrow-down"><span>▼</span></span> Close | |
</a> | |
<ul class="menu" role="tree"> | |
<li class="python-meta "> | |
<a href="/" title="The Python Programming Language">Python</a> | |
</li> | |
<li class="psf-meta "> | |
<a href="/psf-landing/" title="The Python Software Foundation">PSF</a> | |
</li> | |
<li class="docs-meta "> | |
<a href="https://docs.python.org" title="Python Documentation">Docs</a> | |
</li> | |
<li class="pypi-meta "> | |
<a href="https://pypi.python.org/" title="Python Package Index">PyPI</a> | |
</li> | |
<li class="jobs-meta "> | |
<a href="/jobs/" title="Python Job Board">Jobs</a> | |
</li> | |
<li class="shop-meta "> | |
<a href="/community/" title="Python Community">Community</a> | |
</li> | |
</ul> | |
<a id="python-network" class="jump-link" href="#top" aria-hidden="true"> | |
<span aria-hidden="true" class="icon-arrow-up"><span>▲</span></span> The Python Network | |
</a> | |
</nav> | |
</div> | |
<!-- Header elements --> | |
<header class="main-header" role="banner"> | |
<div class="container"> | |
<h1 class="site-headline"> | |
<a href="/"><img class="python-logo" src="/static/img/python-logo.png" alt="python™"></a> | |
</h1> | |
<div class="options-bar do-not-print"> | |
<a id="site-map-link" class="jump-to-menu" href="#site-map"><span class="menu-icon">≡</span> Menu</a><form class="search-the-site" action="/search/" method="get"> | |
<fieldset title="Search Python.org"> | |
<span aria-hidden="true" class="icon-search"></span> | |
<label class="screen-reader-text" for="id-search-field">Search This Site</label> | |
<input id="id-search-field" name="q" type="search" role="textbox" class="search-field placeholder" placeholder="Search" value="" tabindex="1"> | |
<button type="submit" name="submit" id="submit" class="search-button" title="Submit this Search" tabindex="3"> | |
GO | |
</button> | |
<!--[if IE]><input type="text" style="display: none;" disabled="disabled" size="1" tabindex="4"><![endif]--> | |
</fieldset> | |
</form><span class="breaker"></span><div class="adjust-font-size" aria-hidden="true"> | |
<ul class="navigation menu" aria-label="Adjust Text Size on Page"> | |
<li class="tier-1 last" aria-haspopup="true"> | |
<a href="#" class="action-trigger"><strong><small>A</small> A</strong></a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a class="text-shrink" title="Make Text Smaller" href="javascript:;">Smaller</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a class="text-grow" title="Make Text Larger" href="javascript:;">Larger</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a class="text-reset" title="Reset any font size changes I have made" href="javascript:;">Reset</a></li> | |
</ul> | |
</li> | |
</ul> | |
</div><div class="winkwink-nudgenudge"> | |
<ul class="navigation menu" aria-label="Social Media Navigation"> | |
<li class="tier-1 last" aria-haspopup="true"> | |
<a href="#" class="action-trigger">Socialize</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="http://plus.google.com/+Python"><span aria-hidden="true" class="icon-google-plus"></span>Google+</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://www.facebook.com/pythonlang?fref=ts"><span aria-hidden="true" class="icon-facebook"></span>Facebook</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="http://twitter.com/ThePSF"><span aria-hidden="true" class="icon-twitter"></span>Twitter</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/community/irc/"><span aria-hidden="true" class="icon-freenode"></span>Chat on IRC</a></li> | |
</ul> | |
</li> | |
</ul> | |
</div><div class="account-signin"> | |
<ul class="navigation menu" aria-label="Social Media Navigation"> | |
<li class="tier-1 last" aria-haspopup="true"> | |
<a href="/accounts/login/" title="Sign Up or Sign In to Python.org">Sign In</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/accounts/signup/">Sign Up / Register</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/accounts/login/">Sign In</a></li> | |
</ul> | |
</li> | |
</ul> | |
</div> | |
</div><!-- end options-bar --> | |
<nav id="mainnav" class="python-navigation main-navigation do-not-print" role="navigation"> | |
<ul class="navigation menu" role="menubar" aria-label="Main Navigation"> | |
<li id="about" class="tier-1 element-1 with-supernav" aria-haspopup="true"> | |
<a href="/about/" title="" class="">About</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/apps/" title="">Applications</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/quotes/" title="">Quotes</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/gettingstarted/" title="">Getting Started</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/help/" title="">Help</a></li> | |
<li class="tier-2 super-navigation"> | |
<h4>Python is a programming language that lets you work more quickly and integrate your systems more effectively.</h4> | |
<p>You can learn to use Python and see almost immediate gains in productivity and lower maintenance costs. <a href="/about">Learn more about Python.</a>. | |
</p></li></ul> | |
</li> | |
<li id="downloads" class="tier-1 element-2 with-supernav" aria-haspopup="true"> | |
<a href="/downloads/" title="" class="">Downloads</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/downloads/" title="">All releases</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/downloads/source/" title="">Source code</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/downloads/windows/" title="">Windows</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/downloads/mac-osx/" title="">Mac OS X</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/download/other/" title="">Other Platforms</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://docs.python.org/3/license.html" title="">License</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/download/alternatives" title="">Alternative Implementations</a></li> | |
<!-- download supernav from templates/downloads/supernav.html --> | |
<li class="tier-2 super-navigation"> | |
<div class="download-os-mac-osx" style="display: none;"> | |
<h4>Download for Mac OS X</h4> | |
<p> | |
<a class="button" href="https://www.python.org/ftp/python/3.4.3/python-3.4.3-macosx10.6.pkg">Python 3.4.3</a> | |
<a class="button" href="https://www.python.org/ftp/python/2.7.9/python-2.7.9-macosx10.6.pkg">Python 2.7.9</a> | |
</p> | |
<p>Not the OS you are looking for? Python can be used on 21 different operating systems and environments. <a href="/downloads/">View the full list</a>.</p> | |
</div> | |
<div class="download-os-source" style=""> | |
<h3>Python Source</h3> | |
<p> | |
<a class="button" href="https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tar.xz">Python 3.4.3</a> | |
<a class="button" href="https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tar.xz">Python 2.7.9</a> | |
</p> | |
<p>Not the OS you are looking for? Python can be used on 21 different operating systems and environments. <a href="/downloads/">View the full list</a>.</p> | |
</div> | |
<div class="download-os-windows" style="display: none;"> | |
<h4>Download for Windows</h4> | |
<p> | |
<a class="button" href="https://www.python.org/ftp/python/3.4.3/python-3.4.3.msi">Python 3.4.3</a> | |
<a class="button" href="https://www.python.org/ftp/python/2.7.9/python-2.7.9.msi">Python 2.7.9</a> | |
</p> | |
<p>Not the OS you are looking for? Python can be used on 21 different operating systems and environments. <a href="/downloads/">View the full list</a>.</p> | |
</div> | |
<div class="download-unknown" style="display: none;"> | |
<h4>Download Python for Any OS</h4> | |
<p>Python can be used on 21 different operating systems and environments.</p> | |
<p> | |
<a class="button" href="/downloads/operating-systems/">View the Full List</a> | |
</p> | |
</div> | |
</li> | |
</ul> | |
</li> | |
<li id="documentation" class="tier-1 element-3 with-supernav" aria-haspopup="true"> | |
<a href="/doc/" title="" class="">Documentation</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/doc/" title="">Docs</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/doc/av" title="">Audio/Visual Talks</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="https://wiki.python.org/moin/BeginnersGuide" title="">Beginner's Guide</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="https://docs.python.org/devguide/" title="">Developer's Guide</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://docs.python.org/faq/" title="">FAQ</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="http://wiki.python.org/moin/Languages" title="">Non-English Docs</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="http://python.org/dev/peps/" title="">PEP Index</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="https://wiki.python.org/moin/PythonBooks" title="">Python Books</a></li> | |
<li class="tier-2 super-navigation"> | |
<h4>Python’s standard documentation: download, browse or watch a tutorial.</h4> | |
<p>Get started below, or visit the <a href="/doc/versions/">Documentation page to browse by version</a>. </p> | |
<p class="download-buttons"> | |
<a class="button" href="http://docs.python.org/3/">Python 3.x Docs</a> | |
<a class="button" href="http://docs.python.org/2/">Python 2.x Docs</a> | |
</p> | |
<p>See also <a href="https://wiki.python.org/moin/Python2orPython3">Should I use Python 2 or 3</a>? </p> | |
</li></ul> | |
</li> | |
<li id="community" class="tier-1 element-4 with-supernav" aria-haspopup="true"> | |
<a href="/community/" title="" class="">Community</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/community/diversity/" title="">Diversity</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/community/irc/" title="">IRC</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/community/lists/" title="">Mailing Lists</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/community/workshops/" title="">Python Conferences</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/community/sigs/" title="">Special Interest Groups</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://wiki.python.org/moin/" title="">Python Wiki</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/community/logos/" title="">Python Logo</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="/community/merchandise/" title="">Merchandise</a></li> | |
<li class="tier-2 element-9" role="treeitem"><a href="/community/awards" title="">Community Awards</a></li> | |
<li class="tier-2 super-navigation"> | |
<h4>The Python Community</h4> | |
<p>Great software is supported by great people. Our user base is enthusiastic, dedicated to encouraging use of the language, and committed to being diverse and friendly.</p> | |
<!-- | |
<p>Here are some events and groups in your area.</p> | |
<ul class="menu"> | |
<li><time datetime="">9/30<span class="say-no-more">/2012</span></time> <a href="#">Royal Python Society Meetup, Providence RI</a></li> | |
<li><time datetime="">10/4<span class="say-no-more">/2012</span></time> <a href="#">Python Users Group, Boston MA</a></li> | |
<li><time datetime="">10/5<span class="say-no-more">/2012</span></time> <a href="#">Python Enthusiasts, Cambridge MA</a></li> | |
</ul> | |
</li>--></li></ul> | |
</li> | |
<li id="success-stories" class="tier-1 element-5 with-supernav" aria-haspopup="true"> | |
<a href="/about/success/" title="success-stories" class="">Success Stories</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/success/#arts" title="">Arts</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/success/#business" title="">Business</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/success/#education" title="">Education</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/success/#engineering" title="">Engineering</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/about/success/#government" title="">Government</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="/about/success/#scientific" title="">Scientific</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/about/success/#software-development" title="">Software Development</a></li> | |
<!-- successstories supernav from templates/successstories/supernav.html --> | |
<li class="tier-2 super-navigation"> | |
<img src="/m/successstories/Lucasfilm_logo.png" alt="Industrial Light & Magic"> | |
<blockquote class="success-quote"> | |
ILM runs a batch processing environment capable of modeling, rendering and compositing tens of thousands of motion picture frames per day. Thousands of machines running Linux, IRIX, Compaq Tru64, OS X, Solaris, and Windows join together to provide a production pipeline used by ~800 users daily. Speed of development is key, and Python was a faster way to code (and re-code) the programs that control this production pipeline. | |
</blockquote> | |
<p class="quote-by"><cite>Tim Fortenberry</cite>, <a href="http://www.ilm.com/">Industrial Light & Magic</a></p> | |
</li></ul> | |
</li> | |
<li id="news" class="tier-1 element-6 " aria-haspopup="true"> | |
<a href="/blogs/" title="News from around the Python world" class="">News</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/blogs/" title="Python Insider Blog Posts">Python News</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://planetpython.org/" title="Planet Python">Community News</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="http://pyfound.blogspot.com/" title="PSF Blog">PSF News</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="http://pycon.blogspot.com/" title="PyCon Blog">PyCon News</a></li> | |
</ul> | |
</li> | |
<li id="events" class="tier-1 element-7 with-supernav" aria-haspopup="true"> | |
<a href="/events/" title="" class="">Events</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/events/python-events/" title="">Python Events</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/events/python-user-group/" title="">User Group Events</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/events/python-events/past/" title="">Python Events Archive</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/events/python-user-group/past/" title="">User Group Events Archive</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event" title="">Submit an Event</a></li> | |
<li class="tier-2 super-navigation">Find events from the Python Community around the world!</li></ul> | |
</li> | |
</ul> | |
</nav> | |
<div class="header-banner "> <!-- for optional "do-not-print" class --> | |
</div> | |
</div><!-- end .container --> | |
</header> | |
<div id="content" class="content-wrapper"> | |
<!-- Main Content Column --> | |
<div class="container"> | |
<section class="main-content with-left-sidebar" role="main"> | |
<ul class="breadcrumbs menu"> | |
</ul> | |
<article class="text"> | |
<header class="article-header"> | |
<h1 class="page-title">Python Patterns - An Optimization Anecdote</h1> | |
</header> | |
<h1>Python Patterns - An Optimization Anecdote</h1> | |
The other day, a friend asked me a seemingly simple question: what's the | |
best way to convert a list of integers into a string, presuming that the | |
integers are ASCII values. For instance, the list [97, 98, 99] should be | |
converted to the string 'abc'. Let's assume we want to write a function to | |
do this.<p> | |
</p><p>The first version I came up with was totally straightforward:</p> | |
<pre> def f1(list): | |
string = "" | |
for item in list: | |
string = string + chr(item) | |
return string | |
</pre> | |
<p>That can't be the fastest way to do it, said my friend. How about this one:</p> | |
<pre> def f2(list): | |
return reduce(lambda string, item: string + chr(item), list, "") | |
</pre> | |
<p>This version performs exactly the same set of string operations as the | |
first one, but gets rid of the for loop overhead in favor of the faster, | |
implied loop of the reduce() function. | |
</p><p>Sure, I replied, but it does so at the cost of a function call (the lambda | |
function) per list item. I betcha it's slower, since function call | |
overhead in Python is bigger than for loop overhead. | |
</p><p>(OK, so I had already done the comparisons. f2() took 60% more time than f1(). So there :-) | |
</p><p>Hmm, said my friend. I need this to be faster. OK, I said, how about | |
this version: | |
</p><pre> def f3(list): | |
string = "" | |
for character in map(chr, list): | |
string = string + character | |
return string | |
</pre> | |
<p>To both our surprise, f3() clocked <i>twice</i> as fast as f1()! The | |
reason that this surprised us was twofold: first, it uses more storage (the | |
result of map(chr, list) is another list of the same length); second, it | |
contains two loops instead of one (the one implied by the map() function, | |
and the for loop). | |
</p><p>Of course, space versus time is a well-known trade-off, so the first one | |
shouldn't have surprised us. However, how come two loops are faster than | |
one? Two reasons. | |
</p><p>First, in f1(), the built-in function chr() is looked up on every | |
iteration, while in f3() it is only looked up once (as the argument to | |
map()). This look-up is relatively expensive, I told my friend, since | |
Python's dynamic scope rules mean that it is first looked up | |
(unsuccessfully) in the current module's global dictionary, and then in the | |
dictionary of built-in function (where it is found). Worse, unsuccessful | |
dictionary lookups are (on average) a bit slower than successful ones, | |
because of the way the hash chaining works. | |
</p><p>The second reason why f3() is faster than f1() is that the call to | |
chr(item), as executed by the bytecode interpreter, is probably a bit | |
slower than when executed by the map() function - the bytecode interpreter | |
must execute three bytecode instructions for each call (load 'chr', load | |
'item', call), while the map() function does it all in C. | |
</p><p>This led us to consider a compromise, which wouldn't waste extra space, but | |
which would speed up the lookup for the chr() function: | |
</p><pre> def f4(list): | |
string = "" | |
lchr = chr | |
for item in list: | |
string = string + lchr(item) | |
return string | |
</pre> | |
<p>As expected, f4() was slower than f3(), but only by 25%; it was about 40% | |
faster than f1() still. This is because local variable lookups are | |
<i>much</i> faster than global or built-in variable lookups: the Python | |
"compiler" optimizes most function bodies so that for local variables, no | |
dictionary lookup is necessary, but a simple array indexing operation is | |
sufficient. The relative speed of f4() compared to f1() and f3() suggests | |
that both reasons why f3() is faster contribute, but that the first reason | |
(fewer lookups) is a bit more important. (To get more precise data on | |
this, we would have to instrument the interpreter.) | |
</p><p>Still, our best version, f3(), was only twice as fast as the most | |
straightforward version, f1(). Could we do better? | |
</p><p>I was worried that the quadratic behavior of the algorithm was killing | |
us. So far, we had been using a list of 256 integers as test data, | |
since that was what my friend needed the function for. But what if it | |
were applied to a list of two thousand characters? We'd be | |
concatenating longer and longer strings, one character at a time. It | |
is easy to see that, apart from overhead, to create a list of length N | |
in this way, there are 1 + 2 + 3 + ... + (N-1) characters to be | |
copied in total, or N*(N-1)/2, or 0.5*N**2 - 0.5*N. In addition to | |
this, there are N string allocation operations, but for sufficiently | |
large N, the term containing N**2 will take over. Indeed, for a list | |
that's 8 times as long (2048 items), these functions all take much | |
more than 8 times as long; close to 16 times as long, in fact. I | |
didn't dare try a list of 64 times as long. | |
</p><p>There's a general technique to avoid quadratic behavior in algorithms like | |
this. I coded it as follows for strings of exactly 256 items: | |
</p><pre> def f5(list): | |
string = "" | |
for i in range(0, 256, 16): # 0, 16, 32, 48, 64, ... | |
s = "" | |
for character in map(chr, list[i:i+16]): | |
s = s + character | |
string = string + s | |
return string | |
</pre> | |
<p>Unfortunately, for a list of 256 items, this version ran a bit slower | |
(though within 20%) of f3(). Since writing a general version would only | |
slow it down more, we didn't bother to pursue this path any further | |
(except that we also compared it with a variant that didn't use map(), | |
which of course was slower again). | |
</p><p>Finally, I tried a radically different approach: use <i>only</i> implied | |
loops. Notice that the whole operation can be described as follows: apply | |
chr() to each list item; then concatenate the resulting characters. We | |
were already using an implied loop for the first part: map(). Fortunately, | |
there are some string concatenation functions in the string module that are | |
implemented in C. In particular, string.joinfields(list_of_strings, | |
delimiter) concatenates a list of strings, placing a delimiter of choice | |
between each two strings. Nothing stops us from concatenating a list of | |
characters (which are just strings of length one in Python), using the | |
empty string as delimiter. Lo and behold: | |
</p><pre> import string | |
def f6(list): | |
return string.joinfields(map(chr, list), "") | |
</pre> | |
<p>This function ran four to five times as fast as our fastest contender, | |
f3(). Moreover, it doesn't have the quadratic behavior of the other | |
versions. | |
</p><h3>And The Winner Is...</h3> | |
<p>The next day, I remembered an odd corner of Python: the array module. This | |
happens to have an operation to create an array of 1-byte wide integers | |
from a list of Python integers, and every array can be written to a file or | |
converted to a string as a binary data structure. Here's our function | |
implemented using these operations: | |
</p><pre> import array | |
def f7(list): | |
return array.array('B', list).tostring() | |
</pre> | |
<p>This is about three times as fast as f6(), or 12 to 15 times as fast as | |
f3()! it also uses less intermediate storage - it only allocates 2 objects | |
of N bytes (plus fixed overhead), while f6() begins by allocating a list of | |
N items, which usually costs 4N bytes (8N bytes on a 64-bit machine) - | |
assuming the character objects are shared with similar objects elsewhere | |
in the program (like small integers, Python caches strings of length one | |
in most cases). | |
</p><p>Stop, said my friend, before you get into negative times - this is fast | |
enough for my program. I agreed, though I had wanted to try one more | |
approach: write the whole function in C. This could have minimal storage | |
requirements (it would allocate a string of length N right away) and save a | |
few instructions in the C code that I knew were there in the array module, | |
because of its genericity (it supports integer widths of 1, 2, and 4 | |
bytes). However, it wouldn't be able to avoid having to extract the items | |
from the list one at a time, and to extract the C integer from them, both | |
of which are fairly costly operations in the Python-C API, so I expected at | |
most modest speed up compared to f7(). Given the effort of writing and | |
testing an extension (compared to whipping up those Python one-liners), as | |
well as the dependency on a non-standard Python extension, I decided not to | |
pursue this option... | |
</p><h3>Conclusion</h3> | |
<p>If you feel the need for speed, go for built-in functions - you can't beat | |
a loop written in C. Check the library manual for a built-in function that | |
does what you want. If there isn't one, here are some guidelines for loop | |
optimization: | |
</p><ul> | |
<li><i>Rule number one:</i> only optimize when there is a proven speed | |
bottleneck. Only optimize the innermost loop. (This rule is independent | |
of Python, but it doesn't hurt repeating it, since it can save a lot of | |
work. :-) | |
</li><li>Small is beautiful. Given Python's hefty charges for bytecode | |
instructions and variable look-up, it rarely pays off to add extra tests | |
to save a little bit of work. | |
</li><li>Use intrinsic operations. An implied loop in map() is faster than an | |
explicit for loop; a while loop with an explicit loop counter is even | |
slower. | |
</li><li>Avoid calling functions written in Python in your inner loop. This | |
includes lambdas. In-lining the inner loop can save a lot of time. | |
</li><li>Local variables are faster than globals; if you use a global constant | |
in a loop, copy it to a local variable before the loop. And in Python, | |
function names (global or built-in) are also global constants! | |
</li><li>Try to use map(), filter() or reduce() to replace an explicit for loop, | |
but only if you can use a built-in function: map with a built-in function | |
beats for loop, but a for loop with in-line code beats map with a lambda | |
function! | |
</li><li>Check your algorithms for quadratic behavior. But notice that a more | |
complex algorithm only pays off for large N - for small N, the complexity | |
doesn't pay off. In our case, 256 turned out to be small enough that the | |
simpler version was still a tad faster. Your mileage may vary - this is | |
worth investigating. | |
</li><li>And last but not least: collect data. Python's excellent profile | |
module can quickly show the bottleneck in your code. if you're considering | |
different versions of an algorithm, test it in a tight loop using the | |
time.clock() function. | |
</li></ul> | |
<p>By the way, here's the timing function that I used. it calls a function f | |
n*10 times with argument a, and prints the function name followed by the | |
time it took, rounded to milliseconds. The 10 repeated calls are done to | |
minimize the loop overhead of the timing function itself. You could go | |
even further and make 100 calls... Also note that the expression range(n) | |
is calculated outside the timing brackets - another trick to minimize the | |
overhead caused by the timing function. If you are worried about this | |
overhead, you can calibrate it by calling the timing function with a | |
do-nothing function. | |
</p><pre> import time | |
def timing(f, n, a): | |
print f.__name__, | |
r = range(n) | |
t1 = time.clock() | |
for i in r: | |
f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a) | |
t2 = time.clock() | |
print round(t2-t1, 3) | |
</pre> | |
<h3>Epilogue</h3> | |
<p>A few days later, my friend was back with the question: how do you do | |
the reverse operation? I.e. create a list of integer ASCII values | |
from a string. Oh no, here we go again, it flashed through my | |
mind... | |
</p><p>But this time, it was relatively painless. There are two candidates, | |
the obvious: | |
</p><pre> def g1(string): | |
return map(ord, string) | |
</pre> | |
<p>and the somewhat less obvious: | |
</p><pre> import array | |
def g2(string): | |
return array.array('b', string).tolist() | |
</pre> | |
<p>Timing these reveals that g2() is about five times as fast as g1(). | |
There's a catch though: g2() returns integers in the range -128..127, | |
while g1() returns integers in the range 0..255. If you need the | |
positive integers, g1() is going to be faster than anything | |
postprocessing you could do on the result from g2(). (Note: since | |
this essay was written, the 'B' typecode was added to the array | |
module, which stores unsigned bytes, so there's no reason to prefer | |
g1() any more.) | |
</p><h3>Sample Code</h3> | |
<ul> | |
<li><a href="f.py">timing f1() through f7()</a> | |
</li><li><a href="g.py">timing g1() and g2()</a> | |
</li></ul> | |
</article> | |
</section> | |
<aside class="left-sidebar" role="secondary"> | |
<div class="twitter-widget sidebar-widget" data-twttr-id="twttr-sandbox-0"> | |
<iframe id="twitter-widget-0" scrolling="no" frameborder="0" allowtransparency="true" class="twitter-timeline twitter-timeline-rendered" allowfullscreen="" style="border: none; max-width: 100%; min-width: 180px; margin: 0px; padding: 0px; display: inline-block; position: static; visibility: visible; width: 224px;" title="Twitter Timeline" height="600"></iframe> | |
<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+"://platform.twitter.com/widgets.js";fjs.parentNode.insertBefore(js,fjs);}}(document,"script","twitter-wjs");</script> | |
</div> | |
<div class="psf-sidebar-widget sidebar-widget"> | |
<h3 class="widget-title">The PSF</h3> | |
<p>The Python Software Foundation is the organization behind Python. Become a member of the PSF and help advance the software and our mission. </p> | |
</div> | |
</aside> | |
</div><!-- end .container --> | |
</div><!-- end #content .content-wrapper --> | |
<!-- Footer and social media list --> | |
<footer id="site-map" class="main-footer" role="contentinfo"> | |
<div class="main-footer-links"> | |
<div class="container"> | |
<a id="back-to-top-1" class="jump-link" href="#python-network"><span aria-hidden="true" class="icon-arrow-up"><span>▲</span></span> Back to Top</a> | |
<ul class="sitemap navigation menu do-not-print" role="tree" id="container" style="position: relative; height: 490.21875px;"> | |
<li class="tier-1 element-1" style="position: absolute; left: 0px; top: 0px;"> | |
<a href="/about/">About</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/apps/" title="">Applications</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/quotes/" title="">Quotes</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/gettingstarted/" title="">Getting Started</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/help/" title="">Help</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-2" style="position: absolute; left: 163px; top: 0px;"> | |
<a href="/downloads/">Downloads</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/downloads/" title="">All releases</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/downloads/source/" title="">Source code</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/downloads/windows/" title="">Windows</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/downloads/mac-osx/" title="">Mac OS X</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/download/other/" title="">Other Platforms</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://docs.python.org/3/license.html" title="">License</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/download/alternatives" title="">Alternative Implementations</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-3" style="position: absolute; left: 327px; top: 0px;"> | |
<a href="/doc/">Documentation</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/doc/" title="">Docs</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/doc/av" title="">Audio/Visual Talks</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="https://wiki.python.org/moin/BeginnersGuide" title="">Beginner's Guide</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="https://docs.python.org/devguide/" title="">Developer's Guide</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://docs.python.org/faq/" title="">FAQ</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="http://wiki.python.org/moin/Languages" title="">Non-English Docs</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="http://python.org/dev/peps/" title="">PEP Index</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="https://wiki.python.org/moin/PythonBooks" title="">Python Books</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-4" style="position: absolute; left: 490px; top: 0px;"> | |
<a href="/community/">Community</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/community/diversity/" title="">Diversity</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/community/irc/" title="">IRC</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/community/lists/" title="">Mailing Lists</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/community/workshops/" title="">Python Conferences</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/community/sigs/" title="">Special Interest Groups</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://wiki.python.org/moin/" title="">Python Wiki</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/community/logos/" title="">Python Logo</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="/community/merchandise/" title="">Merchandise</a></li> | |
<li class="tier-2 element-9" role="treeitem"><a href="/community/awards" title="">Community Awards</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-5" style="position: absolute; left: 654px; top: 0px;"> | |
<a href="/about/success/" title="success-stories">Success Stories</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/success/#arts" title="">Arts</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/success/#business" title="">Business</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/success/#education" title="">Education</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/success/#engineering" title="">Engineering</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/about/success/#government" title="">Government</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="/about/success/#scientific" title="">Scientific</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/about/success/#software-development" title="">Software Development</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-6" style="position: absolute; left: 817px; top: 0px;"> | |
<a href="/blogs/" title="News from around the Python world">News</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/blogs/" title="Python Insider Blog Posts">Python News</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://planetpython.org/" title="Planet Python">Community News</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="http://pyfound.blogspot.com/" title="PSF Blog">PSF News</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="http://pycon.blogspot.com/" title="PyCon Blog">PyCon News</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-7" style="position: absolute; left: 0px; top: 213px;"> | |
<a href="/events/">Events</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/events/python-events/" title="">Python Events</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/events/python-user-group/" title="">User Group Events</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/events/python-events/past/" title="">Python Events Archive</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/events/python-user-group/past/" title="">User Group Events Archive</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event" title="">Submit an Event</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-8" style="position: absolute; left: 817px; top: 213px;"> | |
<a href="/dev/">Contributing</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="http://docs.python.org/devguide/" title="">Developer's Guide</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://bugs.python.org/" title="">Issue Tracker</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="https://mail.python.org/mailman/listinfo/python-dev" title="">python-dev list</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="http://pythonmentors.com/" title="">Core Mentorship</a></li> | |
</ul> | |
</li> | |
</ul> | |
<a id="back-to-top-2" class="jump-link" href="#python-network"><span aria-hidden="true" class="icon-arrow-up"><span>▲</span></span> Back to Top</a> | |
</div><!-- end .container --> | |
</div> <!-- end .main-footer-links --> | |
<div class="site-base"> | |
<div class="container"> | |
<ul class="footer-links navigation menu do-not-print" role="tree"> | |
<li class="tier-1 element-1"><a href="/about/help/">Help & <span class="say-no-more">General</span> Contact</a></li> | |
<li class="tier-1 element-2"><a href="/community/diversity/">Diversity <span class="say-no-more">Initiatives</span></a></li> | |
<li class="tier-1 element-3"><a href="https://github.com/python/pythondotorg/issues">Submit Website Bug</a></li> | |
<!--<li class="tier-1 element-3"><a href="#"><span class="say-no-more">Website</span> Colophon</a></li>--> | |
</ul> | |
<div class="copyright"> | |
<p><small> | |
<span class="pre">Copyright ©2001-2015.</span> | |
<span class="pre"><a href="/psf-landing/">Python Software Foundation</a></span> | |
<span class="pre"><a href="/about/legal/">Legal Statements</a></span> | |
<span class="pre"><a href="/privacy/">Privacy Policy</a></span> | |
</small></p> | |
</div> | |
</div><!-- end .container --> | |
</div><!-- end .site-base --> | |
</footer> | |
</div><!-- end #touchnav-wrapper --> | |
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script> | |
<script>window.jQuery || document.write('<script src="/static/js/libs/jquery-1.8.2.min.js"><\/script>')</script> | |
<script src="/static/js/libs/masonry.pkgd.min.js"></script> | |
<script type="text/javascript" src="/static/js/main-min.js" charset="utf-8"></script> | |
<!--[if lte IE 7]> | |
<script type="text/javascript" src="/static/js/plugins/IE8-min.js" charset="utf-8"></script> | |
<![endif]--> | |
<!--[if lte IE 8]> | |
<script type="text/javascript" src="/static/js/plugins/getComputedStyle-min.js" charset="utf-8"></script> | |
<![endif]--> | |
<iframe id="rufous-sandbox" scrolling="no" frameborder="0" allowtransparency="true" style="display: none;"></iframe><script id="jsbin-javascript"> | |
var flatmap, neigh, readable; | |
flatmap = (function(_this) { | |
return function(l, f) { | |
var r; | |
r = []; | |
l.forEach(function(e) { | |
if (Array.isArray(e)) { | |
return Array.prototype.push.apply(r, flatmap(e, f)); | |
} else { | |
return r.push(f(e)); | |
} | |
}); | |
return r; | |
}; | |
})(this); | |
neigh = (function(_this) { | |
return function(d) { | |
var ps; | |
ps = $(d).parents().get(); | |
return flatmap(ps, function(e) { | |
return $(e).siblings(); | |
}); | |
}; | |
})(this); | |
readable = (function(_this) { | |
return function(d) { | |
var l, n; | |
n = neigh(d); | |
l = neigh.length; | |
if (l > 1000) { | |
return alert("" + l + " siblings!"); | |
} else { | |
return n.forEach(function(e) { | |
return $(e).toggle(); | |
}); | |
} | |
}; | |
})(this); | |
</script> | |
<script id="jsbin-source-html" type="text/html"><html class="js no-touch geolocation fontface generatedcontent svg formvalidation placeholder boxsizing no-retina" lang="en" dir="ltr" style=""><\!--<![endif]--><head> | |
<meta charset="utf-8"> | |
<meta http-equiv="X-UA-Compatible" content="IE=edge"> | |
<link rel="prefetch" href="//ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"> | |
<meta name="application-name" content="Python.org"> | |
<meta name="msapplication-tooltip" content="The official home of the Python Programming Language"> | |
<meta name="apple-mobile-web-app-title" content="Python.org"> | |
<meta name="apple-mobile-web-app-capable" content="yes"> | |
<meta name="apple-mobile-web-app-status-bar-style" content="black"> | |
<meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
<meta name="HandheldFriendly" content="True"> | |
<meta name="format-detection" content="telephone=no"> | |
<meta http-equiv="cleartype" content="on"> | |
<meta http-equiv="imagetoolbar" content="false"> | |
<script id="twitter-wjs" src="https://platform.twitter.com/widgets.js"><\/script><script type="text/javascript" async="" src="https://ssl.google-analytics.com/ga.js"><\/script><script src="/static/js/libs/modernizr.js"><\/script> | |
<link href="/static/stylesheets/style.css" rel="stylesheet" type="text/css" title="default"> | |
<link href="/static/stylesheets/mq.css" rel="stylesheet" type="text/css" media="not print, braille, embossed, speech, tty"> | |
<\!--[if (lte IE 8)&(!IEMobile)]> | |
<link href="/static/stylesheets/no-mq.css" rel="stylesheet" type="text/css" media="screen" /> | |
<![endif]--> | |
<link rel="icon" type="image/x-icon" href="/static/favicon.ico"> | |
<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/static/apple-touch-icon-144x144-precomposed.png"> | |
<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/static/apple-touch-icon-114x114-precomposed.png"> | |
<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/static/apple-touch-icon-72x72-precomposed.png"> | |
<link rel="apple-touch-icon-precomposed" href="/static/apple-touch-icon-precomposed.png"> | |
<link rel="apple-touch-icon" href="/static/apple-touch-icon-precomposed.png"> | |
<meta name="msapplication-TileImage" content="/static/metro-icon-144x144-precomposed.png"><\!-- white shape --> | |
<meta name="msapplication-TileColor" content="#3673a5"><\!-- python blue --> | |
<meta name="msapplication-navbutton-color" content="#3673a5"> | |
<meta property="og:site_name" content="Python.org"> | |
<meta property="og:type" content="website"> | |
<title>Python Patterns - An Optimization Anecdote | Python.org</title> | |
<meta property="og:title" content="Welcome to Python.org"> | |
<meta name="description" content="The official home of the Python Programming Language"> | |
<meta name="og:description" content="The official home of the Python Programming Language"> | |
<meta name="keywords" content="Python programming language object oriented web free open source software license documentation download community"> | |
<meta property="og:tag" content="Python programming language object oriented web free open source software license documentation download community"> | |
<meta property="og:published_time" content=""> | |
<meta property="og:modified_time" content=""> | |
<meta property="og:author" content=""> | |
<meta property="og:section" content=""> | |
<meta property="og:url" content=""> | |
<meta property="og:image" content=""> | |
<meta property="og:video" content=""> | |
<link rel="author" href="/static/humans.txt"> | |
<script type="application/ld+json"> | |
{ | |
"@context": "http://schema.org", | |
"@type": "WebSite", | |
"url": "https://www.python.org/", | |
"potentialAction": { | |
"@type": "SearchAction", | |
"target": "https://www.python.org/search/?q={search_term_string}", | |
"query-input": "required name=search_term_string" | |
} | |
} | |
<\/script> | |
<script type="text/javascript"> | |
var _gaq = _gaq || []; | |
_gaq.push(['_setAccount', 'UA-39055973-1']); | |
_gaq.push(['_trackPageview']); | |
(function() { | |
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; | |
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; | |
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); | |
})(); | |
<\/script> | |
</head> | |
<body class="python pages default-page" data-twttr-rendered="true"> | |
<div id="touchnav-wrapper"> | |
<div id="nojs" class="do-not-print"> | |
<p><strong>Notice:</strong> While Javascript is not essential for this website, your interaction with the content will be limited. Please turn Javascript on for the full experience. </p> | |
</div> | |
<\!--[if lt IE 8]> | |
<div id="oldie-warning" class="do-not-print"> | |
<p><strong>Notice:</strong> Your browser is <em>ancient</em> and <a href="http://www.ie6countdown.com/">Microsoft agrees</a>. <a href="http://browsehappy.com/">Upgrade to a different browser</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to experience a better web.</p> | |
</div> | |
<![endif]--> | |
<\!-- Sister Site Links --> | |
<div id="top" class="top-bar do-not-print"> | |
<nav class="meta-navigation container" role="navigation"> | |
<div class="skip-link screen-reader-text"> | |
<a href="#content" title="Skip to content">Skip to content</a> | |
</div> | |
<a id="close-python-network" class="jump-link" href="#python-network" aria-hidden="true"> | |
<span aria-hidden="true" class="icon-arrow-down"><span>▼</span></span> Close | |
</a> | |
<ul class="menu" role="tree"> | |
<li class="python-meta "> | |
<a href="/" title="The Python Programming Language">Python</a> | |
</li> | |
<li class="psf-meta "> | |
<a href="/psf-landing/" title="The Python Software Foundation">PSF</a> | |
</li> | |
<li class="docs-meta "> | |
<a href="https://docs.python.org" title="Python Documentation">Docs</a> | |
</li> | |
<li class="pypi-meta "> | |
<a href="https://pypi.python.org/" title="Python Package Index">PyPI</a> | |
</li> | |
<li class="jobs-meta "> | |
<a href="/jobs/" title="Python Job Board">Jobs</a> | |
</li> | |
<li class="shop-meta "> | |
<a href="/community/" title="Python Community">Community</a> | |
</li> | |
</ul> | |
<a id="python-network" class="jump-link" href="#top" aria-hidden="true"> | |
<span aria-hidden="true" class="icon-arrow-up"><span>▲</span></span> The Python Network | |
</a> | |
</nav> | |
</div> | |
<\!-- Header elements --> | |
<header class="main-header" role="banner"> | |
<div class="container"> | |
<h1 class="site-headline"> | |
<a href="/"><img class="python-logo" src="/static/img/python-logo.png" alt="python™"></a> | |
</h1> | |
<div class="options-bar do-not-print"> | |
<a id="site-map-link" class="jump-to-menu" href="#site-map"><span class="menu-icon">≡</span> Menu</a><form class="search-the-site" action="/search/" method="get"> | |
<fieldset title="Search Python.org"> | |
<span aria-hidden="true" class="icon-search"></span> | |
<label class="screen-reader-text" for="id-search-field">Search This Site</label> | |
<input id="id-search-field" name="q" type="search" role="textbox" class="search-field placeholder" placeholder="Search" value="" tabindex="1"> | |
<button type="submit" name="submit" id="submit" class="search-button" title="Submit this Search" tabindex="3"> | |
GO | |
</button> | |
<\!--[if IE]><input type="text" style="display: none;" disabled="disabled" size="1" tabindex="4"><![endif]--> | |
</fieldset> | |
</form><span class="breaker"></span><div class="adjust-font-size" aria-hidden="true"> | |
<ul class="navigation menu" aria-label="Adjust Text Size on Page"> | |
<li class="tier-1 last" aria-haspopup="true"> | |
<a href="#" class="action-trigger"><strong><small>A</small> A</strong></a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a class="text-shrink" title="Make Text Smaller" href="javascript:;">Smaller</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a class="text-grow" title="Make Text Larger" href="javascript:;">Larger</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a class="text-reset" title="Reset any font size changes I have made" href="javascript:;">Reset</a></li> | |
</ul> | |
</li> | |
</ul> | |
</div><div class="winkwink-nudgenudge"> | |
<ul class="navigation menu" aria-label="Social Media Navigation"> | |
<li class="tier-1 last" aria-haspopup="true"> | |
<a href="#" class="action-trigger">Socialize</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="http://plus.google.com/+Python"><span aria-hidden="true" class="icon-google-plus"></span>Google+</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://www.facebook.com/pythonlang?fref=ts"><span aria-hidden="true" class="icon-facebook"></span>Facebook</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="http://twitter.com/ThePSF"><span aria-hidden="true" class="icon-twitter"></span>Twitter</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/community/irc/"><span aria-hidden="true" class="icon-freenode"></span>Chat on IRC</a></li> | |
</ul> | |
</li> | |
</ul> | |
</div><div class="account-signin"> | |
<ul class="navigation menu" aria-label="Social Media Navigation"> | |
<li class="tier-1 last" aria-haspopup="true"> | |
<a href="/accounts/login/" title="Sign Up or Sign In to Python.org">Sign In</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/accounts/signup/">Sign Up / Register</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/accounts/login/">Sign In</a></li> | |
</ul> | |
</li> | |
</ul> | |
</div> | |
</div><\!-- end options-bar --> | |
<nav id="mainnav" class="python-navigation main-navigation do-not-print" role="navigation"> | |
<ul class="navigation menu" role="menubar" aria-label="Main Navigation"> | |
<li id="about" class="tier-1 element-1 with-supernav" aria-haspopup="true"> | |
<a href="/about/" title="" class="">About</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/apps/" title="">Applications</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/quotes/" title="">Quotes</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/gettingstarted/" title="">Getting Started</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/help/" title="">Help</a></li> | |
<li class="tier-2 super-navigation"> | |
<h4>Python is a programming language that lets you work more quickly and integrate your systems more effectively.</h4> | |
<p>You can learn to use Python and see almost immediate gains in productivity and lower maintenance costs. <a href="/about">Learn more about Python.</a>. | |
</p></li></ul> | |
</li> | |
<li id="downloads" class="tier-1 element-2 with-supernav" aria-haspopup="true"> | |
<a href="/downloads/" title="" class="">Downloads</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/downloads/" title="">All releases</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/downloads/source/" title="">Source code</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/downloads/windows/" title="">Windows</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/downloads/mac-osx/" title="">Mac OS X</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/download/other/" title="">Other Platforms</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://docs.python.org/3/license.html" title="">License</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/download/alternatives" title="">Alternative Implementations</a></li> | |
<\!-- download supernav from templates/downloads/supernav.html --> | |
<li class="tier-2 super-navigation"> | |
<div class="download-os-mac-osx" style="display: none;"> | |
<h4>Download for Mac OS X</h4> | |
<p> | |
<a class="button" href="https://www.python.org/ftp/python/3.4.3/python-3.4.3-macosx10.6.pkg">Python 3.4.3</a> | |
<a class="button" href="https://www.python.org/ftp/python/2.7.9/python-2.7.9-macosx10.6.pkg">Python 2.7.9</a> | |
</p> | |
<p>Not the OS you are looking for? Python can be used on 21 different operating systems and environments. <a href="/downloads/">View the full list</a>.</p> | |
</div> | |
<div class="download-os-source" style=""> | |
<h3>Python Source</h3> | |
<p> | |
<a class="button" href="https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tar.xz">Python 3.4.3</a> | |
<a class="button" href="https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tar.xz">Python 2.7.9</a> | |
</p> | |
<p>Not the OS you are looking for? Python can be used on 21 different operating systems and environments. <a href="/downloads/">View the full list</a>.</p> | |
</div> | |
<div class="download-os-windows" style="display: none;"> | |
<h4>Download for Windows</h4> | |
<p> | |
<a class="button" href="https://www.python.org/ftp/python/3.4.3/python-3.4.3.msi">Python 3.4.3</a> | |
<a class="button" href="https://www.python.org/ftp/python/2.7.9/python-2.7.9.msi">Python 2.7.9</a> | |
</p> | |
<p>Not the OS you are looking for? Python can be used on 21 different operating systems and environments. <a href="/downloads/">View the full list</a>.</p> | |
</div> | |
<div class="download-unknown" style="display: none;"> | |
<h4>Download Python for Any OS</h4> | |
<p>Python can be used on 21 different operating systems and environments.</p> | |
<p> | |
<a class="button" href="/downloads/operating-systems/">View the Full List</a> | |
</p> | |
</div> | |
</li> | |
</ul> | |
</li> | |
<li id="documentation" class="tier-1 element-3 with-supernav" aria-haspopup="true"> | |
<a href="/doc/" title="" class="">Documentation</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/doc/" title="">Docs</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/doc/av" title="">Audio/Visual Talks</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="https://wiki.python.org/moin/BeginnersGuide" title="">Beginner's Guide</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="https://docs.python.org/devguide/" title="">Developer's Guide</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://docs.python.org/faq/" title="">FAQ</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="http://wiki.python.org/moin/Languages" title="">Non-English Docs</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="http://python.org/dev/peps/" title="">PEP Index</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="https://wiki.python.org/moin/PythonBooks" title="">Python Books</a></li> | |
<li class="tier-2 super-navigation"> | |
<h4>Python’s standard documentation: download, browse or watch a tutorial.</h4> | |
<p>Get started below, or visit the <a href="/doc/versions/">Documentation page to browse by version</a>. </p> | |
<p class="download-buttons"> | |
<a class="button" href="http://docs.python.org/3/">Python 3.x Docs</a> | |
<a class="button" href="http://docs.python.org/2/">Python 2.x Docs</a> | |
</p> | |
<p>See also <a href="https://wiki.python.org/moin/Python2orPython3">Should I use Python 2 or 3</a>? </p> | |
</li></ul> | |
</li> | |
<li id="community" class="tier-1 element-4 with-supernav" aria-haspopup="true"> | |
<a href="/community/" title="" class="">Community</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/community/diversity/" title="">Diversity</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/community/irc/" title="">IRC</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/community/lists/" title="">Mailing Lists</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/community/workshops/" title="">Python Conferences</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/community/sigs/" title="">Special Interest Groups</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://wiki.python.org/moin/" title="">Python Wiki</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/community/logos/" title="">Python Logo</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="/community/merchandise/" title="">Merchandise</a></li> | |
<li class="tier-2 element-9" role="treeitem"><a href="/community/awards" title="">Community Awards</a></li> | |
<li class="tier-2 super-navigation"> | |
<h4>The Python Community</h4> | |
<p>Great software is supported by great people. Our user base is enthusiastic, dedicated to encouraging use of the language, and committed to being diverse and friendly.</p> | |
<\!-- | |
<p>Here are some events and groups in your area.</p> | |
<ul class="menu"> | |
<li><time datetime="">9/30<span class="say-no-more">/2012</span></time> <a href="#">Royal Python Society Meetup, Providence RI</a></li> | |
<li><time datetime="">10/4<span class="say-no-more">/2012</span></time> <a href="#">Python Users Group, Boston MA</a></li> | |
<li><time datetime="">10/5<span class="say-no-more">/2012</span></time> <a href="#">Python Enthusiasts, Cambridge MA</a></li> | |
</ul> | |
</li>--></li></ul> | |
</li> | |
<li id="success-stories" class="tier-1 element-5 with-supernav" aria-haspopup="true"> | |
<a href="/about/success/" title="success-stories" class="">Success Stories</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/success/#arts" title="">Arts</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/success/#business" title="">Business</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/success/#education" title="">Education</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/success/#engineering" title="">Engineering</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/about/success/#government" title="">Government</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="/about/success/#scientific" title="">Scientific</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/about/success/#software-development" title="">Software Development</a></li> | |
<\!-- successstories supernav from templates/successstories/supernav.html --> | |
<li class="tier-2 super-navigation"> | |
<img src="/m/successstories/Lucasfilm_logo.png" alt="Industrial Light & Magic"> | |
<blockquote class="success-quote"> | |
ILM runs a batch processing environment capable of modeling, rendering and compositing tens of thousands of motion picture frames per day. Thousands of machines running Linux, IRIX, Compaq Tru64, OS X, Solaris, and Windows join together to provide a production pipeline used by ~800 users daily. Speed of development is key, and Python was a faster way to code (and re-code) the programs that control this production pipeline. | |
</blockquote> | |
<p class="quote-by"><cite>Tim Fortenberry</cite>, <a href="http://www.ilm.com/">Industrial Light & Magic</a></p> | |
</li></ul> | |
</li> | |
<li id="news" class="tier-1 element-6 " aria-haspopup="true"> | |
<a href="/blogs/" title="News from around the Python world" class="">News</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/blogs/" title="Python Insider Blog Posts">Python News</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://planetpython.org/" title="Planet Python">Community News</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="http://pyfound.blogspot.com/" title="PSF Blog">PSF News</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="http://pycon.blogspot.com/" title="PyCon Blog">PyCon News</a></li> | |
</ul> | |
</li> | |
<li id="events" class="tier-1 element-7 with-supernav" aria-haspopup="true"> | |
<a href="/events/" title="" class="">Events</a> | |
<ul class="subnav menu" role="menu" aria-hidden="true"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/events/python-events/" title="">Python Events</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/events/python-user-group/" title="">User Group Events</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/events/python-events/past/" title="">Python Events Archive</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/events/python-user-group/past/" title="">User Group Events Archive</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event" title="">Submit an Event</a></li> | |
<li class="tier-2 super-navigation">Find events from the Python Community around the world!</li></ul> | |
</li> | |
</ul> | |
</nav> | |
<div class="header-banner "> <\!-- for optional "do-not-print" class --> | |
</div> | |
</div><\!-- end .container --> | |
</header> | |
<div id="content" class="content-wrapper"> | |
<\!-- Main Content Column --> | |
<div class="container"> | |
<section class="main-content with-left-sidebar" role="main"> | |
<ul class="breadcrumbs menu"> | |
</ul> | |
<article class="text"> | |
<header class="article-header"> | |
<h1 class="page-title">Python Patterns - An Optimization Anecdote</h1> | |
</header> | |
<h1>Python Patterns - An Optimization Anecdote</h1> | |
The other day, a friend asked me a seemingly simple question: what's the | |
best way to convert a list of integers into a string, presuming that the | |
integers are ASCII values. For instance, the list [97, 98, 99] should be | |
converted to the string 'abc'. Let's assume we want to write a function to | |
do this.<p> | |
</p><p>The first version I came up with was totally straightforward:</p> | |
<pre> def f1(list): | |
string = "" | |
for item in list: | |
string = string + chr(item) | |
return string | |
</pre> | |
<p>That can't be the fastest way to do it, said my friend. How about this one:</p> | |
<pre> def f2(list): | |
return reduce(lambda string, item: string + chr(item), list, "") | |
</pre> | |
<p>This version performs exactly the same set of string operations as the | |
first one, but gets rid of the for loop overhead in favor of the faster, | |
implied loop of the reduce() function. | |
</p><p>Sure, I replied, but it does so at the cost of a function call (the lambda | |
function) per list item. I betcha it's slower, since function call | |
overhead in Python is bigger than for loop overhead. | |
</p><p>(OK, so I had already done the comparisons. f2() took 60% more time than f1(). So there :-) | |
</p><p>Hmm, said my friend. I need this to be faster. OK, I said, how about | |
this version: | |
</p><pre> def f3(list): | |
string = "" | |
for character in map(chr, list): | |
string = string + character | |
return string | |
</pre> | |
<p>To both our surprise, f3() clocked <i>twice</i> as fast as f1()! The | |
reason that this surprised us was twofold: first, it uses more storage (the | |
result of map(chr, list) is another list of the same length); second, it | |
contains two loops instead of one (the one implied by the map() function, | |
and the for loop). | |
</p><p>Of course, space versus time is a well-known trade-off, so the first one | |
shouldn't have surprised us. However, how come two loops are faster than | |
one? Two reasons. | |
</p><p>First, in f1(), the built-in function chr() is looked up on every | |
iteration, while in f3() it is only looked up once (as the argument to | |
map()). This look-up is relatively expensive, I told my friend, since | |
Python's dynamic scope rules mean that it is first looked up | |
(unsuccessfully) in the current module's global dictionary, and then in the | |
dictionary of built-in function (where it is found). Worse, unsuccessful | |
dictionary lookups are (on average) a bit slower than successful ones, | |
because of the way the hash chaining works. | |
</p><p>The second reason why f3() is faster than f1() is that the call to | |
chr(item), as executed by the bytecode interpreter, is probably a bit | |
slower than when executed by the map() function - the bytecode interpreter | |
must execute three bytecode instructions for each call (load 'chr', load | |
'item', call), while the map() function does it all in C. | |
</p><p>This led us to consider a compromise, which wouldn't waste extra space, but | |
which would speed up the lookup for the chr() function: | |
</p><pre> def f4(list): | |
string = "" | |
lchr = chr | |
for item in list: | |
string = string + lchr(item) | |
return string | |
</pre> | |
<p>As expected, f4() was slower than f3(), but only by 25%; it was about 40% | |
faster than f1() still. This is because local variable lookups are | |
<i>much</i> faster than global or built-in variable lookups: the Python | |
"compiler" optimizes most function bodies so that for local variables, no | |
dictionary lookup is necessary, but a simple array indexing operation is | |
sufficient. The relative speed of f4() compared to f1() and f3() suggests | |
that both reasons why f3() is faster contribute, but that the first reason | |
(fewer lookups) is a bit more important. (To get more precise data on | |
this, we would have to instrument the interpreter.) | |
</p><p>Still, our best version, f3(), was only twice as fast as the most | |
straightforward version, f1(). Could we do better? | |
</p><p>I was worried that the quadratic behavior of the algorithm was killing | |
us. So far, we had been using a list of 256 integers as test data, | |
since that was what my friend needed the function for. But what if it | |
were applied to a list of two thousand characters? We'd be | |
concatenating longer and longer strings, one character at a time. It | |
is easy to see that, apart from overhead, to create a list of length N | |
in this way, there are 1 + 2 + 3 + ... + (N-1) characters to be | |
copied in total, or N*(N-1)/2, or 0.5*N**2 - 0.5*N. In addition to | |
this, there are N string allocation operations, but for sufficiently | |
large N, the term containing N**2 will take over. Indeed, for a list | |
that's 8 times as long (2048 items), these functions all take much | |
more than 8 times as long; close to 16 times as long, in fact. I | |
didn't dare try a list of 64 times as long. | |
</p><p>There's a general technique to avoid quadratic behavior in algorithms like | |
this. I coded it as follows for strings of exactly 256 items: | |
</p><pre> def f5(list): | |
string = "" | |
for i in range(0, 256, 16): # 0, 16, 32, 48, 64, ... | |
s = "" | |
for character in map(chr, list[i:i+16]): | |
s = s + character | |
string = string + s | |
return string | |
</pre> | |
<p>Unfortunately, for a list of 256 items, this version ran a bit slower | |
(though within 20%) of f3(). Since writing a general version would only | |
slow it down more, we didn't bother to pursue this path any further | |
(except that we also compared it with a variant that didn't use map(), | |
which of course was slower again). | |
</p><p>Finally, I tried a radically different approach: use <i>only</i> implied | |
loops. Notice that the whole operation can be described as follows: apply | |
chr() to each list item; then concatenate the resulting characters. We | |
were already using an implied loop for the first part: map(). Fortunately, | |
there are some string concatenation functions in the string module that are | |
implemented in C. In particular, string.joinfields(list_of_strings, | |
delimiter) concatenates a list of strings, placing a delimiter of choice | |
between each two strings. Nothing stops us from concatenating a list of | |
characters (which are just strings of length one in Python), using the | |
empty string as delimiter. Lo and behold: | |
</p><pre> import string | |
def f6(list): | |
return string.joinfields(map(chr, list), "") | |
</pre> | |
<p>This function ran four to five times as fast as our fastest contender, | |
f3(). Moreover, it doesn't have the quadratic behavior of the other | |
versions. | |
</p><h3>And The Winner Is...</h3> | |
<p>The next day, I remembered an odd corner of Python: the array module. This | |
happens to have an operation to create an array of 1-byte wide integers | |
from a list of Python integers, and every array can be written to a file or | |
converted to a string as a binary data structure. Here's our function | |
implemented using these operations: | |
</p><pre> import array | |
def f7(list): | |
return array.array('B', list).tostring() | |
</pre> | |
<p>This is about three times as fast as f6(), or 12 to 15 times as fast as | |
f3()! it also uses less intermediate storage - it only allocates 2 objects | |
of N bytes (plus fixed overhead), while f6() begins by allocating a list of | |
N items, which usually costs 4N bytes (8N bytes on a 64-bit machine) - | |
assuming the character objects are shared with similar objects elsewhere | |
in the program (like small integers, Python caches strings of length one | |
in most cases). | |
</p><p>Stop, said my friend, before you get into negative times - this is fast | |
enough for my program. I agreed, though I had wanted to try one more | |
approach: write the whole function in C. This could have minimal storage | |
requirements (it would allocate a string of length N right away) and save a | |
few instructions in the C code that I knew were there in the array module, | |
because of its genericity (it supports integer widths of 1, 2, and 4 | |
bytes). However, it wouldn't be able to avoid having to extract the items | |
from the list one at a time, and to extract the C integer from them, both | |
of which are fairly costly operations in the Python-C API, so I expected at | |
most modest speed up compared to f7(). Given the effort of writing and | |
testing an extension (compared to whipping up those Python one-liners), as | |
well as the dependency on a non-standard Python extension, I decided not to | |
pursue this option... | |
</p><h3>Conclusion</h3> | |
<p>If you feel the need for speed, go for built-in functions - you can't beat | |
a loop written in C. Check the library manual for a built-in function that | |
does what you want. If there isn't one, here are some guidelines for loop | |
optimization: | |
</p><ul> | |
<li><i>Rule number one:</i> only optimize when there is a proven speed | |
bottleneck. Only optimize the innermost loop. (This rule is independent | |
of Python, but it doesn't hurt repeating it, since it can save a lot of | |
work. :-) | |
</li><li>Small is beautiful. Given Python's hefty charges for bytecode | |
instructions and variable look-up, it rarely pays off to add extra tests | |
to save a little bit of work. | |
</li><li>Use intrinsic operations. An implied loop in map() is faster than an | |
explicit for loop; a while loop with an explicit loop counter is even | |
slower. | |
</li><li>Avoid calling functions written in Python in your inner loop. This | |
includes lambdas. In-lining the inner loop can save a lot of time. | |
</li><li>Local variables are faster than globals; if you use a global constant | |
in a loop, copy it to a local variable before the loop. And in Python, | |
function names (global or built-in) are also global constants! | |
</li><li>Try to use map(), filter() or reduce() to replace an explicit for loop, | |
but only if you can use a built-in function: map with a built-in function | |
beats for loop, but a for loop with in-line code beats map with a lambda | |
function! | |
</li><li>Check your algorithms for quadratic behavior. But notice that a more | |
complex algorithm only pays off for large N - for small N, the complexity | |
doesn't pay off. In our case, 256 turned out to be small enough that the | |
simpler version was still a tad faster. Your mileage may vary - this is | |
worth investigating. | |
</li><li>And last but not least: collect data. Python's excellent profile | |
module can quickly show the bottleneck in your code. if you're considering | |
different versions of an algorithm, test it in a tight loop using the | |
time.clock() function. | |
</li></ul> | |
<p>By the way, here's the timing function that I used. it calls a function f | |
n*10 times with argument a, and prints the function name followed by the | |
time it took, rounded to milliseconds. The 10 repeated calls are done to | |
minimize the loop overhead of the timing function itself. You could go | |
even further and make 100 calls... Also note that the expression range(n) | |
is calculated outside the timing brackets - another trick to minimize the | |
overhead caused by the timing function. If you are worried about this | |
overhead, you can calibrate it by calling the timing function with a | |
do-nothing function. | |
</p><pre> import time | |
def timing(f, n, a): | |
print f.__name__, | |
r = range(n) | |
t1 = time.clock() | |
for i in r: | |
f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a); f(a) | |
t2 = time.clock() | |
print round(t2-t1, 3) | |
</pre> | |
<h3>Epilogue</h3> | |
<p>A few days later, my friend was back with the question: how do you do | |
the reverse operation? I.e. create a list of integer ASCII values | |
from a string. Oh no, here we go again, it flashed through my | |
mind... | |
</p><p>But this time, it was relatively painless. There are two candidates, | |
the obvious: | |
</p><pre> def g1(string): | |
return map(ord, string) | |
</pre> | |
<p>and the somewhat less obvious: | |
</p><pre> import array | |
def g2(string): | |
return array.array('b', string).tolist() | |
</pre> | |
<p>Timing these reveals that g2() is about five times as fast as g1(). | |
There's a catch though: g2() returns integers in the range -128..127, | |
while g1() returns integers in the range 0..255. If you need the | |
positive integers, g1() is going to be faster than anything | |
postprocessing you could do on the result from g2(). (Note: since | |
this essay was written, the 'B' typecode was added to the array | |
module, which stores unsigned bytes, so there's no reason to prefer | |
g1() any more.) | |
</p><h3>Sample Code</h3> | |
<ul> | |
<li><a href="f.py">timing f1() through f7()</a> | |
</li><li><a href="g.py">timing g1() and g2()</a> | |
</li></ul> | |
</article> | |
</section> | |
<aside class="left-sidebar" role="secondary"> | |
<div class="twitter-widget sidebar-widget" data-twttr-id="twttr-sandbox-0"> | |
<iframe id="twitter-widget-0" scrolling="no" frameborder="0" allowtransparency="true" class="twitter-timeline twitter-timeline-rendered" allowfullscreen="" style="border: none; max-width: 100%; min-width: 180px; margin: 0px; padding: 0px; display: inline-block; position: static; visibility: visible; width: 224px;" title="Twitter Timeline" height="600"></iframe> | |
<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+"://platform.twitter.com/widgets.js";fjs.parentNode.insertBefore(js,fjs);}}(document,"script","twitter-wjs");<\/script> | |
</div> | |
<div class="psf-sidebar-widget sidebar-widget"> | |
<h3 class="widget-title">The PSF</h3> | |
<p>The Python Software Foundation is the organization behind Python. Become a member of the PSF and help advance the software and our mission. </p> | |
</div> | |
</aside> | |
</div><\!-- end .container --> | |
</div><\!-- end #content .content-wrapper --> | |
<\!-- Footer and social media list --> | |
<footer id="site-map" class="main-footer" role="contentinfo"> | |
<div class="main-footer-links"> | |
<div class="container"> | |
<a id="back-to-top-1" class="jump-link" href="#python-network"><span aria-hidden="true" class="icon-arrow-up"><span>▲</span></span> Back to Top</a> | |
<ul class="sitemap navigation menu do-not-print" role="tree" id="container" style="position: relative; height: 490.21875px;"> | |
<li class="tier-1 element-1" style="position: absolute; left: 0px; top: 0px;"> | |
<a href="/about/">About</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/apps/" title="">Applications</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/quotes/" title="">Quotes</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/gettingstarted/" title="">Getting Started</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/help/" title="">Help</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-2" style="position: absolute; left: 163px; top: 0px;"> | |
<a href="/downloads/">Downloads</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/downloads/" title="">All releases</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/downloads/source/" title="">Source code</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/downloads/windows/" title="">Windows</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/downloads/mac-osx/" title="">Mac OS X</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/download/other/" title="">Other Platforms</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://docs.python.org/3/license.html" title="">License</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/download/alternatives" title="">Alternative Implementations</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-3" style="position: absolute; left: 327px; top: 0px;"> | |
<a href="/doc/">Documentation</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/doc/" title="">Docs</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/doc/av" title="">Audio/Visual Talks</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="https://wiki.python.org/moin/BeginnersGuide" title="">Beginner's Guide</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="https://docs.python.org/devguide/" title="">Developer's Guide</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://docs.python.org/faq/" title="">FAQ</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="http://wiki.python.org/moin/Languages" title="">Non-English Docs</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="http://python.org/dev/peps/" title="">PEP Index</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="https://wiki.python.org/moin/PythonBooks" title="">Python Books</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-4" style="position: absolute; left: 490px; top: 0px;"> | |
<a href="/community/">Community</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/community/diversity/" title="">Diversity</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/community/irc/" title="">IRC</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/community/lists/" title="">Mailing Lists</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/community/workshops/" title="">Python Conferences</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/community/sigs/" title="">Special Interest Groups</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="https://wiki.python.org/moin/" title="">Python Wiki</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/community/logos/" title="">Python Logo</a></li> | |
<li class="tier-2 element-8" role="treeitem"><a href="/community/merchandise/" title="">Merchandise</a></li> | |
<li class="tier-2 element-9" role="treeitem"><a href="/community/awards" title="">Community Awards</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-5" style="position: absolute; left: 654px; top: 0px;"> | |
<a href="/about/success/" title="success-stories">Success Stories</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/about/success/#arts" title="">Arts</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/about/success/#business" title="">Business</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/about/success/#education" title="">Education</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/about/success/#engineering" title="">Engineering</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="/about/success/#government" title="">Government</a></li> | |
<li class="tier-2 element-6" role="treeitem"><a href="/about/success/#scientific" title="">Scientific</a></li> | |
<li class="tier-2 element-7" role="treeitem"><a href="/about/success/#software-development" title="">Software Development</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-6" style="position: absolute; left: 817px; top: 0px;"> | |
<a href="/blogs/" title="News from around the Python world">News</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/blogs/" title="Python Insider Blog Posts">Python News</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://planetpython.org/" title="Planet Python">Community News</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="http://pyfound.blogspot.com/" title="PSF Blog">PSF News</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="http://pycon.blogspot.com/" title="PyCon Blog">PyCon News</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-7" style="position: absolute; left: 0px; top: 213px;"> | |
<a href="/events/">Events</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="/events/python-events/" title="">Python Events</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="/events/python-user-group/" title="">User Group Events</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="/events/python-events/past/" title="">Python Events Archive</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="/events/python-user-group/past/" title="">User Group Events Archive</a></li> | |
<li class="tier-2 element-5" role="treeitem"><a href="https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event" title="">Submit an Event</a></li> | |
</ul> | |
</li> | |
<li class="tier-1 element-8" style="position: absolute; left: 817px; top: 213px;"> | |
<a href="/dev/">Contributing</a> | |
<ul class="subnav menu"> | |
<li class="tier-2 element-1" role="treeitem"><a href="http://docs.python.org/devguide/" title="">Developer's Guide</a></li> | |
<li class="tier-2 element-2" role="treeitem"><a href="http://bugs.python.org/" title="">Issue Tracker</a></li> | |
<li class="tier-2 element-3" role="treeitem"><a href="https://mail.python.org/mailman/listinfo/python-dev" title="">python-dev list</a></li> | |
<li class="tier-2 element-4" role="treeitem"><a href="http://pythonmentors.com/" title="">Core Mentorship</a></li> | |
</ul> | |
</li> | |
</ul> | |
<a id="back-to-top-2" class="jump-link" href="#python-network"><span aria-hidden="true" class="icon-arrow-up"><span>▲</span></span> Back to Top</a> | |
</div><\!-- end .container --> | |
</div> <\!-- end .main-footer-links --> | |
<div class="site-base"> | |
<div class="container"> | |
<ul class="footer-links navigation menu do-not-print" role="tree"> | |
<li class="tier-1 element-1"><a href="/about/help/">Help & <span class="say-no-more">General</span> Contact</a></li> | |
<li class="tier-1 element-2"><a href="/community/diversity/">Diversity <span class="say-no-more">Initiatives</span></a></li> | |
<li class="tier-1 element-3"><a href="https://github.com/python/pythondotorg/issues">Submit Website Bug</a></li> | |
<\!--<li class="tier-1 element-3"><a href="#"><span class="say-no-more">Website</span> Colophon</a></li>--> | |
</ul> | |
<div class="copyright"> | |
<p><small> | |
<span class="pre">Copyright ©2001-2015.</span> | |
<span class="pre"><a href="/psf-landing/">Python Software Foundation</a></span> | |
<span class="pre"><a href="/about/legal/">Legal Statements</a></span> | |
<span class="pre"><a href="/privacy/">Privacy Policy</a></span> | |
</small></p> | |
</div> | |
</div><\!-- end .container --> | |
</div><\!-- end .site-base --> | |
</footer> | |
</div><\!-- end #touchnav-wrapper --> | |
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"><\/script> | |
<script>window.jQuery || document.write('<script src="/static/js/libs/jquery-1.8.2.min.js"><\/script>')<\/script> | |
<script src="/static/js/libs/masonry.pkgd.min.js"><\/script> | |
<script type="text/javascript" src="/static/js/main-min.js" charset="utf-8"><\/script> | |
<\!--[if lte IE 7]> | |
<script type="text/javascript" src="/static/js/plugins/IE8-min.js" charset="utf-8"><\/script> | |
<![endif]--> | |
<\!--[if lte IE 8]> | |
<script type="text/javascript" src="/static/js/plugins/getComputedStyle-min.js" charset="utf-8"><\/script> | |
<![endif]--> | |
<iframe id="rufous-sandbox" scrolling="no" frameborder="0" allowtransparency="true" style="display: none;"></iframe></body></html></script> | |
<script id="jsbin-source-javascript" type="text/javascript"> | |
addlib = (url) => | |
console.log "." | |
e = document.createElement "script" | |
e.type = "text/javascript" | |
e.src = url | |
console.log e | |
document.body.appendChild e | |
x = -> | |
o = 1e3 | |
console.log "Waiting", o, "ms, to check", e.src, "loading status." | |
return -> | |
console.log "Waited", o, "ms. jQuery.fn.jquery ->", jQuery.fn.jquery | |
setTimeout x(), o | |
# addlib "https://code.jquery.com/jquery-2.1.4.min.js" | |
flatmap = (l,f) => | |
r = [] | |
l.forEach (e) => | |
if Array.isArray e | |
then Array.prototype.push.apply r, (flatmap e,f) | |
else r.push (f e) | |
r | |
neigh = (d) => | |
ps = $(d).parents().get() | |
flatmap ps, (e) => $(e).siblings() | |
readable = (d) => | |
n = neigh d | |
l = neigh.length | |
if l > 1000 then alert "#{l} siblings!" else n.forEach (e) => $(e).toggle()</script></body></html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var addlib; | |
addlib = (function(_this) { | |
return function(url) { | |
var e, x; | |
console.log("."); | |
e = document.createElement("script"); | |
e.type = "text/javascript"; | |
e.src = url; | |
console.log(e); | |
document.body.appendChild(e); | |
var o; | |
o = 1e3; | |
x = function() { | |
console.log("Waiting", o, "ms, to check", e.src, "loading status."); | |
return function() { | |
return console.log("Waited", o, "ms. jQuery.fn.jquery ->", jQuery.fn.jquery); | |
}; | |
}; | |
return setTimeout(x(), o); | |
}; | |
})(this); | |
addlib("https://code.jquery.com/jquery-2.1.4.min.js"); | |
var flatmap, neigh, readable; | |
flatmap = (function(_this) { | |
return function(l, f) { | |
var r; | |
r = []; | |
l.forEach(function(e) { | |
if (Array.isArray(e)) { | |
return Array.prototype.push.apply(r, flatmap(e, f)); | |
} else { | |
return r.push(f(e)); | |
} | |
}); | |
return r; | |
}; | |
})(this); | |
neigh = (function(_this) { | |
return function(d) { | |
var ps; | |
ps = $(d).parents().get(); | |
return flatmap(ps, function(e) { | |
return $(e).siblings(); | |
}); | |
}; | |
})(this); | |
readable = (function(_this) { | |
return function(d) { | |
var l, n; | |
n = neigh(d); | |
l = neigh.length; | |
if (l > 1000) { | |
return alert("" + l + " siblings!"); | |
} else { | |
return n.forEach(function(e) { | |
return $(e).toggle(); | |
}); | |
} | |
}; | |
})(this); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment