Encode json more compactly.

Optimise json by attempting in the following order:
- using C simplejson module (faster than baseline by 3x)
- using python's built in json module (faster than baseline by 2x)

if both fail use pure Python embedded simplejson (baseline).

Author: retropc

bin/dependencies_b.py

@@ -16,6 +16,7 @@ def check_dependencies():
 
   check_twisted()
   check_win32()
+  i+=check_json()
   i+=check_java()
   i+=check_hg()
 
@@ -99,7 +100,16 @@ def twisted_fail(x, y=None):
     import twisted.words
   except ImportError:
     twisted_fail("words")
-
+    
+def check_json():
+  import qwebirc.util.qjson
+  if qwebirc.util.qjson.slow:
+    warn("simplejson module with C speedups not installed.",
+         "using embedded module (slower); consider installing simplejson from:",
+         "http://pypi.python.org/pypi/simplejson/")
+    return 1
+  return 0
+  
 if __name__ == "__main__":
   import dependencies
   dependencies.check_dependencies()

bin/mkstatic.py

@@ -64,7 +64,7 @@ def main():
 
   compile.main(DEST, produce_debug=False)
 
-  for x in "authgate qwebirc simplejson twisted".split(" "):
+  for x in "authgate qwebirc esimplejson twisted".split(" "):
     copypydir(x, DEST)
   for x in "images panes sound".split(" "):
     copydir(os.path.join("static", x), DEST)

bin/optionsgen.py

@@ -1,5 +1,6 @@
-import config, simplejson
+import config
+import qwebirc.util.qjson as json
 
 def get_options():
   options = dict(networkName=config.NETWORK_NAME, networkServices=[config.AUTH_SERVICE], loginRegex=config.  AUTH_OK_REGEX, appTitle=config.APP_TITLE, baseURL=config.BASE_URL)
-  return simplejson.dumps(options)
+  return json.dumps(options)

simplejson/__init__.py renamed to esimplejson/__init__.py

@@ -106,8 +106,8 @@
 ]
 
 if __name__ == '__main__':
-    from simplejson.decoder import JSONDecoder
-    from simplejson.encoder import JSONEncoder
+    from esimplejson.decoder import JSONDecoder
+    from esimplejson.encoder import JSONEncoder
 else:
     from decoder import JSONDecoder
     from encoder import JSONEncoder

simplejson/decoder.py renamed to esimplejson/decoder.py

@@ -4,9 +4,9 @@
 import re
 import sys
 
-from simplejson.scanner import Scanner, pattern
+from esimplejson.scanner import Scanner, pattern
 try:
-    from simplejson._speedups import scanstring as c_scanstring
+    from esimplejson._speedups import scanstring as c_scanstring
 except ImportError:
     pass
 

simplejson/encoder.py renamed to esimplejson/encoder.py

@@ -4,7 +4,7 @@
 import re
 
 try:
-    from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
+    from esimplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
 except ImportError:
     pass
 

simplejson/scanner.py renamed to esimplejson/scanner.py

@@ -2,11 +2,12 @@
 from twisted.names import client
 from twisted.internet import reactor, error
 from authgateengine import login_optional, getSessionData
-import simplejson, md5, sys, os, time, config, qwebirc.config_options as config_options, traceback, socket
+import md5, sys, os, time, config, qwebirc.config_options as config_options, traceback, socket
 import qwebirc.ircclient as ircclient
 from adminengine import AdminEngineAction
 from qwebirc.util import HitCounter
 import qwebirc.dns as qdns
+import qwebirc.util.qjson as json
 Sessions = {}
 
 def get_session_id():
@@ -25,6 +26,7 @@ class PassthruException(Exception):
   pass
 
 NOT_DONE_YET = None
+EMPTY_JSON_LIST = json.dumps([])
 
 def jsondump(fn):
   def decorator(*args, **kwargs):
@@ -38,7 +40,7 @@ def decorator(*args, **kwargs):
     except PassthruException, e:
       return str(e)
 
-    return simplejson.dumps(x)
+    return json.dumps(x)
   return decorator
 
 def cleanupSession(id):
@@ -79,7 +81,7 @@ def timeout(self, channel):
     if self.schedule:
       return
 
-    channel.write(simplejson.dumps([]))
+    channel.write(EMPTY_JSON_LIST)
     if channel in self.subscriptions:
       self.subscriptions.remove(channel)
 
@@ -105,7 +107,7 @@ def flush(self, scheduled=False):
 
     self.throttle = t + config.UPDATE_FREQ
 
-    encdata = simplejson.dumps(self.buffer)
+    encdata = json.dumps(self.buffer)
     self.buffer = []
     self.buflen = 0
 

qwebirc/engines/ajaxengine.py

@@ -0,0 +1,24 @@
+slow = True
+try:
+  # first try the system module
+  import simplejson as json
+  try:
+    # try to see if the C module is available
+    json._speedups
+  except AttributeError:
+    pass
+  else:
+    slow = False
+except ImportError:
+  # try python 2.6's json library as
+  # it is 2x as fast as simplejson with no C
+  try:
+    import json
+    json.dumps # we don't want the silly third party version
+  except (ImportError, AttributeError):
+    # fallback to the embedded
+    import esimplejson as json
+  
+__SEPS = (',', ':')
+dumps = lambda x: json.dumps(x, encoding="utf8", ensure_ascii=True, check_circular=False, indent=None, separators=__SEPS)
+loads = lambda x: json.loads(x, encoding="utf8")