Render infinite values to valid JSON values. Fixes graphite-project#813

- The 'json' and 'jsonp' formats output float('inf') as 1e9999, float
('-inf') as -1e9999, and float('nan') as null
- The 'dygraph' format (a JSON-based format) outputs the same as
Infinity, -Infinity, and null (see http://dygraphs
.com/tests/gviz-infinity.html)
- Minor updates to documents to make sure commands can be run without errors or problems

Author: rehevkor5

CONTRIBUTING.md

@@ -32,6 +32,7 @@ We are always looking to improve our test coverage.  New tests will be appreciat
 
 If you see a mistake, have a feature, or a performance gain, we'd love to see it.  It's _strongly_ encouraged for contributions that aren't already covered by tests to come with them.  We're not trying to foist work on to you, but it makes it much easier for us to accept your contributions.
 
+To set up your development environment, see the instructions in requirements.txt
 
 ### Documentation
 

requirements.txt

@@ -1,11 +1,13 @@
 # This is a PIP requirements file.
 # To setup a dev environment:
 #
+#   Use Python 2.7
+#
 #   If you use virtualenvwrapper, you can use the misc/virtualenvwrapper hook scripts to
 #   automate most of the following commands
 #
 #   easy_install virtualenv
-#   virtualenv --distribute --no-site-packages --prompt "(graphite venv) " .venv
+#   virtualenv --distribute --no-site-packages --prompt "(graphite venv) " --python=/usr/bin/python2.7 .venv
 #   source .venv/bin/activate
 #
 #   brew install cairo && brew link cairo # on OSX
@@ -23,7 +25,7 @@
 #
 #   mkdir -p .venv/storage/log/webapp /opt/graphite/storage/
 #   sudo chown -R $USER: /opt/graphite
-#   .venv/bin/django-admin.py syncdb --settings=graphite.settings --pythonpath=webapp
+#   .venv/bin/django-admin.py migrate --run-syncdb --settings=graphite.settings --pythonpath=webapp
 #   bin/run-graphite-devel-server.py ./
 #   # or
 #   # cd webapp/graphite && $GRAPHITE_ROOT/.venv/bin/gunicorn_django -b 127.0.0.1:8080

webapp/graphite/render/float_encoder.py

@@ -0,0 +1,59 @@
+import json
+
+
+class FloatEncoder(json.JSONEncoder):
+    def __init__(self, nan_str="null", **kwargs):
+        super(FloatEncoder, self).__init__(**kwargs)
+        self.nan_str = nan_str
+
+    def iterencode(self, o, _one_shot=False):
+        """Encode the given object and yield each string
+        representation as available.
+
+        For example::
+
+            for chunk in JSONEncoder().iterencode(bigobject):
+                mysocket.write(chunk)
+        """
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            _encoder = json.encoder.encode_basestring_ascii
+        else:
+            _encoder = json.encoder.encode_basestring
+        if self.encoding != 'utf-8':
+            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
+                if isinstance(o, str):
+                    o = o.decode(_encoding)
+                return _orig_encoder(o)
+
+        def floatstr(o, allow_nan=self.allow_nan, _repr=json.encoder.FLOAT_REPR,
+                     _inf=json.encoder.INFINITY, _neginf=-json.encoder.INFINITY,
+                     nan_str=self.nan_str):
+            # Check for specials.  Note that this type of test is processor
+            # and/or platform-specific, so do tests which don't depend on the
+            # internals.
+
+            if o != o:
+                text = nan_str
+            elif o == _inf:
+                text = '1e9999'
+            elif o == _neginf:
+                text = '-1e9999'
+            else:
+                return _repr(o)
+
+            if not allow_nan:
+                raise ValueError(
+                    "Out of range float values are not JSON compliant: " +
+                    repr(o))
+
+            return text
+
+        _iterencode = json.encoder._make_iterencode(
+            markers, self.default, _encoder, self.indent, floatstr,
+            self.key_separator, self.item_separator, self.sort_keys,
+            self.skipkeys, _one_shot)
+        return _iterencode(o, 0)

webapp/graphite/render/views.py

@@ -22,6 +22,7 @@
 from urlparse import urlsplit, urlunsplit
 from cgi import parse_qs
 from cStringIO import StringIO
+
 try:
   import cPickle as pickle
 except ImportError:
@@ -36,6 +37,7 @@
 from graphite.render.functions import PieFunctions
 from graphite.render.hashing import hashRequest, hashData
 from graphite.render.glyph import GraphTypes
+from graphite.render.float_encoder import FloatEncoder
 
 from django.http import HttpResponseServerError, HttpResponseRedirect
 from django.template import Context, loader
@@ -172,10 +174,10 @@ def renderView(request):
 
       if 'jsonp' in requestOptions:
         response = HttpResponse(
-          content="%s(%s)" % (requestOptions['jsonp'], json.dumps(series_data)),
+          content="%s(%s)" % (requestOptions['jsonp'], json.dumps(series_data, cls=FloatEncoder)),
           content_type='text/javascript')
       else:
-        response = HttpResponse(content=json.dumps(series_data),
+        response = HttpResponse(content=json.dumps(series_data, cls=FloatEncoder),
                                 content_type='application/json')
 
       if useCache:
@@ -193,7 +195,15 @@ def renderView(request):
         for series in data:
           labels.append(series.name)
           for i, point in enumerate(series):
-            datapoints[i].append(point if point is not None else 'null')
+            if point is None:
+              point = 'null'
+            elif point == float('inf'):
+              point = 'Infinity'
+            elif point == float('-inf'):
+              point = '-Infinity'
+            elif math.isnan(point):
+              point = 'null'
+            datapoints[i].append(point)
         line_template = '[%%s000%s]' % ''.join([', %s'] * len(data))
         lines = [line_template % tuple(points) for points in datapoints]
         result = '{"labels" : %s, "data" : [%s]}' % (json.dumps(labels), ', '.join(lines))

webapp/tests/test_render.py

@@ -2,6 +2,7 @@
 import json
 import os
 import time
+import math
 import logging
 import shutil
 
@@ -90,43 +91,66 @@ def test_render_view(self):
         whisper.create(self.db, [(1, 60)])
 
         ts = int(time.time())
-        whisper.update(self.db, 0.1234567890123456789012, ts - 2)
-        whisper.update(self.db, 0.4, ts - 1)
-        whisper.update(self.db, 0.6, ts)
+        whisper.update(self.db, 0.1234567890123456789012, ts - 5)
+        whisper.update(self.db, 0.4, ts - 4)
+        whisper.update(self.db, 0.6, ts - 3)
+        whisper.update(self.db, float('inf'), ts - 2)
+        whisper.update(self.db, float('-inf'), ts - 1)
+        whisper.update(self.db, float('nan'), ts)
 
         response = self.client.get(url, {'target': 'test', 'format': 'raw'})
         raw_data = ("None,None,None,None,None,None,None,None,None,None,None,"
                     "None,None,None,None,None,None,None,None,None,None,None,"
                     "None,None,None,None,None,None,None,None,None,None,None,"
                     "None,None,None,None,None,None,None,None,None,None,None,"
-                    "None,None,None,None,None,None,None,None,None,None,None,"
-                    "None,None,0.12345678901234568,0.4,0.6")
+                    "None,None,None,None,None,None,None,None,None,None,"
+                    "0.12345678901234568,0.4,0.6,inf,-inf,nan")
         raw_response = "test,%d,%d,1|%s\n" % (ts-59, ts+1, raw_data)
         self.assertEqual(response.content, raw_response)
 
         response = self.client.get(url, {'target': 'test', 'format': 'json'})
+        self.assertIn('[1e9999, ' + str(ts - 2) + ']', response.content)
+        self.assertIn('[-1e9999, ' + str(ts - 1) + ']', response.content)
         data = json.loads(response.content)
-        end = data[0]['datapoints'][-4:]
+        end = data[0]['datapoints'][-7:]
         self.assertEqual(
-            end, [[None, ts - 3], [0.12345678901234568, ts - 2], [0.4, ts - 1], [0.6, ts]])
+            end, [[None, ts - 6],
+                  [0.12345678901234568, ts - 5],
+                  [0.4, ts - 4],
+                  [0.6, ts - 3],
+                  [float('inf'), ts - 2],
+                  [float('-inf'), ts - 1],
+                  [None, ts]])
 
         response = self.client.get(url, {'target': 'test', 'format': 'dygraph'})
+        self.assertIn('[' + str((ts - 2) * 1000) + ', Infinity]', response.content)
+        self.assertIn('[' + str((ts - 1) * 1000) + ', -Infinity]', response.content)
         data = json.loads(response.content)
-        end = data['data'][-4:]
+        end = data['data'][-7:]
         self.assertEqual(end,
-            [[(ts - 3) * 1000, None],
-            [(ts - 2) * 1000, 0.123456789012],
-            [(ts - 1) * 1000, 0.4],
-            [ts * 1000, 0.6]])
+            [[(ts - 6) * 1000, None],
+            [(ts - 5) * 1000, 0.123456789012],
+            [(ts - 4) * 1000, 0.4],
+            [(ts - 3) * 1000, 0.6],
+            [(ts - 2) * 1000, float('inf')],
+            [(ts - 1) * 1000, float('-inf')],
+            [ts * 1000, None]])
 
         response = self.client.get(url, {'target': 'test', 'format': 'rickshaw'})
         data = json.loads(response.content)
-        end = data[0]['datapoints'][-4:]
+        end = data[0]['datapoints'][-7:-1]
         self.assertEqual(end,
-            [{'x': ts - 3, 'y': None},
-            {'x': ts - 2, 'y': 0.12345678901234568},
-            {'x': ts - 1, 'y': 0.4},
-            {'x': ts, 'y': 0.6}])
+            [{'x': ts - 6, 'y': None},
+            {'x': ts - 5, 'y': 0.12345678901234568},
+            {'x': ts - 4, 'y': 0.4},
+            {'x': ts - 3, 'y': 0.6},
+            {'x': ts - 2, 'y': float('inf')},
+            {'x': ts - 1, 'y': float('-inf')}])
+
+        last = data[0]['datapoints'][-1]
+        self.assertEqual(last['x'], ts)
+        self.assertTrue(math.isnan(last['y']))
+
 
     def test_hash_request(self):
         # Requests with the same parameters should hash to the same values,