dumper reproducer

test_dumper.py

#!/usr/bin/env python

import datetime
import io
import json
import os
import shutil
import six

from ansibullbot._text_compat import to_bytes, to_text

meta = {
    u'hello': u'world'
}

mfile = os.path.join(
    u'/tmp',
    u'meta.json'
)
if os.path.exists(mfile):
    os.remove(mfile)


meta[u'time'] = to_text(datetime.datetime.now().isoformat())

# Using in-memory buffer here to work around inability to dump
# json to file directly
tmp_inmemory_json = six.StringIO()
json.dump(meta, tmp_inmemory_json, sort_keys=True, indent=2)

with io.open(mfile, 'w', encoding='utf-8') as f:
    shutil.copyfileobj(tmp_inmemory_json, f, -1)

tmp_inmemory_json.close()
del tmp_inmemory_json


assert os.path.isfile(mfile)
assert json.load(open(mfile, 'r'))

Wow, that's odd:

➜ python test_dumper.py 
Traceback (most recent call last):
  File "test_dumper.py", line 39, in <module>
    assert json.load(open(mfile, 'r'))
  File "/home/wk/.pyenv/versions/3.7.0/lib/python3.7/json/__init__.py", line 296, in load
    parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
  File "/home/wk/.pyenv/versions/3.7.0/lib/python3.7/json/__init__.py", line 348, in loads
    return _default_decoder.decode(s)
  File "/home/wk/.pyenv/versions/3.7.0/lib/python3.7/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/home/wk/.pyenv/versions/3.7.0/lib/python3.7/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

➜ python2 test_dumper.py
Traceback (most recent call last):
  File "test_dumper.py", line 39, in <module>
    assert json.load(open(mfile, 'r'))
  File "/home/wk/.pyenv/versions/2.7.15/lib/python2.7/json/__init__.py", line 291, in load
    **kw)
  File "/home/wk/.pyenv/versions/2.7.15/lib/python2.7/json/__init__.py", line 339, in loads
    return _default_decoder.decode(s)
  File "/home/wk/.pyenv/versions/2.7.15/lib/python2.7/json/decoder.py", line 364, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/home/wk/.pyenv/versions/2.7.15/lib/python2.7/json/decoder.py", line 382, in raw_decode
    raise ValueError("No JSON object could be decoded")
ValueError: No JSON object could be decoded

/tmp/meta.json is an empty file in both cases. I'm in progress of debugging this. To be continued.

I was trying to fix unicode/bytes compat, but forgot to rewind the buffer. This fixes Python 3, but still fails under Python 2:

diff --git a/test_dumper.py.old b/test_dumper.py
index 3b92e3e..dd70eef 100644
--- a/test_dumper.py.old
+++ b/test_dumper.py
@@ -26,7 +26,9 @@ meta[u'time'] = to_text(datetime.datetime.now().isoformat())
 # Using in-memory buffer here to work around inability to dump
 # json to file directly
 tmp_inmemory_json = six.StringIO()
+#import ipdb; ipdb.set_trace()
 json.dump(meta, tmp_inmemory_json, sort_keys=True, indent=2)
+tmp_inmemory_json.seek(0)  # Rewind it, so for shutil.copyfileobj
 
 with io.open(mfile, 'w', encoding='utf-8') as f:
     shutil.copyfileobj(tmp_inmemory_json, f, -1)

This works for both:

diff --git a/test_dumper.py.old b/test_dumper.py
index 3b92e3e..c8c1eeb 100644
--- a/test_dumper.py.old
+++ b/test_dumper.py
@@ -5,7 +5,7 @@ import io
 import json
 import os
 import shutil
-import six
+import io
 
 from ansibullbot._text_compat import to_bytes, to_text
 
@@ -25,8 +25,16 @@ meta[u'time'] = to_text(datetime.datetime.now().isoformat())
 
 # Using in-memory buffer here to work around inability to dump
 # json to file directly
-tmp_inmemory_json = six.StringIO()
-json.dump(meta, tmp_inmemory_json, sort_keys=True, indent=2)
+tmp_inmemory_json = io.StringIO()
+class JSONUnicodeEncoder(json.JSONEncoder):
+    def iterencode(self, o):
+        #import ipdb; ipdb.set_trace()
+        def noop(v): return v
+        transformer = noop if self.ensure_ascii else to_text
+        for chunk in super(JSONUnicodeEncoder, self).iterencode(o):
+            yield transformer(chunk)
+json.dump(meta, tmp_inmemory_json, cls=JSONUnicodeEncoder, ensure_ascii=False, sort_keys=True, indent=2)
+tmp_inmemory_json.seek(0)  # Rewind it, so for shutil.copyfileobj
 
 with io.open(mfile, 'w', encoding='utf-8') as f:
     shutil.copyfileobj(tmp_inmemory_json, f, -1)

Even better:

diff --git a/test_dumper.py.old b/test_dumper.py
index 3b92e3e..4530c9f 100644
--- a/test_dumper.py.old
+++ b/test_dumper.py
@@ -5,7 +5,6 @@ import io
 import json
 import os
 import shutil
-import six
 
 from ansibullbot._text_compat import to_bytes, to_text
 
@@ -23,16 +22,17 @@ if os.path.exists(mfile):
 
 meta[u'time'] = to_text(datetime.datetime.now().isoformat())
 
-# Using in-memory buffer here to work around inability to dump
-# json to file directly
-tmp_inmemory_json = six.StringIO()
-json.dump(meta, tmp_inmemory_json, sort_keys=True, indent=2)
 
-with io.open(mfile, 'w', encoding='utf-8') as f:
-    shutil.copyfileobj(tmp_inmemory_json, f, -1)
+class JSONUnicodeEncoder(json.JSONEncoder):
+    def iterencode(self, o):
+        def noop(v): return v
+        transformer = noop if self.ensure_ascii else to_text
+        for chunk in super(JSONUnicodeEncoder, self).iterencode(o):
+            yield transformer(chunk)
+
 
-tmp_inmemory_json.close()
-del tmp_inmemory_json
+with io.open(mfile, 'w', encoding='utf-8') as f:
+    json.dump(meta, f, cls=JSONUnicodeEncoder, ensure_ascii=False, sort_keys=True, indent=2)
 
 
 assert os.path.isfile(mfile)

I'll send a PR.

ansible/ansibullbot#1078