test_pyfakefs_multiprocessing.py

# -*- coding:utf-8 -*-
"""
Demonstrating problem with pyfakefs==3.5.8 with multiprocessing:

1. In Python 3, multiprocessing tries to close a pipe,
    which isn't supported by pyfakefs.
2. In Python 2 (and probably Python 3 as well), pyfakefs patcher isn't
    applied to the forked Python process.
"""

import multiprocessing
import os

import requests


def download(url, file_path):
    with open(file_path, 'wb') as f:
        f.write(requests.get(url).content)


def download_error(result, url, file_path):
    try:
        download(url=url, file_path=file_path)
    except Exception as err:
        result.put(err)
    else:
        result.put(True)


def download_subprocess(url, file_path):
    result_queue = multiprocessing.Queue()
    p = multiprocessing.Process(
        target=download_error,
        args=(result_queue, url, file_path),
    )
    p.start()
    result = result_queue.get()
    p.join()
    if isinstance(result, Exception):
        raise result
    print(result)


def test_1(fs, requests_mock):
    """
    Python 3 fails as soon as I depend on fs (pyfakefs).
    > /usr/lib/python3.6/multiprocessing/popen_fork.py:77: in _launch
    >   os.close(child_w)
    >   E       OSError: [Errno 9] Bad file descriptor in the fake filesystem: '12'
    Looks like multiprocessing in Python 3 calls os.close()
    on file descriptor of a pipe (child_w), which isn't supported by pyfakefs.

    Python 2 fails because file_path (/fake/dir/file.html)
    isn't found during assertion.
    Probably, it's because pyfakefs cannot monkey-patch forked Python process.
    """
    # setup HTTP
    url = 'https://fake.url/'
    content = b'foo'
    requests_mock.get(url, content=content)
    # setup filesystem
    tmp_dir = '/fake/dir/'
    file_path = os.path.join(tmp_dir, 'file.html')
    fs.create_dir(tmp_dir)

    # This downloads url to file_path in using multiprocessing.
    download_subprocess(url, file_path)

    with open(file_path, 'r') as f:
        assert f.read() == content


def test_2(fs):
    """
    pyfakefs does not support closing file descriptor from file.
    This fails for both Python 2 and Python 3.
    """
    read, write = os.pipe()
    os.close(write)


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('url', help='url to fetch')
    parser.add_argument('file_path', help='file path to save the fetched content to')
    args = parser.parse_args()

    download_subprocess(
        url=args.url,
        file_path=args.file_path,
    )


if __name__ == '__main__':
    main()