output.txt

'<div>\n'
'    <p>This is a bunch of stuff it has new lines everywhere.</p>\n'
"    <p>and we don't need them</p>\n"
'    <pre>But\n'
'    we do\n'
'need these\n'
'</pre>\n'
"    <p>let's see what we can do!</p>\n"
'</div>\n'

pre.py

import pprint
import re

SAFE_PRE = r'(?<!(\<pre\>))({})(?!(\<\/pre\>))'
RE_MULTISPACE = re.compile(SAFE_PRE.format(r"\s{4,}"))
RE_NEWLINE = re.compile(SAFE_PRE.format(r"(\n{2,})"))

input_string = """<div>
    <p>This is a bunch of stuff it has new lines everywhere.</p>
    <p>and we don't need them</p>
    <pre>But


    we do

need these
</pre>

    <p>let's see what we can do!</p>
</div>
"""

#pprint.pprint(RE_MULTISPACE.sub(' ', input_string))
pprint.pprint(RE_NEWLINE.sub('\n', input_string))