Quick script to split a big unreal.py stubs file into multiple smaller ones

split_unreal_stubs.py

# quick script to split a big unreal.py stubs file into multiple smaller ones
# for use as autocompletion in PyCharm (without having to increase the memory allocation)
__author__ = "[email protected]"
__version__ = "1"
  
import os

# path to unreal stubs
unreal_stubs_src_path = r"YOUR_PROJECT_HERE\Intermediate\PythonStub\unreal.py"

# the last folder here should be called "unreal"
output_stubs_folder = os.path.expanduser('~') + "/pycharm_extras/unreal_stubs/unreal"
if not os.path.exists(output_stubs_folder):
    os.makedirs(output_stubs_folder)


# append lines to lists in a master list
output_content = [[]]
with open(unreal_stubs_src_path, "r") as fp:
    
    for line in fp.readlines():
        latest_list = output_content[-1]
        
        # un-private class that is used between modules
        line = line.replace("(_ObjectBase)", "(PrivateObjectBase)").replace("class _ObjectBase(", "class PrivateObjectBase(")
        
        new_file_length_reached = len(latest_list) > 10000
        
        # we've hit the end of a funtion that's not in a class, this should be safe-ish
        next_line_is_safe = line == "    pass\r\n"

        # keep appending until we hit a safe space to stop
        # then add a new list to begin the next file
        if new_file_length_reached:
            if next_line_is_safe:
                output_content.append([])
                continue
                
            else:
                new_class_reached = line.startswith("class ")
                if new_class_reached:
                    output_content.append([])
        
        # append line to latest list
        latest_list = output_content[-1]
        latest_list.append(line)


# write lists to files
for i, output_list in enumerate(output_content):
    
    # add import * from previous stub file
    stub_content = ["from .unreal_stub_{} import *\n".format(i-1)] if i > 0 else []
    stub_content.extend(output_list)
    
    with open(output_stubs_folder + "/unreal_stub_{}.py".format(i), "w") as fp:
        fp.writelines(stub_content)


# create __init__ that imports the last stub file (which will have imported previous stubs)
with open(output_stubs_folder + "/__init__.py", "w") as fp:
    fp.writelines("from .unreal_stub_{} import *\n".format(len(output_content)-1))


print("done, {} files written to: {}".format(len(output_content), output_stubs_folder))