Migrate delft preprocessors to JSON

preprocessor_migration.py

import json
import os
import pathlib
import sys

from delft.sequenceLabelling.preprocess import WordPreprocessor

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("Invalid parameters. Usage: python json_migration.py model directory. "
              "The new preprocessor will be saved in the same directory as the old preprocessor ")
        sys.exit(-1)

    input_directory = sys.argv[1]

    if os.path.isdir(input_directory):
        for file in os.listdir(input_directory):
            if file.endswith("preprocessor.pkl"):
                input_file= pathlib.Path(input_directory, file)
                p = WordPreprocessor().load(input_file)
                output_dict = vars(p)
                output_path = pathlib.Path(input_file.with_suffix('').as_posix() + '.json')
                with open(output_path, 'w') as fp:
                    json.dump(output_dict, fp, sort_keys=False, indent=4)

    elif os.path.isfile(input_directory):
        print("Please specify a directory where the model is located. ")

Steps:

• git checkout 2dad199dbc6e011ae3b270b260e4784b7ffe2d17
• <activate the delft virtual environment>
• python preprocessor_migration.py data/models/sequenceLabelling/my_model_I_want_to_migrate/

You should have a new file preprocessor.json in the data/models/sequenceLabelling/my_model_I_want_to_migrate/ directory