diff.txt

diff monolithic mono_deduped.txt
95,97d94
< function log() {
<   echo "$1" | eval "$LOGGER_CMD"
< }
99,101d95
< function upcase() {
<   echo "$1" | tr '[a-z]' '[A-Z]'
< }
103,121d96
< function is_false() {
<     val=$1
<     if [[ ! -n $val ]]; then
<         echo 0
<         return 0
<     fi
<     if [[ $(upcase $val) == 'NONE' ]]; then
<         echo 0
<         return 0
<     fi
<     if [[ $(upcase $val) == 'FALSE' ]]; then
<         echo 0
<         return 0
<     fi
<     if [[ $(upcase $val) == '0' ]]; then
<         echo 0
<         return 0
<     fi
< }
157,160d131
< function get_json_value() {
<   echo -n $(perl -MJSON -ne "\$value = decode_json(\$_)->$1; \
<     \$value =~ s/([^a-zA-Z0-9])/\$1/g; print \$value" $2)
< }
162,164d132
< function get_user_data_value() {
<   echo -n $(get_json_value $1 $OS_USER_DATA_TMP_FILE)
< }
171,174d138
< function get_user_data_system_cmds() {
<   echo -n $(perl -MJSON -ne "print join(';;', \
<   @{decode_json(\$_)->{bigip}{system_cmds}})" $OS_USER_DATA_TMP_FILE)
< }
176,179d139
< function get_user_data_firstboot_cmds() {
<   echo -n $(perl -MJSON -ne "print join(';;', \
<   @{decode_json(\$_)->{bigip}{firstboot_cmds}})" $OS_USER_DATA_TMP_FILE)
< }
181,186d140
< function get_user_data_network_routes() {
<   echo -n $(perl -MJSON -ne "\$data = decode_json(\$_); \
<   foreach \$route (@{\$data->{'bigip'}->{'network'}->{'routes'}}) { \
<     print \$route->{'destination'}.\";\".\$route->{'gateway'}.\"|\"; \
<   }" $OS_USER_DATA_TMP_FILE)
< }
188,191d141
< function generate_sha512_passwd_hash() {
<   salt=$(openssl rand -base64 8)
<   echo -n $(perl -e "print crypt(q[$1], \"\\\$6\\\$$salt\\\$\")")
< }
489,491d438
< function get_supported_modules() {
<   echo -n $(tmsh list sys provision one-line | awk '/^sys/ { print $3 }')
< }
977,980d923
< function cleanup_user_data() {
<   [[ $OS_USER_DATA_CLEANUP == true ]] && rm -f $OS_USER_DATA_TMP_FILE
<   [[ $OS_META_DATA_CLEANUP == true ]] && rm -f $OS_META_DATA_TMP_FILE
< }
1057,1076d999
< function randomize_base_passwords() {
<   admin_password=`< /dev/urandom tr -dc A-Z | head -c10`
<   root_password=`< /dev/urandom tr -dc A-Z | head -c10`
<
<   /usr/bin/passwd admin $admin_password >/dev/null 2>&1
<   /usr/bin/passwd root $root_password >/dev/null 2>&1
<
<   echo "" >> /dev/kmsg
<   echo "" >> /dev/kmsg
<   echo "########################################################" >> /dev/kmsg
<   echo "#                                                      #" >> /dev/kmsg
<   echo "# random root password:           $root_password           #" >> /dev/kmsg
<   echo "# random admin password:          $admin_password           #" >> /dev/kmsg
<   echo "#                                                      #" >> /dev/kmsg
<   echo "########################################################" >> /dev/kmsg
<   echo "" >> /dev/kmsg
<   echo "" >> /dev/kmsg
<   echo "    r: $root_password   a: $admin_password" >> /etc/issue
<   echo "" >> /etc/issue
< }
1078,1080d1000
< function restore_issue() {
<   cat /etc/issue | head -n 2 > /etc/issue
< }
1214a1135
>

remove_identical_functions.py

#! /usr/bin/env python

monolithic_lines = open('monolithic','r').readlines()

overlapping_set = set(
    [('cleanup_user_data () \n{ \n    [[ $OS_USER_DATA_CLEANUP == true ]] && rm -f $OS_USER_DATA_TMP_FILE;\n    [[ $OS_META_DATA_CLEANUP == true ]] && rm -f $OS_META_DATA_TMP_FILE\n}\n',),
     ('generate_sha512_passwd_hash () \n{ \n    salt=$(openssl rand -base64 8);\n    echo -n $(perl -e "print crypt(q[$1], \\"\\\\\\$6\\\\\\$$salt\\\\\\$\\")")\n}\n',),
     ('get_json_value () \n{ \n    echo -n $(perl -MJSON -ne "\\$value = decode_json(\\$_)->$1;     \\$value =~ s/([^a-zA-Z0-9])/\\$1/g; print \\$value" $2)\n}\n',),
     ("get_supported_modules () \n{ \n    echo -n $(tmsh list sys provision one-line | awk '/^sys/ { print $3 }')\n}\n",),
     ('get_user_data_firstboot_cmds () \n{ \n    echo -n $(perl -MJSON -ne "print join(\';;\',   @{decode_json(\\$_)->{bigip}{firstboot_cmds}})" $OS_USER_DATA_TMP_FILE)\n}\n',),
     ('get_user_data_network_routes () \n{ \n    echo -n $(perl -MJSON -ne "\\$data = decode_json(\\$_);   foreach \\$route (@{\\$data->{\'bigip\'}->{\'network\'}->{\'routes\'}}) {     print \\$route->{\'destination\'}.\\";\\".\\$route->{\'gateway\'}.\\"|\\";   }" $OS_USER_DATA_TMP_FILE)\n}\n',),
     ('get_user_data_system_cmds () \n{ \n    echo -n $(perl -MJSON -ne "print join(\';;\',   @{decode_json(\\$_)->{bigip}{system_cmds}})" $OS_USER_DATA_TMP_FILE)\n}\n',),
     ('get_user_data_value () \n{ \n    echo -n $(get_json_value $1 $OS_USER_DATA_TMP_FILE)\n}\n',),
     ("is_false () \n{ \n    val=$1;\n    if [[ ! -n $val ]]; then\n        echo 0;\n        return 0;\n    fi;\n    if [[ $(upcase $val) == 'NONE' ]]; then\n        echo 0;\n        return 0;\n    fi;\n    if [[ $(upcase $val) == 'FALSE' ]]; then\n        echo 0;\n        return 0;\n    fi;\n    if [[ $(upcase $val) == '0' ]]; then\n        echo 0;\n        return 0;\n    fi\n}\n",),
     ('log () \n{ \n    echo "$1" | eval "$LOGGER_CMD"\n}\n',),
     ('randomize_base_passwords () \n{ \n    admin_password=`< /dev/urandom tr -dc A-Z | head -c10`;\n    root_password=`< /dev/urandom tr -dc A-Z | head -c10`;\n    /usr/bin/passwd admin $admin_password > /dev/null 2>&1;\n    /usr/bin/passwd root $root_password > /dev/null 2>&1;\n    echo "" >> /dev/kmsg;\n    echo "" >> /dev/kmsg;\n    echo "########################################################" >> /dev/kmsg;\n    echo "#                                                      #" >> /dev/kmsg;\n    echo "# random root password:           $root_password           #" >> /dev/kmsg;\n    echo "# random admin password:          $admin_password           #" >> /dev/kmsg;\n    echo "#                                                      #" >> /dev/kmsg;\n    echo "########################################################" >> /dev/kmsg;\n    echo "" >> /dev/kmsg;\n    echo "" >> /dev/kmsg;\n    echo "    r: $root_password   a: $admin_password" >> /etc/issue;\n    echo "" >> /etc/issue\n}\n',),
     ('restore_issue () \n{ \n    cat /etc/issue | head -n 2 > /etc/issue\n}\n',),
     ('upcase () \n{ \n    echo "$1" | tr \'[a-z]\' \'[A-Z]\'\n}\n',)
    ])
names = [x[0].split()[0]+'()' for x in overlapping_set]

inside_dup = False
deduped = ''
for line in monolithic_lines:
    if inside_dup:
        # Inside duplicate function!
        if line.startswith('}'):
            # Last line of dupe.
            inside_dup = False
    else:
        # Previous line not in dupe
        spline = line.split()
        if len(spline) >= 2 and spline[0] == 'function' and spline[1] in names:
            # Entering dupe
            inside_dup = True
        else:
            # Last line not in dupe, and dupe not entered.
            deduped += line

print(deduped)