SUS-1905

check_msg_keys.sh

#!/bin/bash
APP_DIR=$HOME/Wikia/git/app
I18N_FILE=$1

STRINGS=`./get_msg_keys.php ${I18N_FILE}`
STRINGS_COUNT=`./get_msg_keys.php ${I18N_FILE} | wc -l`

echo "Checking i18n strings in $I18N_FILE ($STRINGS_COUNT strings in English)..."

NOT_USED=0

for STRING in $STRINGS
do
	COUNT=`ack-grep --no-filename -i --php --js "$STRING" $APP_DIR | grep -iE "msg|Message|descriptionmsg|Status::" -c`

	if [ $COUNT -eq "0" ];
	then
		echo " -- $STRING message is not used"
		((NOT_USED++))
	fi
done

echo "Done - found ${NOT_USED} not used strings in English (out of ${STRINGS_COUNT})"

cleanup_i18n_file.php

#!/usr/bin/env php
<?php

$BLACKLIST = [
"chat-log-reason-banchange",
"chat-log-reason-banremove",
"chat-ban-already-banned",
"chat-user-menu-give-chat-mod",
"chat-user-menu-kick",
"chat-user-menu-ban",
"chat-user-menu-private-close"
];


/**
 * See https://github.com/Wikia/i18n-tools/blob/master/wikia/crowdin/scripts/JsonToi18n.php
 *
 * @param array $messages
 * @param string $fileName i18n file name
 */
function writeI18nFile( Array $messages, $fileName ) {
	$newPHPContents = "<?php
\$messages = array();

";
	foreach ( $messages as $langCode => $langMessages ) {
		$newPHPContents .= "\$messages['{$langCode}'] = array(\n";
		foreach ( $langMessages as $msgKey => $msgText ) {
			$newPHPContents .= "\t'$msgKey' => " . quoteMessage( $msgText ) . ",\n";
		}
		$newPHPContents .= ");\n\n";
	}
	file_put_contents( $fileName, $newPHPContents );
}

/**
 * Copied from TWN's Translate extension
 *
 * @param string $value
 * @return string
 */
function quoteMessage( $value ) {
	$value = addcslashes( $value, '\\' );
	$single = "'";
	$double = '"';
	$quote = $single;
	if ( strpos( $value, $single ) !== false ) {
		if ( strpos( $value, $double ) === false && !preg_match( '/\$[^0-9]/', $value ) ) {
			$quote = $double;
		} else {
			$doubleEsc = substr_count( $value, $double ) + substr_count( $value, '$' );
			$singleEsc = substr_count( $value, $single );
			if ( $doubleEsc < $singleEsc ) {
				$quote = $double;
				$extra = '$';
			} else {
				$extra = '';
			}
			$value = addcslashes( $value, $quote . $extra );
		}
	}
	return $quote . $value . $quote;
}

/**
 * Reads messages from a given i18n file
 *
 * @param string $file
 * @return array
 */
function get_messages_from_file($file) {
	print "Loading messages from {$file} ...";

	/* @var array $messages */
	include $file;

	$count = count($messages['en']);
	print " got {$count} messages in English\n";

	return $messages;
}

$file = $argv[1];
$messages = get_messages_from_file($file);

// apply blacklist to a i18n file
foreach($messages as $lang => &$strings) {
	foreach($strings as $key => $_) {
		if (in_array($key, $BLACKLIST)) {
			print " -- removed {$key} from '{$lang}' strings\n";
			unset($strings[$key]);
		}
	}
}

writeI18nFile($messages, $file);

get_msg_keys.php

#!/usr/bin/env php
<?php

$file = $argv[1];

require_once($file);

$keys = array_keys($messages['en']);

echo join("\n", $keys);

merge_i18n_files.php

#!/usr/bin/env php
<?php

/**
 * See https://github.com/Wikia/i18n-tools/blob/master/wikia/crowdin/scripts/JsonToi18n.php
 *
 * @param array $messages
 * @param string $fileName i18n file name
 */
function writeI18nFile( Array $messages, $fileName ) {
	$newPHPContents = "<?php
\$messages = array();

";
	foreach ( $messages as $langCode => $langMessages ) {
		$newPHPContents .= "\$messages['{$langCode}'] = array(\n";
		foreach ( $langMessages as $msgKey => $msgText ) {
			$newPHPContents .= "\t'$msgKey' => " . quoteMessage( $msgText ) . ",\n";
		}
		$newPHPContents .= ");\n\n";
	}
	file_put_contents( $fileName, $newPHPContents );
}

/**
 * Copied from TWN's Translate extension
 *
 * @param string $value
 * @return string
 */
function quoteMessage( $value ) {
	$value = addcslashes( $value, '\\' );
	$single = "'";
	$double = '"';
	$quote = $single;
	if ( strpos( $value, $single ) !== false ) {
		if ( strpos( $value, $double ) === false && !preg_match( '/\$[^0-9]/', $value ) ) {
			$quote = $double;
		} else {
			$doubleEsc = substr_count( $value, $double ) + substr_count( $value, '$' );
			$singleEsc = substr_count( $value, $single );
			if ( $doubleEsc < $singleEsc ) {
				$quote = $double;
				$extra = '$';
			} else {
				$extra = '';
			}
			$value = addcslashes( $value, $quote . $extra );
		}
	}
	return $quote . $value . $quote;
}

/**
 * Reads messages from a given i18n file
 *
 * @param string $file
 * @return array
 */
function get_messages_from_file($file) {
	print "Loading messages from {$file} ...";

	/* @var array $messages */
	include $file;

	$count = count($messages['en']);
	print " got {$count} messages in English\n";

	return $messages;
}

$WHITELIST = [
	'autocreatewiki-welcomebody-HTML',
];

// process command line arguments
$files = array_slice($argv, 1);
$baseFile = array_shift($files);

echo "Will merge the following files into {$baseFile}:\n* " . join("\n* ", $files) . "\n";

// read messages
$baseMessages = get_messages_from_file($baseFile);
$filesMessages = array_map(
	function($file) {
		return get_messages_from_file($file);
	},
	$files
);

// merge messages
foreach($filesMessages as &$messages) {
	foreach($messages as $lang => $values) {
		foreach($values as $key => $value) {
			if (empty($WHITELIST) || in_array($key, $WHITELIST)) {
				$baseMessages[$lang][$key] = $value;
				unset($messages[$lang][$key]);

				echo " -- moved '{$key}' to the base file ('{$lang}')\n";
			}
		}
	}
}

writeI18nFile($baseMessages, $baseFile);

// update the source files as well
foreach($files as $n => $file) {
	writeI18nFile($filesMessages[$n], $file);
}

echo "Done\n";