import_ar_data.sh

#!/bin/bash

readonly mysql_command="/home/agustin.gallego/sandboxes/ps_5.7.25/use";

echo "--> Creating tables in MySQL.";
date;

# Create schema and tables needed
$mysql_command -e "CREATE SCHEMA IF NOT EXISTS ar";
$mysql_command -e "DROP TABLE IF EXISTS ar.percona_server_modules";
$mysql_command -e "CREATE TABLE ar.percona_server_modules ( \
  id int(11) NOT NULL AUTO_INCREMENT, \
  file_name varchar(255) DEFAULT NULL, \
  dependency_module_name_full text, \
  dependency_module_name text, \
  dependency_module_file_name text, \
  PRIMARY KEY (id), \
  KEY idx_file_name (file_name) \
) ENGINE=InnoDB";

$mysql_command -e "DROP TABLE IF EXISTS ar.percona_server_jira_commits";
$mysql_command -e "CREATE TABLE ar.percona_server_jira_commits ( \
  id int(11) NOT NULL AUTO_INCREMENT, \
  commit binary(20) NOT NULL, \
  files_changed text, \
  commit_text text, \
  PRIMARY KEY (id), \
  UNIQUE KEY commit (commit) \
) ENGINE=InnoDB";

$mysql_command -e "DROP TABLE IF EXISTS ar.percona_server_other_commits";
$mysql_command -e "CREATE TABLE ar.percona_server_other_commits ( \
  id int(11) NOT NULL AUTO_INCREMENT, \
  commit binary(20) NOT NULL, \
  files_changed text, \
  commit_text text, \
  PRIMARY KEY (id), \
  UNIQUE KEY commit (commit) \
) ENGINE=InnoDB";

echo "--> Iterating through files to get module dependencies.";
echo "--> Check \`tail -f /tmp/ar_modules.out\` for progress.";
date;

# Iterate through files getting dependencies
find . -type f | egrep "\.c$|\.cc$|\.cpp$|\.h$|\.hpp$" | {
  while read file; do {
    echo "file: " $file;
    module_name_list=`grep "^#include" $file | cut -d ' ' -f2`;
    echo "module_name_list: " $module_name_list;
    # Iterate on each module listed as dependency
    for module_name_full in $module_name_list; do {
        echo "module_name_full: " $module_name_full;
        module_name=`echo $module_name_full | sed -e "s/[<>\"]//g"`;
        echo "module_name: " $module_name;
        if [[ ${module_name} =~ .*(/.*)+ ]]; then
            module_file=`find . -path \*${module_name}`;
        else
            module_file=`find . -name ${module_name}`;
        fi

        # If we don't find an exact match, we try with a wildcard at the end
        if [[ ${module_file} =~ ^$ ]]; then
            module_file=`find . -name ${module_name}"*"`;
        fi
        echo "module_file: " $module_file;

        # Insert into table
        ${mysql_command} -e "INSERT INTO ar.percona_server_modules VALUES \
                             (NULL, '${file}', '${module_name_full}', \
                             '${module_name}', '${module_file}')";
    } done;
  } done;
} > /tmp/ar_modules.out 2>&1


echo "--> Iterating through commits to get relevant information.";
echo "--> Check \`tail -f /tmp/ar_commits.out\` for progress."
date;

git rev-list --no-merges HEAD | {
    while read commit; do {
        files_in_commit=`git diff-tree --no-commit-id --name-only -r $commit | tr '\n' ' '`;
        # Check if (case insensitive match) PS-x appears on the commit message,
        # where x is one or more digits. If so, it's most likely a reference to a Jira bug
        git show --pretty=tformat:%B -s $commit | egrep -i "(ps-[0-9]+)" >/dev/null;
        egrep_exit_status=$?;
        commit_text=`git show --pretty=tformat:%B -s $commit | sed -e "s/[\"\']//g"`;
        
        if [[ $egrep_exit_status -eq 0 ]]; then
            # If there is a match, we log that commit to JIRA's table    
            echo "Inserting commit JIRA: " ${commit};
            ${mysql_command} -e "INSERT INTO ar.percona_server_jira_commits VALUES \
                                 (null, unhex('$commit'), '${files_in_commit}', '${commit_text}')";
        else
            # Otherwise, we log that commit to the generic table
            echo "Inserting commit generic: " ${commit};
            ${mysql_command} -e "INSERT INTO ar.percona_server_other_commits VALUES \
                                 (null, unhex('$commit'), '${files_in_commit}', '${commit_text}')";
        fi 
    } done;
} > /tmp/ar_commits.out 2>&1

echo "--> Finished processing modules and commits";
date;

exit 0;

Examples of data stored in each table:

mysql> SELECT * FROM percona_server_modules LIMIT 10;
+----+----------------------------------+-----------------------------+------------------------+----------------------------------+
| id | file_name                        | dependency_module_name_full | dependency_module_name | dependency_module_file_name      |
+----+----------------------------------+-----------------------------+------------------------+----------------------------------+
|  1 | ./storage/archive/azio.c         | "azlib.h"                   | azlib.h                | ./storage/archive/azlib.h        |
|  2 | ./storage/archive/azio.c         | <stdio.h>                   | stdio.h                |                                  |
|  3 | ./storage/archive/azio.c         | <string.h>                  | string.h               |                                  |
|  4 | ./storage/archive/azio.c         | "my_thread_local.h"         | my_thread_local.h      | ./include/my_thread_local.h      |
|  5 | ./storage/archive/azio.c         | "mysql/psi/mysql_file.h"    | mysql/psi/mysql_file.h | ./include/mysql/psi/mysql_file.h |
|  6 | ./storage/archive/archive_test.c | "azlib.h"                   | azlib.h                | ./storage/archive/azlib.h        |
|  7 | ./storage/archive/archive_test.c | <string.h>                  | string.h               |                                  |
|  8 | ./storage/archive/archive_test.c | <assert.h>                  | assert.h               |                                  |
|  9 | ./storage/archive/archive_test.c | <stdio.h>                   | stdio.h                |                                  |
| 10 | ./storage/archive/archive_test.c | <string.h>                  | string.h               |                                  |
+----+----------------------------------+-----------------------------+------------------------+----------------------------------+
10 rows in set (0.00 sec)

mysql> SELECT id, hex(commit) as commit, files_changed, commit_text FROM percona_server_jira_commits LIMIT 1\G
*************************** 1. row ***************************
           id: 1
       commit: 205B48F017A9842C736D346478716763629C044B
files_changed: mysql-test/suite/rpl/r/rpl_stop_slave_partial_trx.result mysql-test/suite/rpl/t/rpl_stop_slave_partial_trx-master.opt mysql-test/suite/rpl/t/rpl_stop_slave_partial_trx-slave.opt mysql-test/suite/rpl/t/rpl_stop_slave_partial_trx.test sql/rpl_binlog_sender.cc sql/rpl_rli.cc sql/rpl_rli.h sql/rpl_rli_pdb.cc sql/rpl_slave.cc
  commit_text: PS-5824: MTS STOP SLAVE takes over a minute when master crashed during event logging

https://jira.percona.com/browse/PS-5824

Bug #96400 : MTS STOP SLAVE takes over a minute when master crashed during event logging
https://bugs.mysql.com/bug.php?id=96400

FB8-105: Stop slave immediately in MTS if partial trx in the relay log can be rollbacked
https://jira.percona.com/browse/FB8-105

In MTS stop slave can take a minute to complete if the last transaction
is partially downloaded from the master. The slave waits for a minute for the
master to send the rest of the trx and then finally gives up. Strictly, this
wait is only required if the partial trx cannot be rollbacked safely (e.g. trx
on non-transactional engine, DDLs etc.). This change checks if there are no jobs
queued in the worker threads and if the partial trx can be rollbacked, if yes,
it immediately stops the slave.

This commit is based on patches from Facebook at:
- https://github.com/facebook/mysql-5.6/commit/cd39ae1
- https://github.com/facebook/mysql-5.6/commit/0623a07
with improvements from Percona at:
- https://github.com/facebook/mysql-5.6/commit/2f459f7
1 row in set (0.00 sec)

mysql> SELECT id, hex(commit) as commit, files_changed, commit_text FROM percona_server_other_commits LIMIT 1\G
*************************** 1. row ***************************
           id: 1
       commit: 31C20F4284B4E912EE263747AFBACE4A38D2B97C
files_changed: storage/innobase/include/os0proc.h storage/innobase/os/os0proc.cc
  commit_text: Bug #94747: 4GB Limit on large_pages shared memory set-up

mysqld --large-pages --innodb_buffer_pool_chunk_size=4G result in 1G chunk allocations instead of 4G.

This patch was originally provided by Daniel Black at https://bugs.mysql.com/bug.php?id=94747

No testcase provided: the server setup requires a specific environment.
1 row in set (0.00 sec)