Bug#30535835 IMPROVE PLUGIN USE OF MySQL SERVICES

Implement wrapper to encapsulate ndbcluster plugins use of MySQL
services, this unifies and documents how and what the plugin uses.

Add support for requesting the MySQL server to shut itself down using
the "host_application_signal" interface. Use the new interface when a
problem is detected during ndbcluster plugin initialization. This
fixes a problem with the testcase for --ndb-cluster-connection-pool-
nodeids which starts the MySQL Server several times with invalid
parameters, the expected result is that the ndbcluster plugin should
write a message to log and then shutdown the MySQL Server. Since the
shutdown is done in the middle of MySQL startup this may cause sporadic
problems due to missing synchronization, when using the new interface
the MySQL Server initialization will be alloowed to finish before
shutdown is performed.

Also use the new interface to shutdown the MySQL Server when failure to
setup events occur, this reduces the ndbcluster plugins usage of
internal functions in favour of the stable service interface.

Move the service initialization to the module which uses it, this
encapsulates related code, reduces dependencies and use of global
variables or functions.

Change-Id: I6f40b1fdc57707eae704bbe80a99811e5cf19223

Author: blaudden

mysql-test/suite/ndb/t/ndb_connection_pool_nodeids.test

@@ -36,47 +36,41 @@ let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.1.expect;
 --source include/wait_until_disconnected.inc
 
 --echo # - Empty nodeid last
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="1,2," $args;
 let SEARCH_PATTERN= Found empty nodeid;
 let SEARCH_FILE= $error_log;
 --source include/search_pattern.inc
 --remove_file $SEARCH_FILE
 
 --echo # - Empty nodeid in the middle
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="1,,3" $args;
 let SEARCH_PATTERN= Found empty nodeid;
 let SEARCH_FILE= $error_log;
 --source include/search_pattern.inc
 --remove_file $SEARCH_FILE
 
 --echo # - Specified nodeid is not a number
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="1,a,3" $args;
 let SEARCH_PATTERN= Could not parse;
 let SEARCH_FILE= $error_log;
 --source include/search_pattern.inc
 --remove_file $SEARCH_FILE
 
 --echo # - Check valid nodeid(> 0)
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="2,0,5" --ndb-nodeid=1 $args;
 let SEARCH_PATTERN= Invalid nodeid 0;
 let SEARCH_FILE= $error_log;
 --source include/search_pattern.inc
 --remove_file $SEARCH_FILE
 
 --echo # - Check valid nodeid(<= 255)
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="2,256,5" --ndb-nodeid=1 $args;
 let SEARCH_PATTERN= Invalid nodeid 256;
 let SEARCH_FILE= $error_log;
 --source include/search_pattern.inc
 --remove_file $SEARCH_FILE
 
 --echo # - Specified nodeid is unique
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="1,1,3" $args;
 let SEARCH_PATTERN= Found duplicate nodeid 1;
 let SEARCH_FILE= $error_log;
@@ -85,15 +79,13 @@ let SEARCH_FILE= $error_log;
 
 --echo # - Number of nodeids in list must match size of pool
 --echo #   (config specifies --ndb-cluster-connection-pool=3)
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="1,3" $args;
 let SEARCH_PATTERN= The size of the cluster connection pool must be;
 let SEARCH_FILE= $error_log;
 --source include/search_pattern.inc
 --remove_file $SEARCH_FILE
 
 --echo # - First nodeid must be equal to --ndb-nodeid(if given)
---error 1
 --exec $MYSQLD_CMD_11 --ndb-cluster-connection-pool-nodeids="2,3,5" --ndb-nodeid=1 $args;
 let SEARCH_PATTERN= The nodeid specified by --ndb-nodeid must be equal;
 let SEARCH_FILE= $error_log;

mysql-test/suite/ndb/t/ndb_startup.test

@@ -34,7 +34,6 @@ let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.1.expect;
 
 --echo # Start server with provoked error in ndbcluster_init()
 --echo # - the mysqld should write an error to the log and shutdown
---error 1
 --exec $MYSQLD_CMD_11 --debug="+d,ndbcluster_init_fail1" $args;
 
 --echo # Check the log file for expected error
@@ -45,7 +44,6 @@ let SEARCH_FILE= $error_log;
 
 --echo # Start server with another provoked error in ndbcluster_init()
 --echo # - the mysqld should write an error to the log and shutdown
---error 1
 --exec $MYSQLD_CMD_11 --debug="+d,ndbcluster_init_fail2" $args;
 
 --echo # Check the log file for expected error

mysql-test/suite/ndb_binlog/r/ndb_binlog_startup_fail.result

@@ -1,23 +1,50 @@
-use test;
+# First test, check behaviour with --ndb-log-fail-terminate=1
+# during metadata synchronization
+# - Create table
+# - Setup mysqld2 with DBUG error to force "binlog setup" failure
+# - Setup mtr.pl to expect mysqld2 to shutdown
+# - Use DUMP 900 to disconnect both mysqld's (node 16 and 49)
+# - The mysqld2 will disconnect and then shut itself down when
+#   hitting the DBUG error during metadata sync
+# - Start mysqld2 again and check things work
 create table t1 (a int primary key) engine=ndb;
 insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
-Prepare for Server2 to fail
-Wait for 'server2' binlog rotate to indicate disconnect
+# Setup mysqld2 DBUG
+# NOTE! since the process will restart there is no need to save value
+set global debug='+d,ndb_binlog_fail_setup';
+# Write file to make mtr.pl expect mysqld2 shutdown, but don't start
+# until it's told to
+# Disconnect both mysqld's with DUMP 900
+# Wait for mysqld2 binlog rotate to indicate disconnect
+# Wait for mysqld1 to reconnect and insert data to check it's alive
 insert into t1 values (11);
-Give 'server2' some time to start, and fail, a binlog_setup
-Wait for Server2 to fail
-Server2 failed as expected
+# Wait for mysqld2 to disconnect
+# mysqld2 disconnected
+# Write file to make mtr.pl start up mysqld2 again
+# Wait for mysqld2 to reconnect and insert data to check it's alive
 insert into t1 values (10);
-drop table test.t1;
-Wait for 'server2' to complete setup and get out of read-only mode
-Prepare for Server2 to fail
-set global debug='-d';
+drop table t1;
+# Second test, check behaviour with --ndb-log-fail-terminate=1
+# during CREATE TABLE
+# - Setup mysqld2 with DBUG error to force "binlog setup" failure
+# - Setup mtr.pl to expect mysqld2 to shutdown
+# - Create table on mysqld1
+# - mysqld2 will fail during schema distribution and shut itself down
+# - Start mysqld2 again and check things work
+# Setup mysqld2 DBUG
+# NOTE! since the process will restart there is no need to save value
 set global debug='+d,ndb_binlog_fail_setup';
-use test;
+# Write file to make mtr.pl expect mysqld2 shutdown, but don't start
+# until it's told to
+# Create table from mysqld1
 create table t1 (a int primary key) engine=ndb;
+Warnings:
+Warning	1296	Node 16: 'Distribution of CREATE TABLE 't1' failed'
 insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
-Wait for Server2 to fail
-Server2 failed as expected
+# Wait for mysqld2 diconnect
+# mysqld2 disconnected
+# Write file to make mtr.pl start up mysqld2 again
+# Wait for mysqld2 to reconnect and insert data to check it's alive
 insert into t1 values (10);
 insert into t1 values (11);
 drop table t1;

mysql-test/suite/ndb_binlog/t/ndb_binlog_startup_fail.test

@@ -1,173 +1,125 @@
 -- source include/have_multi_ndb.inc
 -- source include/have_binlog_format_mixed_or_row.inc
-# We are using some debug-only features in this test
+# Test is using some debug-only features
 --source include/have_debug.inc
 # Test terminates mysqld which causes valgrind warnings
 --source include/not_valgrind.inc
 # Ignore any mysqld failure reporting
 --source include/not_crashrep.inc
 
-connection server1;
-# Find NodeId of the mysqld we are connected to:
---let $node_id= `SHOW STATUS LIKE 'Ndb_cluster_node_id'`
---let $node_id= `SELECT SUBSTRING('$node_id', 20)+0`
+--connection server1
 --disable_query_log
 call mtr.add_suppression("cluster disconnect An incident event");
 --enable_query_log
 
-connection server2;
-# Find NodeId2 of 'server2' we are connected to:
---let $node_id2= `SHOW STATUS LIKE 'Ndb_cluster_node_id'`
---let $node_id2= `SELECT SUBSTRING('$node_id2', 20)+0`
-
-# Ignore server shutdown/startup failure
+--connection server2
+# Ignore server shutdown/startup errors in log
 --disable_query_log
 call mtr.add_suppression("Failed to setup binlog");
 call mtr.add_suppression("Distribution of CREATE TABLE");
 call mtr.add_suppression("Server shutdown in progress");
 call mtr.add_suppression("cluster disconnect An incident event");
-call mtr.add_suppression("Failed setting up binlogging");
 call mtr.add_suppression("mysqld startup an incident event");
-call mtr.add_suppression("Failed to release global schema lock");
+# Reported when the --ndb-log-fail-terminate logic kicks in
+call mtr.add_suppression("Requesting server shutdown");
 --enable_query_log
 
-#
-# 1 create a table
-# 2 Setup error insert on MySQLD2
-# 3 Disconnect MySQLD2 using DUMP code, causing Binlog re-init
-# 4 Observe MySQLD2 fail, then restart (and succeed)
-#
-
-connection server1;
-use test;
+--echo # First test, check behaviour with --ndb-log-fail-terminate=1
+--echo # during metadata synchronization
+--echo # - Create table
+--echo # - Setup mysqld2 with DBUG error to force "binlog setup" failure
+--echo # - Setup mtr.pl to expect mysqld2 to shutdown
+--echo # - Use DUMP 900 to disconnect both mysqld's (node 16 and 49)
+--echo # - The mysqld2 will disconnect and then shut itself down when
+--echo #   hitting the DBUG error during metadata sync
+--echo # - Start mysqld2 again and check things work
 
+--connection server1
 create table t1 (a int primary key) engine=ndb;
 insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
 
-connection server2;
-disable_query_log;
-let $debug_saved = `select @@global.debug`;
-set global debug='-d'; # Switch DEBUG/TRACING OFF
+--connection server2
+--echo # Setup mysqld2 DBUG
+--echo # NOTE! since the process will restart there is no need to save value
 set global debug='+d,ndb_binlog_fail_setup';
-enable_query_log;
-
---echo Prepare for Server2 to fail
-connection server2;
-# Find NodeId2 of 'server2' we are connected to:
---let $node_id2= `SHOW STATUS LIKE 'Ndb_cluster_node_id'`
---let $node_id2= `SELECT SUBSTRING('$node_id2', 20)+0`
-
-#
-# Some magic for MTR to allow failure, taken from include/restart_mysqld.inc
-#
---let $_server_id= `SELECT @@server_id`
---let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.1.expect
-
---exec echo "wait" > $_expect_file_name
-#
-# /end magic part 1
-#
-
-#
-# Cause data nodes to disconnect all API nodes
-# Each API will setup binlog again
-# MySQLD2 will get stuck and restart
-#
-
---exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT
---exec $NDB_MGM -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT
-
-#connection server1;
-#--echo Wait for 'server1' binlog rotate to indicate disconnect
-#--let $wait_binlog_event= binlog.000002
-#--source include/wait_for_binlog_event.inc
-
-connection server2;
---echo Wait for 'server2' binlog rotate to indicate disconnect
+
+--echo # Write file to make mtr.pl expect mysqld2 shutdown, but don't start
+--echo # until it's told to
+--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.1.expect
+wait
+EOF
+
+--echo # Disconnect both mysqld's with DUMP 900
+let $out_file = $MYSQLTEST_VARDIR/tmp/mgm_dump.out;
+--exec $NDB_MGM -e "ALL DUMP 900 16" > $out_file 2>&1
+--exec $NDB_MGM -e "ALL DUMP 900 49" >> $out_file 2>&1
+--remove_file $out_file
+
+--echo # Wait for mysqld2 binlog rotate to indicate disconnect
 --let $wait_binlog_event= binlog.000002
 --source include/wait_for_binlog_event.inc
 
-connection server1;
+--connection server1
+--echo # Wait for mysqld1 to reconnect and insert data to check it's alive
 --source include/ndb_not_readonly.inc
 insert into t1 values (11);
 
---echo Give 'server2' some time to start, and fail, a binlog_setup
-sleep 2;
-
---echo Wait for Server2 to fail
-connection server2;
+--connection server2
+--echo # Wait for mysqld2 to disconnect
 --source include/wait_until_disconnected.inc
+--echo # mysqld2 disconnected
 
---echo Server2 failed as expected
-
-#
-# More magic from include/restart_mysqld.inc
-#
---exec echo "restart" > $_expect_file_name
+--echo # Write file to make mtr.pl start up mysqld2 again
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.1.expect
+restart
+EOF
 
+--echo # Wait for mysqld2 to reconnect and insert data to check it's alive
 --enable_reconnect
 --source include/wait_until_connected_again.inc
 --disable_reconnect
-#
-# /end magic part 2
-#
 
-connection server1;
 insert into t1 values (10);
-drop table test.t1;
-
-connection server2;
---echo Wait for 'server2' to complete setup and get out of read-only mode
---source include/ndb_not_readonly.inc
---remove_file $NDB_TOOLS_OUTPUT
-
-#
-# 1 Setup error insert on MySQLD2
-# 2 create a table on MySQLD1
-# 3 Observe MySQLD2 fail, then restart (and succeed)
-#
-
---echo Prepare for Server2 to fail
---connection server2
-#
-# Some magic for MTR to allow failure, taken from include/restart_mysqld.inc
-#
---let $_server_id= `SELECT @@server_id`
---let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.1.expect
+drop table t1;
 
---exec echo "wait" > $_expect_file_name
-#
-# /end magic part 1
-#
+--echo # Second test, check behaviour with --ndb-log-fail-terminate=1
+--echo # during CREATE TABLE
+--echo # - Setup mysqld2 with DBUG error to force "binlog setup" failure
+--echo # - Setup mtr.pl to expect mysqld2 to shutdown
+--echo # - Create table on mysqld1
+--echo # - mysqld2 will fail during schema distribution and shut itself down
+--echo # - Start mysqld2 again and check things work
 
 --connection server2
-set global debug='-d'; # Switch DEBUG/TRACING OFF
+--echo # Setup mysqld2 DBUG
+--echo # NOTE! since the process will restart there is no need to save value
 set global debug='+d,ndb_binlog_fail_setup';
 
+--echo # Write file to make mtr.pl expect mysqld2 shutdown, but don't start
+--echo # until it's told to
+--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.1.expect
+wait
+EOF
+
 --connection server1
-use test;
---disable_warnings
+--echo # Create table from mysqld1
 create table t1 (a int primary key) engine=ndb;
 insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
 
---echo Wait for Server2 to fail
 --connection server2
+--echo # Wait for mysqld2 diconnect
 --source include/wait_until_disconnected.inc
+--echo # mysqld2 disconnected
 
---echo Server2 failed as expected
---enable_warnings
-
-#
-# More magic from include/restart_mysqld.inc
-#
---exec echo "restart" > $_expect_file_name
+--echo # Write file to make mtr.pl start up mysqld2 again
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.1.expect
+restart
+EOF
 
+--echo # Wait for mysqld2 to reconnect and insert data to check it's alive
 --enable_reconnect
 --source include/wait_until_connected_again.inc
 --disable_reconnect
-#
-# /end magic part 2
-#
 
 --source include/ndb_not_readonly.inc
 insert into t1 values (10);
@@ -177,8 +129,3 @@ insert into t1 values (10);
 insert into t1 values (11);
 drop table t1;
 
---connection server2
-disable_query_log;
-set global debug='+d'; # Switch DEBUG/TRACING ON
-eval set global debug= '$debug_saved';
-enable_query_log;