diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index ad536bfc4a97..ac48791bb060 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -420,6 +420,9 @@ The following options may be given as the first argument: before storage engine initialization, where each plugin is identified as name=library, where name is the plugin name and library is the plugin library in plugin_dir. + --enable-acl-cache-deadlock-detection + Enable deadlock detection on ACL_CACHE MDL lock + (Defaults to on; use --skip-enable-acl-cache-deadlock-detection to disable.) --enable-acl-db-cache Enable ACL db_cache lookup on (ip, user, db). Please issue FLUSH PRIVILEGES for the changes to take effect. @@ -2869,6 +2872,7 @@ disallow-raft FALSE disconnect-on-expired-password TRUE disconnect-slave-event-count 0 div-precision-increment 4 +enable-acl-cache-deadlock-detection TRUE enable-acl-db-cache TRUE enable-acl-fast-lookup FALSE enable-binlog-hlc FALSE diff --git a/mysql-test/suite/sys_vars/r/enable_acl_cache_deadlock_detection_basic.result b/mysql-test/suite/sys_vars/r/enable_acl_cache_deadlock_detection_basic.result new file mode 100644 index 000000000000..4055a0e84a86 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/enable_acl_cache_deadlock_detection_basic.result @@ -0,0 +1,40 @@ +SET @start_global_value = @@GLOBAL.enable_acl_cache_deadlock_detection; +SELECT @start_global_value; +@start_global_value +1 +SET @@GLOBAL.enable_acl_cache_deadlock_detection = on; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = DEFAULT; +select @@GLOBAL.enable_acl_cache_deadlock_detection; +@@GLOBAL.enable_acl_cache_deadlock_detection +1 +FLUSH PRIVILEGES; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = off; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = DEFAULT; +select @@GLOBAL.enable_acl_cache_deadlock_detection; +@@GLOBAL.enable_acl_cache_deadlock_detection +1 +FLUSH PRIVILEGES; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = on; +select @@GLOBAL.enable_acl_cache_deadlock_detection; +@@GLOBAL.enable_acl_cache_deadlock_detection +1 +FLUSH PRIVILEGES; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = off; +select @@GLOBAL.enable_acl_cache_deadlock_detection; +@@GLOBAL.enable_acl_cache_deadlock_detection +0 +FLUSH PRIVILEGES; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = something; +ERROR 42000: Variable 'enable_acl_cache_deadlock_detection' can't be set to the value of 'something' +select @@GLOBAL.enable_acl_cache_deadlock_detection; +@@GLOBAL.enable_acl_cache_deadlock_detection +0 +SET @@GLOBAL.enable_acl_cache_deadlock_detection = somethingelse; +ERROR 42000: Variable 'enable_acl_cache_deadlock_detection' can't be set to the value of 'somethingelse' +select @@GLOBAL.enable_acl_cache_deadlock_detection; +@@GLOBAL.enable_acl_cache_deadlock_detection +0 +SET @@GLOBAL.enable_acl_cache_deadlock_detection = @start_global_value; +SELECT @@GLOBAL.enable_acl_cache_deadlock_detection; +@@GLOBAL.enable_acl_cache_deadlock_detection +1 diff --git a/mysql-test/suite/sys_vars/t/enable_acl_cache_deadlock_detection_basic.test b/mysql-test/suite/sys_vars/t/enable_acl_cache_deadlock_detection_basic.test new file mode 100644 index 000000000000..a6bdf7af7b66 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/enable_acl_cache_deadlock_detection_basic.test @@ -0,0 +1,75 @@ +################ mysql-test\t\enable_acl_cache_deadlock_detection.test ######## +# # +# Variable Name: enable_acl_cache_deadlock_detection # +# Scope: Global # +# # +# Creation Date: 2021-04-13 # +# # +# # +# Description:Test Cases of Dynamic System Variable # +# enable_acl_cache_deadlock_detection that checks the behavior of # +# this variable in the following ways # +# * Default Value # +# * Valid Value # +# * Invalid value # +# # +############################################################################### +SET @start_global_value = @@GLOBAL.enable_acl_cache_deadlock_detection; +SELECT @start_global_value; + +######################################################################## +# Display the DEFAULT value of enable_acl_cache_deadlock_detection # +######################################################################## + +SET @@GLOBAL.enable_acl_cache_deadlock_detection = on; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = DEFAULT; +select @@GLOBAL.enable_acl_cache_deadlock_detection; + +# FLUSH PRIVILEGES not required but just a quick check to see acl cache +# is working properly +FLUSH PRIVILEGES; + +SET @@GLOBAL.enable_acl_cache_deadlock_detection = off; +SET @@GLOBAL.enable_acl_cache_deadlock_detection = DEFAULT; +select @@GLOBAL.enable_acl_cache_deadlock_detection; + +# FLUSH PRIVILEGES not required but just a quick check to see acl cache +# is working properly +FLUSH PRIVILEGES; + +############################################################################### +# change the value of enable_acl_cache_deadlock_detection to a valid value # +############################################################################### + +SET @@GLOBAL.enable_acl_cache_deadlock_detection = on; +select @@GLOBAL.enable_acl_cache_deadlock_detection; + +# FLUSH PRIVILEGES not required but just a quick check to see acl cache +# is working properly +FLUSH PRIVILEGES; + +SET @@GLOBAL.enable_acl_cache_deadlock_detection = off; +select @@GLOBAL.enable_acl_cache_deadlock_detection; + +# FLUSH PRIVILEGES not required but just a quick check to see acl cache +# is working properly +FLUSH PRIVILEGES; + +############################################################################### +# change the value of enable_acl_cache_deadlock_detection to an invalid value # +############################################################################### + +--Error ER_WRONG_VALUE_FOR_VAR +SET @@GLOBAL.enable_acl_cache_deadlock_detection = something; +select @@GLOBAL.enable_acl_cache_deadlock_detection; + +--Error ER_WRONG_VALUE_FOR_VAR +SET @@GLOBAL.enable_acl_cache_deadlock_detection = somethingelse; +select @@GLOBAL.enable_acl_cache_deadlock_detection; + +############################### +# Restore initial value # +############################### + +SET @@GLOBAL.enable_acl_cache_deadlock_detection = @start_global_value; +SELECT @@GLOBAL.enable_acl_cache_deadlock_detection; diff --git a/mysql-test/t/all_persisted_variables.test b/mysql-test/t/all_persisted_variables.test index 10fd7121d7d6..ec5aa863dbaa 100644 --- a/mysql-test/t/all_persisted_variables.test +++ b/mysql-test/t/all_persisted_variables.test @@ -51,9 +51,10 @@ let $total_excluded_vars=`SELECT COUNT(*) FROM performance_schema.global_variabl 'binlog_file_basedir', 'binlog_index_basedir', 'default_collation_for_utf8mb4_init', +'disable_raft_log_repointing', +'enable_acl_cache_deadlock_detection', 'enable_acl_db_cache', 'enable_acl_fast_lookup', -'disable_raft_log_repointing', 'group_replication_plugin_hooks', 'gtid_committed', 'gtid_purged_for_tailing', diff --git a/sql/mdl.cc b/sql/mdl.cc index 0facfc848565..4736f3958092 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -4020,6 +4020,30 @@ bool MDL_lock::visit_subgraph(MDL_ticket *waiting_ticket, MDL_context *src_ctx = waiting_ticket->get_ctx(); bool result = true; +#ifndef EXTRA_CODE_FOR_UNIT_TESTING + if (!enable_acl_cache_deadlock_detection && + key.mdl_namespace() == MDL_key::ACL_CACHE) { + /* + Deadlock detection for ACL_CACHE may lead to bad contention problems + in the connection path. In particular taking rdlock on m_rwlock + while doing deadlock traversal takes longer with more granted and + waiting tickets, but the releasing of these granted tickets are + themselves blocked on taking wrlock on m_rwlock, meanwhile more + connections are trying to come in and grant more tickets and + taking wrlock as well, so this becomes a serious lock contention + that you can't get out of if you keep new connection come in at + a fast pace. A easy way to trigger this is to run FLUSH + PRIVILEGES on a server that has lots of incoming connections with + lots of connection from acl_check_host / has_global_grant. + + For ACL_CACHE, the deadlock detection in most cases doesn't provide + a ton of value in production so this allows turning it off for + ACL_CACHE only. + */ + return false; + } +#endif + mysql_prlock_rdlock(&m_rwlock); /* diff --git a/sql/mysqld.cc b/sql/mysqld.cc index b36868245406..a18d160c6811 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1233,6 +1233,7 @@ double wait_for_hlc_sleep_scaling_factor = 0.75; char *default_collation_for_utf8mb4_init = nullptr; bool enable_blind_replace = false; bool enable_acl_fast_lookup = false; +bool enable_acl_cache_deadlock_detection = false; bool enable_acl_db_cache = true; bool enable_super_log_bin_read_only = false; ulonglong max_tmp_disk_usage{0}; diff --git a/sql/mysqld.h b/sql/mysqld.h index 15004a46e0bc..d871e37b063c 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -299,6 +299,7 @@ extern ulong wait_for_hlc_sleep_threshold_ms; extern double wait_for_hlc_sleep_scaling_factor; extern char *default_collation_for_utf8mb4_init; extern bool enable_acl_fast_lookup; +extern bool enable_acl_cache_deadlock_detection; extern bool enable_acl_db_cache; extern bool enable_super_log_bin_read_only; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 3b19680e20e0..81ec81591703 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -9103,6 +9103,12 @@ static Sys_var_bool Sys_enable_acl_fast_lookup( DEFAULT(false), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_enable_acl_fast_lookup)); +static Sys_var_bool Sys_enable_acl_cache_deadlock_detection( + "enable_acl_cache_deadlock_detection", + "Enable deadlock detection on ACL_CACHE MDL lock", + NON_PERSIST GLOBAL_VAR(enable_acl_cache_deadlock_detection), + CMD_LINE(OPT_ARG), DEFAULT(true)); + static Sys_var_bool Sys_enable_acl_db_cache( "enable_acl_db_cache", "Enable ACL db_cache lookup on (ip, user, db). Please issue "