MySQL Crash 故障记录：Failing assertion: index-＞table-＞stat_initialized

MySQL Crash 故障记录：Failing assertion: index->table->stat_initialized

---

1.错误日志

记录一例 MySQL Crash 故障，数据库版本：MySQL Percona Server 5.7.19，Error log 如下：

-- Crash 部分日志
2024-11-12 17:42:45 0x7fa77256e700  InnoDB: Assertion failure in thread 140357154563840 in file ha_innodb.cc line 14640
InnoDB: Failing assertion: index->table->stat_initialized
InnoDB: We intentionally generate a memory trap.
InnoDB: Submit a detailed bug report to http://bugs.mysql.com.
InnoDB: If you get repeated assertion failures or crashes, even
InnoDB: immediately after the mysqld startup, there may be
InnoDB: corruption in the InnoDB tablespace. Please refer to
InnoDB: http://dev.mysql.com/doc/refman/5.7/en/forcing-innodb-recovery.html
InnoDB: about forcing recovery.
09:42:45 UTC - mysqld got signal 6 ;
This could be because you hit a bug. It is also possible that this binary
or one of the libraries it was linked against is corrupt, improperly built,
or misconfigured. This error can also be caused by malfunctioning hardware.
Attempting to collect some information that could help diagnose the problem.
As this is a crash and something is definitely wrong, the information
collection process might fail.
Please help us make Percona Server better by reporting any
bugs at http://bugs.percona.com/

key_buffer_size=8388608
read_buffer_size=1048576
max_used_connections=2230
max_threads=100001
thread_count=2222
connection_count=2222
It is possible that mysqld could use up to 
key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = 1740864518 K  bytes of memory
Hope that's ok; if not, decrease some variables in the equation.

Thread pointer: 0x7fa708536000
Attempting backtrace. You can use the following information to find out
where mysqld died. If you see no messages after this, something went
terribly wrong...
stack_bottom = 7fa77256dbe8 thread_stack 0x80000
/usr/local/mysql/bin/mysqld(my_print_stacktrace+0x2c)[0xed36bc]
/usr/local/mysql/bin/mysqld(handle_fatal_signal+0x461)[0x7a26e1]
/lib64/libpthread.so.0[0x369ea0f7e0]
/lib64/libc.so.6(gsignal+0x35)[0x369e232625]
/lib64/libc.so.6(abort+0x175)[0x369e233e05]
/usr/local/mysql/bin/mysqld[0x770b56]
/usr/local/mysql/bin/mysqld[0xf4424d]
/usr/local/mysql/bin/mysqld(_ZN11ha_innobase8info_lowEjb+0x3e5)[0xf49045]
/usr/local/mysql/bin/mysqld(_ZN11ha_innobase4openEPKcij+0xa6d)[0xf556ed]
/usr/local/mysql/bin/mysqld(_ZN7handler7ha_openEP5TABLEPKcii+0x33)[0x805143]
/usr/local/mysql/bin/mysqld(_Z21open_table_from_shareP3THDP11TABLE_SHAREPKcjjjP5TABLEb+0x766)[0xd4c966]
/usr/local/mysql/bin/mysqld(_Z10open_tableP3THDP10TABLE_LISTP18Open_table_context+0xf91)[0xc529b1]
/usr/local/mysql/bin/mysqld(_Z11open_tablesP3THDPP10TABLE_LISTPjjP19Prelocking_strategy+0x5f2)[0xc59612]
/usr/local/mysql/bin/mysqld(_Z21open_tables_for_queryP3THDP10TABLE_LISTj+0x4b)[0xc59f2b]
/usr/local/mysql/bin/mysqld[0x760afe]
/usr/local/mysql/bin/mysqld(_Z21mysql_execute_commandP3THDb+0x4816)[0xcb0156]
/usr/local/mysql/bin/mysqld(_Z11mysql_parseP3THDP12Parser_state+0x5d5)[0xcb3385]
/usr/local/mysql/bin/mysqld(_Z16dispatch_commandP3THDPK8COM_DATA19enum_server_command+0xaba)[0xcb3eca]
/usr/local/mysql/bin/mysqld(_Z10do_commandP3THD+0x1b7)[0xcb5917]
/usr/local/mysql/bin/mysqld(_Z26threadpool_process_requestP3THD+0xc7)[0xd5a267]
/usr/local/mysql/bin/mysqld[0xd6979e]
/usr/local/mysql/bin/mysqld(pfs_spawn_thread+0x1b4)[0x123cf24]
/lib64/libpthread.so.0[0x369ea07aa1]
/lib64/libc.so.6(clone+0x6d)[0x369e2e893d]

Trying to get some variables.
Some pointers may be invalid and cause the dump to abort.
Query (7fa70d691030): is an invalid pointer
Connection ID (thread ID): 30441553
Status: NOT_KILLED

You may download the Percona Server operations manual by visiting
http://www.percona.com/software/percona-server/. You may find information
in the manual which will help you identify the cause of the crash.
Writing a core file

2.问题排查

开启 Core File 功能：

# root
shell> echo 'kernel.core_pattern = /mysql/cores/core.%e.%t.%p' >> /etc/sysctl.conf
shell> sysctl -p

# mysql
mysql> set global core_file=on;
mysql> select @@core_file;
+-------------+
| @@core_file |
+-------------+
|           1 |
+-------------+
1 row in set (0.00 sec)

GDB 分析堆栈如下：

shell> gdb /usr/local/mysql/bin/mysqld /mysql/core.mysqld.1731404565.6788
(gdb) bt
...（此处省略相关信息）

(gdb) f 13
#13 open_tables (thd=0x7fa708536000, start=Unhandled dwarf expression opcode 0xf3
) at /mnt/workspace/percona-server-5.7-binaries-release-rocks/label_exp/centos6-64/percona-server-5.7.19-17/sql/sql_base.cc:5794
5794 in /mnt/workspace/percona-server-5.7-binaries-release-rocks/label_exp/centos6-64/percona-server-5.7.19-17/sql/sql_base.cc
(gdb) p thd->m_query_string
$4 = {
str = 0x7fa70d691030 "SELECT * \t\t\t\tFROM \t\t\t\tSO_INFO_0626 \t\t\t\tWHERE ITEM_ID = '120031903101' \t\t\t\t", ' ' <repeats 21 times>, "AND ACTIVE_FLAG= 1", length = 139}

(gdb) f 18
#18 0x0000000000cb3eca in dispatch_command (thd=0x7fa708536000, com_data=Unhandled dwarf expression opcode 0xf3
)
at /mnt/workspace/percona-server-5.7-binaries-release-rocks/label_exp/centos6-64/percona-server-5.7.19-17/sql/sql_parse.cc:1493
1493 in /mnt/workspace/percona-server-5.7-binaries-release-rocks/label_exp/centos6-64/percona-server-5.7.19-17/sql/sql_parse.cc

定位报错点在 index->table->stat_initialized ，报错代码函数为 ha_innodb.cc ，这里与错误日志中记录的信息一致，且找到当时执行的 SQL 为 SELECT * SO_INFO_0626 WHERE ITEM_ID = '120031903101'ANDACTIVE_FLAG= 1 。该语句为简单的 SQL 查询，走索引，且表数据量不足 1000 条，并未发现异常。

查看分析源码，定位报错函数如下（与统计信息有关）：

innodb_rec_per_key(
/*===============*/
	dict_index_t*	index,		/*!< in: dict_index_t structure */
	ulint		i,		/*!< in: the column we are
					calculating rec per key */
	ha_rows		records)	/*!< in: estimated total records */
{
	rec_per_key_t	rec_per_key;
	ib_uint64_t	n_diff;

	ut_a(index->table->stat_initialized);

	ut_ad(i < dict_index_get_n_unique(index));
	ut_ad(!dict_index_is_spatial(index));

	if (records == 0) {
		/* "Records per key" is meaningless for empty tables.
		Return 1.0 because that is most convenient to the Optimizer. */
		return(1.0);
	}

	n_diff = index->stat_n_diff_key_vals[i];

	if (n_diff == 0) {

		rec_per_key = static_cast<rec_per_key_t>(records);
	} else if (srv_innodb_stats_method == SRV_STATS_NULLS_IGNORED) {
		ib_uint64_t	n_null;
		ib_uint64_t	n_non_null;

		n_non_null = index->stat_n_non_null_key_vals[i];

		/* In theory, index->stat_n_non_null_key_vals[i]
		should always be less than the number of records.
		Since this is statistics value, the value could
		have slight discrepancy. But we will make sure
		the number of null values is not a negative number. */
		if (records < n_non_null) {
			n_null = 0;
		} else {
			n_null = records - n_non_null;
		}

		/* If the number of NULL values is the same as or
		large than that of the distinct values, we could
		consider that the table consists mostly of NULL value.
		Set rec_per_key to 1. */
		if (n_diff <= n_null) {
			rec_per_key = 1.0;
		} else {
			/* Need to exclude rows with NULL values from
			rec_per_key calculation */
			rec_per_key
				= static_cast<rec_per_key_t>(records - n_null)
				/ (n_diff - n_null);
		}
	} else {
		DEBUG_SYNC_C("after_checking_for_0");
		rec_per_key = static_cast<rec_per_key_t>(records) / n_diff;
	}

	if (rec_per_key < 1.0) {
		/* Values below 1.0 are meaningless and must be due to the
		stats being imprecise. */
		rec_per_key = 1.0;
	}

	return(rec_per_key);
}

暂时未发现相关异常点，与之类似 BUG 信息反馈如下：
https://jira.mariadb.org/browse/MDEV-17102
https://bugs.mysql.com/bug.php?id=77588

但均未明确定位问题，无法复现。另根据 Facebook 提交的 BUG 分析来看与锁获取有关（https://bugs.mysql.com/bug.php?id=104344 ，很遗憾，无权限访问）:

至此，该问题仍无有效的复现方式。

另，该 Crash 问题未再有出现，本次记录留存，待后续分析。

如有其他朋友遇到类似的问题或有新的发现，也希望能交流下。