【GaussDB】破除gaussdb ugin索引支持中文模糊查询的迷思-字符序

ugin索引是GaussDB 505.2以后专门针对ustore存储引擎推出的。通过创建ugin索引，不需要改造SQL，就支持SQL的模糊like搜索，例如：like '%新年快乐%'。目前只支持集中式，分布式暂时不支持。

结论

之前实验的时候，一直没考虑字符排序的问题，导致测试ugin的时候发现模糊查询一直有一些问题，可以参考上一篇文章

GaussDB默认的字符序是C,在 C字符序下，数据库根本不知道什么是汉字。因为一个中文字符在 UTF-8 编码下占 3 个字节，数据库只把它当作 3 个独立的、没有任何逻辑关联的二进制数字（例如 '新' 被拆成了 \xE6、\x96、\xB0 3个纯数字）。

字符序zh_CN.utf8和en_US.utf8，它们都是基于字符（Character-based）的多字节 UTF-8 字符集，而不是基于字节（Byte-based）的 C 字符序，所以在ugin索引的opclass:ugin_trgm_ops下，可以完美的支持模糊搜索。

测试SQL

sql 复制代码

drop database xzjy_zh_db;
create database xzjy_zh_db  encoding 'utf8' dbcompatibility 'A' ;
\c xzjy_zh_db
CREATE TABLE test_trgm_new(t varchar(100)) WITH (STORAGE_TYPE=USTORE);
INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;
INSERT INTO test_trgm_new SELECT '当前企业名称是中国第七建设集团有限公司第' || seq::text || '分公司' FROM generate_series(1, 50000) AS seq;
INSERT INTO test_trgm_new VALUES('2024新年快乐7days');    
INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;
INSERT INTO test_trgm_new VALUES('2026新年快乐');   
select count(*) from test_trgm_new;
CREATE INDEX trgm_idx_new ON test_trgm_new USING ugin (t ugin_trgm_ops);
ANALYZE test_trgm_new;
explain analyze select * from test_trgm_new where t like '%新年快乐%';

默认LC_COLLATE = 'C'

默认C字符序，ugin索引，模糊查询，依然只能走全表扫描

sql 复制代码

gaussdb=# create database xzjy_zh_db  encoding 'utf8' dbcompatibility 'A' ;
CREATE DATABASE
gaussdb=# \c xzjy_zh_db
Non-SSL connection (SSL connection is recommended when requiring high-security)
You are now connected to database "xzjy_zh_db" as user "rdsAdmin".
xzjy_zh_db=# CREATE TABLE test_trgm_new(t varchar(100)) WITH (STORAGE_TYPE=USTORE);
CREATE TABLE
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;
INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT '当前企业名称是中国第七建设集团有限公司第' || seq::text || '分公司' FROM generate_series(1, 50000) AS seq;
INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new VALUES('2024新年快乐7days');    
INSERT 0 1
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;
INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new VALUES('2026新年快乐');   
INSERT 0 1
-- 统计总行数
xzjy_zh_db=# select count(*) from test_trgm_new;
 count  
--------
 150002
(1 row)
-- 创建ugin索引
xzjy_zh_db=# CREATE INDEX trgm_idx_new ON test_trgm_new USING ugin (t ugin_trgm_ops);
CREATE INDEX
-- 分析表
xzjy_zh_db=# ANALYZE test_trgm_new;
ANALYZE
--  查看执行计划 全表扫描
xzjy_zh_db=# explain analyze select * from test_trgm_new where t like '%新年快乐%';
 id |           operation           | A-time | A-rows | E-rows | Peak Memory | A-width | E-width |     E-costs     
----+-------------------------------+--------+--------+--------+-------------+---------+---------+-----------------
  1 | ->  Seq Scan on test_trgm_new | 69.031 |      2 |     15 | 61KB        |         |      33 | 0.000..2866.025
(1 row)

   Predicate Information (identified by plan id)   
---------------------------------------------------
   1 --Seq Scan on test_trgm_new
         Filter: ((t)::text ~~ '%新年快乐%'::text)
         Rows Removed by Filter: 150000
(3 rows)

       ====== Query Summary =====       
----------------------------------------
 Datanode executor start time: 0.040 ms
 Datanode executor run time: 69.104 ms
 Datanode executor end time: 0.014 ms
 Planner runtime: 0.544 ms
 Query Id: 1974265486663584903
 Total runtime: 69.172 ms
(6 rows)

LC_COLLATE = 'zh_CN.utf8'

字符序LC_COLLATE = 'zh_CN.utf8'，ugin索引，模糊查询，可以走到索引

sql 复制代码

gaussdb=# create database xzjy_zh_db  encoding 'utf8' dbcompatibility 'A' LC_COLLATE = 'zh_CN.utf8' LC_CTYPE = 'zh_CN.utf8';

CREATE DATABASE
gaussdb=# \c xzjy_zh_db
Non-SSL connection (SSL connection is recommended when requiring high-security)
You are now connected to database "xzjy_zh_db" as user "rdsAdmin".
xzjy_zh_db=# CREATE TABLE test_trgm_new(t varchar(100)) WITH (STORAGE_TYPE=USTORE);
CREATE TABLE
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;
INSERT INTO test_trgm_new SELECT '当前企业名称是中国第七建设集团有限公司第' || seq::text || '分公司' FROM generate_series(1, 50000) AS seq;INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT '当前企业名称是中国第七建设集团有限公司第' || seq::text || '分公司' FROM generate_series(1, 50000) AS seq;
INSERT 0 50000

xzjy_zh_db=# INSERT INTO test_trgm_new VALUES('2024新年快乐7days');         
INSERT 0 1
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;                               
INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new VALUES('2026新年快乐'); 
INSERT 0 1
xzjy_zh_db=# select count(*) from test_trgm_new;
 count  
--------
 150002
(1 row)

xzjy_zh_db=# CREATE INDEX trgm_idx_new ON test_trgm_new USING ugin (t ugin_trgm_ops);
CREATE INDEX
xzjy_zh_db=# ANALYZE test_trgm_new;
ANALYZE
-- 查看执行计划 正常走到了索引
xzjy_zh_db=# explain analyze select * from test_trgm_new where t like '%新年快乐%';
 id |                  operation                  | A-time | A-rows | E-rows | Peak Memory | A-width | E-width |    E-costs     
----+---------------------------------------------+--------+--------+--------+-------------+---------+---------+----------------
  1 | ->  Bitmap Heap Scan on test_trgm_new       | 0.055  |      2 |      2 | 27KB        |         |      32 | 20.110..74.761
  2 |    ->  Bitmap Index Scan using trgm_idx_new | 0.025  |      2 |     15 | 73KB        |         |       0 | 0.000..20.109
(2 rows)

      Predicate Information (identified by plan id)      
---------------------------------------------------------
   1 --Bitmap Heap Scan on test_trgm_new
         Recheck Cond: ((t)::text ~~ '%新年快乐%'::text)
   2 --Bitmap Index Scan using trgm_idx_new
         Index Cond: ((t)::text ~~ '%新年快乐%'::text)
(4 rows)

       ====== Query Summary =====       
----------------------------------------
 Datanode executor start time: 0.042 ms
 Datanode executor run time: 0.071 ms
 Datanode executor end time: 0.012 ms
 Planner runtime: 0.249 ms
 Query Id: 1974265486663559932
 Total runtime: 0.137 ms
(6 rows)

LC_COLLATE = 'en_US.utf8'

字符序LC_COLLATE = 'en_US.utf8'，ugin索引，模糊查询，可以走到索引

sql 复制代码

gaussdb=# drop database xzjy_zh_db;
DROP DATABASE
gaussdb=# create database xzjy_zh_db  encoding 'utf8' dbcompatibility 'A' LC_COLLATE = 'en_US.utf8' LC_CTYPE = 'en_US.utf8';
CREATE DATABASE
gaussdb=# \c xzjy_zh_db
Non-SSL connection (SSL connection is recommended when requiring high-security)
You are now connected to database "xzjy_zh_db" as user "rdsAdmin".
xzjy_zh_db=# CREATE TABLE test_trgm_new(t varchar(100)) WITH (STORAGE_TYPE=USTORE);
CREATE TABLE
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;
INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT '当前企业名称是中国第七建设集团有限公司第' || seq::text || '分公司' FROM generate_series(1, 50000) AS seq;
INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new VALUES('2024新年快乐7days'); 
INSERT 0 1
xzjy_zh_db=# INSERT INTO test_trgm_new SELECT 'qwertyu' || LPAD(seq::text, 4, '0') FROM generate_series(1, 50000) AS seq;
INSERT 0 50000
xzjy_zh_db=# INSERT INTO test_trgm_new VALUES('2026新年快乐');   
INSERT 0 1
xzjy_zh_db=# select count(*) from test_trgm_new;
 count  
--------
 150002
(1 row)

xzjy_zh_db=# CREATE INDEX trgm_idx_new ON test_trgm_new USING ugin (t ugin_trgm_ops);
CREATE INDEX
xzjy_zh_db=# ANALYZE test_trgm_new;
ANALYZE
-- 查看执行计划 正常走到索引
xzjy_zh_db=# explain analyze select * from test_trgm_new where t like '%新年快乐%';
 id |                  operation                  | A-time | A-rows | E-rows | Peak Memory | A-width | E-width |    E-costs     
----+---------------------------------------------+--------+--------+--------+-------------+---------+---------+----------------
  1 | ->  Bitmap Heap Scan on test_trgm_new       | 0.055  |      2 |     15 | 27KB        |         |      32 | 20.113..74.764
  2 |    ->  Bitmap Index Scan using trgm_idx_new | 0.029  |      2 |     15 | 73KB        |         |       0 | 0.000..20.109
(2 rows)

      Predicate Information (identified by plan id)      
---------------------------------------------------------
   1 --Bitmap Heap Scan on test_trgm_new
         Recheck Cond: ((t)::text ~~ '%新年快乐%'::text)
   2 --Bitmap Index Scan using trgm_idx_new
         Index Cond: ((t)::text ~~ '%新年快乐%'::text)
(4 rows)

       ====== Query Summary =====       
----------------------------------------
 Datanode executor start time: 0.080 ms
 Datanode executor run time: 0.101 ms
 Datanode executor end time: 0.018 ms
 Planner runtime: 1.563 ms
 Query Id: 1974265486663622404
 Total runtime: 0.211 ms
(6 rows)

注意，无论LC_COLLATE = 'zh_CN.utf8'还是LC_COLLATE = 'en_US.utf8'，ugin_trgm_ops索引基于 pg_trgm 的三元组切词，如果只是2个字符，则依然会走全表扫描。

sql 复制代码

xzjy_zh_db=# explain analyze select * from test_trgm_new where t like '%新年%';

 id |           operation           | A-time | A-rows | E-rows | Peak Memory | A-width | E-width |     E-costs     
----+-------------------------------+--------+--------+--------+-------------+---------+---------+-----------------
  1 | ->  Seq Scan on test_trgm_new | 61.358 |      2 |     15 | 61KB        |         |      32 | 0.000..2866.025
(1 row)

 Predicate Information (identified by plan id) 
-----------------------------------------------
   1 --Seq Scan on test_trgm_new
         Filter: ((t)::text ~~ '%新年%'::text)
         Rows Removed by Filter: 150000
(3 rows)

       ====== Query Summary =====       
----------------------------------------
 Datanode executor start time: 0.035 ms
 Datanode executor run time: 61.425 ms
 Datanode executor end time: 0.018 ms
 Planner runtime: 0.284 ms
 Query Id: 1974265486663628451
 Total runtime: 61.489 ms
(6 rows)

最后，由于后期无法修改数据库的字符序，那么我们建库的时候建议指定字符序为zh_CN.utf8或en_US.utf8。

sql 复制代码

gaussdb=# alter database xzjy_zh_db set LC_COLLATE = 'zh_CN.utf8';

NOTICE:  SET "lc_collate" TO "zh_CN.utf8" will only take effect after restarting session.
ERROR:  parameter "lc_collate" cannot be changed

转载：阿才漫谈

复制代码