MPP
xxxxxxxx.142
xxxxxxxx.143
xxxxxxxx.144
前置环境安装
依赖环境######################################################################################################
#java -version
openjdk version "1.8.0_392"
python -V
Python 3.9.18
#which java
/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.392.b08-3.el9.x86_64/jre/bin/java
设置环境变量
#vi /etc/profile
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.392.b08-3.el9.x86_64/jre
export PATH= J A V A H O M E / b i n : JAVA_HOME/bin: JAVAHOME/bin:PATH
export CLASSPATH=.: J A V A H O M E / l i b / d t . j a r : JAVA_HOME/lib/dt.jar: JAVAHOME/lib/dt.jar:JAVA_HOME/lib/tools.jar
刷新
#source /etc/profile
#mysql -hxxxxmmxxxx.142 -P16603 -uroot -pxxxxmmxxxx==
CREATE DATABASE ranger DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
- 创建用户并设置密码
SET GLOBAL validate_password.policy = LOW;
CREATE USER 'ranger'@'%' IDENTIFIED BY 'xxxxmmxxxx'; - 授予所有权限
GRANT ALL PRIVILEGES ON ranger.* TO 'ranger'@'%';
update mysql.user set Grant_priv='Y',Super_priv='Y' where user='ranger';
SELECT host,user,authentication_string,Grant_priv,Super_priv,authentication_string FROM mysql.user;
- 刷新权限（可选，通常 GRANT 会自动刷新）
FLUSH PRIVILEGES;
mysql -hxxxxmmxxxx.142 -P16603 -uranger -pxxxxmmxxxx
LdapHA安装
######################################################################################################
cat /etc/redhat-release
Rocky Linux release 9.3 (Blue Onyx)
Ldap（主从）安装，两个主机都要root下操作
#######################Ldap主从配置#####################
安装
#yum localinstall -y openldap*
#rpm -qa | grep ldap
sssd-ldap-2.9.1-4.el9_3.x86_64
openldap-2.6.8-4.el9.0.1.x86_64
openldap-devel-2.6.8-4.el9.0.1.x86_64
openldap-servers-2.6.8-4.el9.0.1.x86_64
openldap-clients-2.6.8-4.el9.0.1.x86_64
启动（设置自启动）
systemctl start slapd
systemctl enable slapd
systemctl status slapd
#设置密码
slappasswd -h {md5} -s "xxxxmmxxxx"
{MD5}ZywcmQg9WAs07FG8qRFKMQ==
#vi set_rootpw.ldif
#################################################
dn: olcDatabase={0}config,cn=config
changetype: modify
add: olcRootPW
olcRootPW: {MD5}ZywcmQg9WAs07FG8qRFKMQ==
#################################################
#ldapadd -Y EXTERNAL -H ldapi:/// -f set_rootpw.ldif
问题：additional info: modify/add: olcRootPW: no equality matching rule 解决办法：
修改modify.ldif中对应选项的"add"为"replace"即可
#添加基础配置
ldapadd -Y EXTERNAL -H ldapi:/// -f /etc/openldap/schema/cosine.ldif
ldapadd -Y EXTERNAL -H ldapi:/// -f /etc/openldap/schema/nis.ldif
ldapadd -Y EXTERNAL -H ldapi:/// -f /etc/openldap/schema/inetorgperson.ldif
#添加domain配置
#ldapsearch -Y EXTERNAL -H ldapi:/// -b "cn=config" "olcDatabase=*" dn
dn: olcDatabase={2}mdb,cn=config ← 这才是正确的 DN
vi set_domain.ldif
###################################
dn: olcDatabase={1}monitor,cn=config
changetype: modify
replace: olcAccess
olcAccess: {0}to * by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth"
read by dn.base="cn=admin,dc=gtk,dc=com" read by * none
dn: olcDatabase={2}mdb,cn=config
changetype: modify
replace: olcSuffix
olcSuffix: dc=gtk,dc=com
dn: olcDatabase={2}mdb,cn=config
changetype: modify
replace: olcRootDN
olcRootDN: cn=admin,dc=gtk,dc=com
dn: olcDatabase={2}mdb,cn=config
changetype: modify
add: olcRootPW
olcRootPW: {MD5}ZywcmQg9WAs07FG8qRFKMQ==
dn: olcDatabase={2}mdb,cn=config
changetype: modify
add: olcAccess
olcAccess: {0}to attrs=userPassword,shadowLastChange by
dn="cn=admin,dc=gtk,dc=com" write by anonymous auth by self write by * none
olcAccess: {1}to dn.base="" by * read
olcAccess: {2}to * by dn="cn=admin,dc=gtk,dc=com" write by * read
##########################################################
ldapmodify -Y EXTERNAL -H ldapi:/// -f set_domain.ldif
vi create_basedomain.ldif
##########################################################
dn: dc=gtk,dc=com
objectClass: top
objectClass: dcObject
objectclass: organization
o: Server com
dc: gtk
dn: cn=admin,dc=gtk,dc=com
objectClass: organizationalRole
cn: admin
description: Directory admin
dn: ou=People,dc=gtk,dc=com
objectClass: organizationalUnit
ou: People
dn: ou=Group,dc=gtk,dc=com
objectClass: organizationalUnit
ou: Group
##########################################################
ldapadd -x -D cn=admin,dc=gtk,dc=com -wxxxxmmxxxx -f create_basedomain.ldif
#主从配置
master配置
##########################################################
vi sync_provider_addMode.ldif
###################################
create new
dn: cn=module,cn=config
objectClass: olcModuleList
cn: module
olcModulePath: /usr/lib64/openldap
olcModuleLoad: syncprov.la
####################################
ldapadd -Y EXTERNAL -H ldapi:/// -f sync_provider_addMode.ldif
vi sync_provider.ldif
#############################
create new
dn: olcOverlay=syncprov,olcDatabase={2}mdb,cn=config
objectClass: olcOverlayConfig
objectClass: olcSyncProvConfig
olcOverlay: syncprov
olcSpSessionLog: 100
###############################
ldapadd -Y EXTERNAL -H ldapi:/// -f sync_provider.ldif
slave配置
##########################################################
vi sync_consumer.ldif
##############################
dn: olcDatabase={2}mdb,cn=config
changetype: modify
add: olcSyncRepl
olcSyncRepl: rid=001
provider=ldap://xxxxxxxx.142:389/
bindmethod=simple
binddn="cn=admin,dc=gtk,dc=com"
credentials=xxxxmmxxxx
searchbase="dc=gtk,dc=com"
scope=sub
schemachecking=on
type=refreshAndPersist
retry="30 5 300 3"
interval=00:00:05:00
################################
ldapadd -Y EXTERNAL -H ldapi:/// -f sync_consumer.ldif
其它memberof配置，两个节点都执行
#############################
vi memberof_conf.ldif
#############################
dn: cn=module,cn=config
cn: module
objectClass: olcModuleList
objectClass: top
olcModulePath: /usr/lib64/openldap
olcModuleLoad: memberof.la
############################
ldapadd -Y EXTERNAL -H ldapi:/// -f memberof_conf.ldif
vi memberOfOverlay.ldif
#########################
dn: olcOverlay=memberof,olcDatabase={2}mdb,cn=config
changetype: add
objectClass: olcOverlayConfig
objectClass: olcMemberOf
olcOverlay: {0}memberof
########################
ldapadd -Y EXTERNAL -H ldapi:/// -f memberOfOverlay.ldif
测试，master节点执行
#############################
#创建 demo 用户
#设置密码
slappasswd -h {md5} -s "demo"
{MD5}/gHOKn+6yPr67XyYKgTiKQ==
查询当前最大 uidNumber，然后递增：
ldapsearch -x -b "dc=example,dc=com" "uidNumber=*" uidNumber | grep "^uidNumber:" | sort -n | tail -1
vi create_user_demo.ldif
#################################3
dn: uid=demo,ou=People,dc=gtk,dc=com
objectClass: inetOrgPerson
objectClass: posixAccount
objectClass: shadowAccount
cn: demo
sn: demo
userPassword: {MD5}/gHOKn+6yPr67XyYKgTiKQ==
loginShell: /bin/bash
uidNumber: 10001
gidNumber: 10001
homeDirectory: /home/demo
dn: cn=demo,ou=Group,dc=gtk,dc=com
objectClass: posixGroup
cn: demo
gidNumber: 10001
memberUid: uid=demo,ou=People,dc=gtk,dc=com
########################################
ldapadd -x -D cn=admin,dc=gtk,dc=com -wxxxxmmxxxx -f create_user_demo.ldif
验证创建用户，两个主机验证同步
########################################
ldapsearch -x -b 'ou=People,dc=gtk,dc=com'
ldapsearch -x
ldapsearch -H ldap://xxxxxxxx.142:389 -x -D "cn=admin,dc=gtk,dc=com" -w 'xxxxmmxxxx' -b "dc=gtk,dc=com" -s base
#ldap客户端部署，所有的主机执行
########################################
yum localinstall -y /home/openldap*
-rw-r--r-- 1 root root 262004 Jun 12 14:55 openldap-2.6.8-4.el9.0.1.x86_64.rpm
-rw-r--r-- 1 root root 178205 Jun 12 14:53 openldap-clients-2.6.8-4.el9.0.1.x86_64.rpm
同步命令，所有主机
########################################
authconfig --enableldap --enableldapauth --ldapserver=xxxxxxxx.142:389 --ldapbasedn='dc=gtk,dc=com' --enablemkhomedir --update
非server主机，验证
########################################
ldapsearch -x -b 'ou=People,dc=@@@,dc=com'
ldapwhoami -x -D "uid=ocdp,ou=People,dc=@@@,dc=com" -w '密码'
ldapsearch -x -b 'ou=People,dc=@@@,dc=com'
ldapsearch -x
#sssd部署
########################################
yum install sssd -y
yum install authselect-compat-1.2.6-2.el9.x86_64 -y
authconfig --update --enablesssd
cat > /etc/sssd/sssd.conf << EOF
sssd
config_file_version = 2
services = nss, pam
domains = LDAP
nss
pam
domain/LDAP
id_provider = ldap
auth_provider = ldap
ldap_schema = rfc2307
ldap_uri = ldap://xxxxxxxx.142:389,ldap://xxxxxxxx.143:389
ldap_search_base = dc=gtk,dc=com
ldap禁用匿名绑定必填
ldap_default_bind_dn = cn=admin,dc=gtk,dc=com
ldap_default_authtok_type = password
ldap_default_authtok = xxxxmmxxxx
EOF
chmod 600 /etc/sssd/sssd.conf
rm -rf /etc/sssd/conf.d/
systemctl restart sssd
systemctl enable --now sssd
systemctl status sssd
清理缓存：sss_cache -E
验证用户：id demo
SOLR安装
######################################################################################################
https://mirrors.aliyun.com/apache/solr/solr/10.0.0/?spm=a2c6h.25603864.0.0.51965cd5suq9ws
https://mirrors.aliyun.com/apache/ranger/2.8.0/?spm=a2c6h.25603864.0.0.6043362faGlrpv
tar -zxvf ranger-2.8.0-admin.tar.gz -C /data01
tar -zxvf solr-8.11.1.tgz -C /data01
编辑/data01/ranger-2.8.0-admin/contrib/solr_for_audit_setup/install.properties
#配置JAVA路径
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.392.b08-3.el9.x86_64/jre
#审计日志保存的最大天数，默认为90天
MAX_AUDIT_RETENTION_DAYS=90
#联网下载，默认为false
SOLR_INSTALL=false
#solr线上下载路径（如果SOLR_INSTALL=true，则需要配置SOLR_DOWNLOAD_URL）
#SOLR_DOWNLOAD_URL=http://archive.apache.org/dist/lucene/solr/8.3.0/solr-8.3.0.tgz
solr安装目录
SOLR_INSTALL_FOLDER=/data01/solr-8.11.1
solr对接ranger的服务
SOLR_RANGER_HOME=/data01/solr-8.11.1/ranger_audit_server
solr连接ranger的端口
SOLR_RANGER_PORT=6083
solr部署模式
SOLR_DEPLOYMENT=standalone
solr数据存储目录
SOLR_RANGER_DATA_FOLDER=/data01/solr-8.11.1/ranger_audit_server/data
SOLR_LOG_FOLDER=/data01/solr-8.11.1/ranger_audits
solr单机部署，故为空
SOLR_ZK=
将 Solr 安装目录的所有权赋予该用户
chown -R solr:solr /data01/solr-8.11.1
初始化solr安装脚本
cd /data01/ranger-2.8.0-admin/contrib/solr_for_audit_setup/
./setup.sh
启动单机版solr
cd /data01/solr-8.11.1/ranger_audit_server/scripts/
./start_solr.sh
登陆网页查看：http://xxxxxxxx.142:6083/
mysql -hxxxxxxx.142 -P16603 -uroot -pxxxxmmxxxx==
CREATE DATABASE ranger DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
- 创建用户并设置密码
SET GLOBAL validate_password.policy = LOW;
CREATE USER 'ranger'@'%' IDENTIFIED BY 'xxxxmmxxxx';
ALTER USER 'ranger'@'%' IDENTIFIED WITH mysql_native_password BY 'xxxxmmxxxx';
FLUSH PRIVILEGES;
安装Ranger
######################################################################################################
1. 创建组 groupadd ranger
2. 创建用户 useradd -m -s /bin/bash -g ranger ranger
3. 设置密码 echo "ranger:xxxxmmxxxx" | sudo chpasswd
编辑/data01/ranger-2.8.0-admin/install.properties
setup_mode=SeparateDBA
#mysql驱动
SQL_CONNECTOR_JAR=/opt/doris/jdbc_drivers/mysql-connector-java-8.0.25.jar
#mysql的主机名和root用户的用户名密码
db_root_user=root
db_root_password=xxxxmmxxxx==
db_host=xxxxxxxx.142
db_port=16603
#ranger需要的数据库名和用户信息，需要和之前创建的信息要一一对应
db_name=ranger
db_user=ranger
db_password=xxxxmmxxxx
db_url=jdbc:mysql://xxxxxxxx.142:16603/ranger?useSSL=false&allowPublicKeyRetrieval=true
db_root_url=jdbc:mysql://xxxxxxxx.142:16603/ranger?useSSL=false&allowPublicKeyRetrieval=true
db_driver=com.mysql.cj.jdbc.Driver
#其他ranger admin需要的用户密码（最少8个字符）
rangerAdmin_password=xxxxmmxxxx
rangerTagsync_password=xxxxmmxxxx
rangerUsersync_password=xxxxmmxxxx
keyadmin_password=xxxxmmxxxx
#ranger存储审计日志的路径和url，默认为solr
audit_store=solr
audit_solr_urls=http://xxxxxxxx.142:6083/solr/ranger_audits
#策略管理器的url,rangeradmin安装在哪台机器，主机名就为对应的主机名
policymgr_external_url=http://xxxxxxxx.142:6080
#启动ranger admin进程的linux用户信息
unix_user=ranger
unix_user_pwd=xxxxmmxxxx
unix_group=ranger
初始化ranger-admin脚本
/data01/ranger-2.8.0-admin/setup.sh
/data01/ranger-2.8.0-admin/set_globals.sh
修改时区
vi /data01/ranger-2.8.0-admin/ews/ranger-admin-services.sh
-Duser.timezone=Asia/Shanghai
启动ranger-admin
ranger-admin start
登陆网页查看：http://xxxxxxxx.142:6080 admin : xxxxmmxxxx
######################################################################################################
https://mirrors.aliyun.com/apache/ranger/2.8.0/?spm=a2c6h.25603864.0.0.6043362faGlrpv
tar -zxvf ranger-2.8.0-usersync.tar.gz -C /data01
编辑/data01/ranger-2.8.0-usersync/install.properties
#rangeradmin的url
POLICY_MGR_URL =http://xxxxxxxx.142:6080
#同步间隔时间，单位(分钟)
SYNC_INTERVAL = 1
#运行此进程的linux用户
unix_user=ranger
unix_group=ranger
#rangerUserSync的用户密码，参考rangeradmin中install.properties的配置
rangerUsersync_password=xxxxmmxxxx
初始化ranger-usersync脚本
/data01/ranger-2.8.0-usersync/setup.sh
启动ranger-usersync
ranger-usersync start
登录ranger admin的web-UI界面，查看用户信息
Ranger与Ldap集成
######################################################################################################
ldapsearch -H ldap://xxxxxxxx.142:389 -x -D "cn=admin,dc=gtk,dc=com" -w 'xxxxmmxxxx' -b "dc=gtk,dc=com" -s base
修改设置/data01/ranger-2.8.0-usersync/install.properties
SYNC_SOURCE = ldap
SYNC_LDAP_URL = ldap://xxxxxxxx.142:389
SYNC_LDAP_BIND_DN = cn=admin,dc=gtk,dc=com
SYNC_LDAP_SEARCH_BASE = dc=gtk,dc=com
SYNC_LDAP_USER_SEARCH_BASE = ou=People,dc=gtk,dc=com
SYNC_LDAP_USER_OBJECT_CLASS = posixAccount
SYNC_GROUP_SEARCH_ENABLED=true
SYNC_GROUP_USER_MAP_SYNC_ENABLED=true
SYNC_GROUP_SEARCH_BASE=ou=Group,dc=gtk,dc=com
SYNC_GROUP_OBJECT_CLASS=posixGroup
SYNC_GROUP_MEMBER_ATTRIBUTE_NAME=memberUid
执行/data01/ranger-2.8.0-usersync/setup.sh
ranger-usersync restart
Ranger与Doris集成
######################################################################################################
Ranger安装Doris插件
1、下载 ranger-doris-plugin-3.0.0-SNAPSHOT.jar mysql-connector-java-8.0.25.jar ranger-servicedef-doris.json
2、将下载好的文件放到 Ranger 服务的 ranger-plugins/doris 目录下
mkdir /data01/ranger-2.8.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/doris
chmod -R 777 /data01/ranger-2.8.0-admin/ews/webapp/WEB-INF/classes/ranger-plugins/doris
3、重启 Ranger 服务。
4、下载 ranger-servicedef-doris.json
执行以下命令上传定义文件到 Ranger 服务：
curl -u admin:xxxxmmxxxx -X POST
-H "Accept: application/json"
-H "Content-Type: application/json"
http://xxxxxxxx.142:6080/service/plugins/definitions
-d@ranger-servicedef-doris.json
5、Ranger页面配置接入Doris源：
xxxxxxxx.121 root xxxxmmxxxx jdbc:mysql://xxxxxxxx.121:9030
xxxxxxxx.142/143/144 xxxxxxxx.96 root xxxxmmxxxx== jdbc:mysql://xxxxxxxx.96:9030
/opt/sdb/fe/conf/fe.conf
Doris与Ldap集成
######################################################################################################
在 /opt/doris/fe/conf/fe.conf 中设置认证方式：
authentication_type=ldap
在 /opt/doris/fe/conf/ldap.conf 中配置 LDAP 服务的连接信息：
ldap_authentication_enabled = true
ldap_host = xxxxxxxx.142
ldap_port = 389
ldap_admin_name = cn=admin,dc=gtk,dc=com
ldap_user_basedn = ou=people,dc=gtk,dc=com
ldap_user_filter = (&(uid={login}))
ldap_group_basedn = ou=group,dc=gtk,dc=com
启动 fe 后，使用 root 或 admin 账号登录 Doris，设置 LDAP 管理员密码：
set ldap_admin_password = password('xxxxmmxxxx');
LDAP 认证要求客户端以明文方式发送密码，因此需要启用明文验证插件。
MySQL Client启用明文验证插件，设置环境变量（永久生效）
echo "export LIBMYSQL_ENABLE_CLEARTEXT_PLUGIN=1" >> ~/.bash_profile && source ~/.bash_profile
登录时添加参数（单次生效）mysql -hxxxxxxx.121 -P9030 -uroot -pxxxxmmxxxx --enable-cleartext-plugin
mysql -hxxxxxxx.121 -P9030 -udemo -pdemo --enable-cleartext-plugin
JDBC Client需要Doris开启SSL
fe.conf 中添加 enable_ssl=true
jdbcUrl = "jdbc:mysql://xxxxxxxx.121:9030/?useSSL=true&sslMode=REQUIRED"
######################################################################################################
更改Doris配置使用Ranger权限管理
1、在 fe/conf/fe.conf 文件中配置鉴权方式为 ranger :
access_controller_type=ranger-doris
2、在所有 FE 的 conf 目录创建 ranger-doris-security.xml 文件，内容如下：
<?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> ranger.plugin.doris.policy.cache.dir /data01/ranger/cache/ ranger.plugin.doris.policy.pollIntervalMs 30000 ranger.plugin.doris.policy.rest.client.connection.timeoutMs 60000 ranger.plugin.doris.policy.rest.client.read.timeoutMs 60000 ranger.plugin.doris.policy.rest.url http://xxxxxxxx.142:6080 ranger.plugin.doris.policy.source.impl org.apache.ranger.admin.client.RangerAdminRESTClient ranger.plugin.doris.service.name WFMPP
其中需要将 ranger.plugin.doris.policy.cache.dir 和 ranger.plugin.doris.policy.rest.url 改为实际值。
3、重启集群
######################################################################################################
权限示例
在 Ldap中 中创建 user1，在 Doris 与 Ranger 中均有user1.