Centos8下 K8S 安装配置Mysql8 多主MGR

192.168.37.128 k8s1

192.168.37.130 k8s2

192.168.37.131 k8s3

192.168.37.129 k8s4

拉取MySQL最新的镜像
docker pull mysql

cat namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
 name: mysqldb

cat service.yaml
apiVersion: v1
kind: Service
metadata:
name: mgrtest
 namespace: mysqldb
spec:
clusterIP: None
 selector:
  name: mysql-mgr  
ports:
 - name: foo
   port: 3306
   targetPort: 3306

节点1：
cat mysql-mgr-cnf-0.yaml
apiVersion: v1
kind: ConfigMap
metadata:
 name: mysql-mgr-0-cnf
namespace: mysqldb
data:
 mysql-mgr-0.cnf: |
  [mysqld]
   port = 3306
   character_set_server = utf8
   server_id = 1
   log_bin = mysql-bin
  relay-log = relay-bin
   #back_log = 500
  #max_connections = 3000
   #wait_timeout = 5022397
  interactive_timeout = 5022397
   max_connect_errors = 1000
  relay-log-recovery=1
   #max_allowed_packet = 32M
  sort_buffer_size = 4M
   read_buffer_size = 4M
  join_buffer_size = 8M
   thread_cache_size = 64
  #tmp_table_size = 256M
   log_slave_updates=1
  long_query_time = 1
   slow_query_log = 1
  skip-name-resolve
   innodb_buffer_pool_size=700M
  #innodb_data_file_path = ibdata1:1024M:autoextend
  innodb_flush_log_at_trx_commit=1
   innodb_log_buffer_size = 16M
   innodb_log_file_size = 256M
  innodb_log_files_in_group = 2
   innodb_max_dirty_pages_pct = 50
   sync_binlog=1
   master_info_repository=TABLE
  relay_log_info_repository=TABLE
   log_timestamps=SYSTEM
  gtid_mode = ON
   enforce_gtid_consistency = ON
  master_info_repository = TABLE
   relay_log_info_repository = TABLE
   log_slave_updates = ON
   binlog_checksum = NONE
   log_slave_updates = ON
  slave_parallel_type=LOGICAL_CLOCK
  slave_parallel_workers=8
   slave-preserve-commit-order=on
  #group_replication_compression_threshold=200000
  plugin_load_add=group_replication.so
  transaction_write_set_extraction = XXHASH64
  loose-group_replication_group_name="01e5fb97-be64-41f7-bafd-3afc7a6ab555"
  loose-group_replication_start_on_boot=off
  loose-group_replication_local_address="k8s2.mgrtest.mysqldb.svc.cluster.local:13306"
  loose-group_replication_group_seeds="k8s2.mgrtest.mysqldb.svc.cluster.local:13306,k8s3.mgrtest.mysqldb.svc.cluster.local:13306,k8s4.mgrtest.mysqldb.svc.cluster.local:13306"
  loose-group_replication_bootstrap_group = off
  loose-group_replication_ip_whitelist=10.96.0.0/16,172.16.0.0/16,192.168.0.0/16,172.17.0.0/16,127.0.0.1/8
  report_host = k8s2.mgrtest.mysqldb.svc.cluster.local
  [mysqldump]
   quick
   max_allowed_packet = 32M

cat mysql-mgr-cnf-1.yaml  
apiVersion: v1
kind: ConfigMap
metadata:
 name: mysql-mgr-1-cnf
namespace: mysqldb
data:
 mysql-mgr-1.cnf: |
  [mysqld]
   port = 3306
   character_set_server = utf8
   server_id = 2
   log_bin = mysql-bin
  relay-log = relay-bin
   #back_log = 500
  #max_connections = 3000
   #wait_timeout = 5022397
  interactive_timeout = 5022397
   max_connect_errors = 1000
  relay-log-recovery=1
   #max_allowed_packet = 32M
  sort_buffer_size = 4M
   read_buffer_size = 4M
  join_buffer_size = 8M
   thread_cache_size = 64
  #tmp_table_size = 256M
   log_slave_updates=1
  long_query_time = 1
   slow_query_log = 1
  skip-name-resolve
   innodb_buffer_pool_size=700M
  #innodb_data_file_path = ibdata1:1024M:autoextend
  innodb_flush_log_at_trx_commit=1
   innodb_log_buffer_size = 16M
   innodb_log_file_size = 256M
  innodb_log_files_in_group = 2
   innodb_max_dirty_pages_pct = 50
   sync_binlog=1
   master_info_repository=TABLE
  relay_log_info_repository=TABLE
   log_timestamps=SYSTEM
  gtid_mode = ON
   enforce_gtid_consistency = ON
  master_info_repository = TABLE
   relay_log_info_repository = TABLE
   log_slave_updates = ON
   binlog_checksum = NONE
   log_slave_updates = ON
  slave_parallel_type=LOGICAL_CLOCK
  slave_parallel_workers=8
   slave-preserve-commit-order=on
  #group_replication_compression_threshold=200000
  transaction_write_set_extraction = XXHASH64
  loose-group_replication_group_name="01e5fb97-be64-41f7-bafd-3afc7a6ab555"
  loose-group_replication_start_on_boot=off
  loose-group_replication_local_address="k8s3.mgrtest.mysqldb.svc.cluster.local:13306"
  loose-group_replication_group_seeds="k8s2.mgrtest.mysqldb.svc.cluster.local:13306,k8s3.mgrtest.mysqldb.svc.cluster.local:13306,k8s4.mgrtest.mysqldb.svc.cluster.local:13306"
  loose-group_replication_bootstrap_group = off
  loose-group_replication_ip_whitelist=10.96.0.0/16,172.16.0.0/16,192.168.0.0/16,172.17.0.0/16,127.0.0.1/8
  report_host = k8s3.mgrtest.mysqldb.svc.cluster.local
  [mysqldump]
   quick
   max_allowed_packet = 32M

cat mysql-mgr-cnf-2.yaml  
apiVersion: v1
kind: ConfigMap
metadata:
 name: mysql-mgr-2-cnf
namespace: mysqldb
data:
 mysql-mgr-2.cnf: |
  [mysqld]
   port = 3306
   character_set_server = utf8
   server_id = 3
   log_bin = mysql-bin
  relay-log = relay-bin
   #back_log = 500
  #max_connections = 3000
   #wait_timeout = 5022397
  interactive_timeout = 5022397
   max_connect_errors = 1000
  relay-log-recovery=1
   #max_allowed_packet = 32M
  sort_buffer_size = 4M
   read_buffer_size = 4M
  join_buffer_size = 8M
   thread_cache_size = 64
  #tmp_table_size = 256M
   log_slave_updates=1
  long_query_time = 1
   slow_query_log = 1
  skip-name-resolve
   innodb_buffer_pool_size=700M
  #innodb_data_file_path = ibdata1:1024M:autoextend
  innodb_flush_log_at_trx_commit=1
   innodb_log_buffer_size = 16M
   innodb_log_file_size = 256M
  innodb_log_files_in_group = 2
   innodb_max_dirty_pages_pct = 50
   sync_binlog=1
   master_info_repository=TABLE
  relay_log_info_repository=TABLE
   log_timestamps=SYSTEM
  gtid_mode = ON
   enforce_gtid_consistency = ON
  master_info_repository = TABLE
   relay_log_info_repository = TABLE
   log_slave_updates = ON
   binlog_checksum = NONE
   log_slave_updates = ON
  slave_parallel_type=LOGICAL_CLOCK
  slave_parallel_workers=8
   slave-preserve-commit-order=on
  #group_replication_compression_threshold=200000
  transaction_write_set_extraction = XXHASH64
  loose-group_replication_group_name="01e5fb97-be64-41f7-bafd-3afc7a6ab555"
  loose-group_replication_start_on_boot=off
  loose-group_replication_local_address="k8s4.mgrtest.mysqldb.svc.cluster.local:13306"
  loose-group_replication_group_seeds="k8s2.mgrtest.mysqldb.svc.cluster.local:13306,k8s3.mgrtest.mysqldb.svc.cluster.local:13306,k8s4.mgrtest.mysqldb.svc.cluster.local:13306"
  loose-group_replication_bootstrap_group = off
  loose-group_replication_ip_whitelist=10.96.0.0/16,172.16.0.0/16,192.168.0.0/16,172.17.0.0/16,127.0.0.1/8
  report_host = k8s4.mgrtest.mysqldb.svc.cluster.local
  [mysqldump]
   quick
   max_allowed_packet = 32M

cat node1_pod.yaml
apiVersion: v1
kind: Pod
metadata:
name: k8s2
 namespace: mysqldb
 labels:
  name: mysql-mgr
spec:
 nodeName: k8s2
 hostname: k8s2
 subdomain: mgrtest
 containers:
 - image: mysql:latest
   name: mysql-mgr-0
   imagePullPolicy: IfNotPresent    
   command: [ "/bin/bash", "-ce", "cd /usr/bin && mysqld --initialize-insecure --user=mysql && mysqld_safe && tail -f /dev/null" ]
   ports:
     - containerPort: 3306
  volumeMounts:
   - name: tz-config
     mountPath: /etc/localtime
   - name: mysql-data
     mountPath: /data/mysql/data/
   - name: mysql-config
    mountPath: /etc/my.cnf
     subPath: my.cnf
 volumes:
  - name: tz-config
     hostPath:
       path: /etc/localtime
   - name: mysql-data
     hostPath:
      path: /data/mysql/data/
   - name: mysql-config
    configMap:
       name: mysql-mgr-0-cnf
      items:
         - key: mysql-mgr-0.cnf
          path: my.cnf

cat node2_pod.yaml  
apiVersion: v1
kind: Pod
metadata:
name: k8s3
 namespace: mysqldb
 labels:
  name: mysql-mgr
spec:
 nodeName: k8s3
 hostname: k8s3
 subdomain: mgrtest
 containers:
 - image: mysql:latest
   name: mysql-mgr-1
   imagePullPolicy: IfNotPresent
   command: [ "/bin/bash", "-ce", "cd /usr/bin && mysqld --initialize-insecure --user=mysql && mysqld_safe && tail -f /dev/null" ]
   ports:
     - containerPort: 3306
  volumeMounts:
   - name: tz-config
     mountPath: /etc/localtime
   - name: mysql-data
     mountPath: /data/mysql/data
   - name: mysql-config
    mountPath: /etc/my.cnf
     subPath: my.cnf
 volumes:
  - name: tz-config
     hostPath:
       path: /etc/localtime
   - name: mysql-data
     hostPath:
      path: /data/mysql/data/
   - name: mysql-config
    configMap:
       name: mysql-mgr-1-cnf
      items:
         - key: mysql-mgr-1.cnf
          path: my.cnf

cat node3_pod.yaml  
apiVersion: v1
kind: Pod
metadata:
name: k8s4
 namespace: mysqldb
 labels:
  name: mysql-mgr
spec:
 nodeName: k8s4
 hostname: k8s4
 subdomain: mgrtest
 containers:
 - image: mysql:latest
   name: mysql-mgr-2
   imagePullPolicy: IfNotPresent
   command: [ "/bin/bash", "-ce", "cd /usr/bin && mysqld --initialize-insecure --user=mysql && mysqld_safe && tail -f /dev/null" ]
   ports:
     - containerPort: 3306
  volumeMounts:
   - name: tz-config
     mountPath: /etc/localtime
   - name: mysql-data
     mountPath: /data/mysql/data
   - name: mysql-config
    mountPath: /etc/my.cnf
     subPath: my.cnf
 volumes:
  - name: tz-config
     hostPath:
       path: /etc/localtime
   - name: mysql-data
     hostPath:
      path: /data/mysql/data/
   - name: mysql-config
    configMap:
       name: mysql-mgr-2-cnf
      items:
         - key: mysql-mgr-2.cnf
          path: my.cnf

kubectl create -f namespace.yaml
kubectl create -f service.yaml
kubectl create -f mysql-mgr-cnf-0.yaml
kubectl create -f mysql-mgr-cnf-1.yaml
kubectl create -f mysql-mgr-cnf-2.yaml
kubectl create -f node1_pod.yaml
kubectl create -f node2_pod.yaml
kubectl create -f node3_pod.yaml

1.查看pods状态(看到node节点的mysql已经在运行)
# kubectl get pods -n mysqldb -o wide

NAME  READY   STATUS    RESTARTS   AGE    IP               NODE   NOMINATED NODE   READINESS GATES

k8s2  1/1     Running   0          112s   172.16.109.65    k8s2            

k8s3  1/1     Running   0          112s   172.16.219.1     k8s3            

k8s4  1/1     Running   0          111s   172.16.106.129   k8s4            

kubectl exec -it k8s2 -n mysqldb /bin/bash

alter user root@localhost identified by Mysql123!@#;
set sql_log_bin=0;
create user rpl_user@% identified by Rpl_pass@123;
grant replication slave on *.* to rpl_user@%;
flush privileges;
set sql_log_bin=1;
change master to master_user=rpl_user,master_password=Rpl_pass@123 for channel group_replication_recovery;
install PLUGIN group_replication SONAME group_replication.so;

reset master;
set global group_replication_single_primary_mode=FALSE;
set global group_replication_enforce_update_everywhere_checks=TRUE;
set global group_replication_bootstrap_group=ON;
start group_replication;
set global group_replication_bootstrap_group=OFF;
select * from performance_schema.replication_group_members;

1节点执行完后可以看到状态图

kubectl exec -it k8s3 -n mysqldb /bin/bash

set sql_log_bin=0;
create user rpl_user@% identified by Rpl_pass@123;
grant replication slave on *.* to rpl_user@%;
flush privileges;
set sql_log_bin=1;
change master to master_user=rpl_user,master_password=Rpl_pass@123 for channel group_replication_recovery;
install PLUGIN group_replication SONAME group_replication.so;

reset master;
set global group_replication_single_primary_mode=FALSE;
set global group_replication_enforce_update_everywhere_checks=TRUE;
set global group_replication_recovery_get_public_key=on;
start group_replication;
select * from performance_schema.replication_group_members;

kubectl exec -it k8s4 -n mysqldb /bin/bash

set sql_log_bin=0;
create user rpl_user@% identified by Rpl_pass@123;
grant replication slave on *.* to rpl_user@%;
flush privileges;
set sql_log_bin=1;
change master to master_user=rpl_user,master_password=Rpl_pass@123 for channel group_replication_recovery;
install PLUGIN group_replication SONAME group_replication.so;

reset master;
set global group_replication_single_primary_mode=FALSE;
set global group_replication_enforce_update_everywhere_checks=TRUE;
set global group_replication_recovery_get_public_key=on;
start group_replication;
select * from performance_schema.replication_group_members;

kubectl create -f namespace.yaml

Error from server (AlreadyExists): error when creating "namespace.yaml": object is being deleted: namespaces "mysqldb" already exists

查看状态也是Terminating的

1.获取mysqldb的namespace下json格式文件

# kubectl get namespace mysqldb -o json > /tmp/mysqldb.json

2.删除掉spec里面指定的内容

3.新开一个终端窗口执行kubectl proxy --port=8081,监听本地

# kubectl proxy --port=8081

Starting to serve on 127.0.0.1:8081

4.执行

# curl -k -H "Content-Type: application/json" -X PUT --data-binary @mysqldb.json http://127.0.0.1:8081/api/v1/namespaces/mysqldb/finalize

注意以上替换mysqldb名字

5.执行完后,查看namespace,mysqldb的Terminating状态已经没有了