GlusterFS(中)

4.常用选项

1)限制IP连接

设置auth.reject选项

[root@node2 ~]# gluster volume set vol_distributed_dispersed auth.reject 10.0.0.11
volume set: success

测试

[root@node1 ~]# mkdir /mnt/distributed_dispersed
[root@node1 ~]# mount -t glusterfs node2:/vol_distributed_dispersed /mnt/distributed_dispersed/
Mount failed. Check the log file  for more details.

恢复auth.reject选项

[root@node2 ~]# gluster volume reset vol_distributed_dispersed auth.reject
volume reset: success: reset volume successful

2)ACL

挂载的时候添加acl参数

[root@node1 ~]# mount -t glusterfs -o acl node2:/vol_distributed_dispersed /mnt/distributed_dispersed
[root@node1 ~]# useradd acl_test
[root@node1 ~]# set -u m:acl_test:rwx /mnt/distributed_dispersed/

3)配额

创建要配置配额的文件夹

[root@node1 ~]# mkdir /mnt/distributed_dispersed/quota

配置卷的配额

[root@node2 ~]# gluster volume quota vol_distributed_dispersed enable
volume quota : success
[root@node2 ~]# gluster volume quota vol_distributed_dispersed limit-usage /quota 10MB
volume quota : success
[root@node2 ~]# gluster volume quota vol_distributed_dispersed default-soft-limit 90%
volume quota : success
[root@node2 ~]# gluster volume quota vol_distributed_dispersed list
                  Path                   Hard-limit  Soft-limit      Used  Available  Soft-limit exceeded? Hard-limit exceeded?
-------------------------------------------------------------------------------------------------------------------------------
/quota                                    10.0MB     90%(9.0MB)   0Bytes  10.0MB              No                   No

4)扩容和缩小卷

扩容卷

[root@node2 ~]# gluster v status vol_distributed
Status of volume: vol_distributed
Gluster process                             TCP Port  RDMA Port  Online  Pid
------------------------------------------------------------------------------
Brick node2:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick1/distributed     49160     0          Y       2569 
Brick node3:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick2/distributed     49160     0          Y       2335 
Brick node4:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick3/distributed     49153     0          Y       2302 

Task Status of Volume vol_distributed
------------------------------------------------------------------------------
There are no active volume tasks
# 增加brick
[root@node2 ~]# gluster volume add-brick vol_distributed node4:/data/brick4/distributed_ex
volume add-brick: success
# 平衡数据
[root@node2 ~]# gluster volume rebalance vol_distributed start
volume rebalance: vol_distributed: success: Rebalance on vol_distributed has been started successfully. Use rebalance status command to check status of the rebalance process.
ID: 41a5f864-999b-4b95-b353-5d5f0ffd5a5f
[root@node2 ~]# gluster volume rebalance vol_distributed status
                                    Node Rebalanced-files          size       scanned      failures       skipped               status  run time in h:m:s
                               ---------      -----------   -----------   -----------   -----------   -----------         ------------     --------------
                                   node3                0        0Bytes             0             0             0            completed        0:00:00
                                   node4                1       14Bytes             1             0             0            completed        0:00:00
                               localhost                0        0Bytes             0             0             0            completed        0:00:00
volume rebalance: vol_distributed: success
[root@node2 ~]# gluster v status vol_distributed
Status of volume: vol_distributed
Gluster process                             TCP Port  RDMA Port  Online  Pid
------------------------------------------------------------------------------
Brick node2:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick1/distributed     49160     0          Y       2569 
Brick node3:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick2/distributed     49160     0          Y       2335 
Brick node4:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick3/distributed     49153     0          Y       2302 
Brick node4:/data/brick4/distributed_ex     49160     0          Y       2982 

Task Status of Volume vol_distributed
------------------------------------------------------------------------------
Task                 : Rebalance           
ID                   : 41a5f864-999b-4b95-b353-5d5f0ffd5a5f
Status               : completed

缩小卷

[root@node2 ~]# gluster volume remove-brick vol_distributed node4:/data/brick4/distributed_ex force
Remove-brick force will not migrate files from the removed bricks, so they will no longer be available on the volume.
Do you want to continue? (y/n) y
volume remove-brick commit force: success
[root@node2 ~]# gluster v status vol_distributed
Status of volume: vol_distributed
Gluster process                             TCP Port  RDMA Port  Online  Pid
------------------------------------------------------------------------------
Brick node2:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick1/distributed     49160     0          Y       2569 
Brick node3:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick2/distributed     49160     0          Y       2335 
Brick node4:/run/gluster/snaps/82859b1c0c15
48ae8ac3b974679170f6/brick3/distributed     49153     0          Y       2302 

Task Status of Volume vol_distributed
------------------------------------------------------------------------------
There are no active volume tasks

5.快照

1)前提设置

所有节点配置

[root@node2 ~]# vim /etc/glusterfs/glusterd.vol
[root@node2 ~]# cat /etc/glusterfs/glusterd.vol
...略
    option max-port  60999
    option rpc-auth-allow-insecure on  # 增加
end-volume
[root@node2 ~]# systemctl restart glusterd

设置卷

[root@node2 ~]# gluster v set vol_distributed server.allow-insecure on
volume set: success

2)创建快照

[root@node2 ~]# gluster snapshot create vol_distributed_snap vol_distributed no-timestamp
snapshot create: success: Snap vol_distributed_snap created successfully
[root@node2 ~]# gluster snapshot list
vol_distributed_snap
[root@node2 ~]# gluster snapshot info vol_distributed_snap
Snapshot                  : vol_distributed_snap
Snap UUID                 : 85b4121a-c2ec-4447-af51-327cb42c24f7
Created                   : 2020-07-13 06:29:10
Snap Volumes:

  Snap Volume Name          : 82859b1c0c1548ae8ac3b974679170f6
  Origin Volume name        : vol_distributed
  Snaps taken for vol_distributed      : 1
  Snaps available for vol_distributed  : 255
  Status                    : Stopped

[root@node2 ~]# gluster snapshot status vol_distributed_snap

Snap Name : vol_distributed_snap
Snap UUID : 85b4121a-c2ec-4447-af51-327cb42c24f7

  Brick Path        :   node2:/run/gluster/snaps/82859b1c0c1548ae8ac3b974679170f6/brick1/distributed
  Volume Group      :   N/A (Deactivated Snapshot)
  Brick Running     :   No
  Brick PID         :   N/A
  Data Percentage   :   N/A
  LV Size           :   N/A

  Brick Path        :   node3:/run/gluster/snaps/82859b1c0c1548ae8ac3b974679170f6/brick2/distributed
  Volume Group      :   N/A (Deactivated Snapshot)
  Brick Running     :   No
  Brick PID         :   N/A
  Data Percentage   :   N/A
  LV Size           :   N/A

  Brick Path        :   node4:/run/gluster/snaps/82859b1c0c1548ae8ac3b974679170f6/brick3/distributed
  Volume Group      :   N/A (Deactivated Snapshot)
  Brick Running     :   No
  Brick PID         :   N/A
  Data Percentage   :   N/A
  LV Size           :   N/A

no-timestamp 表示不带时间戳

3)对快照进行限制

[root@node2 ~]# gluster snapshot config vol_distributed snap-max-hard-limit 100
Changing snapshot-max-hard-limit will limit the creation of new snapshots if they exceed the new limit.
Do you want to continue? (y/n) y
snapshot config: snap-max-hard-limit for vol_distributed set successfully

4)激活快照

[root@node2 ~]# gluster snapshot activate vol_distributed_snap
Snapshot activate: vol_distributed_snap: Snap activated successfully
[root@node2 ~]# gluster snapshot status vol_distributed_snap

Snap Name : vol_distributed_snap
Snap UUID : 85b4121a-c2ec-4447-af51-327cb42c24f7

  Brick Path        :   node2:/run/gluster/snaps/82859b1c0c1548ae8ac3b974679170f6/brick1/distributed
  Volume Group      :   vol
  Brick Running     :   Yes
  Brick PID         :   2269
  Data Percentage   :   0.22
  LV Size           :   10.00g


  Brick Path        :   node3:/run/gluster/snaps/82859b1c0c1548ae8ac3b974679170f6/brick2/distributed
  Volume Group      :   vol
  Brick Running     :   Yes
  Brick PID         :   2089
  Data Percentage   :   0.22
  LV Size           :   10.00g


  Brick Path        :   node4:/run/gluster/snaps/82859b1c0c1548ae8ac3b974679170f6/brick3/distributed
  Volume Group      :   vol
  Brick Running     :   Yes
  Brick PID         :   2078
  Data Percentage   :   0.22
  LV Size           :   10.00g

5)访问快照

跟卷一样挂载

[root@node1 ~]# mkdir /mnt/distributed_snap
[root@node1 ~]# mount -t glusterfs node2:/snaps/vol_distributed_snap/vol_distributed /mnt/distributed_snap
[root@node1 ~]# df -Th
Filesystem                                        Type            Size  Used Avail Use% Mounted on
...略
node2:/snaps/vol_distributed_snap/vol_distributed fuse.glusterfs   30G  437M   30G   2% /mnt/distributed_snap
[root@node1 ~]# cat /mnt/distributed_snap/welcome.txt 
Here is node1

6)恢复快照

删除数据

[root@node1 ~]# rm -f /mnt/distributed/welcome.txt 
[root@node1 ~]# umount /mnt/distributed

恢复数据

[root@node2 ~]# gluster volume stop vol_distributed
Stopping volume will make its data inaccessible. Do you want to continue? (y/n) y
volume stop: vol_distributed: success
[root@node2 ~]# gluster snapshot restore vol_distributed_snap
Restore operation will replace the original volume with the snapshotted volume. Do you still want to continue? (y/n) y
Snapshot restore: vol_distributed_snap: Snap restored successfully
[root@node2 ~]# gluster volume start vol_distributed
volume start: vol_distributed: success

重新挂载查看数据

[root@node1 ~]# mount -t glusterfs node2:/vol_distributed /mnt/distributed/
[root@node1 ~]# cat /mnt/distributed/welcome.txt 
Here is node1

7)计划任务创建快照

# 启用自动删除快照(超过之前设置的最大数量)
[root@node2 ~]# gluster snapshot config auto-delete enable
snapshot config: auto-delete successfully set
# 启用共享卷
[root@node2 ~]# gluster volume set all cluster.enable-shared-storage enable
volume set: success
# 初始化调度功能
[root@node2 ~]# snap_scheduler.py init
snap_scheduler: Successfully initialised snapshot scheduler for this node
[root@node2 ~]# snap_scheduler.py enable
snap_scheduler: Snapshot scheduling is enabled
[root@node2 ~]# snap_scheduler.py status
snap_scheduler: Snapshot scheduling status: Enabled
# 创建计划
[root@node2 ~]# snap_scheduler.py add "auto_take_snap_for_distributed_volume" "00 03 * * *" "vol_distributed"
snap_scheduler: Successfully added snapshot schedule
[root@node2 ~]# snap_scheduler.py list
JOB_NAME                                  SCHEDULE                                  OPERATION                                 VOLUME NAME                               
------------------------------------------------------------------------------------------------------------------------------------------------------------------------
auto_take_snap_for_distributed_volume     00 03 * * *                               Snapshot Create                           vol_distributed

6.CTDB

实现SAMBA高可用
在实验开始之前,将环境回退到创建完卷的位置;

所有节点安装ctdb和samba包

[root@node2 ~]# yum install -y ctdb samba samba-vfs-glusterfs 
...略

启用通过SMB自动导出Gluster卷

[root@node2 ~]# gluster volume set vol_replicated user.smb enable 
volume set: success

创建CTDB的meta volume

[root@node2 ~]# gluster volume create ctdb replica 3 node2:/data/brick2/ctdb node3:/data/brick3/ctdb node4:/data/brick4/ctdb 
volume create: ctdb: success: please start the volume to access data

所有节点修改META值

[root@node2 ~]# sed -i 's#META="all"#META="ctdb"#g' /var/lib/glusterd/hooks/1/start/post/S29CTDBsetup.sh 
[root@node2 ~]# sed -i 's#META="all"#META="ctdb"#g' /var/lib/glusterd/hooks/1/stop/pre/S29CTDB-teardown.sh

启用ctdb卷

[root@node2 ~]# gluster vol start ctdb 
volume start: ctdb: success

所有节点修改smb.conf,并验证ctdb是否挂载在节点上

[root@node2 ~]# vim /etc/samba/smb.conf 
[root@node2 ~]# cat /etc/samba/smb.conf 
...略 
[global]
     workgroup = SAMBA
     security = user
     
     passdb backend = tdbsam 
     
     printing = cups 
     printcap name = cups 
     load printers = yes 
     cups options = raw 
     
     clustering = yes  # 增加 
     idmap backend = tdb2  # 增加 
...略 
[root@node2 ~]# df -Th 
Filesystem                   Type            Size  Used Avail Use% Mounted on 
...略 
node2.newbie.local:/ctdb.tcp fuse.glusterfs   10G  136M  9.9G   2% /gluster/lock 
[root@node2 ~]# cat /etc/fstab 
...略 
node2.newbie.local:/ctdb /gluster/lock glusterfs _netdev,transport=tcp,xlator-option=*client*.ping-timeout=10 0 

所有节点创建配置文件

[root@node2 ~]# vim /etc/ctdb/nodes 
[root@node2 ~]# cat /etc/ctdb/nodes 
10.0.0.12 
10.0.0.13 
10.0.0.14 
[root@node2 ~]# vim /etc/ctdb/public_addresses 
[root@node2 ~]# cat /etc/ctdb/public_addresses  # 注意这个是虚拟IP,保持同网段即可 
10.0.0.10/24 enp0s3 
[root@node2 ~]# scp /etc/ctdb/nodes node3:/etc/ctdb/nodes 
root@node3's password: 
nodes                                                                                                                                      100%   30    47.8KB/s   00:00     
[root@node2 ~]# scp /etc/ctdb/nodes node4:/etc/ctdb/nodes 
root@node4's password: 
nodes                                                                                                                                      100%   30    27.4KB/s   00:00     
[root@node2 ~]# scp /etc/ctdb/public_addresses node3:/etc/ctdb/public_addresses 
root@node3's password: 
public_addresses                                                                                                                           100%   20     9.8KB/s   00:00     
[root@node2 ~]# scp /etc/ctdb/public_addresses node4:/etc/ctdb/public_addresses 
root@node4's password: 
public_addresses                                                                                                                           100%   20    10.7KB/s   00:00

所有节点禁用CTDB检查共享

[root@node2 ~]# echo "CTDB_SAMBA_SKIP_SHARE_CHECK=yes" >> /etc/ctdb/script.options 
[root@node2 ~]# scp /etc/ctdb/script.options node3:/etc/ctdb/script.options 
root@node3's password: 
script.options                                                                                                                             100%   32    37.5KB/s   00:00     
[root@node2 ~]# scp /etc/ctdb/script.options node4:/etc/ctdb/script.options 
root@node4's password: 
script.options                                                                                                                             100%   32    39.0KB/s   00:00

确保IO一致性

[root@node2 ~]# gluster volume set ctdb storage.batch-fsync-delay-usec 0 
volume set: success

所有节点再次修改smb.conf文件,添加配置

[root@node2 ~]# vim /etc/samba/smb.conf 
[root@node2 ~]# cat /etc/samba/smb.conf 
...略 
[global]
     workgroup = SAMBA      
     security = user
     
     passdb backend = tdbsam
     
     printing = cups 
     printcap name = cups 
     load printers = yes 
     cups options = raw
     
     clustering = yes 
     idmap backend = tdb2
     
     kernel share modes = no
     kernel oplocks = no 
     map archive = no 
     map hidden = no 
     map read only = no 
     map system = no 
     store dos attributes = yes 
...略 
[root@node2 ~]# scp /etc/samba/smb.conf node3:/etc/samba/smb.conf 
root@node3's password: 
smb.conf                                                                                                                                   100% 1171   562.8KB/s   00:00     
[root@node2 ~]# scp /etc/samba/smb.conf node4:/etc/samba/smb.conf 
root@node4's password: 
smb.conf                                                                                                                                   100% 1171     2.0MB/s   00:00

配置卷

[root@node2 ~]# gluster vol set vol_replicated group samba
volume set: success
[root@node2 ~]# gluster vol set vol_replicated client.event-threads 4
volume set: success
[root@node2 ~]# gluster vol set vol_replicated server.event-threads 4
volume set: success

启动CTDB

[root@node2 ~]# mkdir /var/lib/ctdb/{volatile,persistent,state}
[root@node2 ~]# mkdir /etc/ctdb/events/legacy
[root@node2 ~]# cp /usr/share/ctdb/events/legacy/* /etc/ctdb/events/legacy/
[root@node2 ~]# cd /etc/ctdb/events/legacy/
[root@node2 legacy]# rm -f 31.clamd.script 40.vsftpd.script 41.httpd.script 49.winbind.script 50.samba.script 60.nfs.script 70.iscsi.script 91.lvs.script 
[root@node2 legacy]# ls
00.ctdb.script  01.reclock.script  05.system.script  06.nfs.script  10.interface.script  11.natgw.script  11.routing.script  13.per_ip_routing.script  20.multipathd.script
[root@node2 legacy]# cd ~
[root@node2 ~]# vim /etc/ctdb/ctdb.conf 
[root@node2 ~]# grep -v -E "#|^$" /etc/ctdb/ctdb.conf 
[logging]
[cluster]
  recovery lock = /gluster/lock/lockfile
[legacy]
        realtime scheduling = false
[root@node2 ~]# systemctl start ctdb
[root@node2 ~]# systemctl enable ctdb

查看CTDB状态

[root@node2 ~]# ctdb status
Number of nodes:3
pnn:0 10.0.0.12        OK (THIS NODE)
pnn:1 10.0.0.13        OK
pnn:2 10.0.0.14        OK
Generation:908450427
Size:3
hash:0 lmaster:0
hash:1 lmaster:1
hash:2 lmaster:2
Recovery mode:NORMAL (0)
Recovery master:0

查看虚拟IP由哪台主机接管

[root@node2 ~]# ctdb -v ip
Public IPs on node 0
10.0.0.10 node[0] active[enp0s3] available[enp0s3] configured[enp0s3]
[root@node2 ~]# ip addr | grep 10.0.0.10
    inet 10.0.0.10/24 brd 10.0.0.255 scope global secondary enp0s3

所有节点启动Samba服务,并新建Samba用户

[root@node2 ~]# systemctl start smb
[root@node2 ~]# systemctl enable smb
[root@node2 ~]# adduser ctdber
[root@node2 ~]# smbpasswd -a ctdber
New SMB password:
Retype new SMB password:
Added user ctdber.

客户端准备挂载

[root@node1 ~]# yum install -y samba-client glusterfs glusterfs-fuse
[root@node1 ~]# smbclient -L 10.0.0.10 -U ctdber%123456

  Sharename       Type      Comment
  ---------       ----      -------
  print$          Disk      Printer Drivers
  gluster-vol_replicated Disk      For samba share of volume vol_replicated
  IPC$            IPC       IPC Service (Samba 4.10.4)
  ctdber          Disk      Home Directories
Reconnecting with SMB1 for workgroup listing.

  Server               Comment
  ---------            -------

  Workgroup            Master
  ---------            -------
[root@node1 ~]# mkdir /mnt/ctdb_temp
[root@node1 ~]# mount -t glusterfs node2:/vol_replicated /mnt/ctdb_temp/
[root@node1 ~]# chmod 777 /mnt/ctdb_temp/
[root@node1 ~]# umount /mnt/ctdb_temp/

客户端挂载

[root@node1 ~]# mkdir /mnt/ctdb
[root@node1 ~]# mount -t cifs -o username=ctdber,password=123456 //10.0.0.10/gluster-vol_replicated /mnt/ctdb
[root@node1 ~]# df -Th
Filesystem                         Type      Size  Used Avail Use% Mounted on
...略
//10.0.0.10/gluster-vol_replicated cifs       10G  136M  9.9G   2% /mnt/ctdb
[root@node1 ~]# echo "Here is node1" >> /mnt/ctdb/welcome.txt
[root@node1 ~]# cat /mnt/ctdb/welcome.txt 
Here is node1

本文转载自公众号:开源Ops
本文链接:https://mp.weixin.qq.com/s/_91iLpCDsmEAmMVKdFsuYA

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值