Purpose of this setup:
This setup is to test out how nfs-ganesha can be running as a docker process to connect a ceph cluster in docker, get cephfs then export to 3rd host as the nfs server.
## add the nfs-ganesha host to existing ceph cluster’s host file
## on “10.8.100.27 ceph01”, do:
[root@ip-10-8-100-27 ~]# cat /etc/hosts | grep ceph
10.8.100.27 ceph01
10.8.100.176 ceph02
10.8.100.139 ceph03
[root@ip-10-8-100-27 ~]# echo “10.8.100.202 ganesha_01” >> /etc/hosts
[root@ip-10-8-100-27 ~]# cat /etc/hosts | grep 10
10.8.100.27 ceph01
10.8.100.176 ceph02
10.8.100.139 ceph03
10.8.100.202 ganesha_01
[root@ip-10-8-100-27 ~]# for i in {176,139,202}; do scp /etc/hosts 10.8.100.$i:/etc/hosts; done;
## setup ssh key trust in between cluster and ganesha:
[root@ip-10-8-100-27 ~]# ssh-copy-id ganesha_01
[root@ip-10-8-100-176 ~]# ssh-copy-id ganesha_01
[root@ip-10-8-100-139 ~]# ssh-copy-id ganesha_01
[root@ip-10-8-100-202 ~]# ssh-keygen
[root@ip-10-8-100-202 ~]# for i in {27,176,139,202}; do ssh-copy-id 10.8.100.$i; done;
## copy ceph config from ceph01 to ganesha node:
[root@ip-10-8-100-27 ~]# ssh root@ganesha_01 mkdir -p /var/lib/ceph
[root@ip-10-8-100-27 ~]# ssh root@ganesha_01 mkdir -p /var/ceph
[root@ip-10-8-100-27 ~]# scp -r /etc/ceph/* root@ganesha_01:/etc/ceph/
ceph.conf 100% 220 224.9KB/s 00:00
ceph.client.admin.keyring 100% 159 195.1KB/s 00:00
ceph.mon.keyring 100% 688 528.8KB/s 00:00
[root@ip-10-8-100-27 ~]# scp -r /var/lib/ceph/bootstrap* root@ganesha_01:/var/lib/ceph/
ceph.keyring 100% 113 72.6KB/s 00:00
ceph.keyring 100% 113 61.7KB/s 00:00
ceph.keyring 100% 113 89.0KB/s 00:00
ceph.keyring 100% 113 91.9KB/s 00:00
## create ganesha docker run on ganesha node:
[root@ip-10-8-100-202 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
ceph/daemon HEAD-mimic-centos-7-aarch64 2d790517ca63 4 hours ago 823MB
ceph/daemon-base HEAD-mimic-centos-7-aarch64 560e662102f5 4 hours ago 693MB
centos 7 7a51de8a65d5 3 months ago 238MB
[root@ip-10-8-100-202 ~]# docker run -d –net=host -v /etc/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ 2d790517ca63 ganesha
9d83b14bbc3efbba3a97876f29cf0be691cfce5217807568eb84ca6ad9d6a1c8
[root@ip-10-8-100-202 ~]# docker run -d –net=host -v /etc/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ 2d790517ca63 ceph-client
1decd18089e27cedc139c69450629b450df48b9b2b0abc7a67c53be4dd733609
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# docker container ls –all
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
1decd18089e2 2d790517ca63 “/entrypoint.sh ceph…” 4 seconds ago Exited (0) 3 seconds ago relaxed_mendeleev
9d83b14bbc3e 2d790517ca63 “/entrypoint.sh gane…” 4 minutes ago Exited (0) 4 minutes ago goofy_roentgen
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-27 ~]# docker run -d –net=host -v /etc/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ ceph-13.2.6-daemon/cento7 nfs
350d4b357061b984770230a0ae8ca9a9c36f2fb0ac097aeb5f7b3e9efa32350d
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
350d4b357061 ceph-13.2.6-daemon/cento7 “/entrypoint.sh nfs” 7 seconds ago Up 7 seconds zealous_mcnulty
60e13885c463 ceph-13.2.6-daemon/cento7 “/entrypoint.sh rgw” 2 weeks ago Up 2 weeks pensive_ramanujan
d0876c3f6699 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mds” 2 weeks ago Up 2 weeks wonderful_dijkstra
95230ffafa12 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mgr” 2 weeks ago Up 2 weeks stoic_bell
7c980d1930ef ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks frosty_blackwell
52d084554e82 ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks boring_bhaskara
3869b330fc6d ceph-13.2.6-daemon/cento7 “/entrypoint.sh mon” 2 weeks ago Up 2 weeks elegant_bardeen
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]# docker exec -it 350d4b357061 ceph -s
cluster:
id: a00f505c-c342-4208-a000-8e4444c53bd6
health: HEALTH_OK
services:
mon: 3 daemons, quorum ip-10-8-100-27,ip-10-8-100-139,ip-10-8-100-176
mgr: ip-10-8-100-27(active)
mds: cephfs-1/1/1 up {0=ip-10-8-100-27=up:active}
osd: 6 osds: 6 up, 6 in
rgw: 1 daemon active
data:
pools: 7 pools, 156 pgs
objects: 241 objects, 3.4 KiB
usage: 14 GiB used, 342 GiB / 356 GiB avail
pgs: 156 active+clean
[root@ip-10-8-100-27 ~]# docker run -d –net=host -v /etc/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ ceph-13.2.6-daemon/cento7 ganesha
e04a1bc3ebe88fb8d9b09a90faf93371a934ced0ee4ef419e7feb3dc37ef7ff4
[root@ip-10-8-100-27 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
350d4b357061 ceph-13.2.6-daemon/cento7 “/entrypoint.sh nfs” 6 minutes ago Up 6 minutes zealous_mcnulty
60e13885c463 ceph-13.2.6-daemon/cento7 “/entrypoint.sh rgw” 2 weeks ago Up 2 weeks pensive_ramanujan
d0876c3f6699 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mds” 2 weeks ago Up 2 weeks wonderful_dijkstra
95230ffafa12 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mgr” 2 weeks ago Up 2 weeks stoic_bell
7c980d1930ef ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks frosty_blackwell
52d084554e82 ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks boring_bhaskara
3869b330fc6d ceph-13.2.6-daemon/cento7 “/entrypoint.sh mon” 2 weeks ago Up 2 weeks elegant_bardeen
[root@ip-10-8-100-27 ~]# docker container ls –all
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
e04a1bc3ebe8 ceph-13.2.6-daemon/cento7 “/entrypoint.sh gane…” 2 minutes ago Exited (0) 2 minutes ago infallible_shtern
350d4b357061 ceph-13.2.6-daemon/cento7 “/entrypoint.sh nfs” 7 minutes ago Up 7 minutes zealous_mcnulty
60e13885c463 ceph-13.2.6-daemon/cento7 “/entrypoint.sh rgw” 2 weeks ago Up 2 weeks pensive_ramanujan
d0876c3f6699 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mds” 2 weeks ago Up 2 weeks wonderful_dijkstra
95230ffafa12 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mgr” 2 weeks ago Up 2 weeks stoic_bell
7c980d1930ef ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks frosty_blackwell
52d084554e82 ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks boring_bhaskara
3869b330fc6d ceph-13.2.6-daemon/cento7 “/entrypoint.sh mon” 2 weeks ago Up 2 weeks elegant_bardeen
[root@ip-10-8-100-27 ~]#
### we can see that the ganesha docker cannot be created sucessfully for some reason, the status is “Exited”, need to find out !!!
### however, the same image can be used to creat a mon, which works well:
#
[root@ip-10-8-100-202 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
ceph/daemon HEAD-mimic-centos-7-aarch64 2d790517ca63 5 hours ago 823MB
ceph/daemon-base HEAD-mimic-centos-7-aarch64 560e662102f5 5 hours ago 693MB
centos 7 7a51de8a65d5 3 months ago 238MB
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# docker run -d –net=host -v /etc/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -e MON_IP=10.8.100.202 -e CEPH_PUBLIC_NETWORK=10.1.100.0/24 2d790517ca63 mon
510ed14031ae62cc7d242ec8b4b140272012645414acde4b581327c4b8b1af44
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
510ed14031ae 2d790517ca63 “/entrypoint.sh mon” 6 seconds ago Up 5 seconds adoring_albattani
[root@ip-10-8-100-202 ~]# docker exec -it 510ed14031ae ceph -s
cluster:
id: a00f505c-c342-4208-a000-8e4444c53bd6
health: HEALTH_OK
services:
mon: 4 daemons, quorum ip-10-8-100-27,ip-10-8-100-139,ip-10-8-100-176,ip-10-8-100-202
mgr: ip-10-8-100-27(active)
mds: cephfs-1/1/1 up {0=ip-10-8-100-27=up:active}
osd: 6 osds: 6 up, 6 in
rgw: 1 daemon active
data:
pools: 7 pools, 156 pgs
objects: 241 objects, 3.4 KiB
usage: 14 GiB used, 342 GiB / 356 GiB avail
pgs: 156 active+clean
[root@ip-10-8-100-202 ~]#
## verify pkgs installed:
[root@ip-10-8-100-202 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
510ed14031ae 2d790517ca63 “/entrypoint.sh mon” 8 minutes ago Up 8 minutes adoring_albattani
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# docker exec -it 510ed14031ae yum list installed | grep ganesha
[root@ip-10-8-100-202 ~]# docker exec -it 510ed14031ae rpm -qa | grep ganesha
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-27 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
350d4b357061 ceph-13.2.6-daemon/cento7 “/entrypoint.sh nfs” 36 minutes ago Up 36 minutes zealous_mcnulty
60e13885c463 ceph-13.2.6-daemon/cento7 “/entrypoint.sh rgw” 2 weeks ago Up 2 weeks pensive_ramanujan
d0876c3f6699 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mds” 2 weeks ago Up 2 weeks wonderful_dijkstra
95230ffafa12 ceph-13.2.6-daemon/cento7 “/entrypoint.sh mgr” 2 weeks ago Up 2 weeks stoic_bell
7c980d1930ef ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks frosty_blackwell
52d084554e82 ceph-13.2.6-daemon/cento7 “/entrypoint.sh osd_…” 2 weeks ago Up 2 weeks boring_bhaskara
3869b330fc6d ceph-13.2.6-daemon/cento7 “/entrypoint.sh mon” 2 weeks ago Up 2 weeks elegant_bardeen
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]# docker exec -it 3869b330fc6d yum list installed | grep ganesha
nfs-ganesha.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-ceph.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-debuginfo.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-gpfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-lttng.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mem.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mount-9P.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-nullfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-proxy.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rados-grace.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rgw.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-vfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-xfs.aarch64 2.7.1-0.1.el7 installed
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]# docker exec -it 350d4b357061 yum list installed | grep ganesha
nfs-ganesha.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-ceph.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-debuginfo.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-gpfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-lttng.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mem.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mount-9P.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-nullfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-proxy.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rados-grace.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rgw.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-vfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-xfs.aarch64 2.7.1-0.1.el7 installed
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]# docker exec -it 350d4b357061 rpm -qa | grep ganesha
nfs-ganesha-2.7.1-0.1.el7.aarch64
nfs-ganesha-gpfs-2.7.1-0.1.el7.aarch64
nfs-ganesha-mem-2.7.1-0.1.el7.aarch64
nfs-ganesha-proxy-2.7.1-0.1.el7.aarch64
nfs-ganesha-rgw-2.7.1-0.1.el7.aarch64
nfs-ganesha-xfs-2.7.1-0.1.el7.aarch64
nfs-ganesha-debuginfo-2.7.1-0.1.el7.aarch64
nfs-ganesha-ceph-2.7.1-0.1.el7.aarch64
nfs-ganesha-lttng-2.7.1-0.1.el7.aarch64
nfs-ganesha-nullfs-2.7.1-0.1.el7.aarch64
nfs-ganesha-rados-grace-2.7.1-0.1.el7.aarch64
nfs-ganesha-vfs-2.7.1-0.1.el7.aarch64
nfs-ganesha-mount-9P-2.7.1-0.1.el7.aarch64
[root@ip-10-8-100-27 ~]#
#########################################################################################
## cook 5 files below to make “ganesha/aarch64-cento7” docker image:
#
[root@ip-10-8-100-202 ~]# vi kubernetes.repo
[root@ip-10-8-100-202 ~]# vi ganesha.repo
[root@ip-10-8-100-202 ~]# vi ceph.repo
[root@ip-10-8-100-202 ~]# vi dockerfile
[root@ip-10-8-100-202 ~]# vi build_ganesha_aarch64_cento7_docker_image.sh
[root@ip-10-8-100-202 ~]# ls -ltr
total 20
-rw-r–r– 1 root root 264 Jul 15 21:28 kubernetes.repo
-rw-r–r– 1 root root 79 Jul 15 21:28 ganesha.repo
-rw-r–r– 1 root root 537 Jul 15 21:28 ceph.repo
-rw-r–r– 1 root root 2400 Jul 15 23:44 dockerfile
-rw-r–r– 1 root root 41 Jul 15 23:56 build_ganesha_aarch64_cento7_docker_image.sh
[root@ip-10-8-100-202 ~]# cat ganesha.repo
[ganesha]
name=ganesha
baseurl=http://10.8.100.27/ganesha
gpgcheck=0
enabled=1
[root@ip-10-8-100-202 ~]# cat kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-aarch64
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# cat ceph.repo
[ceph]
name=Ceph packages for \$basearch
baseurl=https://download.ceph.com/rpm-mimic/el7/\$basearch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://download.ceph.com/rpm-mimic/el7/noarch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
[ceph-source]
name=Ceph source packages
baseurl=https://download.ceph.com/rpm-mimic/el7/SRPMS
enabled=0
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# cat build_ganesha_aarch64_cento7_docker_image.sh
docker build -t ganesha/aarch64-cento7 .
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# cat dockerfile
# CEPH DAEMON IMAGE
# CEPH VERSION: Mimic
# CEPH VERSION DETAIL: 13.2.6
FROM centos:7
MAINTAINER Benjamin Lu “Benjamin.lu@gokeuslabcom”
RUN yum update -y
ENV CEPH_VERSION 13.2.6
ENV CONFD_VERSION 0.10.0
ENV KUBECTL_VERSION v1.6.0
CMD [“/usr/sbin/init”]
# install prerequisites
##############################################
# setup ceph.repo, kubernetes.repo
ADD ceph.repo /etc/yum.repos.d/
ADD kubernetes.repo /etc/yum.repos.d/
ADD ganesha.repo /etc/yum.repos.d/
##############################################
# install ceph, ganesha and repends
RUN yum -y update && yum install -y wget which unzip rpm uuid-runtime python-setuptools udev dmsetup localedef && \
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
yum install -y yum-utils device-mapper-persistent-data lvm2 && \
yum install -y libtirpc nfs-utils rpcbind libwbclient lttng-tools && \
rpm –import ‘https://download.ceph.com/keys/release.asc’ && \
yum clean all && \
rm -rf /var/cache/yum && \
yum install -y ceph-common ceph-base libcephfs libcephfs-devel libcephfs2 python-cephfs ceph-fuse nfs-ganesha && \
which ceph && \
ceph –version && \
curl -L http://10.8.100.27/ganesha/rpm-V2.7-stable/mimic/aarch64/ganesha-rpms-list > ganesha-rpms-list && \
bash -c ‘ \
if [ -n “ganesha-rpms-list” ]; then \
for i in `cat ganesha-rpms-list`; do rpm -Uvh http://10.8.100.27/ganesha/rpm-V2.7-stable/mimic/aarch64/$i; done; \
fi ‘ && \
yum list installed | grep ganesha && \
rpm -qa | grep ganesha && \
#######################################################
# Install kubenetes
yum makecache fast && \
yum install -y kubelet kubeadm kubectl && \
which kubelet kubeadm kubectl && \
chmod +x /usr/bin/kubectl && \
# set rpcbind, portmap, dbus
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
mkdir -p /run/rpcbind /export /var/run/dbus && \
touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr && \
chmod 755 /run/rpcbind/* && \
#chown messagebus:messagebus /var/run/dbus && \
# set locale
yum reinstall -y glibc-common && \
localedef -i en_US -f UTF-8 en_US.UTF-8
# Add startup script
COPY start.sh /
# Add volumes for Ceph config and data
VOLUME [“/etc/ceph”,”/var/lib/ceph”, “/etc/ganesha”]
# NFS ports and portmapper
EXPOSE 2049 38465-38467 662 111/udp 111
# Start Ganesha NFS daemon by default
CMD [“/start.sh”]
LABEL version=”1.0″
[root@ip-10-8-100-202 ~]#
## making the nfs-ganesha docker images now:
[root@ip-10-8-100-202 ~]# docker build -t ganesha/aarch64-cento7 .
……….
……….
Installed:
glibc-common.aarch64 0:2.17-260.el7_6.6
Complete!
Removing intermediate container 8dd283e800d5
—> 7f7e24e1f168
Step 12/16 : COPY start.sh /
—> 96250be9b1d4
Step 13/16 : VOLUME [“/etc/ceph”,”/var/lib/ceph”, “/etc/ganesha”]
—> Running in 7de4edbe18b1
Removing intermediate container 7de4edbe18b1
—> f4f1205a245b
Step 14/16 : EXPOSE 2049 38465-38467 662 111/udp 111
—> Running in bd568d76beda
Removing intermediate container bd568d76beda
—> f5b8a0c85f9c
Step 15/16 : CMD [“/start.sh”]
—> Running in 0e349c672aa8
Removing intermediate container 0e349c672aa8
—> f798624743b0
Step 16/16 : LABEL version=”1.0″
—> Running in 81345ed03c02
Removing intermediate container 81345ed03c02
—> 716a66a9fc7e
Successfully built 716a66a9fc7e
Successfully tagged ganesha/aarch64-cento7:latest
[root@ip-10-8-100-202 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
ganesha/aarch64-cento7 latest 716a66a9fc7e 28 seconds ago 1.12GB
centos 7 7a51de8a65d5 4 months ago 238MB
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# docker run -it ganesha/aarch64-cento7 /bin/bash
[root@7d53be711b2a /]#
[root@7d53be711b2a /]# which ceph
/usr/bin/ceph
[root@7d53be711b2a /]#
[root@7d53be711b2a /]#
[root@7d53be711b2a /]# ceph –version
ceph version 13.2.6 (7b695f835b03642f85998b2ae7b6dd093d9fbce4) mimic (stable)
[root@7d53be711b2a /]#
[root@7d53be711b2a /]#
[root@7d53be711b2a /]# yum list installed | grep nfs
libnfsidmap.aarch64 0.25-19.el7 @base
nfs-ganesha.aarch64 2.7.1-0.1.el7 @ganesha
nfs-ganesha-ceph.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-debuginfo.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-gpfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-lttng.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mem.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mount-9P.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-nullfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-proxy.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rados-grace.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rgw.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-vfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-xfs.aarch64 2.7.1-0.1.el7 installed
nfs-utils.aarch64 1:1.3.0-0.61.el7 @base
[root@7d53be711b2a /]#
#####################################################################################################
#This section make cephfs export through libcephfs2
#test export ceph ganesha nfs to 3rd server:
#
[root@ip-10-8-100-27 ceph]# vi /etc/ceph/ceph.conf
[root@ip-10-8-100-27 ceph]# cat /etc/ceph/ceph.conf
[global]
fsid = a00f505c-c342-4208-a000-8e4444c53bd6
mon initial members = ip-10-8-100-27,ip-10-8-100-176,ip-10-8-100-139
mon host = 10.8.100.27,10.8.100.176,10.8.100.139
public network = 10.8.100.0/24
cluster network = 10.8.100.0/24
osd journal size = 100
log file = /dev/null
[client]
mon_host = 10.8.100.27:6789,10.8.100.176:6789,10.8.100.139:6789
[root@ip-10-8-100-27 ceph]#
[root@ip-10-8-100-27 ceph]# for i in {176,139,202}; do scp /etc/ceph/ceph.conf 10.8.100.$i:/etc/ceph/; done;
[root@ip-10-8-100-202 ceph]# vi /etc/ganesha/ganesha.conf
[root@ip-10-8-100-202 ceph]# cat /etc/ganesha/ganesha.conf
EXPORT
{
Export_Id = 2;
Path = “/”;
Pseudo = “/”;
Protocols = 3,4;
Transport_Protocols = UDP,TCP;
Access_Type = RW;
Squash = root_squash;
## Allowed security types for this export
Sectype = sys,krb5,krb5i,krb5p;
## Exporting FSAL
FSAL {
Name = VFS;
}
FSAL {
Name = CEPH;
}
}
# Config RGW for ceph
RGW
{
ceph_conf = “/etc/ceph/ceph.conf”;
name = “name”;
cluster = “ceph”;
}
# Config block for FSAL_CEPH
CEPH
{
# Path to a ceph.conf file for this cluster.
ceph_conf = “/etc/ceph/ceph.conf”;
# User file-creation mask. These bits will be masked off from the unix
# permissions on newly-created inodes.
umask = 0;
}
# Config NFSV4
NFSV4
{
Allow_Numeric_Owners = true;
Only_Numeric_Owners = true;
}
[root@ip-10-8-100-202 ceph]#
495 vi setup_firewall.sh
496 cat setup_firewall.sh
497 chmod 755 setup_firewall.sh
498 ./setup_firewall.sh
509 ceph-authtool -p /etc/ceph/ceph.client.admin.keyring > ceph.key
511 chmod 600 ceph.key
516 sudo mount -t ceph 10.8.100.27:6789:/ /mnt/cephfs-nfs -o name=admin,secretfile=ceph.key
518 umount /mnt/cephfs-nfs
519 df -h
520 sudo mount -t ceph 10.8.100.27:6789:/ /mnt/cephfs-nfs -o name=admin,secretfile=ceph.key
526 cd /mnt/cephfs-nfs
527 ls -ltr
528 mkdir test_ganesha_wr
593 netstat -an | grep 6789
## test export the ganesha export from ceph to 3rd server at 10.8.100.32:
#
[root@ip-10-8-100-32 ~]# mount -t nfs -o nfsvers=4.1,proto=tcp 10.8.100.202:/ /root/from_nfs_ganesha_export_dir
[root@ip-10-8-100-32 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 1.9G 0 1.9G 0% /dev
tmpfs 1.9G 0 1.9G 0% /dev/shm
tmpfs 1.9G 428K 1.9G 1% /run
tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
/dev/nvme0n1p1 60G 2.9G 58G 5% /
/dev/nvme0n1p128 10M 3.7M 6.3M 38% /boot/efi
tmpfs 380M 0 380M 0% /run/user/1001
10.8.100.202:/ 105G 0 105G 0% /root/from_nfs_ganesha_export_dir
[root@ip-10-8-100-32 ~]# cd /root/from_nfs_ganesha_export_dir
[root@ip-10-8-100-32 from_nfs_ganesha_export_dir]# ls -ltr
total 1
drwxr-xr-x 2 root root 0 Jul 17 22:24 test_ganesha_wr
[root@ip-10-8-100-32 from_nfs_ganesha_export_dir]#
[root@ip-10-8-100-32 ~]# docker run –privileged ganesha/aarch64-cento7 &
[root@ip-10-8-100-27 ~]# scp -r /var/lib/ceph/bootstrap* root@ganesha_02:/var/lib/ceph
ceph.keyring 100% 113 36.8KB/s 00:00
ceph.keyring 100% 113 90.6KB/s 00:00
ceph.keyring 100% 113 37.5KB/s 00:00
ceph.keyring 100% 113 82.1KB/s 00:00
[root@ip-10-8-100-27 ~]#
[root@ip-10-8-100-27 ~]# scp -r /etc/ceph root@ganesha_02:/etc/ceph
ceph.client.admin.keyring 100% 159 159.7KB/s 00:00
ceph.mon.keyring 100% 688 841.9KB/s 00:00
ceph.conf 100% 351 413.4KB/s 00:00
ceph.conf.orig.BKUP
[root@ip-10-8-100-32 ~]# ls -l /etc/ceph
total 20
-rw——- 1 root root 159 Jul 19 22:32 ceph.client.admin.keyring
-rw-r–r– 1 root root 351 Jul 19 22:32 ceph.conf
-rw-r–r– 1 root root 220 Jul 19 22:32 ceph.conf.orig.BKUP
-rw-r–r– 1 root root 41 Jul 19 23:22 ceph.key
-rw——- 1 root root 688 Jul 19 22:32 ceph.mon.keyring
[root@ip-10-8-100-32 ~]#
[root@ip-10-8-100-32 ~]# ls -l /var/lib/ceph/bootstrap*
/var/lib/ceph/bootstrap-mds:
total 4
-rw——- 1 root root 113 Jul 19 22:32 ceph.keyring
/var/lib/ceph/bootstrap-osd:
total 4
-rw——- 1 root root 113 Jul 19 22:32 ceph.keyring
/var/lib/ceph/bootstrap-rbd:
total 4
-rw——- 1 root root 113 Jul 19 22:32 ceph.keyring
/var/lib/ceph/bootstrap-rgw:
total 4
-rw——- 1 root root 113 Jul 19 22:32 ceph.keyring
[root@ip-10-8-100-32 ~]# docker cp /etc/ceph/ceph.client.admin.keyring 4592ec428182:/etc/ceph/
[root@ip-10-8-100-32 ~]# docker exec 4592ec428182 ceph-authtool -p /etc/ceph/ceph.client.admin.keyring | tee /etc/ceph/ceph.key ## this line does not work
[root@ip-10-8-100-32 ~]# docker exec 4592ec428182 ceph -s
cluster:
id: a00f505c-c342-4208-a000-8e4444c53bd6
health: HEALTH_WARN
1/4 mons down, quorum ip-10-8-100-27,ip-10-8-100-139,ip-10-8-100-176
services:
mon: 4 daemons, quorum ip-10-8-100-27,ip-10-8-100-139,ip-10-8-100-176, out of quorum: ip-10-8-100-202
mgr: ip-10-8-100-27(active)
mds: cephfs-1/1/1 up {0=ip-10-8-100-27=up:active}
osd: 6 osds: 6 up, 6 in
rgw: 1 daemon active
data:
pools: 7 pools, 156 pgs
objects: 241 objects, 8.6 KiB
usage: 14 GiB used, 342 GiB / 356 GiB avail
pgs: 156 active+clean
[root@ip-10-8-100-32 ~]# docker exec 4592ec428182 mkdir /mnt/cephfs
[root@ip-10-8-100-32 ~]# docker exec 4592ec428182 ls -l /mnt/cephfs
total 0
[root@ip-10-8-100-32 ~]# docker exec 4592ec428182 yum list installed | grep nfs
libnfsidmap.aarch64 0.25-19.el7 @base
nfs-ganesha.aarch64 2.7.1-0.1.el7 @ganesha
nfs-ganesha-ceph.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-debuginfo.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-gpfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-lttng.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mem.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-mount-9P.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-nullfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-proxy.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rados-grace.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-rgw.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-vfs.aarch64 2.7.1-0.1.el7 installed
nfs-ganesha-xfs.aarch64 2.7.1-0.1.el7 installed
nfs-utils.aarch64 1:1.3.0-0.61.el7 @base
[root@ip-10-8-100-32 ~]#
https://github.com/mitcdh/docker-nfs-ganesha
## test write from docker on ceph client -ganesha1 ( 10.8.100.202)
# create docker name it as “ganasha” with mount point /export inside docker to use cephfs-nfs from ceph cluster:
#
[root@ip-10-8-100-202 ~]# docker run -d –name ganesha -v /mnt/cephfs-nfs:/export ganesha/aarch64-cento7
69a29172aa0264a93cfa62f3c3396388752c6e55c13c0027aa5ecc100b8c6d9d
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
69a29172aa02 ganesha/aarch64-cento7 “/start.sh” 7 seconds ago Up 6 seconds 2049/tcp, 38465-38467/tcp ganesha
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# docker exec 69a29172aa02 df -h
Filesystem Size Used Avail Use% Mounted on
overlay 60G 3.7G 57G 7% /
tmpfs 64M 0 64M 0% /dev
tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
10.8.100.27:/ 104G 0 104G 0% /export
/dev/nvme0n1p1 60G 3.7G 57G 7% /etc/ceph
shm 64M 8.0K 64M 1% /dev/shm
tmpfs 1.9G 0 1.9G 0% /proc/acpi
tmpfs 1.9G 0 1.9G 0% /proc/scsi
tmpfs 1.9G 0 1.9G 0% /sys/firmware
[root@ip-10-8-100-202 ~]#
## test tounch a file inside docker on the cephfs mount /export:
#
[root@ip-10-8-100-202 ~]# docker exec 69a29172aa02 touch /export/test_ganesha_wr/testfile
[root@ip-10-8-100-202 ~]# docker exec 69a29172aa02 ls -l /export/test_ganesha_wr
total 0
-rw-r–r– 1 root root 0 Jul 23 00:06 testfile
[root@ip-10-8-100-202 ~]# docker exec 69a29172aa02 touch /export/test_ganesha_wr/testfile2
[root@ip-10-8-100-202 ~]# docker exec 69a29172aa02 touch /export/test_ganesha_wr/testfile3
[root@ip-10-8-100-202 ~]# docker exec 69a29172aa02 ls -l /export/test_ganesha_wr
total 0
-rw-r–r– 1 root root 0 Jul 23 00:06 testfile
-rw-r–r– 1 root root 0 Jul 23 19:22 testfile2
-rw-r–r– 1 root root 0 Jul 23 19:25 testfile3
## check 3 files in local dir if 3 files are visiable
[root@ip-10-8-100-202 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 1.9G 0 1.9G 0% /dev
tmpfs 1.9G 0 1.9G 0% /dev/shm
tmpfs 1.9G 556K 1.9G 1% /run
tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
/dev/nvme0n1p1 60G 3.7G 57G 7% /
/dev/nvme0n1p128 10M 3.7M 6.3M 38% /boot/efi
10.8.100.27:/ 104G 0 104G 0% /mnt/cephfs-nfs
overlay 60G 3.7G 57G 7% /var/lib/docker/overlay2/6671136a87233f9370b1ef13d3ed97cdef6ae0e433ee95c514d52cdddaeb4aab/merged
shm 64M 8.0K 64M 1% /var/lib/docker/containers/69a29172aa0264a93cfa62f3c3396388752c6e55c13c0027aa5ecc100b8c6d9d/mounts/shm
tmpfs 380M 0 380M 0% /run/user/0
[root@ip-10-8-100-202 ~]#
[root@ip-10-8-100-202 ~]# ls -l /mnt/cephfs-nfs
total 0
drwxr-xr-x 1 root root 3 Jul 23 19:25 test_ganesha_wr
[root@ip-10-8-100-202 ~]# ls -l /mnt/cephfs-nfs/test_ganesha_wr/
total 0
-rw-r–r– 1 root root 0 Jul 23 00:06 testfile
-rw-r–r– 1 root root 0 Jul 23 19:22 testfile2
-rw-r–r– 1 root root 0 Jul 23 19:25 testfile3
[root@ip-10-8-100-202 ~]#
## Let’s verify 3 files on docker that ruuning on other ceph client node: ganesha2 (10.8.100.32):
# create a docker fist:
[root@ip-10-8-100-32 ~]# docker run -d –name ganesha -v /root/from_nfs_ganesha_export_dir:/export ganesha/aarch64-cento7
fde05b4a10d687539530c483f24c658f4e18ae373b1c77b095322be1eccd5d3f
[root@ip-10-8-100-32 ~]#
[root@ip-10-8-100-32 ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
fde05b4a10d6 ganesha/aarch64-cento7 “/start.sh” 6 seconds ago Up 5 seconds 2049/tcp, 38465-38467/tcp ganesha
[root@ip-10-8-100-32 ~]#
[root@ip-10-8-100-32 ~]# docker exec fde05b4a10d6 df -h
Filesystem Size Used Avail Use% Mounted on
overlay 60G 4.0G 57G 7% /
tmpfs 64M 0 64M 0% /dev
tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
10.8.100.202:/ 104G 0 104G 0% /export
/dev/nvme0n1p1 60G 4.0G 57G 7% /etc/ceph
shm 64M 8.0K 64M 1% /dev/shm
tmpfs 1.9G 0 1.9G 0% /proc/acpi
tmpfs 1.9G 0 1.9G 0% /proc/scsi
tmpfs 1.9G 0 1.9G 0% /sys/firmware
[root@ip-10-8-100-32 ~]#
[root@ip-10-8-100-32 ~]# docker exec fde05b4a10d6 ls -l /export/test_ganesha_wr
total 0
-rw-r–r– 1 root root 0 Jul 23 00:06 testfile
-rw-r–r– 1 root root 0 Jul 23 19:22 testfile2
-rw-r–r– 1 root root 0 Jul 23 19:25 testfile3
[root@ip-10-8-100-32 ~]#
## however, test found that 3 files is not permiited to write contains from docker runing on ceph client node: ganesha2 (10.8.100.32):
#
[root@ip-10-8-100-32 ~]# docker exec fde05b4a10d6 echo ‘This is a test write on server 32 docker, it will show up in server 202 docker’ > /export/test_ganesha_wr/testfile
-bash: /export/test_ganesha_wr/testfile: No such file or directory
[root@ip-10-8-100-32 ~]# docker exec fde05b4a10d6 echo ‘This is a test write on server 32 docker, it will show up in server 202 docker’ | tee /export/test_ganesha_wr/testfile
tee: /export/test_ganesha_wr/testfile: No such file or directory
This is a test write on server 32 docker, it will show up in server 202 docker
[root@ip-10-8-100-32 ~]# docker exec fde05b4a10d6 cat /export/test_ganesha_wr/testfile
[root@ip-10-8-100-32 ~]#
## let’s go inside the docker to try write again:
[root@ip-10-8-100-32 ~]# docker exec -it fde05b4a10d6 /bin/bash
[root@fde05b4a10d6 /]#
[root@fde05b4a10d6 /]# df -h
Filesystem Size Used Avail Use% Mounted on
overlay 60G 4.0G 57G 7% /
tmpfs 64M 0 64M 0% /dev
tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
10.8.100.202:/ 104G 0 104G 0% /export
/dev/nvme0n1p1 60G 4.0G 57G 7% /etc/ceph
shm 64M 8.0K 64M 1% /dev/shm
tmpfs 1.9G 0 1.9G 0% /proc/acpi
tmpfs 1.9G 0 1.9G 0% /proc/scsi
tmpfs 1.9G 0 1.9G 0% /sys/firmware
[root@fde05b4a10d6 /]# cd /export/
[root@fde05b4a10d6 export]# ls
test_ganesha_wr
[root@fde05b4a10d6 export]# cd test_ganesha_wr/
[root@fde05b4a10d6 test_ganesha_wr]# ls -l
total 0
-rw-r–r– 1 root root 0 Jul 23 00:06 testfile
-rw-r–r– 1 root root 0 Jul 23 19:22 testfile2
-rw-r–r– 1 root root 0 Jul 23 19:25 testfile3
…. this doc will be updated later based on test….