최신 버전 Rook v1.20.7 — 2026-09-20 확인. https://github.com/rook/rook/releases/latest . 현행 배포는
deploy/examples에서crds.yaml·common.yaml·csi-operator.yaml·operator.yaml·cluster.yaml순이다 (csi-operator.yaml이 추가됐다). 아래는 옛 버전 기록이다.
[root@nn01:~/project/test:]# nnctl get nodes --show-labels
NAME STATUS ROLES AGE VERSION LABELS
dt01 Ready <none> 7d14h v1.22.2 beta.nnrnetes.io/arch=amd64,beta.nnrnetes.io/os=linux,nnrnetes.io/arch=amd64,nnrnetes.io/hostname=dt01,nnrnetes.io/os=linux
dt02 Ready <none> 7d14h v1.22.2 beta.nnrnetes.io/arch=amd64,beta.nnrnetes.io/os=linux,nnrnetes.io/arch=amd64,nnrnetes.io/hostname=dt02,nnrnetes.io/os=linux
dt03 Ready <none> 7d14h v1.22.2 beta.nnrnetes.io/arch=amd64,beta.nnrnetes.io/os=linux,nnrnetes.io/arch=amd64,nnrnetes.io/hostname=dt03,nnrnetes.io/os=linux
swmeainn01 Ready control-plane,master 7d14h v1.20.4 beta.nnrnetes.io/arch=amd64,beta.nnrnetes.io/os=linux,nnrnetes.io/arch=amd64,nnrnetes.io/hostname=swmeainn01,nnrnetes.io/os=linux,node-role.nnrnetes.io/control-plane=,node-role.nnrnetes.io/master=,node.nnrnetes.io/exclude-from-external-load-balancers=
Ref : https://github.com/rook/rook/blob/master/Documentation/quickstart.md
root@nn01:~/project/rook:]# git clone --single-branch --branch release-1.7 https://github.com/rook/rook.git
Cloning into 'rook'...
remote: Enumerating objects: 68726, done.
remote: Counting objects: 100% (926/926), done.
remote: Compressing objects: 100% (532/532), done.
remote: Total 68726 (delta 479), reused 670 (delta 374), pack-reused 67800
Receiving objects: 100% (68726/68726), 40.03 MiB | 13.64 MiB/s, done.
Resolving deltas: 100% (48081/48081), done.
root@nn01:~/project/rook:]# cd rook/cluster/examples/nnrnetes/ceph/
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# ll
합계 1052
-rw-r--r-- 1 root root 436 9월 17 13:33 ceph-client.yaml
-rw-r--r-- 1 root root 1066 9월 17 13:33 cluster-external-management.yaml
-rw-r--r-- 1 root root 1242 9월 17 13:33 cluster-external.yaml
-rw-r--r-- 1 root root 6988 9월 17 13:33 cluster-on-local-pvc.yaml
-rw-r--r-- 1 root root 8338 9월 17 13:33 cluster-on-pvc.yaml
-rw-r--r-- 1 root root 5240 9월 17 13:33 cluster-stretched-aws.yaml
-rw-r--r-- 1 root root 3238 9월 17 13:33 cluster-stretched.yaml
-rw-r--r-- 1 root root 1476 9월 17 13:33 cluster-test.yaml
-rw-r--r-- 1 root root 14428 9월 17 13:33 cluster.yaml
-rw-r--r-- 1 root root 2341 9월 17 13:33 common-external.yaml
-rw-r--r-- 1 root root 3714 9월 17 13:33 common-second-cluster.yaml
-rw-r--r-- 1 root root 32886 9월 17 13:33 common.yaml
-rw-r--r-- 1 root root 690480 9월 17 13:33 crds.yaml
-rw-r--r-- 1 root root 58560 9월 17 13:33 create-external-cluster-resources.py
-rw-r--r-- 1 root root 3615 9월 17 13:33 create-external-cluster-resources.sh
drwxr-xr-x 5 root root 47 9월 17 13:33 csi
-rw-r--r-- 1 root root 410 9월 17 13:33 dashboard-external-http.yaml
-rw-r--r-- 1 root root 411 9월 17 13:33 dashboard-external-https.yaml
-rw-r--r-- 1 root root 954 9월 17 13:33 dashboard-ingress-https.yaml
-rw-r--r-- 1 root root 413 9월 17 13:33 dashboard-loadbalancer.yaml
-rw-r--r-- 1 root root 1789 9월 17 13:33 direct-mount.yaml
-rw-r--r-- 1 root root 3432 9월 17 13:33 filesystem-ec.yaml
-rw-r--r-- 1 root root 1215 9월 17 13:33 filesystem-mirror.yaml
-rw-r--r-- 1 root root 777 9월 17 13:33 filesystem-test.yaml
-rw-r--r-- 1 root root 5232 9월 17 13:33 filesystem.yaml
drwxr-xr-x 2 root root 115 9월 17 13:33 flex
-rw-r--r-- 1 root root 368 9월 17 13:33 images.txt
-rw-r--r-- 1 root root 4977 9월 17 13:33 import-external-cluster.sh
drwxr-xr-x 2 root root 4096 9월 17 13:33 monitoring
-rw-r--r-- 1 root root 745 9월 17 13:33 nfs-test.yaml
-rw-r--r-- 1 root root 1832 9월 17 13:33 nfs.yaml
-rw-r--r-- 1 root root 587 9월 17 13:33 object-bucket-claim-delete.yaml
-rw-r--r-- 1 root root 587 9월 17 13:33 object-bucket-claim-retain.yaml
-rw-r--r-- 1 root root 3665 9월 17 13:33 object-ec.yaml
-rw-r--r-- 1 root root 814 9월 17 13:33 object-external.yaml
-rw-r--r-- 1 root root 1804 9월 17 13:33 object-multisite-pull-realm.yaml
-rw-r--r-- 1 root root 1282 9월 17 13:33 object-multisite.yaml
-rw-r--r-- 1 root root 5913 9월 17 13:33 object-openshift.yaml
-rw-r--r-- 1 root root 685 9월 17 13:33 object-test.yaml
-rw-r--r-- 1 root root 770 9월 17 13:33 object-user.yaml
-rw-r--r-- 1 root root 5620 9월 17 13:33 object.yaml
-rw-r--r-- 1 root root 24215 9월 17 13:33 operator-openshift.yaml
-rw-r--r-- 1 root root 22872 9월 17 13:33 operator.yaml
-rw-r--r-- 1 root root 2759 9월 17 13:33 osd-purge.yaml
-rw-r--r-- 1 root root 1100 9월 17 13:33 pool-ec.yaml
-rw-r--r-- 1 root root 525 9월 17 13:33 pool-test.yaml
-rw-r--r-- 1 root root 3394 9월 17 13:33 pool.yaml
drwxr-xr-x 2 root root 23 9월 17 13:33 pre-k8s-1.16
-rw-r--r-- 1 root root 1470 9월 17 13:33 rbdmirror.yaml
-rw-r--r-- 1 root root 525 9월 17 13:33 rgw-external.yaml
-rw-r--r-- 1 root root 2558 9월 17 13:33 scc.yaml
-rw-r--r-- 1 root root 737 9월 17 13:33 storageclass-bucket-delete.yaml
-rw-r--r-- 1 root root 736 9월 17 13:33 storageclass-bucket-retain.yaml
drwxr-xr-x 2 root root 29 9월 17 13:33 test-data
-rw-r--r-- 1 root root 1738 9월 17 13:33 toolbox-job.yaml
-rw-r--r-- 1 root root 1474 9월 17 13:33 toolbox.yaml
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f crds.yaml -f common.yaml -f operator.yaml
customresourcedefinition.apiextensions.k8s.io/cephblockpools.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephclients.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephclusters.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystemmirrors.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephfilesystems.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephnfses.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectrealms.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectstores.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectstoreusers.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectzonegroups.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephobjectzones.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/cephrbdmirrors.ceph.rook.io created
customresourcedefinition.apiextensions.k8s.io/objectbucketclaims.objectbucket.io created
customresourcedefinition.apiextensions.k8s.io/objectbuckets.objectbucket.io created
customresourcedefinition.apiextensions.k8s.io/volumereplicationclasses.replication.storage.openshift.io created
customresourcedefinition.apiextensions.k8s.io/volumereplications.replication.storage.openshift.io created
customresourcedefinition.apiextensions.k8s.io/volumes.rook.io created
namespace/rook-ceph created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-object-bucket created
serviceaccount/rook-ceph-admission-controller created
clusterrole.rbac.authorization.k8s.io/rook-ceph-admission-controller-role created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-admission-controller-rolebinding created
clusterrole.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created
clusterrole.rbac.authorization.k8s.io/rook-ceph-system created
role.rbac.authorization.k8s.io/rook-ceph-system created
clusterrole.rbac.authorization.k8s.io/rook-ceph-global created
clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created
clusterrole.rbac.authorization.k8s.io/rook-ceph-object-bucket created
serviceaccount/rook-ceph-system created
rolebinding.rbac.authorization.k8s.io/rook-ceph-system created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-system created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-global created
serviceaccount/rook-ceph-osd created
serviceaccount/rook-ceph-mgr created
serviceaccount/rook-ceph-cmd-reporter created
role.rbac.authorization.k8s.io/rook-ceph-osd created
clusterrole.rbac.authorization.k8s.io/rook-ceph-osd created
clusterrole.rbac.authorization.k8s.io/rook-ceph-mgr-system created
role.rbac.authorization.k8s.io/rook-ceph-mgr created
role.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created
rolebinding.rbac.authorization.k8s.io/rook-ceph-cluster-mgmt created
rolebinding.rbac.authorization.k8s.io/rook-ceph-osd created
rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr created
rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-system created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-cluster created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-osd created
rolebinding.rbac.authorization.k8s.io/rook-ceph-cmd-reporter created
podsecuritypolicy.policy/00-rook-privileged created
clusterrole.rbac.authorization.k8s.io/psp:rook created
clusterrolebinding.rbac.authorization.k8s.io/rook-ceph-system-psp created
rolebinding.rbac.authorization.k8s.io/rook-ceph-default-psp created
rolebinding.rbac.authorization.k8s.io/rook-ceph-osd-psp created
rolebinding.rbac.authorization.k8s.io/rook-ceph-mgr-psp created
rolebinding.rbac.authorization.k8s.io/rook-ceph-cmd-reporter-psp created
serviceaccount/rook-csi-cephfs-plugin-sa created
serviceaccount/rook-csi-cephfs-provisioner-sa created
role.rbac.authorization.k8s.io/cephfs-external-provisioner-cfg created
rolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role-cfg created
clusterrole.rbac.authorization.k8s.io/cephfs-csi-nodeplugin created
clusterrole.rbac.authorization.k8s.io/cephfs-external-provisioner-runner created
clusterrolebinding.rbac.authorization.k8s.io/rook-csi-cephfs-plugin-sa-psp created
clusterrolebinding.rbac.authorization.k8s.io/rook-csi-cephfs-provisioner-sa-psp created
clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-nodeplugin created
clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-provisioner-role created
serviceaccount/rook-csi-rbd-plugin-sa created
serviceaccount/rook-csi-rbd-provisioner-sa created
role.rbac.authorization.k8s.io/rbd-external-provisioner-cfg created
rolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role-cfg created
clusterrole.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
clusterrole.rbac.authorization.k8s.io/rbd-external-provisioner-runner created
clusterrolebinding.rbac.authorization.k8s.io/rook-csi-rbd-plugin-sa-psp created
clusterrolebinding.rbac.authorization.k8s.io/rook-csi-rbd-provisioner-sa-psp created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role created
role.rbac.authorization.k8s.io/rook-ceph-purge-osd created
rolebinding.rbac.authorization.k8s.io/rook-ceph-purge-osd created
serviceaccount/rook-ceph-purge-osd created
configmap/rook-ceph-operator-config created
deployment.apps/rook-ceph-operator created
yum install lvm2 -y
rpm -qa|grep lvm
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# rpm -qa|grep lvm
lvm2-libs-2.02.187-6.el7.x86_64
lvm2-2.02.187-6.el7.x86_64
llvm-private-7.0.1-1.el7.x86_64
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
xvda 202:0 0 64G 0 disk
├─xvda1 202:1 0 1G 0 part /boot
└─xvda2 202:2 0 63G 0 part
├─centos-root 253:0 0 61G 0 lvm /
└─centos-swap 253:1 0 2G 0 lvm
xvdb 202:16 0 100G 0 disk
cd cluster/examples/nnrnetes/ceph
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f crds.yaml -f common.yaml -f operator.yaml
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f cluster.yaml
cephcluster.ceph.rook.io/rook-ceph created
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl get pod -n rook-ceph
NAME READY STATUS RESTARTS AGE
csi-cephfsplugin-4gzpr 3/3 Running 0 3m23s
csi-cephfsplugin-8mhdm 3/3 Running 0 3m23s
csi-cephfsplugin-fpzk4 3/3 Running 0 3m23s
csi-cephfsplugin-provisioner-7dcc8ff54d-kdncb 6/6 Running 0 3m22s
csi-cephfsplugin-provisioner-7dcc8ff54d-vh4j2 6/6 Running 0 3m22s
csi-cephfsplugin-wqrjg 3/3 Running 0 3m23s
csi-rbdplugin-25bjp 3/3 Running 0 3m26s
csi-rbdplugin-jglwx 3/3 Running 0 3m26s
csi-rbdplugin-jz8sr 3/3 Running 0 3m26s
csi-rbdplugin-lmzhh 3/3 Running 0 3m25s
csi-rbdplugin-provisioner-d8bcc5fc4-hcr6t 6/6 Running 0 3m24s
csi-rbdplugin-provisioner-d8bcc5fc4-m9vwm 6/6 Running 0 3m24s
rook-ceph-crashcollector-nn02-5b6986f89c-sfpt8 1/1 Running 0 81s
rook-ceph-crashcollector-nn03-6b695ddd84-gd9xw 1/1 Running 0 86s
rook-ceph-crashcollector-nn04-5d79fcbcc-nlvfg 1/1 Running 0 88s
rook-ceph-mgr-a-d866b6d57-2j62k 1/1 Running 0 88s
rook-ceph-mon-a-77cfc59bf9-rqrnn 1/1 Running 0 3m23s
rook-ceph-mon-b-85bcd97979-qspdx 1/1 Running 0 2m7s
rook-ceph-mon-c-858db8699b-xptht 1/1 Running 0 116s
rook-ceph-operator-7c55c7cdcb-7ctwg 1/1 Running 0 6m14s
rook-ceph-osd-prepare-nn01-hdtzq 0/1 Completed 0 81s
rook-ceph-osd-prepare-nn02-5tbqf 0/1 Completed 0 81s
rook-ceph-osd-prepare-nn03-d6z6j 0/1 Completed 0 81s
rook-ceph-osd-prepare-nn04-52ws8 0/1 Completed 0 81s
/root/project/rook/rook/cluster/examples/nnrnetes/ceph/nnctl create -f toolbox.yaml
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/csi-cephfsplugin-provisioner 2/2 2 2 4h14m
deployment.apps/csi-rbdplugin-provisioner 2/2 2 2 4h14m
deployment.apps/rook-ceph-crashcollector-nn01 1/1 1 1 4h6m
deployment.apps/rook-ceph-crashcollector-nn02 1/1 1 1 4h6m
deployment.apps/rook-ceph-crashcollector-nn03 1/1 1 1 4h6m
deployment.apps/rook-ceph-crashcollector-nn04 1/1 1 1 4h7m
deployment.apps/rook-ceph-mgr-a 1/1 1 1 4h7m
deployment.apps/rook-ceph-mon-a 1/1 1 1 4h14m
deployment.apps/rook-ceph-mon-c 1/1 1 1 4h9m
deployment.apps/rook-ceph-mon-d 1/1 1 1 4h5m
deployment.apps/rook-ceph-operator 1/1 1 1 4h30m
deployment.apps/rook-ceph-osd-0 1/1 1 1 4h6m
deployment.apps/rook-ceph-osd-1 1/1 1 1 4h6m
deployment.apps/rook-ceph-osd-2 1/1 1 1 4h6m
deployment.apps/rook-ceph-osd-3 1/1 1 1 4h6m
deployment.apps/rook-ceph-tools 1/1 1 1 3m47s ###
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl -n rook-ceph exec -it pod/rook-ceph-tools-54fc95f4f4-hhjr5 -- bash
[root@rook-ceph-tools-54fc95f4f4-hhjr5 /]# ceph -s
cluster:
id: e3900df1-70ec-4304-98cb-69a372af415f
health: HEALTH_OK
services:
mon: 3 daemons, quorum a,c,d (age 4h)
mgr: a(active, since 4h)
osd: 4 osds: 4 up (since 4h), 4 in (since 4h)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 20 MiB used, 400 GiB / 400 GiB avail
pgs: 1 active+clean
[root@rook-ceph-tools-54fc95f4f4-hhjr5 /]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.39075 root default
-9 0.09769 host nn01
3 ssd 0.09769 osd.3 up 1.00000 1.00000
-3 0.09769 host nn02
0 ssd 0.09769 osd.0 up 1.00000 1.00000
-5 0.09769 host nn03
1 ssd 0.09769 osd.1 up 1.00000 1.00000
-7 0.09769 host nn04
2 ssd 0.09769 osd.2 up 1.00000 1.00000
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
xvda 202:0 0 64G 0 disk
├─xvda1 202:1 0 1G 0 part /boot
└─xvda2 202:2 0 63G 0 part
├─centos-root 253:0 0 61G 0 lvm /
└─centos-swap 253:1 0 2G 0 lvm
xvdb 202:16 0 100G 0 disk
└─ceph--7399ccbd--4cff--4abe--8a63--6e6aeded73a9-osd--block--a7168dd8--704a--4154--842c--5a4412d5cb63 253:2 0 100G 0 lvm
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl get svc -n rook-ceph
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
csi-cephfsplugin-metrics ClusterIP 10.233.49.15 <none> 8080/TCP,8081/TCP 4h32m
csi-rbdplugin-metrics ClusterIP 10.233.37.4 <none> 8080/TCP,8081/TCP 4h32m
rook-ceph-mgr ClusterIP 10.233.6.163 <none> 9283/TCP 4h25m
rook-ceph-mgr-dashboard ClusterIP 10.233.15.140 <none> 8443/TCP 4h25m
rook-ceph-mon-a ClusterIP 10.233.25.46 <none> 6789/TCP,3300/TCP 4h32m
rook-ceph-mon-c ClusterIP 10.233.26.206 <none> 6789/TCP,3300/TCP 4h28m
rook-ceph-mon-d ClusterIP 10.233.38.114 <none> 6789/TCP,3300/TCP 4h24m
nodeport
https://jx2lee.github.io/cloud-export_rook_ceph_dashboard/
root@nn01:~/project/rook/dashboard:]# cat dashboard-external-https.yaml
apiVersion: v1
kind: Service
metadata:
name: rook-ceph-mgr-dashboard-external-https
namespace: rook-ceph
labels:
app: rook-ceph-mgr
rook_cluster: rook-ceph
spec:
ports:
- name: dashboard
port: 8443
protocol: TCP
targetPort: 8443
selector:
app: rook-ceph-mgr
rook_cluster: rook-ceph
sessionAffinity: None
type: NodePort
root@nn01:~/project/rook/dashboard:]# nnctl create -f dashboard-external-https.yaml
service/rook-ceph-mgr-dashboard-external-https created
root@nn01:~/project/rook/dashboard:]# nnctl get service -n rook-ceph
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
csi-cephfsplugin-metrics ClusterIP 10.233.49.15 <none> 8080/TCP,8081/TCP 4h45m
csi-rbdplugin-metrics ClusterIP 10.233.37.4 <none> 8080/TCP,8081/TCP 4h45m
rook-ceph-mgr ClusterIP 10.233.6.163 <none> 9283/TCP 4h38m
rook-ceph-mgr-dashboard ClusterIP 10.233.15.140 <none> 8443/TCP 4h38m
rook-ceph-mgr-dashboard-external-https NodePort 10.233.12.187 <none> 8443:31801/TCP 19s
rook-ceph-mon-a ClusterIP 10.233.25.46 <none> 6789/TCP,3300/TCP 4h46m
rook-ceph-mon-c ClusterIP 10.233.26.206 <none> 6789/TCP,3300/TCP 4h41m
rook-ceph-mon-d ClusterIP 10.233.38.114 <none> 6789/TCP,3300/TCP 4h37m
root@nn01:~/project/rook/dashboard:]# nnctl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo
h-M.jeQK5D2P?X=p*$$_
admin // h-M.jeQK5D2P?X=p*$$_
(이미지 없음: picture131-1.png)
/root/project/rook/rook/cluster/examples/nnrnetes/ceph/csi/rbd
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph/csi/rbd:]# nnctl create -f storageclass.yaml
cephblockpool.ceph.rook.io/replicapool created
storageclass.storage.k8s.io/rook-ceph-block created
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: test-pvc
spec:
storageClassName: rook-ceph-block
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
nnctl create -f test-pvc.yaml
root@nn01:~/project/rook/pvc:]# nnctl get pv,pvc
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/pvc-13c7dbdb-07cc-43d8-9ff9-7d5f2ff78097 5Gi RWO Delete Bound default/test-pvc rook-ceph-block 87s
persistentvolume/pvc-2effc491-c15e-48f0-b6ea-4dd5cd68615b 2Gi RWO Delete Bound default/data1 local 3d23h
persistentvolume/pvc-5b872b63-b15a-4772-bd12-fa713af88fb8 1Gi RWX Delete Bound nn-system/cephfs-pvc csi-cephfs 17h
persistentvolume/pvc-6360cc35-fae8-4110-a9c5-2cdbf56df636 20Gi RWO Delete Bound nnsphere-monitoring-system/prometheus-k8s-db-prometheus-k8s-0 local 21d
persistentvolume/pvc-e8bb06a0-d3d3-4365-b047-3685d8b97e55 500Gi RWO Delete Bound default/minio local 18d
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/data1 Bound pvc-2effc491-c15e-48f0-b6ea-4dd5cd68615b 2Gi RWO local 3d23h
persistentvolumeclaim/minio Bound pvc-e8bb06a0-d3d3-4365-b047-3685d8b97e55 500Gi RWO local 18d
persistentvolumeclaim/test-pvc Bound pvc-13c7dbdb-07cc-43d8-9ff9-7d5f2ff78097 5Gi RWO rook-ceph-block 89s
root@nn01:~/project/rook/pvc:]# nnctl get sc/rook-ceph-block -o json
{
"allowVolumeExpansion": true,
"apiVersion": "storage.k8s.io/v1",
"kind": "StorageClass",
"metadata": {
"annotations": {
"storageclass.nnsphere.io/support-snapshot": "false"
},
"creationTimestamp": "2021-09-28T01:49:02Z",
"managedFields": [
{
"apiVersion": "storage.k8s.io/v1",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:storageclass.nnsphere.io/support-snapshot": {}
}
}
},
"manager": "controller-manager",
"operation": "Update",
"time": "2021-09-28T01:49:02Z"
},
{
"apiVersion": "storage.k8s.io/v1",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:allowVolumeExpansion": {},
"f:parameters": {
".": {},
"f:clusterID": {},
"f:csi.storage.k8s.io/controller-expand-secret-name": {},
"f:csi.storage.k8s.io/controller-expand-secret-namespace": {},
"f:csi.storage.k8s.io/fstype": {},
"f:csi.storage.k8s.io/node-stage-secret-name": {},
"f:csi.storage.k8s.io/node-stage-secret-namespace": {},
"f:csi.storage.k8s.io/provisioner-secret-name": {},
"f:csi.storage.k8s.io/provisioner-secret-namespace": {},
"f:imageFeatures": {},
"f:imageFormat": {},
"f:pool": {}
},
"f:provisioner": {},
"f:reclaimPolicy": {},
"f:volumeBindingMode": {}
},
"manager": "nnctl-create",
"operation": "Update",
"time": "2021-09-28T01:49:02Z"
}
],
"name": "rook-ceph-block",
"resourceVersion": "5242688",
"uid": "acb43996-32f1-45ac-bed3-052002b6641b"
},
"parameters": {
"clusterID": "rook-ceph",
"csi.storage.k8s.io/controller-expand-secret-name": "rook-csi-rbd-provisioner",
"csi.storage.k8s.io/controller-expand-secret-namespace": "rook-ceph",
"csi.storage.k8s.io/fstype": "ext4",
"csi.storage.k8s.io/node-stage-secret-name": "rook-csi-rbd-node",
"csi.storage.k8s.io/node-stage-secret-namespace": "rook-ceph",
"csi.storage.k8s.io/provisioner-secret-name": "rook-csi-rbd-provisioner",
"csi.storage.k8s.io/provisioner-secret-namespace": "rook-ceph",
"imageFeatures": "layering",
"imageFormat": "2",
"pool": "replicapool"
},
"provisioner": "rook-ceph.rbd.csi.ceph.com",
"reclaimPolicy": "Delete",
"volumeBindingMode": "Immediate"
}
root@nn01:~/project/rook/pvc:]# nnctl -n rook-ceph exec -it pod/rook-ceph-tools-54fc95f4f4-hhjr5 -- bash
[root@rook-ceph-tools-54fc95f4f4-hhjr5 /]# ceph osd pool ls
device_health_metrics
myfs-metadata
myfs-data0
replicapool
[root@rook-ceph-tools-54fc95f4f4-hhjr5 /]# rbd ls -p replicapool
csi-vol-193aab72-2001-11ec-8907-de94e8249db4
[root@rook-ceph-tools-54fc95f4f4-hhjr5 /]# rbd info csi-vol-193aab72-2001-11ec-8907-de94e8249db4 -p replicapool
rbd image 'csi-vol-193aab72-2001-11ec-8907-de94e8249db4':
size 5 GiB in 1280 objects
order 22 (4 MiB objects)
snapshot_count: 0
id: ea502e6d279d
block_name_prefix: rbd_data.ea502e6d279d
format: 2
features: layering
op_features:
flags:
create_timestamp: Tue Sep 28 02:09:25 2021
access_timestamp: Tue Sep 28 02:09:25 2021
modify_timestamp: Tue Sep 28 02:09:25 2021
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f object.yaml -n rook-ceph
cephobjectstore.ceph.rook.io/my-store created
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f storageclass-bucket-delete.yaml -n rook-ceph
storageclass.storage.k8s.io/rook-ceph-delete-bucket created
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f object-bucket-claim-delete.yaml -n rook-ceph
objectbucketclaim.objectbucket.io/ceph-delete-bucket created
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f rgw-external.yaml
service/rook-ceph-rgw-my-store-external created
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl get svc -n rook-ceph
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
csi-cephfsplugin-metrics ClusterIP 10.233.49.15 <none> 8080/TCP,8081/TCP 28h
csi-rbdplugin-metrics ClusterIP 10.233.37.4 <none> 8080/TCP,8081/TCP 28h
rook-ceph-mgr ClusterIP 10.233.6.163 <none> 9283/TCP 28h
rook-ceph-mgr-dashboard ClusterIP 10.233.15.140 <none> 8443/TCP 28h
rook-ceph-mgr-dashboard-external-https NodePort 10.233.12.187 <none> 8443:31801/TCP 23h
rook-ceph-mon-a ClusterIP 10.233.25.46 <none> 6789/TCP,3300/TCP 28h
rook-ceph-mon-c ClusterIP 10.233.26.206 <none> 6789/TCP,3300/TCP 28h
rook-ceph-mon-d ClusterIP 10.233.38.114 <none> 6789/TCP,3300/TCP 28h
rook-ceph-rgw-my-store ClusterIP 10.233.26.254 <none> 80/TCP 5m38s
rook-ceph-rgw-my-store-external NodePort 10.233.29.218 <none> 80:31247/TCP 13s
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl get all -l app=rook-ceph-rgw -n rook-ceph
NAME READY STATUS RESTARTS AGE
pod/rook-ceph-rgw-my-store-a-7cdfd7587c-mw2d2 1/1 Running 0 20m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/rook-ceph-rgw-my-store ClusterIP 10.233.26.254 <none> 80/TCP 20m
service/rook-ceph-rgw-my-store-external NodePort 10.233.29.218 <none> 80:31247/TCP 14m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/rook-ceph-rgw-my-store-a 1/1 1 1 20m
NAME DESIRED CURRENT READY AGE
replicaset.apps/rook-ceph-rgw-my-store-a-7cdfd7587c 1 1 1 20m
# user 생성
# https://yeti.tistory.com/247
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl create -f object-user.yaml
cephobjectstoreuser.ceph.rook.io/my-user created
# 보안토큰 조회
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl -n rook-ceph describe secret rook-ceph-object-user-my-store-my-user
Name: rook-ceph-object-user-my-store-my-user
Namespace: rook-ceph
Labels: app=rook-ceph-rgw
rook_cluster=rook-ceph
rook_object_store=my-store
user=my-user
Annotations: <none>
Type: nnrnetes.io/rook
Data
====
AccessKey: 20 bytes
Endpoint: 46 bytes
SecretKey: 40 bytes
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o yaml | grep AccessKey |head -1 |awk '{print $2}' | base64 --decode
UTI9K4I4P0FMQXHW3Z9J
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# nnctl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o yaml | grep SecretKey |head -1 |awk '{print $2}' | base64 --decode
mXmNnBqCPK3zcVs5EKYVNPk50fjZAHfJK6YP4oRA
Ref: https://github.com/rook/rook/blob/master/Documentation/helm-operator.md
helm repo add rook-release https://charts.rook.io/release
helm install --create-namespace --namespace rook-ceph rook-ceph rook-release/rook-ceph
root@nn01:~/project/rook/rook/cluster/examples/nnrnetes/ceph:]# sudo yum update -y && sudo yum install ceph-deploy python2-pip -y
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
local-epel | 2.9 kB 00:00:00
local-mariadb10.6 | 2.9 kB 00:00:00
local-mysql8 | 2.9 kB 00:00:00
local-postgresql-13 | 2.9 kB 00:00:00
local_centos | 2.9 kB 00:00:00
local_centos_extras | 2.9 kB 00:00:00
local_dockers | 2.9 kB 00:00:00
local_python_ius | 2.9 kB 00:00:00
Resolving Dependencies
--> Running transaction check
---> Package MariaDB-compat.x86_64 0:10.6.4-1.el7.centos will be obsoleting
--> Processing Dependency: MariaDB-common for package: MariaDB-compat-10.6.4-1.el7.centos.x86_64
---> Package mariadb-libs.x86_64 1:5.5.68-1.el7 will be obsoleted
---> Package mysql-community-libs.x86_64 0:8.0.26-1.el7 will be obsoleting
--> Processing Dependency: mysql-community-client-plugins = 8.0.26-1.el7 for package: mysql-community-libs-8.0.26-1.el7.x86_64
--> Processing Dependency: mysql-community-common(x86-64) >= 8.0.11 for package: mysql-community-libs-8.0.26-1.el7.x86_64
---> Package mysql-community-libs-compat.x86_64 0:8.0.26-1.el7 will be obsoleting
--> Running transaction check
---> Package MariaDB-common.x86_64 0:10.6.4-1.el7.centos will be installed
---> Package mysql-community-client-plugins.x86_64 0:8.0.26-1.el7 will be installed
---> Package mysql-community-common.x86_64 0:8.0.26-1.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
============================================================================================================================================================================================================
Package Arch Version Repository Size
============================================================================================================================================================================================================
Installing:
MariaDB-compat x86_64 10.6.4-1.el7.centos local-mariadb10.6 2.2 M
replacing mariadb-libs.x86_64 1:5.5.68-1.el7
mysql-community-libs x86_64 8.0.26-1.el7 local-mysql8 4.0 M
replacing mariadb-libs.x86_64 1:5.5.68-1.el7
mysql-community-libs-compat x86_64 8.0.26-1.el7 local-mysql8 1.2 M
replacing mariadb-libs.x86_64 1:5.5.68-1.el7
Installing for dependencies:
MariaDB-common x86_64 10.6.4-1.el7.centos local-mariadb10.6 81 k
mysql-community-client-plugins x86_64 8.0.26-1.el7 local-mysql8 4.5 M
mysql-community-common x86_64 8.0.26-1.el7 local-mysql8 620 k
Transaction Summary
============================================================================================================================================================================================================
Install 3 Packages (+3 Dependent packages)
Total download size: 13 M
Downloading packages:
(1/6): MariaDB-common-10.6.4-1.el7.centos.x86_64.rpm | 81 kB 00:00:00
(2/6): mysql-community-common-8.0.26-1.el7.x86_64.rpm | 620 kB 00:00:00
(3/6): MariaDB-compat-10.6.4-1.el7.centos.x86_64.rpm | 2.2 MB 00:00:00
(4/6): mysql-community-client-plugins-8.0.26-1.el7.x86_64.rpm | 4.5 MB 00:00:00
(5/6): mysql-community-libs-8.0.26-1.el7.x86_64.rpm | 4.0 MB 00:00:00
(6/6): mysql-community-libs-compat-8.0.26-1.el7.x86_64.rpm | 1.2 MB 00:00:00
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Total 43 MB/s | 13 MB 00:00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : MariaDB-compat-10.6.4-1.el7.centos.x86_64 1/7
Installing : MariaDB-common-10.6.4-1.el7.centos.x86_64 2/7
Installing : mysql-community-common-8.0.26-1.el7.x86_64 3/7
Installing : mysql-community-client-plugins-8.0.26-1.el7.x86_64 4/7
Installing : mysql-community-libs-8.0.26-1.el7.x86_64 5/7
Installing : mysql-community-libs-compat-8.0.26-1.el7.x86_64 6/7
Erasing : 1:mariadb-libs-5.5.68-1.el7.x86_64 7/7
Verifying : MariaDB-common-10.6.4-1.el7.centos.x86_64 1/7
Verifying : mysql-community-libs-compat-8.0.26-1.el7.x86_64 2/7
Verifying : mysql-community-libs-8.0.26-1.el7.x86_64 3/7
Verifying : MariaDB-compat-10.6.4-1.el7.centos.x86_64 4/7
Verifying : mysql-community-client-plugins-8.0.26-1.el7.x86_64 5/7
Verifying : mysql-community-common-8.0.26-1.el7.x86_64 6/7
Verifying : 1:mariadb-libs-5.5.68-1.el7.x86_64 7/7
Installed:
MariaDB-compat.x86_64 0:10.6.4-1.el7.centos mysql-community-libs.x86_64 0:8.0.26-1.el7 mysql-community-libs-compat.x86_64 0:8.0.26-1.el7
Dependency Installed:
MariaDB-common.x86_64 0:10.6.4-1.el7.centos mysql-community-client-plugins.x86_64 0:8.0.26-1.el7 mysql-community-common.x86_64 0:8.0.26-1.el7
Replaced:
mariadb-libs.x86_64 1:5.5.68-1.el7
Complete!
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
Resolving Dependencies
--> Running transaction check
---> Package ceph-deploy.noarch 0:1.5.25-1.el7 will be installed
--> Processing Dependency: python-remoto for package: ceph-deploy-1.5.25-1.el7.noarch
---> Package python2-pip.noarch 0:8.1.2-14.el7 will be installed
--> Processing Dependency: python-setuptools for package: python2-pip-8.1.2-14.el7.noarch
--> Running transaction check
---> Package python-setuptools.noarch 0:0.9.8-7.el7 will be installed
--> Processing Dependency: python-backports-ssl_match_hostname for package: python-setuptools-0.9.8-7.el7.noarch
---> Package python2-remoto.noarch 0:1.0.0-1.el7 will be installed
--> Processing Dependency: python2-execnet >= 1.2.0 for package: python2-remoto-1.0.0-1.el7.noarch
--> Running transaction check
---> Package python-backports-ssl_match_hostname.noarch 0:3.5.0.1-1.el7 will be installed
--> Processing Dependency: python-ipaddress for package: python-backports-ssl_match_hostname-3.5.0.1-1.el7.noarch
--> Processing Dependency: python-backports for package: python-backports-ssl_match_hostname-3.5.0.1-1.el7.noarch
---> Package python-execnet.noarch 0:1.7.1-3.el7 will be installed
--> Processing Dependency: python-apipkg for package: python-execnet-1.7.1-3.el7.noarch
--> Running transaction check
---> Package python-backports.x86_64 0:1.0-8.el7 will be installed
---> Package python-ipaddress.noarch 0:1.0.16-2.el7 will be installed
---> Package python2-apipkg.noarch 0:1.5-5.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
============================================================================================================================================================================================================
Package Arch Version Repository Size
============================================================================================================================================================================================================
Installing:
ceph-deploy noarch 1.5.25-1.el7 local-epel 156 k
python2-pip noarch 8.1.2-14.el7 local-epel 1.7 M
Installing for dependencies:
python-backports x86_64 1.0-8.el7 local_centos 5.8 k
python-backports-ssl_match_hostname noarch 3.5.0.1-1.el7 local_centos 13 k
python-execnet noarch 1.7.1-3.el7 local-epel 260 k
python-ipaddress noarch 1.0.16-2.el7 local_centos 34 k
python-setuptools noarch 0.9.8-7.el7 local_centos 397 k
python2-apipkg noarch 1.5-5.el7 local-epel 17 k
python2-remoto noarch 1.0.0-1.el7 local-epel 42 k
Transaction Summary
============================================================================================================================================================================================================
Install 2 Packages (+7 Dependent packages)
Total size: 2.6 M
Total download size: 2.1 M
Installed size: 11 M
Downloading packages:
(1/5): ceph-deploy-1.5.25-1.el7.noarch.rpm | 156 kB 00:00:00
(2/5): python-execnet-1.7.1-3.el7.noarch.rpm | 260 kB 00:00:00
(3/5): python2-apipkg-1.5-5.el7.noarch.rpm | 17 kB 00:00:00
(4/5): python2-pip-8.1.2-14.el7.noarch.rpm | 1.7 MB 00:00:00
(5/5): python2-remoto-1.0.0-1.el7.noarch.rpm | 42 kB 00:00:00
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Total 15 MB/s | 2.1 MB 00:00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : python-ipaddress-1.0.16-2.el7.noarch 1/9
Installing : python-backports-1.0-8.el7.x86_64 2/9
Installing : python-backports-ssl_match_hostname-3.5.0.1-1.el7.noarch 3/9
Installing : python-setuptools-0.9.8-7.el7.noarch 4/9
Installing : python2-apipkg-1.5-5.el7.noarch 5/9
Installing : python-execnet-1.7.1-3.el7.noarch 6/9
Installing : python2-remoto-1.0.0-1.el7.noarch 7/9
Installing : ceph-deploy-1.5.25-1.el7.noarch 8/9
Installing : python2-pip-8.1.2-14.el7.noarch 9/9
Verifying : python2-apipkg-1.5-5.el7.noarch 1/9
Verifying : python-backports-ssl_match_hostname-3.5.0.1-1.el7.noarch 2/9
Verifying : python2-pip-8.1.2-14.el7.noarch 3/9
Verifying : python-execnet-1.7.1-3.el7.noarch 4/9
Verifying : python-backports-1.0-8.el7.x86_64 5/9
Verifying : python-ipaddress-1.0.16-2.el7.noarch 6/9
Verifying : ceph-deploy-1.5.25-1.el7.noarch 7/9
Verifying : python-setuptools-0.9.8-7.el7.noarch 8/9
Verifying : python2-remoto-1.0.0-1.el7.noarch 9/9
Installed:
ceph-deploy.noarch 0:1.5.25-1.el7 python2-pip.noarch 0:8.1.2-14.el7
Dependency Installed:
python-backports.x86_64 0:1.0-8.el7 python-backports-ssl_match_hostname.noarch 0:3.5.0.1-1.el7 python-execnet.noarch 0:1.7.1-3.el7 python-ipaddress.noarch 0:1.0.16-2.el7
python-setuptools.noarch 0:0.9.8-7.el7 python2-apipkg.noarch 0:1.5-5.el7 python2-remoto.noarch 0:1.0.0-1.el7
Complete!
Ref : https://www.humblec.com/forcefully-delete-stuck-rook-ceph-namespace/
nnctl -n rook-ceph patch cephclusters.ceph.rook.io rook-ceph -p '{"metadata":{"finalizers": []}}' --type=merge
git clone --single-branch --branch release-1.7 https://github.com/rook/rook.git
cd rook/cluster/examples/nnrnetes/ceph/
nnctl create -f crds.yaml -f common.yaml -f operator.yaml
nnctl create -f cluster.yaml
nnctl get pod -n rook-ceph
-- sudo yum update -y && sudo yum install ceph-deploy python2-pip -y
which ceph
ll
cd ..
ll
cd ceph/
ll
find ./ -name ceph -print
cd ..
ll
cd ..
ll
cd charts/
ll
cd rook-ceph
ll
cd .
cd ..
cd rook-ceph-cluster/
ll
cd ..
cd ..
ll
cd rook/
ll
cd rook/
ll
cd ..
mkdir ceph-toolbox
cd ceph-toolbox/
vi toolbox.yaml
nnctl create -f toolbox.yaml
nnctl get deployments -n rook-ceph
nnctl -n rook-ceph rollout status deploy/rook-ceph-tools
nnctl get deployments -n rook-ceph
nnctl -n rook-ceph exec -it deploy/rook-ceph-tools -- bash
ceph status
nnctl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph status
nnctl -n rook-ceph logs rook-ceph-osd-prepare-nn01-hdtzq provision
lsblk /dev/sr0 --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME
nnctl proxy &
curl -k -H "Content-Type: application/json" -X PUT --data-binary @temp.json http://127.0.0.1:8001/api/v1/namespaces/rook-ceph/finalize
curl -k -H "Content-Type: application/json" -X PUT --data-binary @temp.json https://lb.nnsphere.local:6443/api/v1/namespaces/rook-ceph/finalize
nnctl -n rook-ceph patch cephclusters.ceph.rook.io rook-ceph -p '{"metadata":{"finalizers": []}}' --type=merge
root@nn01:~/project/rook/delete_namespace:]# nnctl proxy &
[1] 19370
Starting to serve on 127.0.0.1:8001
nnctl get ns airflow-spark -o json > todelete.json
nnctl replace --raw http://127.0.0.1:8001/api/v1/namespaces/airflow-spark/finalize -f ./todelete.json