{"id":570,"date":"2024-04-07T09:03:48","date_gmt":"2024-04-07T01:03:48","guid":{"rendered":"https:\/\/www.bunnyism.com\/?p=570"},"modified":"2024-12-18T22:04:59","modified_gmt":"2024-12-18T14:04:59","slug":"k8s%e9%9b%86%e6%88%90cephnfs","status":"publish","type":"post","link":"https:\/\/www.bunnyism.com\/?p=570","title":{"rendered":"K8s\u96c6\u6210CephNFS"},"content":{"rendered":"<h2>ceph\u73af\u5883\u51c6\u5907<\/h2>\n<h3>\u521b\u5efa\u5b58\u50a8\u6c60<\/h3>\n<pre class=\"code\"> ceph osd pool create test-k8s<\/pre>\n<h3>\u542f\u7528\u5b58\u50a8\u6c60\u7684\u5e94\u7528\u7c7b\u578b\u4e3arbd<\/h3>\n<pre class=\"code\">ceph osd pool application get test-k8s\r\nceph osd pool application enable test-k8s rbd\r\nceph osd pool application get test-k8s<\/pre>\n<h3>\u521b\u5efa\u5757\u8bbe\u5907<\/h3>\n<pre class=\"code\">rbd create -s 20G test-k8s\/xiao\r\nrbd ls -p yinzhengjie-k8s\r\nrbd info yinzhengjie-k8s\/xiao<\/pre>\n<h2>k8s\u8282\u70b9\u73af\u5883\u51c6\u5907<\/h2>\n<h3>K8S\u6240\u6709\u8282\u70b9\u5b89\u88c5ceph-common<\/h3>\n<pre class=\"code\">apt -y install ceph-common<\/pre>\n<h3>\u62f7\u8d1d\u8ba4\u8bc1\u6587\u4ef6<\/h3>\n<pre class=\"code\">scp \/etc\/ceph\/ceph.client.admin.keyring 10.0.0.231:\/etc\/ceph\r\nscp \/etc\/ceph\/ceph.client.admin.keyring 10.0.0.232:\/etc\/ceph\r\nscp \/etc\/ceph\/ceph.client.admin.keyring 10.0.0.233:\/etc\/ceph<\/pre>\n<h2>rbd\u5b58\u50a8\u5377\u4e4bkeyring\u5b9e\u6218\u6848\u4f8b<\/h2>\n<h3>\u7f16\u5199\u8d44\u6e90\u6e05\u5355<\/h3>\n<pre class=\"code\">cat 01-deploy-xiao-volumes-rbd.yaml \r\napiVersion: apps\/v1\r\nkind: Deployment\r\nmetadata:\r\nname: deploy-xiao-rbd\r\nspec:\r\nreplicas: 1\r\nselector:\r\nmatchLabels:\r\napps: v1\r\ntemplate:\r\nmetadata:\r\nlabels:\r\napps: v1\r\nspec:\r\nvolumes:\r\n- name: data\r\n# \u6307\u5b9a\u5b58\u50a8\u7c7b\u578b\u4e3arbd\r\nrbd:\r\n# \u6307\u5b9aceph\u7684mon\u7ec4\u4ef6\r\nmonitors:\r\n- 10.0.0.141:6789\r\n- 10.0.0.142:6789\r\n- 10.0.0.143:6789\r\n# \u6307\u5b9aceph\u7684\u5b58\u50a8\u6c60\r\npool: xiao-k8s\r\n# \u6307\u5b9a\u5757\u8bbe\u5907\u955c\u50cf\u7684\u540d\u79f0\r\nimage: xiao\r\n# \u6307\u5b9a\u6587\u4ef6\u7cfb\u7edf\u7684\u683c\u5f0f\uff0c\u6bd4\u5982: \"ext4\", \"xfs\", \"ntfs\"\uff0c\u82e5\u4e0d\u6307\u5b9a\uff0c\u5219\u9ed8\u8ba4\u4e3a: ext4\u3002\r\nfsType: xfs\r\n# \u6307\u5b9a\u8fde\u63a5Ceph\u96c6\u7fa4\u7684\u7528\u6237\uff0c\u82e5\u4e0d\u6307\u5b9a\u5219\u4e3aadmin\u3002\r\nuser: admin\r\n# \u6307\u5b9a\u79d8\u94a5\u8ba4\u8bc1\u7684\u8def\u5f84\uff0c\u82e5\u4e0d\u6307\u5b9a\uff0c\u5219\u9ed8\u8ba4\u4e3a: \/etc\/ceph\/keyring\r\nkeyring: \/etc\/ceph\/ceph.client.admin.keyring\r\ncontainers:\r\n- name: c1\r\nimage: nginx\r\nports:\r\n- containerPort: 80\r\nvolumeMounts:\r\n- name: data\r\nmountPath: \/usr\/share\/nginx\/html<\/pre>\n<h3>\u67e5\u770b\u5bb9\u5668\u5185\u90e8\u7684\u6302\u8f7d\u70b9\u5373\u53ef<\/h3>\n<pre class=\"code\">kubectl get pods -o wide\r\n<span class=\"md-plain md-expand\">kubectl exec -it deploy-xiao-rbd-679dbb9dcc-vgg9g -- sh<\/span><\/pre>\n<h3>\u89c2\u5bdfworker\u8282\u70b9<\/h3>\n<pre class=\"code\">rbd showmapped 2&gt;\/dev\/null<\/pre>\n<h2>k8s\u5bf9\u63a5ceph\u96c6\u7fa4rbd\u5b58\u50a8\u5377\u4e4bsecretRef\u5b9e\u6218\u6848\u4f8b ~\u63a8\u8350\u65b9\u5f0f<\/h2>\n<h3>\u67e5\u770bceph\u7684admin\u7528\u6237\u7684key<\/h3>\n<pre class=\"code\"> grep key \/etc\/ceph\/ceph.client.admin.keyring |awk '{printf \"%s\", $NF}' | more<\/pre>\n<h3>rbd\u5b58\u50a8\u5377\u4e4bsecretRef\u5b9e\u6218\u6848\u4f8b<\/h3>\n<h4>\u7f16\u5199\u8d44\u6e90\u6e05\u5355<\/h4>\n<pre class=\"code\">cat 02-deploy-xiao-volumes-rbd-secretRef.yaml\r\napiVersion: v1\r\nkind: Secret\r\nmetadata:\r\nname: ceph-secret\r\ntype: \"kubernetes.io\/rbd\"\r\nstringData:\r\nkey: AQA6fU5nhO+dOBAAx26GykcFxPPJo9kSOTvghQ==\r\n---\r\napiVersion: apps\/v1\r\nkind: Deployment\r\nmetadata:\r\nname: deploy-xiao-secretref\r\nspec:\r\nreplicas: 1\r\nselector:\r\nmatchLabels:\r\napps: v1\r\ntemplate:\r\nmetadata:\r\nlabels:\r\napps: v1\r\nspec:\r\nvolumes:\r\n- name: data\r\n# \u6307\u5b9a\u5b58\u50a8\u7c7b\u578b\u4e3arbd\r\nrbd:\r\n# \u6307\u5b9aceph\u7684mon\u7ec4\u4ef6\r\nmonitors:\r\n- 10.0.0.141:6789\r\n- 10.0.0.142:6789\r\n- 10.0.0.143:6789\r\n# \u6307\u5b9aceph\u7684\u5b58\u50a8\u6c60\r\npool: test-k8s\r\n# \u6307\u5b9a\u5757\u8bbe\u5907\u955c\u50cf\u7684\u540d\u79f0\r\nimage: xiao\r\n# \u6307\u5b9a\u6587\u4ef6\u7cfb\u7edf\u7684\u683c\u5f0f\uff0c\u6bd4\u5982: \"ext4\", \"xfs\", \"ntfs\"\uff0c\u82e5\u4e0d\u6307\u5b9a\uff0c\u5219\u9ed8\u8ba4\u4e3a: ext4\u3002\r\nfsType: xfs\r\n# \u6307\u5b9a\u8fde\u63a5Ceph\u96c6\u7fa4\u7684\u7528\u6237\uff0c\u82e5\u4e0d\u6307\u5b9a\u5219\u4e3aadmin\u3002\r\nuser: admin\r\n# \u6307\u5b9a\u79d8\u94a5\u8ba4\u8bc1\u7684secrets\r\nsecretRef:\r\n# \u6307\u5b9a\u8ba4\u8bc1secret\u7684\u540d\u79f0\r\nname: ceph-secret\r\ncontainers:\r\n- name: c1\r\nimage: nginx\r\nports:\r\n- containerPort: 80\r\nvolumeMounts:\r\n- name: data\r\nmountPath: \/usr\/share\/nginx\/html\r\n<\/pre>\n<h3>\u67e5\u770b\u5bb9\u5668\u5185\u90e8\u7684\u6302\u8f7d\u70b9\u5373\u53ef<\/h3>\n<h2>k8s\u5bf9\u63a5ceph\u96c6\u7fa4pv\u548cpvc\u6848\u4f8b<\/h2>\n<h3>\u7f16\u5199\u8d44\u6e90\u6e05\u5355<\/h3>\n<pre class=\"code\">cat 03-deploy-xiao-pv-pvc-rbd-secretRef.yaml \r\napiVersion: v1\r\nkind: PersistentVolume\r\nmetadata:\r\nname: pv-rbd-01\r\nspec:\r\naccessModes:\r\n- ReadWriteMany\r\npersistentVolumeReclaimPolicy: Retain\r\nrbd:\r\nimage: xiao\r\nmonitors:\r\n- 10.0.0.141:6789\r\n- 10.0.0.142:6789\r\n- 10.0.0.143:6789\r\npool: xiao-k8s\r\nfsType: xfs\r\nsecretRef:\r\nname: ceph-secret\r\nuser: admin\r\ncapacity:\r\nstorage: 2Gi\r\n---\r\napiVersion: v1\r\nkind: PersistentVolumeClaim\r\nmetadata:\r\nname: test-pvc-rbd\r\nspec:\r\n# \u4f7f\u7528\"\"\u8868\u793a\u4e0d\u4f7f\u7528\u9ed8\u8ba4\u7684\u5b58\u50a8\u7c7b\r\nstorageClassName: \"\"\r\n# \u5c06pvc\u5173\u8054\u5230\u6307\u5b9a\u7684pv\r\nvolumeName: pv-rbd-01\r\naccessModes:\r\n- ReadWriteMany\r\nresources:\r\nlimits:\r\nstorage: 2Gi\r\nrequests:\r\nstorage: 1Gi\r\n---\r\napiVersion: v1\r\nkind: Secret\r\nmetadata:\r\nname: ceph-secret\r\ntype: \"kubernetes.io\/rbd\"\r\nstringData:\r\nkey: AQA6fU5nhO+dOBAAx26GykcFxPPJo9kSOTvghQ==\r\n---\r\napiVersion: apps\/v1\r\nkind: Deployment\r\nmetadata:\r\nname: deploy-xiao-secretref\r\nspec:\r\nreplicas: 1\r\nselector:\r\nmatchLabels:\r\napps: v1\r\ntemplate:\r\nmetadata:\r\nlabels:\r\napps: v1\r\nspec:\r\nterminationGracePeriodSeconds: 3\r\nvolumes:\r\n- name: data\r\npersistentVolumeClaim:\r\nclaimName: test-pvc-rbd\r\ncontainers:\r\n- name: c1\r\nimage: nginx\r\nports:\r\n- containerPort: 80\r\nvolumeMounts:\r\n- name: data\r\nmountPath: \/usr\/share\/nginx\/html\r\n\r\n<\/pre>\n<h3>\u6d4b\u8bd5\u9a8c\u8bc1<\/h3>\n<pre class=\"code\">kubectl apply -f 03-deploy-xiao-pv-pvc-rbd-secretRef.yaml\r\nkubectl get pods -o wide\r\ncurl ip\r\n<\/pre>\n<h3>k8s\u4f7f\u7528rbd\u7684\u52a8\u6001\u5b58\u50a8\u7c7b\u5173\u8054ceph\u96c6\u7fa4<\/h3>\n<p>\u4e0b\u8f7d cloud-computing-stack.git<\/p>\n<p>\u4fee\u6539\u914d\u7f6e\u4fe1\u606f<\/p>\n<blockquote><p>csi-rbd-secret.yaml<\/p>\n<p>csi-config-map.yaml<\/p>\n<p>storageclass.yaml<\/p>\n<p>#\u96c6\u7fa4\u7684ID\u9700\u8981\u52a0\u53cc\u5f15\u53f7\u54df<\/p><\/blockquote>\n<h3>\u5b89\u88c5rbd\u7684sc<\/h3>\n<pre class=\"code\">kubectl apply -f deploy\/rbd\/kubernetes\/<\/pre>\n<h3>\u67e5\u770bpod\u4fe1\u606f<\/h3>\n<pre class=\"code\">kubectl get pods -o wide<\/pre>\n<h3>\u89c2\u5bdfrbd\u662f\u5426\u52a8\u6001\u521b\u5efapv<\/h3>\n<pre class=\"code\">rbd ls -l -p test-k8s<\/pre>\n<h3>\u6d4b\u8bd5\u9a8c\u8bc1<\/h3>\n<pre class=\"code\">cat pvc.yaml \r\napiVersion: v1\r\nkind: PersistentVolumeClaim\r\nmetadata:\r\nname: rbd-pvc01\r\nspec:\r\naccessModes:\r\n- ReadWriteOnce\r\nresources:\r\nrequests:\r\nstorage: 2Gi\r\nstorageClassName: csi-rbd-sc\r\n---\r\napiVersion: v1\r\nkind: PersistentVolumeClaim\r\nmetadata:\r\nname: rbd-pvc02\r\nspec:\r\naccessModes:\r\n- ReadWriteOnce\r\nresources:\r\nrequests:\r\nstorage: 4Gi\r\nstorageClassName: csi-rbd-sc<\/pre>\n<h3>\u67e5\u770bsc<\/h3>\n<pre class=\"code\">kubectl get sc csi-rbd-sc\r\nkubectl get pvc rbd-pvc01 rbd-pvc02<\/pre>\n<h2>k8s\u4f7f\u7528cephFS\u6307\u5b9a\u79d8\u94a5\u4e4bsecretRef<\/h2>\n<h3>\u7f16\u5199\u8d44\u6e90\u6e05\u5355<\/h3>\n<pre class=\"code\">cat 04-deploy-volumes-cephFS-secretRef.yaml\r\napiVersion: v1\r\nkind: Secret\r\nmetadata:\r\nname: ceph-secret\r\ntype: \"kubernetes.io\/rbd\"\r\nstringData:\r\nkey: AQA6fU5nhO+dOBAAx26GykcFxPPJo9kSOTvghQ==\r\n---\r\napiVersion: apps\/v1\r\nkind: Deployment\r\nmetadata:\r\nname: deploy-xiao-secretref\r\nspec:\r\nreplicas: 1\r\nselector:\r\nmatchLabels:\r\napps: v1\r\ntemplate:\r\nmetadata:\r\nlabels:\r\napps: v1\r\nspec:\r\nterminationGracePeriodSeconds: 3\r\nvolumes:\r\n- name: data\r\ncephfs:\r\nmonitors:\r\n- 10.0.0.141:6789\r\n- 10.0.0.142:6789\r\n- 10.0.0.143:6789\r\nreadOnly: true\r\nuser: admin\r\nsecretRef:\r\nname: ceph-secret\r\ncontainers:\r\n- name: c1\r\nimage: nginx\r\nports:\r\n- containerPort: 80\r\nvolumeMounts:\r\n- name: data\r\nmountPath: \/usr\/share\/nginx\/html<\/pre>\n<p style=\"padding-left: 40px;\"><strong>sc\u81ea\u52a8\u521b\u5efapv~<\/strong><\/p>\n","protected":false},"excerpt":{"rendered":"<p>ceph\u73af\u5883\u51c6\u5907 \u521b\u5efa\u5b58\u50a8\u6c60 ceph osd pool create test-k8s \u542f\u7528\u5b58\u50a8\u6c60\u7684\u5e94\u7528\u7c7b\u578b [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":667,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-570","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-k8s"],"_links":{"self":[{"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=\/wp\/v2\/posts\/570","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=570"}],"version-history":[{"count":32,"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=\/wp\/v2\/posts\/570\/revisions"}],"predecessor-version":[{"id":665,"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=\/wp\/v2\/posts\/570\/revisions\/665"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=\/wp\/v2\/media\/667"}],"wp:attachment":[{"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=570"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=570"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.bunnyism.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=570"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}