nodeSelector: {} # Constraint rook-ceph-operator Deployment to nodes with label `disktype: ssd`. # For more info, see https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector # disktype: ssd
# Tolerations for the rook-ceph-operator to allow it to run on nodes with particular taints tolerations: []
# Delay to use in node.kubernetes.io/unreachable toleration unreachableNodeTolerationSeconds: 5
## Rook 的 Agent配置污点容忍、节点亲和 ## toleration: NoSchedule, PreferNoSchedule or NoExecute ## tolerationKey: Set this to the specific key of the taint to tolerate ## tolerations: Array of tolerations in YAML format which will be added to agent deployment ## nodeAffinity: Set to labels of the node to match ## flexVolumeDirPath: The path where the Rook agent discovers the flex volume plugins ## libModulesDirPath: The path where the Rook agent can find kernel modules # agent: # toleration: NoSchedule # tolerationKey: key # tolerations: # - key: key # operator: Exists # effect: NoSchedule # nodeAffinity: key1=value1,value2; key2=value3 # mountSecurityMode: Any ## For information on FlexVolume path, please refer to https://rook.io/docs/rook/master/flexvolume.html # flexVolumeDirPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ # libModulesDirPath: /lib/modules # mounts: mount1=/host/path:/container/path,/host/path2:/container/path2
## Rook 的 Discover 配置污点容忍、节点亲和 ## toleration: NoSchedule, PreferNoSchedule or NoExecute ## tolerationKey: Set this to the specific key of the taint to tolerate ## tolerations: Array of tolerations in YAML format which will be added to agent deployment ## nodeAffinity: Set to labels of the node to match # discover: # toleration: NoSchedule # tolerationKey: key # tolerations: # - key: key # operator: Exists # effect: NoSchedule # nodeAffinity: key1=value1,value2; key2=value3 # podLabels: "key1=value1,key2=value2"
# In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). # Disable it here if you have similar issues. # For more details see https://github.com/rook/rook/issues/2417 enableSelinuxRelabeling: false
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used # 启用OBC监视Operator命名空间 enableOBCWatchOperatorNamespace: true
admissionController: # Set tolerations and nodeAffinity for admission controller pod. # The admission controller would be best to start on the same nodes as other ceph daemons. # tolerations: # - key: key # operator: Exists # effect: NoSchedule # nodeAffinity: key1=value1,value2; key2=value3
$ vim rook-ceph-cluster.yaml -----------------$ 下面内容是我修改后的 特殊需要可以根据values.yaml修改 $------------------ cephClusterSpec: mgr: count: 2 dashboard: enabled: true port: 7001 ssl: false storage: # 是否使用所有节点和节点上面的所有可用磁盘，设置为false时需要手动指定节点选择和磁盘信息过滤 useAllNodes: false useAllDevices: false config: metadataDevice: "vdf"# 我元数据存储的ssd盘，不使用ssd存储元数据会比较慢 use it as block db device of bluestore. databaseSizeMB: "1024"# uncomment if the disks are smaller than 100 GB journalSizeMB: "1024"# uncomment if the disks are 20 GB or smaller
$ kubectl logs -f -n rook-ceph rook-ceph-operator-fdb564699-rv9cg 2021-07-16 02:54:46.417047 I | op-k8sutil: Reporting Event rook-ceph:rook-ceph Normal:ReconcileSucceeded:cluster has been configured successfully I0716 02:54:46.417161 6 manager.go:118] objectbucket.io/provisioner-manager "msg"="starting provisioner""name"="rook-ceph.ceph.rook.io/bucket" 2021-07-16 02:54:48.670028 I | ceph-cluster-controller: Disabling the insecure global ID as no legacy clients are currently connected. If you still require the insecure connections, see the CVE to suppress the health warning and re-enable the insecure connections. https://docs.ceph.com/en/latest/security/CVE-2021-20288/ 2021-07-16 02:54:50.333134 I | ceph-cluster-controller: insecure global ID is now disabled 2021-07-16 02:55:32.278545 I | op-mon: checking if multiple mons are on the same node
# 主机上保存配置文件的路径。必须指定。 # 重要提示：如果重新安装群集，请确保从每个主机上删除此目录，否则mons将无法在新群集上启动。 # 在Minikube中，'/data'目录被配置为在重新启动时保持不变。在Minikube环境中使用“/data/rook”。 dataDirHostPath: /var/lib/rook skipUpgradeChecks: false # 升级期间PG不干净是否继续 continueUpgradeAfterChecksEvenIfNotHealthy: false waitTimeoutForHealthyOSDInMinutes: 10 mon: # 设置要启动的mons数。必须是奇数，通常建议为3 count: 3 # 是否允许mon的pod一个节点上启动多个，mons应该位于唯一的节点上。因此，对于生产，建议至少使用3个节点。 # 对于可以接受数据丢失的测试环境，应该只允许在同一节点上使用Mons。 allowMultiplePerNode: false mgr: # 当需要mgr高可用性时，将计数设置到2。 # 在这种情况下，一个mgr将处于活动状态，另一个处于备用状态。当Ceph更新 # mgr处于活动状态，Rook将更新mgr服务以匹配活动的mgr。 count: 2 modules: - name: pg_autoscaler enabled: true # 开启ceph的dashboard来显示ceph集群状态 dashboard: enabled: true # 当ssl为true时dashboard默认使用8443端口，不需要配置prot: "" # port: 8443 # 当ssl为false时必须指定prot: 7000，否则无法访问dashboard port: 7001 # 是否开启dashboard的ssl ssl: false #network: # 开启主机网络访问 #provider: host # EXPERIMENTAL: enable the Multus network provider #provider: multus #selectors: #public: public-conf --> NetworkAttachmentDefinition object name in Multus #cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus # Provide internet protocol version. IPv6, IPv4 or empty string are valid options. Empty string would mean IPv4 #ipFamily: "IPv6" # Ceph daemons to listen on both IPv4 and Ipv6 networks #dualStack: false # enable the crash collector for ceph daemon crash collection crashCollector: disable: false # Uncomment daysToRetain to prune ceph crash entries older than the # specified number of days. #daysToRetain: 30 # enable log collector, daemons will log on files and rotate # logCollector: # enabled: true # periodicity: 24h # SUFFIX may be 'h' for hours or 'd' for days. # automate [data cleanup process](https://github.com/rook/rook/blob/master/Documentation/ceph-teardown.md#delete-the-data-on-hosts) in cluster destruction. # 数据清除策略 cleanupPolicy: confirmation: "" # sanitizeDisks表示在删除群集时对OSD磁盘进行清理的设置 sanitizeDisks: method: quick dataSource: zero iteration: 1 allowUninstallWithVolumes: false # To control where various services will be scheduled by kubernetes, use the placement configuration sections below. # 如果指定特定的k8s集群节点当做存储的节点，配置以下节点选择器和污点容忍规则 # The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node' and # 1.对特定的存储节点打上节点标签 "role=storage-node" # 2.配置存储节点的污点容忍，带有key是"storage-node"的污点容忍 # tolerate taints with a key of 'storage-node'. # placement: # all: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: # - matchExpressions: # - key: role # operator: In # values: # - storage-node # podAffinity: # podAntiAffinity: # topologySpreadConstraints: # tolerations: # - key: storage-node # operator: Exists # The above placement information can also be specified for mon, osd, and mgr components # mon: # osd: # mgr: # cleanup: #annotations: # all: # mon: # osd: # cleanup: # prepareosd: # If no mgr annotations are set, prometheus scrape annotations will be set by default. # mgr: #labels: # all: # mon: # osd: # cleanup: # mgr: # prepareosd: # monitoring is a list of key-value pairs. It is injected into all the monitoring resources created by operator. # These labels can be passed as LabelSelector to Prometheus # monitoring: # # 针对mgr mon osd 各个组件pod配置资源限制 #resources: # The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory # mgr: # limits: # cpu: "500m" # memory: "1024Mi" # requests: # cpu: "500m" # memory: "1024Mi" # The above example requests/limits can also be added to the other components # mon: # osd: # prepareosd: # mgr-sidecar: # crashcollector: # logcollector: # cleanup: # 自动删除已退出并且可以安全销毁的OSD的选项。 removeOSDsIfOutAndSafeToRemove: false # priorityClassNames: # all: rook-ceph-default-priority-class # mon: rook-ceph-mon-priority-class # osd: rook-ceph-osd-priority-class # mgr: rook-ceph-mgr-priority-class # 群集级存储配置和选择 storage: # cluster level storage configuration and selection # 是否使用所有节点和节点上面的所有可用磁盘，设置为false时需要手动指定节点选择和磁盘信息过滤 useAllNodes: false useAllDevices: false #deviceFilter: "" config: # crushRoot: "custom-root" # specify a non-default root label for the CRUSH map metadataDevice: "vdf"# specify a non-rotational storage so ceph-volume will use it as block db device of bluestore. databaseSizeMB: "1024"# uncomment if the disks are smaller than 100 GB journalSizeMB: "1024"# uncomment if the disks are 20 GB or smaller # osdsPerDevice: "1" # this value can be overridden at the node or device level # encryptedDevice: "true" # the default value for this option is "false"
# 也可以指定单个节点及其配置，但上面的“useAllNodes”必须设置为false。那么，只有 # 下面的节点将用作存储资源。每个节点的“name”字段应与其“kubernetes.io/hostname”标签匹配。 nodes: - name: "cn-zhangjiakou.172.16.1.149" devices: # specific devices to use for storage can be specified for each node - name: "vdb" - name: "vdc"# multiple osds can be created on high performance devices - name: "vdd"# multiple osds can be created on high performance devices - name: "vde"# multiple osds can be created on high performance devices - name: "cn-zhangjiakou.172.16.1.150" devices: # specific devices to use for storage can be specified for each node - name: "vdb" - name: "vdc"# multiple osds can be created on high performance devices - name: "vdd"# multiple osds can be created on high performance devices - name: "vde"# multiple osds can be created on high performance devices - name: "cn-zhangjiakou.172.16.1.151" devices: # specific devices to use for storage can be specified for each node - name: "vdb" - name: "vdc"# multiple osds can be created on high performance devices - name: "vdd"# multiple osds can be created on high performance devices - name: "vde"# multiple osds can be created on high performance devices # config: # osdsPerDevice: "5" # - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths # config: # configuration can be specified at the node level which overrides the cluster level config # - name: "172.17.4.301" # deviceFilter: "^sd." # The section for configuring management of daemon disruptions during upgrade or fencing. disruptionManagement: # If true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically # via the strategy outlined in the [design](https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md). The operator will # block eviction of OSDs by default and unblock them safely when drains are detected. managePodBudgets: true # A duration in minutes that determines how long an entire failureDomain like `region/zone/host` will be held in `noout` (in addition to the # default DOWN/OUT interval) when it is draining. This is only relevant when `managePodBudgets` is `true`. The default value is `30` minutes. osdMaintenanceTimeout: 30 # A duration in minutes that the operator will wait for the placement groups to become healthy (active+clean) after a drain was completed and OSDs came back up. # Operator will continue with the next drain if the timeout exceeds. It only works if `managePodBudgets` is `true`. # No values or 0 means that the operator will wait until the placement groups are healthy before unblocking the next drain. pgHealthCheckTimeout: 0 # If true, the operator will create and manage MachineDisruptionBudgets to ensure OSDs are only fenced when the cluster is healthy. # Only available on OpenShift. manageMachineDisruptionBudgets: false # Namespace in which to watch for the MachineDisruptionBudgets. machineDisruptionBudgetNamespace: openshift-machine-api
# healthChecks # Valid values for daemons are 'mon', 'osd', 'status' healthCheck: daemonHealth: mon: disabled: false interval: 45s osd: disabled: false interval: 60s status: disabled: false interval: 60s # Change pod liveness probe, it works for all mon,mgr,osd daemons livenessProbe: mon: disabled: false mgr: disabled: false osd: disabled: false
$ kubectl apply -f rbd-storageclass.yaml cephblockpool.ceph.rook.io/k8s-rbd-test-pool created storageclass.storage.k8s.io/local-ceph-rbs-storageclass created
# 查看pod和pvc $ kubectl get pod,pvc NAME READY STATUS RESTARTS AGE pod/test-ceph-rdb-deploy-5b7cb7d44-k779p 1/1 Running 0 55s
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/local-ceph-test-rbd-pvc Bound pvc-ad88bb0b-c848-4b82-a7f7-904b6e4223b9 20Gi RWO local-ceph-rbs-storageclass 55s
