---
kind: PersistentVolume
apiVersion: v1
metadata:
  name: k8s-master-sdb
  labels:
    osd: "true"
spec:
  volumeMode: Block
  capacity:
    storage: 10Gi
  local:
    path: /dev/sdb
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-master
---
kind: PersistentVolume
apiVersion: v1
metadata:
  name: k8s-node-sdb
  labels:
    osd: "true"
spec:
  volumeMode: Block
  capacity:
    storage: 10Gi
  local:
    path: /dev/sdb
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - k8s-node

[root]# git clone https://github.com/rook/rook.git
[root]# cd rook/cluster/examples/kubernetes/ceph
[root]# git co v1.4.5
[root]# kubectl apply -f common.yaml

--- a/cluster/examples/kubernetes/ceph/operator.yaml
+++ b/cluster/examples/kubernetes/ceph/operator.yaml
@@ -27,7 +27,7 @@ data:
   # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml
   ROOK_CSI_ENABLE_CEPHFS: "true"
   # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below.
-  ROOK_CSI_ENABLE_RBD: "true"
+  ROOK_CSI_ENABLE_RBD: "false"
   ROOK_CSI_ENABLE_GRPC_METRICS: "true"

   # Set logging level for csi containers.

--- a/cluster/examples/kubernetes/ceph/operator.yaml
+++ b/cluster/examples/kubernetes/ceph/operator.yaml
@@ -399,7 +399,7 @@ spec:
         # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
         # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
         - name: ROOK_ENABLE_DISCOVERY_DAEMON
-          value: "true"
+          value: "false"

         # Time to wait until the node controller will move Rook pods to other
         # nodes after detecting an unreachable node.

Warning  FailedMount             4m42s (x12 over 29m)  kubelet, k8s-master      (combined from similar events): MountVolume.MountDevice failed for volume "pvc-991cbae4-311f-4c8a-bfa9-6af99dde2575" : rpc error: code = Internal desc = an error (exit status 32) occurred while running mount args: [-t ceph 10.0.170.252:6789:/volumes/csi/csi-vol-68a7996a-de9e-11ea-8dcc-e2634a1533d6/ad3ed71f-e4a3-4a2a-9f41-ed3f274a4b54 /var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc-991cbae4-311f-4c8a-bfa9-6af99dde2575/globalmount -o name=csi-cephfs-node,secretfile=/tmp/csi/keys/keyfile-909027452,mds_namespace=shared-fs,_netdev]

--- a/cluster/examples/kubernetes/ceph/operator.yaml
+++ b/cluster/examples/kubernetes/ceph/operator.yaml
@@ -38,7 +38,7 @@ data:
   # If you disable the kernel client, your application may be disrupted during upgrade.
   # See the upgrade guide: https://rook.io/docs/rook/master/ceph-upgrade.html
   # NOTE! cephfs quota is not supported in kernel version < 4.17
-  CSI_FORCE_CEPHFS_KERNEL_CLIENT: "true"
+  CSI_FORCE_CEPHFS_KERNEL_CLIENT: "false"

   # (Optional) Allow starting unsupported ceph-csi image
   ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false"

[root]# kubectl apply -f operator.yaml

[root]# kubectl get po -n rook-ceph
NAME                                            READY   STATUS      RESTARTS   AGE
rook-ceph-operator-7c6fb4bf5f-8zn54             1/1     Running     1          4d3h

kind: ConfigMap
apiVersion: v1
metadata:
  name: rook-config-override
  namespace: rook-ceph
data:
  config: |
    [global]
    osd pool default size = 2

--- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml
+++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml
@@ -17,20 +17,8 @@ metadata:
 spec:
   dataDirHostPath: /var/lib/rook
   mon:
-    count: 3
+    count: 1
     allowMultiplePerNode: false
-    # A volume claim template can be specified in which case new monitors (and
-    # monitors created during fail over) will construct a PVC based on the
-    # template for the monitor's primary storage. Changes to the template do not
-    # affect existing monitors. Log data is stored on the HostPath under
-    # dataDirHostPath. If no storage requirement is specified, a default storage
-    # size appropriate for monitor data will be used.
-    volumeClaimTemplate:
-      spec:
-        storageClassName: gp2
-        resources:
-          requests:
-            storage: 10Gi
   cephVersion:
     image: ceph/ceph:v15.2.4
     allowUnsupported: false
@@ -49,7 +37,7 @@ spec:
     storageClassDeviceSets:
     - name: set1
       # The number of OSDs to create from this device set
-      count: 3
+      count: 2
       # IMPORTANT: If volumes specified by the storageClassName are not portable across nodes
       # this needs to be set to false. For example, if using the local storage provisioner
       # this should be false.
@@ -116,11 +104,12 @@ spec:
       volumeClaimTemplates:
       - metadata:
           name: data
           # if you are looking at giving your OSD a different CRUSH device class than the one detected by Ceph
           # annotations:
           #   crushDeviceClass: hybrid
         spec:
+          selector:
+            matchLabels:
+              osd: 'true'
           resources:
             requests:
               storage: 10Gi
-          # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2)
-          storageClassName: gp2
           volumeMode: Block
           accessModes:
             - ReadWriteOnce

[root]# kubectl apply -f cluster-on-pvc.yaml

[root]# kubectl get po -n rook-ceph
NAME                                            READY   STATUS      RESTARTS   AGE
csi-cephfsplugin-provisioner-7478b9dccf-6gnlp   6/6     Running     0          9m43s
csi-cephfsplugin-provisioner-7478b9dccf-w2f6q   6/6     Running     6          8m58s
csi-cephfsplugin-qdsl2                          3/3     Running     0          8m18s
csi-cephfsplugin-jl2qj                          3/3     Running     0          7m51s
rook-ceph-mgr-a-77cf85dc48-rhr4q                1/1     Running     0          6m36s
rook-ceph-mon-a-7bc4d4d86b-jjbfc                1/1     Running     0          18s
rook-ceph-operator-7c6fb4bf5f-8zn54             1/1     Running     1          4d21h
rook-ceph-osd-0-74bb499d8-dnktb                 1/1     Running     0          3m34s
rook-ceph-osd-1-844dccbc49-cvj7t                1/1     Running     1          4m9s
rook-ceph-osd-prepare-set1-data-0-fzhq5-vjbs6   0/1     Completed   0          4d23h

--- a/cluster/examples/kubernetes/ceph/filesystem.yaml
+++ b/cluster/examples/kubernetes/ceph/filesystem.yaml
@@ -13,7 +13,7 @@ spec:
   # The metadata pool spec. Must use replication.
   metadataPool:
     replicated:
-      size: 3
+      size: 2
       requireSafeReplicaSize: true
     parameters:
       # Inline compression mode for the data pool
@@ -26,7 +26,7 @@ spec:
   dataPools:
     - failureDomain: host
       replicated:
-        size: 3
+        size: 2
         # Disallow setting pool with replica 1, this could lead to data loss without recovery.
         # Make sure you're *ABSOLUTELY CERTAIN* that is what you want
         requireSafeReplicaSize: true

[root]# kubectl apply -f filesystem.yaml

[root]# kubectl get po -n rook-ceph
NAME                                            READY   STATUS      RESTARTS   AGE
csi-cephfsplugin-provisioner-7478b9dccf-6gnlp   6/6     Running     16         8h
csi-cephfsplugin-provisioner-7478b9dccf-w2f6q   6/6     Running     9          8h
csi-cephfsplugin-qdsl2                          3/3     Running     0          8h
csi-cephfsplugin-jl2qj                          3/3     Running     0          8h
rook-ceph-mds-myfs-a-9cdd75c7d-qhpj9            1/1     Running     4          3d18h
rook-ceph-mds-myfs-b-74fdc8f896-mdnps           1/1     Running     2          3d18h
rook-ceph-mgr-a-77cf85dc48-rhr4q                1/1     Running     2          8h
rook-ceph-mon-a-7bc4d4d86b-jjbfc                1/1     Running     0          8h
rook-ceph-operator-7c6fb4bf5f-8zn54             1/1     Running     1          5d6h
rook-ceph-osd-0-74bb499d8-dnktb                 1/1     Running     0          8h
rook-ceph-osd-1-844dccbc49-cvj7t                1/1     Running     1          8h
rook-ceph-osd-prepare-set1-data-0-fzhq5-vjbs6   0/1     Completed   0          5d8h

[root]# kubectl port-forward -n rook-ceph --address 0.0.0.0 svc/rook-ceph-mgr-dashboard  8443:8443

[root]# kubectl get -n rook-ceph secret rook-ceph-dashboard-password -o jsonpath='{.data.password}'| base64 -d && echo

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-cephfs
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
  # clusterID is the namespace where operator is deployed.
  clusterID: rook-ceph

  # CephFS filesystem name into which the volume shall be created
  fsName: myfs

  # Ceph pool into which the volume shall be created
  # Required for provisionVolume: "true"
  pool: myfs-data0

  # Root path of an existing CephFS volume
  # Required for provisionVolume: "false"
  # rootPath: /absolute/path

  # The secrets contain Ceph admin credentials. These are generated automatically by the operator
  # in the same namespace as the cluster.
  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph

  # (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel)
  # If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse
  # or by setting the default mounter explicitly via --volumemounter command-line argument.
  # mounter: kernel
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
  # uncomment the following line for debugging
  #- debug

[root]# kubectl apply -f csi/cephfs/storageclass.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: cephfs-pvc
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: rook-cephfs

[root]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                         STORAGECLASS   REASON   AGE
k8s-master-sdb                             10Gi       RWO            Retain           Bound    rook-ceph/set1-data-0-fzhq5                           6d8h
k8s-node-sdb                               10Gi       RWO            Retain           Bound    rook-ceph/set1-data-1-f6bm6                           6d8h
pvc-c041509d-96da-4975-b3fc-51e0e66983a1   1Gi        RWX            Delete           Bound    default/cephfs-pvc            rook-cephfs             4d17h

---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: psp:priv
rules:
- apiGroups:
  - policy
  resourceNames:
  - privileged
  resources:
  - podsecuritypolicies
  verbs:
  - use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: default:psp:privileged
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: psp:priv
subjects:
- kind: ServiceAccount
  name: default
  namespace: default
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: postgresql-deployment
spec:
  selector:
    matchLabels:
      app: postgresql
  replicas: 1
  template:
    metadata:
      labels:
        app: postgresql
    spec:
      containers:
      - name: postgres
        image: postgres:11.9
        ports:
        - containerPort: 5432
        env:
        - name: POSTGRES_PASSWORD
          value: admin
        - name: PGDATA
          value: /var/lib/postgresql/data/pgdata
        volumeMounts:
        - name: cephfs-pvc
          mountPath: /var/lib/postgresql/data
      volumes:
      - name: cephfs-pvc
        persistentVolumeClaim:
          claimName: cephfs-pvc

[root]# kubectl get po
NAME                                     READY   STATUS    RESTARTS   AGE
postgresql-deployment-68ff869bcb-8jtn4   1/1     Running   2          3d6h

[root]# kubectl exec -it postgresql-deployment-68ff869bcb-8jtn4 -- bash
root@postgresql-deployment-68ff869bcb-8jtn4:/# psql -U postgres postgres
psql (11.9 (Debian 11.9-1.pgdg90+1))
Type "help" for help.

postgres=# create table test (id integer, name varchar(10));
CREATE TABLE
postgres=# insert into test values (1, 'hogehoge');
INSERT 0 1
postgres=# select * from test;
 id |   name
----+----------
  1 | hogehoge
(1 row)

postgres=#