From 10f60a39457b7fc4cd5ee4a4ee71531d3cc61b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 21 Feb 2022 17:29:36 +0800 Subject: [PATCH 01/22] noload option appended --- internal/rbd/nodeserver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index dd8b4d4c29b..289f4462ebb 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -459,7 +459,9 @@ func (ns *NodeServer) mountVolumeToStagePath(ctx context.Context, req *csi.NodeS } } if csicommon.MountOptionContains(opt, rOnly) { + // readOnly mount with noload option readOnly = true + opt = append(opt, "noload") } if fsType == "xfs" { From 4cf63bedf387b65944d4559666a0920c593c6015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Tue, 8 Mar 2022 16:40:01 +0800 Subject: [PATCH 02/22] test for write block --- internal/rbd/controllerserver.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 14f81928a3e..9f22459f34b 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1131,7 +1131,13 @@ func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *cs ro = "true" } - return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro}}, nil + block := "false" + + if _, ok := req.GetVolumeContext()["writeBlock"]; ok { + block = "true" + } + + return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro, "writeBlock": block}}, nil } // ControllerUnpublishVolume does nothing. From 4cb6cd2b2e3ac6cff223161eed0720080d5b2e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Tue, 8 Mar 2022 18:12:33 +0800 Subject: [PATCH 03/22] test write block --- internal/rbd/controllerserver.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 9f22459f34b..79aad13c6ac 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1131,10 +1131,12 @@ func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *cs ro = "true" } - block := "false" + volctx := req.GetVolumeContext() + var block string - if _, ok := req.GetVolumeContext()["writeBlock"]; ok { - block = "true" + if _, ok := volctx["writeBlock"]; ok { + + block = volctx["writeBlock"] } return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro, "writeBlock": block}}, nil From 539bceb91aff2edb3e7464689427d891e1c0a037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Fri, 11 Mar 2022 16:57:42 +0800 Subject: [PATCH 04/22] test for mounting condition under same node --- internal/rbd/nodeserver.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 289f4462ebb..e6581d62f36 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -131,6 +131,12 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol disableInUseChecks = true } + if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + if req.GetPublishContext()[readonlyAttachmentKey] == "true" { + return nil, fmt.Errorf("failed to proceed with RW volume") + } + } + volID := req.GetVolumeId() cr, err := util.NewUserCredentials(req.GetSecrets()) From 03b06b1be34313fc520ac0c7ef9faad8f3c1642e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Fri, 11 Mar 2022 17:43:36 +0800 Subject: [PATCH 05/22] test for RW mount condition checking --- internal/rbd/nodeserver.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index e6581d62f36..b0be35559a3 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -117,6 +117,12 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, err } + if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + if req.GetPublishContext()[readonlyAttachmentKey] == "true" { + return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") + } + } + disableInUseChecks := req.GetPublishContext()[readonlyAttachmentKey] == "true" isBlock := req.GetVolumeCapability().GetBlock() != nil // disableInUseChecks := false @@ -131,12 +137,6 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol disableInUseChecks = true } - if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - if req.GetPublishContext()[readonlyAttachmentKey] == "true" { - return nil, fmt.Errorf("failed to proceed with RW volume") - } - } - volID := req.GetVolumeId() cr, err := util.NewUserCredentials(req.GetSecrets()) From 7eb91b955483b72b5520368d7f0d555867bff888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Fri, 11 Mar 2022 17:58:39 +0800 Subject: [PATCH 06/22] fix status.Error --- internal/rbd/nodeserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index b0be35559a3..904bd894db2 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -131,7 +131,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { if !isBlock { util.WarningLog(ctx, "MULTI_NODE_MULTI_WRITER currently only supported with volumes of access type `block`, invalid AccessMode for volume: %v", req.GetVolumeId()) - return nil, status.Error(codes.InvalidArgument, "rbd: RWX access mode request is only valid for volumes with access type `block`") + return nil, status.Error(codes.Internal, "rbd: RWX access mode request is only valid for volumes with access type `block`") } disableInUseChecks = true From 51dee3340cb47a0ed2407fe3e7b2d54c23f5203b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 14 Mar 2022 14:01:01 +0800 Subject: [PATCH 07/22] fixing volcap for nodestage --- internal/rbd/nodeserver.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 904bd894db2..4e7f8885fd9 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -117,10 +117,15 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, err } - if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + // same with external-attacher's volcap ? + // if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { + req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER if req.GetPublishContext()[readonlyAttachmentKey] == "true" { return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") } + } else { + req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY } disableInUseChecks := req.GetPublishContext()[readonlyAttachmentKey] == "true" From d5385350a279c5c52eb3703cc504955a124438f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 14 Mar 2022 14:39:11 +0800 Subject: [PATCH 08/22] fixes for volcap --- internal/rbd/controllerserver.go | 14 +++++++------- internal/rbd/nodeserver.go | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 79aad13c6ac..19e2f3b9e6e 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1130,16 +1130,16 @@ func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *cs if req.GetReadonly() { ro = "true" } + // volctx := req.GetVolumeContext() + // var block string - volctx := req.GetVolumeContext() - var block string + // if _, ok := volctx["writeBlock"]; ok { - if _, ok := volctx["writeBlock"]; ok { + // block = volctx["writeBlock"] + // } - block = volctx["writeBlock"] - } - - return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro, "writeBlock": block}}, nil + // return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro, "writeBlock": block}}, nil + return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro}}, nil } // ControllerUnpublishVolume does nothing. diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 4e7f8885fd9..4b273feaf0a 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -120,13 +120,13 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol // same with external-attacher's volcap ? // if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { - req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER + // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER if req.GetPublishContext()[readonlyAttachmentKey] == "true" { return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") } - } else { - req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY - } + // } else { + // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY + // } disableInUseChecks := req.GetPublishContext()[readonlyAttachmentKey] == "true" isBlock := req.GetVolumeCapability().GetBlock() != nil From 9dd915d39fd9f7beee86d7791d81fd403559d6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 14 Mar 2022 14:51:46 +0800 Subject: [PATCH 09/22] typo --- internal/rbd/nodeserver.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 4b273feaf0a..10419bafc5a 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -124,9 +124,9 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol if req.GetPublishContext()[readonlyAttachmentKey] == "true" { return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") } - // } else { - // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY - // } + // } else { + // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY + } disableInUseChecks := req.GetPublishContext()[readonlyAttachmentKey] == "true" isBlock := req.GetVolumeCapability().GetBlock() != nil From f9d6c719c310b16b1a96588eb1799d6025d12de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 14 Mar 2022 17:04:10 +0800 Subject: [PATCH 10/22] add volcap ROX --- internal/rbd/controllerserver.go | 3 ++- internal/rbd/driver.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 19e2f3b9e6e..5e2cac6978c 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -677,7 +677,8 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req } for _, capability := range req.VolumeCapabilities { - if capability.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + if capability.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER || + capability.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil } } diff --git a/internal/rbd/driver.go b/internal/rbd/driver.go index 48605a5c916..3c4df83a55f 100644 --- a/internal/rbd/driver.go +++ b/internal/rbd/driver.go @@ -142,7 +142,7 @@ func (r *Driver) Run(conf *util.Config) { // will work those as follow up features r.cd.AddVolumeCapabilityAccessModes( []csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY}) } // Create GRPC servers From a8d3cfd04cdd8c5f3903ea8027048eea39290d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 14 Mar 2022 17:31:01 +0800 Subject: [PATCH 11/22] test for valid volcap --- internal/rbd/nodeserver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 10419bafc5a..0df55339f19 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -118,8 +118,8 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } // same with external-attacher's volcap ? - // if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { + if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + // if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER if req.GetPublishContext()[readonlyAttachmentKey] == "true" { return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") From 40ad988e5d7f51a7adf9160312be57f100e80477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 14 Mar 2022 18:39:45 +0800 Subject: [PATCH 12/22] nodepublish test --- internal/rbd/nodeserver.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 0df55339f19..d4e70b3cc5f 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -117,16 +117,16 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, err } - // same with external-attacher's volcap ? - if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - // if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { - // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER - if req.GetPublishContext()[readonlyAttachmentKey] == "true" { - return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") - } - // } else { - // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY - } + // // same with external-attacher's volcap ? + // // if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + // if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { + // // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER + // if req.GetPublishContext()[readonlyAttachmentKey] == "true" { + // return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") + // } + // // } else { + // // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY + // } disableInUseChecks := req.GetPublishContext()[readonlyAttachmentKey] == "true" isBlock := req.GetVolumeCapability().GetBlock() != nil @@ -412,6 +412,14 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis volID := req.GetVolumeId() stagingPath += "/" + volID + if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER + if req.GetPublishContext()[readonlyAttachmentKey] == "true" { + return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") + } + // } else { + // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY + } if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired { util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) From 8228452c2ceb1b7d8a046c7d3c641c5e3ba7a796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Mon, 14 Mar 2022 20:26:16 +0800 Subject: [PATCH 13/22] seperate branch with external attacher export all info --- internal/rbd/controllerserver.go | 12 +++--------- internal/rbd/nodeserver.go | 19 ------------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 5e2cac6978c..45072930861 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1131,16 +1131,10 @@ func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *cs if req.GetReadonly() { ro = "true" } - // volctx := req.GetVolumeContext() - // var block string - // if _, ok := volctx["writeBlock"]; ok { - - // block = volctx["writeBlock"] - // } - - // return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro, "writeBlock": block}}, nil - return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro}}, nil + mode := req.GetVolumeCapability().GetAccessMode().GetMode() + am := csi.VolumeCapability_AccessMode_Mode_name[int32(mode)] + return &csi.ControllerPublishVolumeResponse{PublishContext: map[string]string{readonlyAttachmentKey: ro, "Capability": am}}, nil } // ControllerUnpublishVolume does nothing. diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index d4e70b3cc5f..23148763db0 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -117,17 +117,6 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, err } - // // same with external-attacher's volcap ? - // // if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - // if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { - // // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER - // if req.GetPublishContext()[readonlyAttachmentKey] == "true" { - // return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") - // } - // // } else { - // // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY - // } - disableInUseChecks := req.GetPublishContext()[readonlyAttachmentKey] == "true" isBlock := req.GetVolumeCapability().GetBlock() != nil // disableInUseChecks := false @@ -412,14 +401,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis volID := req.GetVolumeId() stagingPath += "/" + volID - if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER - if req.GetPublishContext()[readonlyAttachmentKey] == "true" { - return nil, status.Error(codes.InvalidArgument, "vol has already been mount as ReadWrite on another node") - } - // } else { - // req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY - } if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired { util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) From 0acff448f0ce2d7a5314b14b108cf324dc787f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Tue, 15 Mar 2022 10:24:09 +0800 Subject: [PATCH 14/22] error type rollback --- internal/rbd/nodeserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 23148763db0..289f4462ebb 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -125,7 +125,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { if !isBlock { util.WarningLog(ctx, "MULTI_NODE_MULTI_WRITER currently only supported with volumes of access type `block`, invalid AccessMode for volume: %v", req.GetVolumeId()) - return nil, status.Error(codes.Internal, "rbd: RWX access mode request is only valid for volumes with access type `block`") + return nil, status.Error(codes.InvalidArgument, "rbd: RWX access mode request is only valid for volumes with access type `block`") } disableInUseChecks = true From 7b0cccb08eb0e65a676e6e9b52d2654cde088525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Tue, 15 Mar 2022 13:58:58 +0800 Subject: [PATCH 15/22] sync volcap with va's accessmode --- internal/rbd/nodeserver.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 289f4462ebb..e859cc3e8d5 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -117,6 +117,13 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, err } + // sync volcap with va's accessmode + if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { + req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER + } else { + req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY + } + disableInUseChecks := req.GetPublishContext()[readonlyAttachmentKey] == "true" isBlock := req.GetVolumeCapability().GetBlock() != nil // disableInUseChecks := false From cc75291819197039f25c42d590893983188aba43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Tue, 15 Mar 2022 14:31:21 +0800 Subject: [PATCH 16/22] map key fixed --- internal/rbd/nodeserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index e859cc3e8d5..33824eb1aeb 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -118,7 +118,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } // sync volcap with va's accessmode - if req.GetPublishContext()["accessmode"] == csi.VolumeCapability_AccessMode_Mode_name[1] { + if req.GetPublishContext()["Capability"] == csi.VolumeCapability_AccessMode_Mode_name[1] { req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER } else { req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY From 97cd80576850a038c180eba481360a47d5d354a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Tue, 15 Mar 2022 15:47:48 +0800 Subject: [PATCH 17/22] valid volcap fixed --- internal/rbd/controllerserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 45072930861..883dc011d64 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -677,7 +677,7 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req } for _, capability := range req.VolumeCapabilities { - if capability.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER || + if capability.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER && capability.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil } From 780c4684c0e3b41499d89bfd9880213954c33695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Fri, 8 Apr 2022 17:50:05 +0800 Subject: [PATCH 18/22] nodepublish block design --- deploy/rbd/kubernetes/csidriver.yaml | 2 +- internal/rbd/nodeserver.go | 18 +++++++++++ internal/util/readonly.go | 46 ++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 internal/util/readonly.go diff --git a/deploy/rbd/kubernetes/csidriver.yaml b/deploy/rbd/kubernetes/csidriver.yaml index f371e9c938f..24e6fa1e263 100644 --- a/deploy/rbd/kubernetes/csidriver.yaml +++ b/deploy/rbd/kubernetes/csidriver.yaml @@ -7,4 +7,4 @@ metadata: name: rbd.csi.ceph.com spec: attachRequired: true - podInfoOnMount: false + podInfoOnMount: true diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 33824eb1aeb..93369f13122 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -408,6 +408,24 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis volID := req.GetVolumeId() stagingPath += "/" + volID + // set podinfoonmount = true to get the actual pod info + podName := req.GetVolumeContext()["csi.storage.k8s.io/pod.name"] + podNamespace := req.GetVolumeContext()["csi.storage.k8s.io/pod.namespace"] + + pod, err := util.GetPod(podName, podNamespace) + if err != nil { + return nil, err + } + + ro, err := util.CheckIfReadonlyMount(pod) + if err != nil { + req.Readonly = ro + } + + if !ro && req.GetPublishContext()[readonlyAttachmentKey] == "true" { + return nil, status.Errorf(codes.Aborted, "cannot mount readwrite pod for multi readonly capability", volID) + } + if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired { util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) diff --git a/internal/util/readonly.go b/internal/util/readonly.go new file mode 100644 index 00000000000..b9595f15d19 --- /dev/null +++ b/internal/util/readonly.go @@ -0,0 +1,46 @@ +package util + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" +) + +func CheckIfReadonlyMount(po *v1.Pod) (bool, error) { + for _, vol := range po.Spec.Volumes { + if vol.PersistentVolumeClaim != nil { + + if !vol.PersistentVolumeClaim.ReadOnly { + for _, con := range po.Spec.Containers { + if con.VolumeMounts == nil { + continue + } + for _, vm := range con.VolumeMounts { + if vm.Name == vol.Name && vm.ReadOnly { + return true, nil + } + } + } + return false, nil + } + return true, nil + + } + } + return false, fmt.Errorf("no matching conditions") +} + +func GetPod(name string, namespace string) (*v1.Pod, error) { + c := NewK8sClient() + pod, err := c.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + + if err != nil { + klog.V(6).Infof("Can't get pod %s namespace %s: %v", name, namespace, err) + return nil, err + } + + return pod, nil +} From c65df593cb5471e1b50c6195db11a196a828e6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Fri, 21 Oct 2022 23:43:27 +1100 Subject: [PATCH 19/22] fix: treat old pv with readonly mount --- charts/ceph-csi-rbd/templates/csidriver-crd.yaml | 2 +- internal/rbd/nodeserver.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/charts/ceph-csi-rbd/templates/csidriver-crd.yaml b/charts/ceph-csi-rbd/templates/csidriver-crd.yaml index 4991bd4fc2f..e34a86e7482 100644 --- a/charts/ceph-csi-rbd/templates/csidriver-crd.yaml +++ b/charts/ceph-csi-rbd/templates/csidriver-crd.yaml @@ -8,4 +8,4 @@ metadata: name: {{ .Values.driverName }} spec: attachRequired: true - podInfoOnMount: false + podInfoOnMount: true diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 93369f13122..91a9f13513a 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -408,7 +408,7 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis volID := req.GetVolumeId() stagingPath += "/" + volID - // set podinfoonmount = true to get the actual pod info + // set podInfoOnMount = true to get the actual pod info podName := req.GetVolumeContext()["csi.storage.k8s.io/pod.name"] podNamespace := req.GetVolumeContext()["csi.storage.k8s.io/pod.namespace"] @@ -422,8 +422,10 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis req.Readonly = ro } - if !ro && req.GetPublishContext()[readonlyAttachmentKey] == "true" { - return nil, status.Errorf(codes.Aborted, "cannot mount readwrite pod for multi readonly capability", volID) + if req.GetPublishContext()[readonlyAttachmentKey] == "true" || req.GetPublishContext()[readonlyAttachmentKey] == "" { + if !ro { + return nil, status.Errorf(codes.Aborted, "cannot mount readwrite pod for multi readonly capability", volID) + } } if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired { From 5ba0f50bdfd83a18108d9aa0d1fe7ca2b800614d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Wed, 2 Nov 2022 16:41:33 +1100 Subject: [PATCH 20/22] remove tcmu-runner in dockerfile --- deploy/cephcsi/image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/cephcsi/image/Dockerfile b/deploy/cephcsi/image/Dockerfile index 26e3b18b9ab..ae5eee1bdd2 100644 --- a/deploy/cephcsi/image/Dockerfile +++ b/deploy/cephcsi/image/Dockerfile @@ -36,7 +36,7 @@ RUN sudo sed -e 's|^mirrorlist=|#mirrorlist=|g' \ /etc/yum.repos.d/CentOS-Stream-PowerTools.repo \ /etc/yum.repos.d/CentOS-Stream-RealTime.repo -RUN dnf config-manager --disable apache-arrow-centos || true +RUN dnf config-manager --disable apache-arrow-centos,tcmu-runner,tcmu-runner-source,tcmu-runner-noarch || true RUN yum clean all && yum makecache From 8e9ed48998bb18b7fc7527cb12a2428fe3ec3f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Fri, 4 Nov 2022 17:07:43 +1100 Subject: [PATCH 21/22] attachment key replacement --- internal/rbd/nodeserver.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 91a9f13513a..70d418b1ea2 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -421,8 +421,13 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if err != nil { req.Readonly = ro } + if _, ok := req.GetPublishContext()[readonlyAttachmentKey]; !ok { + if !ro { + return nil, status.Errorf(codes.Aborted, "cannot mount readwrite pod for multi readonly capability", volID) + } + } - if req.GetPublishContext()[readonlyAttachmentKey] == "true" || req.GetPublishContext()[readonlyAttachmentKey] == "" { + if req.GetPublishContext()[readonlyAttachmentKey] == "true" { if !ro { return nil, status.Errorf(codes.Aborted, "cannot mount readwrite pod for multi readonly capability", volID) } From b6e831b3a32ba20dc224a429ab4d3e745601b549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E6=87=BF=E6=9D=B0?= Date: Tue, 6 Dec 2022 20:40:30 +1100 Subject: [PATCH 22/22] logic fixed for old pvc --- internal/rbd/nodeserver.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 70d418b1ea2..d73057b7578 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -421,11 +421,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if err != nil { req.Readonly = ro } - if _, ok := req.GetPublishContext()[readonlyAttachmentKey]; !ok { - if !ro { - return nil, status.Errorf(codes.Aborted, "cannot mount readwrite pod for multi readonly capability", volID) - } - } if req.GetPublishContext()[readonlyAttachmentKey] == "true" { if !ro {