diff --git a/cli/client/client_helper.go b/cli/client/client_helper.go new file mode 100644 index 00000000..8ce792da --- /dev/null +++ b/cli/client/client_helper.go @@ -0,0 +1,157 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + + corev1 "k8s.io/api/core/v1" + k8string "k8s.io/utils/strings" + + "huawei-csi-driver/cli/helper" + xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" + "huawei-csi-driver/utils/log" +) + +// CommonCallHandler common call handler +type CommonCallHandler[T any] struct { + client KubernetesClient +} + +// ListResult list query result +type ListResult[T any] struct { + Items []T `json:"items"` +} + +// NewCommonCallHandler init common call handler +func NewCommonCallHandler[T any](client KubernetesClient) *CommonCallHandler[T] { + return &CommonCallHandler[T]{client: client} +} + +// Create resource +func (r *CommonCallHandler[T]) Create(t T) error { + return r.commonOperateResource(t, Create) +} + +// Update resource +func (r *CommonCallHandler[T]) Update(t T) error { + return r.commonOperateResource(t, Apply) +} + +// QueryByName query resource by name +func (r *CommonCallHandler[T]) QueryByName(namespace, name string) (T, error) { + return commonQuery[T, T](r.client, namespace, name) +} + +// QueryList query resource list +func (r *CommonCallHandler[T]) QueryList(namespace string, names ...string) ([]T, error) { + if len(names) == 1 { + t, err := r.QueryByName(namespace, names[0]) + if err != nil { + return []T{}, err + } + return safeToArray(t), nil + } + + result, err := commonQuery[ListResult[T], T](r.client, namespace, names...) + if err != nil { + return []T{}, err + } + return result.Items, nil +} + +// DeleteByNames delete resource by names +func (r *CommonCallHandler[T]) DeleteByNames(namespace string, names ...string) error { + var qualifiedNames []string + resourceType, err := GetResourceTypeByT[T]() + if err != nil { + return err + } + for _, name := range names { + qualifiedNames = append(qualifiedNames, k8string.JoinQualifiedName(string(resourceType), name)) + } + _, err = r.client.DeleteResourceByQualifiedNames(qualifiedNames, namespace) + return err +} + +// commonQuery common query resource +// T is return Type +// R is Resource struct +func commonQuery[T any, R any](client KubernetesClient, namespace string, names ...string) (T, error) { + var t T + resourceType, err := GetResourceTypeByT[R]() + if err != nil { + return t, err + } + jsonBytes, err := client.GetResource(names, namespace, "json", resourceType) + if err != nil || len(jsonBytes) == 0 { + return t, err + } + + if err := json.Unmarshal(jsonBytes, &t); err != nil { + return t, err + } + + return t, nil +} + +// commonQuery common query resource +// T is resource struct +func (r *CommonCallHandler[T]) commonOperateResource(t T, operateType string) error { + bytes, err := helper.StructToYAML(t) + if err != nil { + log.Errorf("%s resource failed, error: %v", operateType, err) + return err + } + return r.client.OperateResourceByYaml(string(bytes), operateType, false) +} + +// GetResourceTypeByT get resource type +func GetResourceTypeByT[T any]() (ResourceType, error) { + var t T + resourceType := parseType(t) + if resourceType == "" { + return "", errors.New(fmt.Sprintf("Unsupported query type: %s", reflect.TypeOf(t).Name())) + } + return resourceType, nil +} + +func parseType(target interface{}) ResourceType { + switch target.(type) { + case corev1.Secret: + return Secret + case corev1.ConfigMap: + return ConfigMap + case xuanwuV1.StorageBackendClaim: + return Storagebackendclaim + case xuanwuV1.StorageBackendContent: + return StoragebackendclaimContent + default: + return "" + } +} + +func safeToArray[T any](t T) []T { + var emptyStruct T + if reflect.DeepEqual(t, emptyStruct) { + return []T{} + } + return []T{t} +} diff --git a/cli/config/config.go b/cli/config/config.go index 47c94523..b440b7f2 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -22,7 +22,7 @@ import ( const ( //CliVersion oceanctl version - CliVersion = "v4.1.0" + CliVersion = "v4.1.1" // DefaultMaxClientThreads default max client threads DefaultMaxClientThreads = "30" diff --git a/cli/resources/backend_helper.go b/cli/resources/backend_helper.go new file mode 100644 index 00000000..47c52194 --- /dev/null +++ b/cli/resources/backend_helper.go @@ -0,0 +1,373 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resources + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "reflect" + "strconv" + "strings" + + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8string "k8s.io/utils/strings" + + "huawei-csi-driver/cli/helper" + xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" +) + +const ( + ApiVersion = "v1" + XuanWuApiVersion = "xuanwu.huawei.io/v1" + KindSecret = "Secret" + KindConfigMap = "ConfigMap" + KindStorageBackendClaim = "StorageBackendClaim" + YamlSeparator = "---" +) + +// BackendConfiguration backend config +type BackendConfiguration struct { + Name string `json:"name,omitempty" yaml:"name"` + NameSpace string `json:"namespace,omitempty" yaml:"namespace"` + Storage string `json:"storage,omitempty" yaml:"storage"` + VstoreName string `json:"vstoreName,omitempty" yaml:"vstoreName"` + AccountName string `json:"accountName,omitempty" yaml:"accountName"` + Urls []string `json:"urls,omitempty" yaml:"urls"` + Pools []string `json:"pools,omitempty" yaml:"pools"` + MetrovStorePairID string `json:"metrovStorePairID,omitempty" yaml:"metrovStorePairID"` + MetroBackend string `json:"metroBackend,omitempty" yaml:"metroBackend"` + SupportedTopologies []map[string]interface{} `json:"supportedTopologies,omitempty" yaml:"supportedTopologies"` + MaxClientThreads string `json:"maxClientThreads,omitempty" yaml:"maxClientThreads"` + Configured bool `json:"-" yaml:"configured"` + Provisioner string `json:"provisioner,omitempty" yaml:"provisioner"` + Parameters struct { + Protocol string `json:"protocol,omitempty" yaml:"protocol"` + ParentName string `json:"parentname" yaml:"parentname"` + Portals []string `json:"portals,omitempty" yaml:"portals"` + Alua []map[string][]map[string]interface{} `json:"ALUA,omitempty" yaml:"ALUA"` + } `json:"parameters,omitempty" yaml:"parameters"` +} + +// BackendShowWide the content echoed by executing the oceanctl get backend -o wide +type BackendShowWide struct { + Namespace string `show:"NAMESPACE"` + Name string `show:"NAME"` + Protocol string `show:"PROTOCOL"` + StorageType string `show:"STORAGETYPE"` + Sn string `show:"SN"` + Status string `show:"STATUS"` + Online string `show:"ONLINE"` + Url string `show:"Url"` + VendorName string `show:"VENDORNAME"` + StorageBackendContentName string `show:"STORAGEBACKENDCONTENTNAME"` +} + +// BackendShow the content echoed by executing the oceanctl get backend +type BackendShow struct { + Namespace string `show:"NAMESPACE"` + Name string `show:"NAME"` + Protocol string `show:"PROTOCOL"` + StorageType string `show:"STORAGETYPE"` + Sn string `show:"SN"` + Status string `show:"STATUS"` + Online string `show:"ONLINE"` + Url string `show:"Url"` +} + +// BackendConfigShow the content echoed by executing the oceanctl create backend +type BackendConfigShow struct { + Number string `show:"NUMBER"` + Configured string `show:"CONFIGURED"` + Name string `show:"NAME"` + Storage string `show:"STORAGE"` + Urls string `show:"URLS"` +} + +// StorageBackendClaimConfig used to create a storageBackendClaim object +type StorageBackendClaimConfig struct { + Name string + Namespace string + ConfigmapMeta string + SecretMeta string + MaxClientThreads string + Provisioner string +} + +// SecretConfig used to create a secret object +type SecretConfig struct { + Name string + Namespace string + User string + Pwd string +} + +// ConfigMapConfig used to create a configmap object +type ConfigMapConfig struct { + Name string + Namespace string + JsonData string +} + +// ShowWithContentOption set StorageBackendContent value for BackendShowWide +func (b *BackendShowWide) ShowWithContentOption(content xuanwuv1.StorageBackendContent) *BackendShowWide { + b.StorageBackendContentName = content.Name + if content.Status != nil { + b.Online = strconv.FormatBool(content.Status.Online) + b.VendorName = content.Status.VendorName + b.Sn = content.Status.SN + } + return b +} + +// ShowWithConfigOption set BackendConfiguration value for BackendShowWide +func (b *BackendShowWide) ShowWithConfigOption(configuration BackendConfiguration) *BackendShowWide { + b.Url = strings.Join(configuration.Urls, "\n") + return b +} + +// ShowWithClaimOption set StorageBackendClaim value for BackendShowWide +func (b *BackendShowWide) ShowWithClaimOption(claim xuanwuv1.StorageBackendClaim) *BackendShowWide { + b.Namespace = claim.Namespace + b.Name = claim.Name + if claim.Status != nil { + b.StorageType = claim.Status.StorageType + b.Protocol = claim.Status.Protocol + b.Status = string(claim.Status.Phase) + } + return b +} + +// ToBackendShow convert BackendShowWide to BackendShow +func (b *BackendShowWide) ToBackendShow() BackendShow { + return BackendShow{ + Namespace: b.Namespace, + Name: b.Name, + Protocol: b.Protocol, + StorageType: b.StorageType, + Sn: b.Sn, + Status: b.Status, + Online: b.Online, + Url: b.Url, + } +} + +// ToStorageBackendClaimConfig covert backend to StorageBackendClaimConfig +func (b *BackendConfiguration) ToStorageBackendClaimConfig() StorageBackendClaimConfig { + return StorageBackendClaimConfig{ + Name: b.Name, + Namespace: b.NameSpace, + ConfigmapMeta: k8string.JoinQualifiedName(b.NameSpace, b.Name), + SecretMeta: k8string.JoinQualifiedName(b.NameSpace, b.Name), + MaxClientThreads: b.MaxClientThreads, + Provisioner: b.Provisioner, + } +} + +// ToConfigMapConfig convert backend to helper.ConfigMapConfig +func (b *BackendConfiguration) ToConfigMapConfig() (ConfigMapConfig, error) { + config := struct { + Backends BackendConfiguration `json:"backends"` + }{*b} + + output, err := json.MarshalIndent(&config, "", " ") + if err != nil { + return ConfigMapConfig{}, helper.LogErrorf(" json.MarshalIndent failed: %v", err) + } + + return ConfigMapConfig{ + Name: b.Name, + Namespace: b.NameSpace, + JsonData: string(output), + }, nil +} + +// ToSecretConfig convert backend to helper.SecretConfig +// If start stdin failed, an error will return. +func (b *BackendConfiguration) ToSecretConfig() (SecretConfig, error) { + userName, password, err := helper.StartStdInput() + if err != nil { + return SecretConfig{}, err + } + + return SecretConfig{ + Name: b.Name, + Namespace: b.NameSpace, + User: userName, + Pwd: password, + }, nil +} + +// ToConfigMap convert ConfigMapConfig to ConfigMap resource +func (c *ConfigMapConfig) ToConfigMap() corev1.ConfigMap { + return corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: ApiVersion, + Kind: KindConfigMap, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name, + Namespace: c.Namespace, + }, + Data: map[string]string{ + "csi.json": c.JsonData, + }, + } +} + +// ToSecret convert SecretConfig to Secret resource +func (c *SecretConfig) ToSecret() corev1.Secret { + return corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: ApiVersion, + Kind: KindSecret, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name, + Namespace: c.Namespace, + }, + StringData: map[string]string{ + "password": c.Pwd, + "user": c.User, + }, + Type: "Opaque", + } +} + +// ToStorageBackendClaim convert StorageBackendClaimConfig to Secret StorageBackendClaim +func (c *StorageBackendClaimConfig) ToStorageBackendClaim() xuanwuv1.StorageBackendClaim { + return xuanwuv1.StorageBackendClaim{ + TypeMeta: metav1.TypeMeta{ + APIVersion: XuanWuApiVersion, + Kind: KindStorageBackendClaim, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name, + Namespace: c.Namespace, + }, + Spec: xuanwuv1.StorageBackendClaimSpec{ + Provider: c.Provisioner, + ConfigMapMeta: c.ConfigmapMeta, + SecretMeta: c.SecretMeta, + MaxClientThreads: c.MaxClientThreads, + }, + } +} + +// LoadBackendsFromJson load backend from json bytes +func LoadBackendsFromJson(jsonData []byte) (map[string]*BackendConfiguration, error) { + result := make(map[string]*BackendConfiguration) + + configmap := corev1.ConfigMap{} + err := json.Unmarshal(jsonData, &configmap) + if err != nil { + return result, err + } + + return LoadBackendsFromConfigMap(configmap) +} + +// LoadBackendsFromConfigMap load backend from configmap resource +func LoadBackendsFromConfigMap(configmap corev1.ConfigMap) (map[string]*BackendConfiguration, error) { + result := make(map[string]*BackendConfiguration) + jsonStr, ok := configmap.Data["csi.json"] + if !ok { + return result, errors.New("not found csi.json config") + } + + backendContent, err := AnalyseBackendExist(jsonStr) + if err != nil { + return nil, err + } + + var backends []*BackendConfiguration + if _, ok = backendContent.([]interface{}); ok { + backends, err = LoadMultipleBackendFromConfigmap(jsonStr) + } else { + backends, err = LoadSingleBackendFromConfigmap(jsonStr) + } + if err != nil { + return nil, err + } + + for _, backend := range backends { + result[backend.Name] = backend + } + return result, nil +} + +//AnalyseBackendExist analyse backend,an error is returned if backends not exist +func AnalyseBackendExist(jsonStr string) (interface{}, error) { + var config map[string]interface{} + if err := json.Unmarshal([]byte(jsonStr), &config); err != nil { + return nil, err + } + backendContent, ok := config["backends"] + if !ok { + return nil, errors.New("not found backends config") + } + return backendContent, nil +} + +// LoadSingleBackendFromConfigmap load single backend +func LoadSingleBackendFromConfigmap(jsonStr string) ([]*BackendConfiguration, error) { + config := struct { + Backends *BackendConfiguration `json:"backends"` + }{} + if err := json.Unmarshal([]byte(jsonStr), &config); err != nil { + return nil, err + } + + return []*BackendConfiguration{config.Backends}, nil +} + +// LoadMultipleBackendFromConfigmap load multiple backend +func LoadMultipleBackendFromConfigmap(jsonStr string) ([]*BackendConfiguration, error) { + config := struct { + Backends []*BackendConfiguration `json:"backends"` + }{} + if err := json.Unmarshal([]byte(jsonStr), &config); err != nil { + return nil, err + } + + return config.Backends, nil +} + +// LoadBackendsFromYaml load backend from yaml +func LoadBackendsFromYaml(yamlData []byte) (map[string]*BackendConfiguration, error) { + cleanYamlData := strings.Trim(strings.TrimSpace(string(yamlData)), YamlSeparator) + decoder := yaml.NewDecoder(bytes.NewReader([]byte(cleanYamlData))) + + var backends = map[string]*BackendConfiguration{} + config := &BackendConfiguration{} + err := decoder.Decode(config) + for err == nil { + if !reflect.DeepEqual(*config, BackendConfiguration{}) { + backends[config.Name] = config + } + config = &BackendConfiguration{} + err = decoder.Decode(config) + } + + if !errors.Is(err, io.EOF) { + return backends, err + } + return backends, nil +} diff --git a/csi/backend/plugin/oceanstor-dtree.go b/csi/backend/plugin/oceanstor-dtree.go new file mode 100644 index 00000000..e1aa554b --- /dev/null +++ b/csi/backend/plugin/oceanstor-dtree.go @@ -0,0 +1,301 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package plugin + +import ( + "context" + "errors" + "fmt" + + "huawei-csi-driver/storage/oceanstor/client" + "huawei-csi-driver/storage/oceanstor/volume" + "huawei-csi-driver/utils" + "huawei-csi-driver/utils/log" +) + +const ( + DTreeStorage = "oceanstor-dtree" +) + +type OceanstorDTreePlugin struct { + OceanstorPlugin + + portal string + parentName string +} + +func init() { + RegPlugin(DTreeStorage, &OceanstorDTreePlugin{}) +} + +func (p *OceanstorDTreePlugin) NewPlugin() Plugin { + return &OceanstorDTreePlugin{} +} + +func (p *OceanstorDTreePlugin) Init(config, parameters map[string]interface{}, keepLogin bool) error { + protocol, exist := utils.ToStringWithFlag(parameters["protocol"]) + if !exist || protocol != "nfs" { + return errors.New("protocol must be provided and be \"nfs\" for oceanstor-dtree backend") + } + + if _, ok := parameters["portals"]; !ok { + return errors.New("portals must be provided for oceanstor-dtree backend and just support one portal") + } + portals, exist := parameters["portals"].([]interface{}) + if !exist || len(portals) != 1 { + return errors.New("portals must be provided for oceanstor-dtree backend and just support one portal") + } + p.portal, _ = utils.ToStringWithFlag(portals[0]) + + p.parentName, _ = utils.ToStringWithFlag(parameters["parentname"]) + + err := p.init(config, keepLogin) + if err != nil { + log.AddContext(context.Background()).Errorf("init dtree plugin failed, data:") + return err + } + + return nil +} + +func (p *OceanstorDTreePlugin) getDTreeObj() *volume.DTree { + return volume.NewDTree(p.cli) +} + +func (p *OceanstorDTreePlugin) CreateVolume(ctx context.Context, name string, parameters map[string]interface{}) ( + utils.Volume, error) { + if p == nil { + return nil, errors.New("empty dtree plugin") + } + if parameters == nil { + return nil, errors.New("empty parameters") + } + + size, ok := parameters["size"].(int64) + if !ok || !utils.IsCapacityAvailable(size, SectorSize) { + msg := fmt.Sprintf("Create Volume: the capacity %d is not an integer multiple of 512.", size) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + parameters["vstoreId"] = p.vStoreId + parameters["parentname"] = p.parentName + params := p.getParams(ctx, name, parameters) + + volObj, err := p.getDTreeObj().Create(ctx, params) + if err != nil { + return nil, err + } + volObj.SetDTreeParentName(p.parentName) + + return volObj, nil +} + +func (p *OceanstorDTreePlugin) QueryVolume(ctx context.Context, name string, parameters map[string]interface{}) ( + utils.Volume, error) { + + return nil, errors.New(" not implement") +} + +func (p *OceanstorDTreePlugin) DeleteDTreeVolume(ctx context.Context, params map[string]interface{}) error { + if p == nil { + return errors.New("empty dtree plugin") + } + if params == nil { + return errors.New("empty parameters") + } + params["vstoreid"] = p.vStoreId + params["parentname"] = p.parentName + + return p.getDTreeObj().Delete(ctx, params) + +} + +func (p *OceanstorDTreePlugin) ExpandDTreeVolume(ctx context.Context, params map[string]interface{}) (bool, error) { + dTree := p.getDTreeObj() + + dTreeName, _ := utils.ToStringWithFlag(params["name"]) + spaceHardQuota, ok := params["spacehardquota"].(int64) + if !ok { + log.AddContext(ctx).Errorln("expand dTree volume failed, spacehardquota is not found") + return false, errors.New("spacehardquota not found") + } + + if !utils.IsCapacityAvailable(spaceHardQuota, SectorSize) { + msg := fmt.Sprintf("Create Volume: the capacity %d is not an integer multiple of 512.", spaceHardQuota) + log.AddContext(ctx).Errorln(msg) + return false, errors.New(msg) + } + + parentName, _ := utils.ToStringWithFlag(params["parentname"]) + err := dTree.Expand(ctx, parentName, dTreeName, p.vStoreId, 0, spaceHardQuota) + if err != nil { + log.AddContext(ctx).Errorf("expand dTree volume failed, ") + return false, err + } + log.AddContext(ctx).Infof("expand dTree volume success, parentName: %v, dTreeName: %v,"+ + " vStoreId: %v, spaceHardQuota: %v", params, dTreeName, p.vStoreId, spaceHardQuota) + return false, nil +} + +func (p *OceanstorDTreePlugin) DeleteVolume(ctx context.Context, name string) error { + return errors.New("not implement") + +} + +func (p *OceanstorDTreePlugin) ExpandVolume(ctx context.Context, name string, size int64) (bool, error) { + return false, errors.New("not implement") +} + +func (p *OceanstorDTreePlugin) Validate(ctx context.Context, param map[string]interface{}) error { + log.AddContext(ctx).Infoln("Start to validate OceanstorDTreePlugin parameters.") + + clientConfig, err := p.getNewClientConfig(ctx, param) + if err != nil { + return err + } + + err = verifyOceanstorDTreeParam(ctx, param) + if err != nil { + return err + } + + // Login verification + cli := client.NewClient(clientConfig) + err = cli.ValidateLogin(ctx) + if err != nil { + return err + } + + cli.Logout(ctx) + + return nil +} + +func verifyOceanstorDTreeParam(ctx context.Context, config map[string]interface{}) error { + // verify storage + storage, exist := utils.ToStringWithFlag(config["storage"]) + if !exist || storage != DTreeStorage { + msg := fmt.Sprintf("Verify storage: [%v] failed. \nstorage must be %s", config["storage"], DTreeStorage) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + // verify parameters + parameters, exist := config["parameters"].(map[string]interface{}) + if !exist { + msg := fmt.Sprintf("Verify parameters: [%v] failed. \nparameters must be provided", config["parameters"]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + // verify parent name + parentName, exist := utils.ToStringWithFlag(parameters["parentname"]) + if !exist || parentName == "" { + msg := fmt.Sprintf("Verify parentname: [%v] failed. \nParentname must be provided for "+ + "oceanstor-dtree backend\n", parameters["parentname"]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + // verify protocol portals + protocol, exist := utils.ToStringWithFlag(parameters["protocol"]) + if !exist || protocol != "nfs" { + msg := fmt.Sprintf("Verify protocol: [%v] failed. \nProtocol must be provided and must be \"nfs\" for "+ + "oceanstor-dtree backend\n", parameters["protocol"]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + if _, ok := parameters["portals"]; !ok { + msg := fmt.Sprintf("Verify portals: [%v] failed. \nportals must be provided for oceanstor-dtree backend "+ + "and just support one portal\n", parameters["portals"]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + portals, exist := parameters["portals"].([]interface{}) + if !exist || len(portals) != 1 { + msg := fmt.Sprintf("Verify portals: [%v] failed. \nportals must be provided for oceanstor-dtree backend "+ + "and just support one portal\n", parameters["portals"]) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + return nil +} + +func (p *OceanstorDTreePlugin) CreateSnapshot(ctx context.Context, s, s2 string) (map[string]interface{}, error) { + return nil, errors.New("not implement") + +} + +func (p *OceanstorDTreePlugin) DeleteSnapshot(ctx context.Context, s, s2 string) error { + return errors.New("not implement") +} + +func (p *OceanstorDTreePlugin) UpdateBackendCapabilities() (map[string]interface{}, map[string]interface{}, error) { + capabilities, specifications, err := p.OceanstorPlugin.UpdateBackendCapabilities() + if err != nil { + return nil, nil, err + } + + err = p.updateNFS4Capability(capabilities) + if err != nil { + return nil, nil, err + } + + return capabilities, specifications, nil +} + +func (p *OceanstorDTreePlugin) UpdatePoolCapabilities(poolNames []string) (map[string]interface{}, error) { + capabilities := make(map[string]interface{}) + + defaultMap := map[string]interface{}{ + "FreeCapacity": 0, + } + for _, poolName := range poolNames { + capabilities[poolName] = defaultMap + } + return capabilities, nil + +} + +func (p *OceanstorDTreePlugin) updateNFS4Capability(capabilities map[string]interface{}) error { + if capabilities == nil { + capabilities = make(map[string]interface{}) + } + + nfsServiceSetting, err := p.cli.GetNFSServiceSetting(context.Background()) + if err != nil { + return err + } + // NFS3 is enabled by default. + capabilities["SupportNFS3"] = true + capabilities["SupportNFS4"] = false + capabilities["SupportNFS41"] = false + + if !nfsServiceSetting["SupportNFS3"] { + capabilities["SupportNFS3"] = false + } + if nfsServiceSetting["SupportNFS4"] { + capabilities["SupportNFS4"] = true + } + if nfsServiceSetting["SupportNFS41"] { + capabilities["SupportNFS41"] = true + } + + return nil +} diff --git a/csi/main.go b/csi/main.go index cb6e04cc..3a7efc62 100644 --- a/csi/main.go +++ b/csi/main.go @@ -54,7 +54,7 @@ const ( controllerLogFile = "huawei-csi-controller" nodeLogFile = "huawei-csi-node" - csiVersion = "4.1.0" + csiVersion = "4.1.1" endpointDirPerm = 0755 ) diff --git a/examples/backend/backend-dtree.yaml b/examples/backend/backend-dtree.yaml new file mode 100644 index 00000000..ce88ca87 --- /dev/null +++ b/examples/backend/backend-dtree.yaml @@ -0,0 +1,12 @@ +storage: "oceanstor-dtree" +name: +namespace: +urls: + - "https://*.*.*.*:8088" + - "https://*.*.*.*:8088" +parameters: + protocol: + parentname: + portals: + - portal1 +maxClientThreads: "30" \ No newline at end of file diff --git a/examples/sc-dtree.yaml b/examples/sc-dtree.yaml new file mode 100644 index 00000000..78e3461e --- /dev/null +++ b/examples/sc-dtree.yaml @@ -0,0 +1,10 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: mysc +provisioner: csi.huawei.com +parameters: + backend: nfs_dtree + volumeType: dtree + allocType: thin + authClient: "*" \ No newline at end of file diff --git a/go.mod b/go.mod index c9c5420f..c1dd1d51 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( golang.org/x/sys v0.5.0 google.golang.org/grpc v1.41.0 google.golang.org/protobuf v1.28.1 - gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.26.1 k8s.io/apimachinery v0.26.1 k8s.io/client-go v0.26.1 @@ -76,6 +75,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/component-base v0.19.0 // indirect k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect diff --git a/helm/esdk/Chart.yaml b/helm/esdk/Chart.yaml index 1e965fb1..0fe7bcdb 100644 --- a/helm/esdk/Chart.yaml +++ b/helm/esdk/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 4.1.0 +version: 4.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. # It is strongly recommended not to modify this parameter -appVersion: "4.1.0" +appVersion: "4.1.1" home: https://github.com/Huawei/eSDK_K8S_Plugin sources: diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml index f3b44bcc..95ed51c5 100644 --- a/helm/esdk/values.yaml +++ b/helm/esdk/values.yaml @@ -1,8 +1,8 @@ images: # Images provided by Huawei - huaweiCSIService: huawei-csi:4.1.0 - storageBackendSidecar: storage-backend-sidecar:4.1.0 - storageBackendController: storage-backend-controller:4.1.0 + huaweiCSIService: huawei-csi:4.1.1 + storageBackendSidecar: storage-backend-sidecar:4.1.1 + storageBackendController: storage-backend-controller:4.1.1 # CSI-related sidecar images provided by the Kubernetes community. # These must match the appropriate Kubernetes version. diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 060e3663..c46abba1 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -18,7 +18,7 @@ package constants type FileType string const ( - ProviderVersion = "4.1.0" + ProviderVersion = "4.1.1" ProviderVendorName = "Huawei" EndpointDirPermission = 0755 diff --git a/pkg/constants/error.go b/pkg/constants/error.go new file mode 100644 index 00000000..1f145d3b --- /dev/null +++ b/pkg/constants/error.go @@ -0,0 +1,22 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package constants is related with provider constants +package constants + +import "errors" + +var ( + TimeoutError = errors.New("timeout") +) diff --git a/pkg/constants/utils.go b/pkg/constants/utils.go new file mode 100644 index 00000000..be4db4b5 --- /dev/null +++ b/pkg/constants/utils.go @@ -0,0 +1,21 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package constants is related with provider constants + +package constants + +const ( + NotMountStr = "not mounted" +) diff --git a/storage/fusionstorage/types/types.go b/storage/fusionstorage/types/types.go new file mode 100644 index 00000000..854a7539 --- /dev/null +++ b/storage/fusionstorage/types/types.go @@ -0,0 +1,65 @@ +package types + +const ( + MaxIopsOfConvergedQoS = 1073741824000 + MaxMbpsOfConvergedQoS = 1073741824 + + QosScaleNamespace = 0 + QosScaleClient = 1 + QosScaleAccount = 2 + + QosModeManual = 3 + + NoQoSPolicyId = -1 + + DefaultAccountName = "system" + DefaultAccountId = 0 +) + +type CreateConvergedQoSReq struct { + // (Mandatory) Upper limit control dimension. + // The value can be: + // 0:"NAMESPACE": namespace. + // 1:"CLIENT": client. + // 2:"ACCOUNT": account. + QosScale int + // (Mandatory) Name of a QoS policy. + // When "qos_scale" is set to "NAMESPACE" or "CLIENT", the value is a string of 1 to 63 characters, including + // digits, letters, hyphens (-), and underscores (_), and must start with a letter or digit. + // When "qos_scale" is set to "ACCOUNT", the value is an account ID and is an integer ranging from 0 to 4294967293. + Name string + // (Mandatory) QoS mode. + // When "qos_scale" is set to "NAMESPACE", the value can be "1" (by_usage), "2" (by_package), or "3" (manual). + // When "qos_scale" is set to "CLIENT" or "ACCOUNT", the value can be "3" (manual). + QosMode int + // (Conditionally Mandatory) Bandwidth upper limit. + // This parameter is mandatory when "qos_mode" is set to "manual". + // The value is an integer ranging from 0 to 1073741824000(0 indicates no limit), in Mbit/s. + MaxMbps int + // (Conditionally Mandatory) OPS upper limit. + // This parameter is mandatory when "qos_mode" is set to "manual". + // The value is an integer ranging from 0 to 1073741824000(0 indicates no limit). + MaxIops int +} + +// AssociateConvergedQoSWithVolumeReq used to AssociateConvergedQoSWithVolume request +type AssociateConvergedQoSWithVolumeReq struct { + // (Mandatory) qos_scale, Upper limit control dimension. + // The value can be: + // 0:"NAMESPACE": namespace. + // 1:"CLIENT": client. + // 2:"ACCOUNT": account. + // 3:"USER": user. + // 5:"HIDDEN_FS": hidden namespace. + QosScale int + + // (Mandatory) object_name, Name of the associated object. + // while qos_scale is NAMESPACE: + // The associated object is a namespace. The value is a string of 1 to 255 characters. Only digits, letters, + // underscores (_), periods (.), and hyphens (-) are supported. + ObjectName string + + // (Mandatory) qos_policy_id, QoS policy ID. + // The value is an integer ranging from 1 to 2147483647. + QoSPolicyID int +} diff --git a/storage/oceanstor/client/client.go b/storage/oceanstor/client/client.go index 397ca38c..b8684e4c 100644 --- a/storage/oceanstor/client/client.go +++ b/storage/oceanstor/client/client.go @@ -492,15 +492,15 @@ func (cli *BaseClient) setDataFromRespData(ctx context.Context, resp Response) e } vStoreName, exist := respData["vstoreName"].(string) - if !exist { + vStoreID, idExist := respData["vstoreId"].(string) + if !exist && !idExist { log.AddContext(ctx).Infof("storage client login response vstoreName is empty, set it to default %s", defaultVStore) cli.VStoreName = defaultVStore - } else { + } else if exist { cli.VStoreName = vStoreName } - vStoreID, idExist := respData["vstoreId"].(string) if !idExist { log.AddContext(ctx).Infof("storage client login response vstoreID is empty, set it to default %s", defaultVStoreID) diff --git a/storage/oceanstor/client/client_dtree.go b/storage/oceanstor/client/client_dtree.go new file mode 100644 index 00000000..74108a54 --- /dev/null +++ b/storage/oceanstor/client/client_dtree.go @@ -0,0 +1,116 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "errors" + "fmt" + + "huawei-csi-driver/utils" +) + +const ( + ParentTypeFS int = 40 + ParentTypeDTree int = 16445 + + QuotaSwitchOpen bool = true + QuotaSwitchClose bool = false + + SecurityStyleUnix int = 3 +) + +type DTree interface { + // CreateDTree use for create a dTree + CreateDTree(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) + // GetDTreeByName use for get dTree information + GetDTreeByName(ctx context.Context, parentID, parentName, vStoreID, name string) (map[string]interface{}, error) + // DeleteDTreeByID use for delete a dTree + DeleteDTreeByID(ctx context.Context, vStoreID, dTreeID string) error + // DeleteDTreeByName use for delete a dTree by name + DeleteDTreeByName(ctx context.Context, parentName, dTreeName, vStoreID string) error +} + +// CreateDTree use for create a dTree +func (cli *BaseClient) CreateDTree(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { + resp, err := cli.Post(ctx, "/QUOTATREE", params) + if err != nil { + return nil, err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return nil, fmt.Errorf("create dtree failed,data: %+v error: %s", params, resp.Error["description"]) + } + + return cli.getResponseDataMap(ctx, resp.Data) +} + +// GetDTreeByName use for get dTree information +func (cli *BaseClient) GetDTreeByName(ctx context.Context, parentID, parentName, vStoreID, name string) (map[string]interface{}, error) { + url := fmt.Sprintf("/QUOTATREE?PARENTNAME=%s&NAME=%s&vstoreId=%s", parentName, name, vStoreID) + + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return nil, err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return nil, fmt.Errorf("get dtree by name failed, dtree: %+v error: %s", name, resp.Error["description"]) + } + if resp.Data == nil { + return nil, nil + } + return cli.getResponseDataMap(ctx, resp.Data) + +} + +// DeleteDTreeByID use for delete a dTree +func (cli *BaseClient) DeleteDTreeByID(ctx context.Context, vStoreID, dTreeID string) error { + url := fmt.Sprintf("/QUOTATREE") + resp, err := cli.Delete(ctx, url, map[string]interface{}{ + "ID": dTreeID, + "vstoreId": vStoreID, + }) + if err != nil { + return err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return errors.New(fmt.Sprintf("%s", resp.Error["description"])) + } + + return nil +} + +// DeleteDTreeByName use for delete a dTree +func (cli *BaseClient) DeleteDTreeByName(ctx context.Context, parentName, dTreeName, vStoreID string) error { + url := fmt.Sprintf("/QUOTATREE") + resp, err := cli.Delete(ctx, url, map[string]interface{}{ + "PARENTNAME": parentName, + "vstoreId": vStoreID, + "NAME": dTreeName, + }) + if err != nil { + return err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return errors.New(fmt.Sprintf("%s", resp.Error["description"])) + } + + return nil +} diff --git a/storage/oceanstor/client/client_quota.go b/storage/oceanstor/client/client_quota.go new file mode 100644 index 00000000..6e59a21a --- /dev/null +++ b/storage/oceanstor/client/client_quota.go @@ -0,0 +1,122 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "fmt" + + "huawei-csi-driver/utils" +) + +const ( + QuotaTypeDir int = 1 + QuotaTypeUser int = 2 + QuotaTypeUserGroup int = 3 + + SpaceUnitTypeGB int = 3 + + ForceFlagTrue bool = true + ForceFlagFalse bool = false +) + +type OceanStorQuota interface { + // CreateQuota use for create quota for dTree or file system + CreateQuota(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) + // UpdateQuota use for update a quota + UpdateQuota(ctx context.Context, quotaID string, params map[string]interface{}) error + // GetQuota use for get quota information + GetQuota(ctx context.Context, quotaID, vStoreID string, spaceUnitType uint32) (map[string]interface{}, error) + // BatchGetQuota use for get quota information + BatchGetQuota(ctx context.Context, params map[string]interface{}) ([]interface{}, error) + // DeleteQuota use for delete a quota + DeleteQuota(ctx context.Context, quotaID, vStoreID string, forceFlag bool) error +} + +func (cli *BaseClient) CreateQuota(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { + resp, err := cli.Post(ctx, "/FS_QUOTA", params) + if err != nil { + return nil, err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return nil, fmt.Errorf("create quota failed, params: %+v error: %v", params, resp.Error["description"]) + } + + return cli.getResponseDataMap(ctx, resp.Data) +} + +func (cli *BaseClient) UpdateQuota(ctx context.Context, quotaID string, params map[string]interface{}) error { + resp, err := cli.Put(ctx, fmt.Sprintf("/FS_QUOTA/%v", quotaID), params) + if err != nil { + return err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return fmt.Errorf("update quota failed, params: %+v error: %v", params, resp.Error["description"]) + } + + return nil +} + +func (cli *BaseClient) GetQuota(ctx context.Context, quotaID, vStoreID string, spaceUnitType uint32) (map[string]interface{}, error) { + resp, err := cli.Get(ctx, fmt.Sprintf("/FS_QUOTA/%v", quotaID), map[string]interface{}{ + "SPACEUNITTYPE": spaceUnitType, + "vstoreId": vStoreID, + }) + if err != nil { + return nil, err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return nil, fmt.Errorf("get quota failed, quotaID: %v error: %s", quotaID, resp.Error["description"]) + } + + return cli.getResponseDataMap(ctx, resp.Data) +} + +func (cli *BaseClient) BatchGetQuota(ctx context.Context, params map[string]interface{}) ([]interface{}, error) { + url := fmt.Sprintf("/FS_QUOTA?PARENTTYPE=%v&PARENTID=%v&range=%v&vstoreId=%v&QUERYTYPE=%v&SPACEUNITTYPE=%v", + params["PARENTTYPE"], params["PARENTID"], params["range"], params["vstoreId"], + params["QUERYTYPE"], params["SPACEUNITTYPE"]) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return nil, err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return nil, fmt.Errorf("get quota failed, params: %v error: %s", params, resp.Error["description"]) + } + + return cli.getResponseDataList(ctx, resp.Data) +} + +func (cli *BaseClient) DeleteQuota(ctx context.Context, quotaID, vStoreID string, forceFlag bool) error { + resp, err := cli.Delete(ctx, fmt.Sprintf("/FS_QUOTA/%v", quotaID), map[string]interface{}{ + "forceFlag": forceFlag, + "vstoreId": vStoreID, + }) + if err != nil { + return err + } + + if utils.ResCodeExist(resp.Error["code"]) { + return fmt.Errorf("delete quota failed, quotaID: %v error: %s", quotaID, resp.Error["description"]) + } + + return nil +} diff --git a/storage/oceanstor/volume/dtree.go b/storage/oceanstor/volume/dtree.go new file mode 100644 index 00000000..7dcbcd7c --- /dev/null +++ b/storage/oceanstor/volume/dtree.go @@ -0,0 +1,653 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package volume + +import ( + "context" + "errors" + "fmt" + "strings" + + "huawei-csi-driver/storage/oceanstor/client" + "huawei-csi-driver/utils" + "huawei-csi-driver/utils/log" + "huawei-csi-driver/utils/taskflow" +) + +type DTree struct { + Base +} + +func NewDTree(cli client.BaseClientInterface) *DTree { + return &DTree{ + Base: Base{ + cli: cli, + metroRemoteCli: nil, + replicaRemoteCli: nil, + product: "", + }, + } +} + +func (p *DTree) preCreate(ctx context.Context, params map[string]interface{}) error { + _, flag := utils.ToStringWithFlag(params["authclient"]) + if !flag { + msg := "authclient must be provided for filesystem" + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + err := p.setWorkLoadID(ctx, p.cli, params) + if err != nil { + return err + } + + // all_squash all_squash: 0 no_all_squash: 1 + val, exist := utils.ToStringWithFlag(params["allsquash"]) + if !exist || val == "" { + params["allsquash"] = noAllSquash + } else { + if strings.EqualFold(val, noAllSquashString) { + params["allsquash"] = noAllSquash + } else if strings.EqualFold(val, allSquashString) { + params["allsquash"] = allSquash + } else { + return utils.Errorf(ctx, "parameter allSquash [%v] in sc must be %s or %s.", + val, allSquashString, noAllSquashString) + } + } + + // root_squash + val, exist = utils.ToStringWithFlag(params["rootsquash"]) + if !exist || val == "" { + params["rootsquash"] = noRootSquash + } else { + if strings.EqualFold(val, noRootSquashString) { + params["rootsquash"] = noRootSquash + } else if strings.EqualFold(val, rootSquashString) { + params["rootsquash"] = rootSquash + } else { + return utils.Errorf(ctx, "parameter rootSquash [%v] in sc must be %s or %s.", + val, rootSquashString, noRootSquashString) + } + } + + return nil +} + +func (p *DTree) Create(ctx context.Context, params map[string]interface{}) (utils.Volume, error) { + err := p.preCreate(ctx, params) + if err != nil { + return nil, err + } + + taskFlow := taskflow.NewTaskFlow(ctx, "Create-FileSystem-DTree-Volume") + taskFlow.AddTask("Check-FS", p.checkFSExist, nil) + taskFlow.AddTask("Create-DTree", p.createDtree, p.revertDtree) + taskFlow.AddTask("Create-Share", p.createShare, p.revertShare) + taskFlow.AddTask("Allow-Share-Access", p.allowShareAccess, p.revertShareAccess) + taskFlow.AddTask("Create-Quota", p.createQuota, p.revertQuota) + + _, err = taskFlow.Run(params) + if err != nil { + taskFlow.Revert() + return nil, err + } + + volObj := p.prepareVolObj(ctx, params, nil) + return volObj, nil +} + +func (p *DTree) Delete(ctx context.Context, params map[string]interface{}) error { + var err error + + taskFlow := taskflow.NewTaskFlow(ctx, "Delete-FileSystem-DTree-Volume") + taskFlow.AddTask("Check-DTree", p.checkDtreeExist, nil) + taskFlow.AddTask("Delete-Quota", p.deleteQuota, nil) + taskFlow.AddTask("Delete-Share", p.deleteShare, nil) + taskFlow.AddTask("Delete-DTree", p.deleteDtree, nil) + + _, err = taskFlow.Run(params) + if err != nil { + taskFlow.Revert() + return err + } + + return nil +} + +func (p *DTree) Expand(ctx context.Context, parentName, dTreeName, vstoreID string, spaceSoftQuota, + spaceHardQuota int64) error { + dTreeID, err := p.getDtreeID(ctx, parentName, vstoreID, dTreeName) + if err != nil { + return err + } + + req := map[string]interface{}{ + "PARENTTYPE": client.ParentTypeDTree, + "PARENTID": dTreeID, + "range": "[0-100]", + "vstoreId": vstoreID, + "QUERYTYPE": "2", + "SPACEUNITTYPE": client.SpaceUnitTypeGB, + } + quotaInfos, err := p.cli.BatchGetQuota(ctx, req) + if err != nil { + log.AddContext(ctx).Errorf("get quota arrays failed, params: %+v, error: %v", req, err) + return err + } + if len(quotaInfos) == 0 { + log.AddContext(ctx).Infof("get empty quota arrays params: %+v", req) + data := make(map[string]interface{}) + data["PARENTTYPE"] = client.ParentTypeDTree + data["PARENTNAME"] = dTreeName + data["QUOTATYPE"] = client.QuotaTypeDir + data["SPACEUNITTYPE"] = client.SpaceUnitTypeGB + data["SPACEHARDQUOTA"] = spaceHardQuota + data["vstoreId"] = vstoreID + _, err = p.cli.CreateQuota(ctx, data) + if err != nil { + log.AddContext(ctx).Errorf("create dtree quota failed, params: %+v, error: %v", data, err) + return err + } + return nil + } + + quotaInfo, ok := quotaInfos[0].(map[string]interface{}) + if !ok { + log.AddContext(ctx).Errorf("quota arrays data is not valid, quotaInfos[0]: %+v", quotaInfos[0]) + return errors.New("data in response is not valid") + } + quotaID, _ := utils.ToStringWithFlag(quotaInfo["ID"]) + err = p.cli.UpdateQuota(ctx, quotaID, map[string]interface{}{ + "SPACEHARDQUOTA": spaceHardQuota, + "vstoreId": vstoreID, + }) + if err != nil { + log.AddContext(ctx).Errorf("update quota failed, SPACEHARDQUOTA :%v, SPACEUNITTYPE: %v vstoreId: %v, err: %v", + spaceHardQuota, client.SpaceUnitTypeGB, vstoreID, err) + return err + } + return nil +} + +func (p *DTree) createDtree(ctx context.Context, + params, taskResult map[string]interface{}) (map[string]interface{}, error) { + + // format request param + data := make(map[string]interface{}) + if params["fspermission"] != nil && params["fspermission"] != "" { + data["unixPermissions"] = params["fspermission"] + } + if _, ok := params["name"]; ok { + data["NAME"] = params["name"] + } + if _, ok := params["parentname"]; ok { + data["PARENTNAME"] = params["parentname"] + } + if _, ok := params["vstoreid"]; ok { + data["vstoreId"] = params["vstoreid"] + } + data["PARENTTYPE"] = client.ParentTypeFS + data["QUOTASWITCH"] = client.QuotaSwitchOpen + data["securityStyle"] = client.SecurityStyleUnix + + res, err := p.cli.CreateDTree(ctx, data) + if err != nil { + log.AddContext(ctx).Errorf("create dtree failed, params: %+v, err: %v", data, err) + return nil, err + } + + dTreeID, _ := utils.ToStringWithFlag(res["ID"]) + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + + log.AddContext(ctx).Infof("create dtree success, dtreeID: %v vstoreID: %v", dTreeID, vStoreID) + + return map[string]interface{}{ + "dTreeId": dTreeID, + "vstoreid": vStoreID, + }, nil +} + +func (p *DTree) checkDtreeExist(ctx context.Context, params, + taskResult map[string]interface{}) (map[string]interface{}, error) { + + dTreeName, _ := utils.ToStringWithFlag(params["name"]) + fsName, _ := utils.ToStringWithFlag(params["parentname"]) + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + + dTreeInfo, err := p.cli.GetDTreeByName(ctx, "0", fsName, vStoreID, dTreeName) + if err != nil { + msg := fmt.Sprintf("get dtree failed, params: %+v, error:%v", params, err) + log.AddContext(ctx).Errorf(msg) + return nil, errors.New(msg) + } + if dTreeInfo == nil { + log.AddContext(ctx).Infof("delete dtree finish, dtree not found. params :%+v", params) + return nil, nil + } + + return map[string]interface{}{ + "dTreeId": dTreeInfo["ID"], + }, nil +} + +func (p *DTree) checkFSExist(ctx context.Context, params, + taskResult map[string]interface{}) (map[string]interface{}, error) { + parentFS, _ := utils.ToStringWithFlag(params["parentname"]) + + fs, err := p.cli.GetFileSystemByName(ctx, parentFS) + if err != nil { + log.AddContext(ctx).Errorf("Get filesystem by name %s error: %v", parentFS, err) + return nil, err + } + if fs == nil { + msg := fmt.Sprintf("Filesystem %s does not exist", parentFS) + log.AddContext(ctx).Errorf(msg) + return nil, errors.New(msg) + } + var fsID string + if _, ok := fs["ID"]; ok { + fsID, _ = utils.ToStringWithFlag(fs["ID"]) + } + + log.AddContext(ctx).Infof("parentName %s is exist", parentFS) + + return map[string]interface{}{ + "fsId": fsID, + }, nil +} + +func (p *DTree) revertDtree(ctx context.Context, taskResult map[string]interface{}) error { + vStoreID, _ := utils.ToStringWithFlag(taskResult["vstoreid"]) + dTreeID, _ := utils.ToStringWithFlag(taskResult["dTreeId"]) + + err := p.cli.DeleteDTreeByID(ctx, vStoreID, dTreeID) + if err != nil { + log.AddContext(ctx).Errorf("revert dtree failed, dTreeID: %v vStoreID: %v, error: %v", dTreeID, vStoreID, err) + return err + } + + log.AddContext(ctx).Infof("revert create dTree success,dTreeID: %s, vStoreID: %s", dTreeID, vStoreID) + return nil +} + +func (p *DTree) deleteDtree(ctx context.Context, params, + taskResult map[string]interface{}) (map[string]interface{}, error) { + + parentName, _ := utils.ToStringWithFlag(params["parentname"]) + dTreeName, _ := utils.ToStringWithFlag(params["name"]) + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + + err := p.cli.DeleteDTreeByName(ctx, parentName, dTreeName, vStoreID) + if err != nil { + log.AddContext(ctx).Errorf("delete dTree failed, parentName: %s, dTreeName: %s, vStoreID: %s", + parentName, dTreeName, vStoreID) + } + + log.AddContext(ctx).Infof("delete create dTree success, parentName: %s, dTreeName: %s, vStoreID: %s", + parentName, dTreeName, vStoreID) + return nil, err +} + +func (p *DTree) allowShareAccess(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { + + shareID, _ := utils.ToStringWithFlag(taskResult["shareId"]) + authClient, _ := utils.ToStringWithFlag(params["authclient"]) + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + + accesses, err := p.getCurrentShareAccess(ctx, shareID, vStoreID, p.cli) + if err != nil { + log.AddContext(ctx).Errorf("Get current access of share %s error: %v", shareID, err) + return nil, err + } + + for _, i := range strings.Split(authClient, ";") { + _, exist := accesses[i] + delete(accesses, i) + + if exist { + continue + } + + req := &client.AllowNfsShareAccessRequest{ + Name: i, + ParentID: shareID, + AccessVal: 1, + Sync: 0, + AllSquash: params["allsquash"].(int), + RootSquash: params["rootsquash"].(int), + VStoreID: vStoreID, + } + err = p.cli.AllowNfsShareAccess(ctx, req) + if err != nil { + log.AddContext(ctx).Errorf("Allow nfs share access %v failed. error: %v", req, err) + return nil, err + } + } + + // Remove all other extra access + for _, i := range accesses { + access := i.(map[string]interface{}) + accessID, _ := utils.ToStringWithFlag(access["ID"]) + + err = p.cli.DeleteNfsShareAccess(ctx, accessID, vStoreID) + if err != nil { + log.AddContext(ctx).Warningf("Delete extra nfs share access %s error: %v", accessID, err) + } + } + + return map[string]interface{}{ + "authClient": authClient, + }, nil +} + +func (p *DTree) getCurrentShareAccess(ctx context.Context, shareID, vStoreID string, + cli client.BaseClientInterface) (map[string]interface{}, error) { + count, err := cli.GetNfsShareAccessCount(ctx, shareID, vStoreID) + if err != nil { + return nil, err + } + + accesses := make(map[string]interface{}) + + var i int64 = 0 + for ; i < count; i += 100 { // Query per page 100 + clients, err := cli.GetNfsShareAccessRange(ctx, shareID, vStoreID, i, i+100) + if err != nil { + return nil, err + } + if clients == nil { + break + } + + for _, c := range clients { + clientTemp, ok := c.(map[string]interface{}) + if !ok { + continue + } + name, _ := utils.ToStringWithFlag(clientTemp["NAME"]) + accesses[name] = c + } + } + + return accesses, nil +} + +func (p *DTree) revertShareAccess(ctx context.Context, taskResult map[string]interface{}) error { + shareID, _ := utils.ToStringWithFlag(taskResult["shareId"]) + authClient, exist := utils.ToStringWithFlag(taskResult["authClient"]) + if !exist { + return nil + } + + vStoreID, _ := utils.ToStringWithFlag(taskResult["vstoreid"]) + accesses, err := p.getCurrentShareAccess(ctx, shareID, vStoreID, p.cli) + if err != nil { + log.AddContext(ctx).Errorf("Get current access of share %s error: %v", shareID, err) + return err + } + + for _, i := range strings.Split(authClient, ";") { + if _, exist := accesses[i]; !exist { + continue + } + access := accesses[i].(map[string]interface{}) + accessID, _ := utils.ToStringWithFlag(access["ID"]) + err := p.cli.DeleteNfsShareAccess(ctx, accessID, vStoreID) + if err != nil { + log.AddContext(ctx).Warningf("Delete extra nfs share access %s error: %v", accessID, err) + } + } + return nil +} + +func (p *DTree) deleteShareAccess(ctx context.Context, params, + taskResult map[string]interface{}) (map[string]interface{}, error) { + // get nfs share id + + parentName, _ := utils.ToStringWithFlag(params["parentname"]) + dTreaName, _ := utils.ToStringWithFlag(params["name"]) + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + sharePath := fmt.Sprintf("/%s/%s", parentName, dTreaName) + + nfsInfo, err := p.cli.GetNfsShareByPath(ctx, sharePath, vStoreID) + if err != nil { + log.AddContext(ctx).Errorf("get nfs share failed, path: %v, vstoreID: %v, error: %v", sharePath, vStoreID, err) + return nil, err + } + if nfsInfo == nil { + log.AddContext(ctx).Infof("delete share access finish, nfs not exist. path: %v, vstoreID: %v", sharePath, vStoreID) + return nil, nil + } + + shareID, _ := utils.ToStringWithFlag(nfsInfo["ID"]) + authClient, exist := utils.ToStringWithFlag(params["authclient"]) + if !exist { + log.AddContext(ctx).Infof("delete share access finish, authClient not exists") + return nil, nil + } + + accesses, err := p.getCurrentShareAccess(ctx, shareID, vStoreID, p.cli) + if err != nil { + log.AddContext(ctx).Errorf("Get current access of share %s error: %v", shareID, err) + return map[string]interface{}{ + "shareID": shareID, + }, err + } + + for _, i := range strings.Split(authClient, ";") { + if _, exist := accesses[i]; !exist { + continue + } + access := accesses[i].(map[string]interface{}) + accessID, _ := utils.ToStringWithFlag(access["ID"]) + err := p.cli.DeleteNfsShareAccess(ctx, accessID, vStoreID) + if err != nil { + log.AddContext(ctx).Warningf("Delete extra nfs share access %s error: %v", accessID, err) + } + } + + return map[string]interface{}{ + "shareID": shareID, + }, nil +} + +func (p *DTree) createQuota(ctx context.Context, + params, taskResult map[string]interface{}) (map[string]interface{}, error) { + + var spaceHardQuota int64 + if _, ok := params["capacity"]; ok { + spaceHardQuota, ok = params["capacity"].(int64) + if !ok { + log.AddContext(ctx).Errorf("get quota capacity failed, capacity: %v", params["capacity"]) + return nil, errors.New("capacity is invalid") + } + } + + data := make(map[string]interface{}) + data["PARENTTYPE"] = client.ParentTypeDTree + data["PARENTID"] = taskResult["dTreeId"] + data["QUOTATYPE"] = client.QuotaTypeDir + data["SPACEHARDQUOTA"] = spaceHardQuota * 512 + data["vstoreId"] = params["vstoreid"] + + quota, err := p.cli.CreateQuota(ctx, data) + if err != nil { + log.AddContext(ctx).Errorf("create quota failed, data: %+v, err: %v", data, err) + return nil, err + } + + quotaID, _ := utils.ToStringWithFlag(quota["ID"]) + return map[string]interface{}{ + "quotaId": quotaID, + }, nil + +} + +func (p *DTree) revertQuota(ctx context.Context, taskResult map[string]interface{}) error { + quotaID, _ := utils.ToStringWithFlag(taskResult["quotaId"]) + vStoreID, _ := utils.ToStringWithFlag(taskResult["vstoreid"]) + + err := p.cli.DeleteQuota(ctx, quotaID, vStoreID, client.ForceFlagFalse) + if err != nil { + log.AddContext(ctx).Errorf("revert quota failed, quotaID: %v, vStoreID: %v, err: %v", quotaID, vStoreID, err) + return err + } + + log.AddContext(ctx).Errorf("revert quota success, quotaID: %v, vStoreID: %v", quotaID, vStoreID) + return nil +} + +func (p *DTree) deleteQuota(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { + req := map[string]interface{}{ + "PARENTTYPE": client.ParentTypeDTree, + "PARENTID": taskResult["dTreeId"], + "range": "[0-100]", + "vstoreId": params["vstoreid"], + "QUERYTYPE": "2", + "SPACEUNITTYPE": client.SpaceUnitTypeGB, + } + quotaInfos, err := p.cli.BatchGetQuota(ctx, req) + if err != nil { + log.AddContext(ctx).Errorf("get quota arrays failed, params: %+v, error: %v", req, err) + return nil, err + } + if len(quotaInfos) == 0 { + log.AddContext(ctx).Infof("get empty quota arrays params: %+v", req) + return nil, nil + } + + quotaInfo, ok := quotaInfos[0].(map[string]interface{}) + if !ok { + log.AddContext(ctx).Errorf("quota arrays data is not valid, quotaInfos[0]: %+v", quotaInfos[0]) + return nil, errors.New("data in response is not valid") + } + + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + quotaID, _ := utils.ToStringWithFlag(quotaInfo["ID"]) + err = p.cli.DeleteQuota(ctx, quotaID, vStoreID, client.ForceFlagFalse) + if err != nil { + log.AddContext(ctx).Errorf("delete quota failed, quotaID: %v, vStoreID: %v, err: %v", quotaID, vStoreID, err) + } + return nil, err +} + +func (p *DTree) createShare(ctx context.Context, params, + taskResult map[string]interface{}) (map[string]interface{}, error) { + + parentName, _ := utils.ToStringWithFlag(params["parentname"]) + dTreeName, _ := utils.ToStringWithFlag(params["name"]) + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + sharePath := fmt.Sprintf("/%s/%s", parentName, dTreeName) + + share, err := p.cli.GetNfsShareByPath(ctx, sharePath, vStoreID) + if err != nil { + log.AddContext(ctx).Errorf("Get dTree nfs share by path %s error: %v", sharePath, err) + return nil, err + } + + if share == nil { + fsID, _ := utils.ToStringWithFlag(taskResult["fsId"]) + description, _ := utils.ToStringWithFlag(params["description"]) + shareParams := map[string]interface{}{ + "sharepath": sharePath, + "fsid": fsID, + "description": description, + "vStoreID": vStoreID, + "DTREEID": taskResult["dTreeId"], + } + + share, err = p.cli.CreateNfsShare(ctx, shareParams) + if err != nil { + log.AddContext(ctx).Errorf("Create dTree nfs share %v error: %v", shareParams, err) + return nil, err + } + } + + var shareID string + if _, ok := share["ID"]; ok { + shareID, _ = utils.ToStringWithFlag(share["ID"]) + } + + log.AddContext(ctx).Infof("create nfs share success, shareID: %v", shareID) + + return map[string]interface{}{ + "shareId": shareID, + }, nil +} + +func (p *DTree) revertShare(ctx context.Context, taskResult map[string]interface{}) error { + shareID, exist := utils.ToStringWithFlag(taskResult["shareId"]) + if !exist || len(shareID) == 0 { + log.AddContext(ctx).Errorf("revert nfs share failed, shareID not exists, shareID: %v", shareID) + return nil + } + vStoreID, _ := utils.ToStringWithFlag(taskResult["vstoreid"]) + + err := p.cli.DeleteNfsShare(ctx, shareID, vStoreID) + if err != nil { + log.AddContext(ctx).Errorf("revert nfs share failed, vStoreID: %v shareID: %v err: %v", vStoreID, shareID, err) + return err + } + log.AddContext(ctx).Infof("revert nfs share success, vStoreID: %v shareID: %v", vStoreID, shareID) + return nil +} + +func (p *DTree) deleteShare(ctx context.Context, params, + taskResult map[string]interface{}) (map[string]interface{}, error) { + + parentName, _ := utils.ToStringWithFlag(params["parentname"]) + dTreeName, _ := utils.ToStringWithFlag(params["name"]) + vStoreID, _ := utils.ToStringWithFlag(params["vstoreid"]) + sharePath := fmt.Sprintf("/%s/%s", parentName, dTreeName) + + share, err := p.cli.GetNfsShareByPath(ctx, sharePath, vStoreID) + if err != nil { + log.AddContext(ctx).Errorf("Get nfs share by path %s error: %v", sharePath, err) + return nil, err + } + + if share != nil { + shareID, _ := utils.ToStringWithFlag(share["ID"]) + err = p.cli.DeleteNfsShare(ctx, shareID, vStoreID) + if err != nil { + log.AddContext(ctx).Errorf("Delete share %s error: %v", shareID, err) + return nil, err + } + } + log.AddContext(ctx).Infof("delete share success, shareID: %v, err: %v", share["ID"], err) + return nil, nil +} + +func (p *DTree) getDtreeID(ctx context.Context, parentName, vstoreID, dTreeName string) (string, error) { + // get dtree id + dTreeInfo, err := p.cli.GetDTreeByName(ctx, "", parentName, vstoreID, dTreeName) + if err != nil { + log.AddContext(ctx).Errorf("get dTree by name failed, parentName :%s, vstoreID: %s, dTreeName: %s err: %v", + parentName, vstoreID, dTreeInfo, err) + return "", err + } + if dTreeInfo == nil { + log.AddContext(ctx).Errorf("get empty dtree finish,parentName :%s, vstoreID: %s, dTreeName: %s", + parentName, vstoreID, dTreeInfo) + return "", errors.New("empty dTree") + } + dTreeID, _ := utils.ToStringWithFlag(dTreeInfo["ID"]) + + return dTreeID, nil +}