DAOS-14698 crtl: Create container with attributes

src/client/dfs/duns.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2019-2023 Intel Corporation.
+ * (C) Copyright 2019-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -899,8 +899,9 @@ duns_link_lustre_path(const char *pool, const char *cont, daos_cont_layout_t typ
 }
 #endif
 
-int
-duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp)
+static int
+create_path(daos_handle_t poh, const char *path, int count, char const *const names[],
+	    void const *const values[], size_t const sizes[], struct duns_attr_t *attrp)
 {
 	char			type[10];
 	daos_pool_info_t	info = {0};
@@ -1080,6 +1081,31 @@ duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp)
 		D_GOTO(err_link, rc);
 	}
 
+	if (count > 0) {
+		daos_handle_t coh;
+
+		D_ASSERT(names != NULL);
+		D_ASSERT(values != NULL);
+		D_ASSERT(sizes != NULL);
+
+		rc = daos_cont_open(poh, attrp->da_cont, DAOS_COO_RW, &coh, NULL, NULL);
+		if (rc != 0) {
+			DL_ERROR(rc, "Failed to open container %s", attrp->da_cont);
+			D_GOTO(err_cont, rc = daos_der2errno(rc));
+		}
+		rc  = daos_cont_set_attr(coh, count, names, values, sizes, NULL);
+		rc2 = daos_cont_close(coh, NULL);
+		if (rc != 0) {
+			DL_ERROR(rc, "Failed to set user attribute of container %s",
+				 attrp->da_cont);
+			D_GOTO(err_cont, rc = daos_der2errno(rc));
+		}
+		if (rc2 != 0) {
+			DL_ERROR(rc2, "failed to close container %s", attrp->da_cont);
+			D_GOTO(err_cont, rc = daos_der2errno(rc2));
+		}
+	}
+
 	/** store the daos attributes in the path xattr */
 	len =
 	    snprintf(str, DUNS_MAX_XATTR_LEN, DUNS_XATTR_FMT, type, attrp->da_pool, attrp->da_cont);
@@ -1128,6 +1154,19 @@ duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp)
 	return rc;
 }
 
+int
+duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp)
+{
+	return create_path(poh, path, 0, NULL, NULL, NULL, attrp);
+}
+
+int
+duns_create_path_attr(daos_handle_t poh, const char *path, int count, char const *const names[],
+		      void const *const values[], size_t const sizes[], struct duns_attr_t *attrp)
+{
+	return create_path(poh, path, count, names, values, sizes, attrp);
+}
+
 int
 duns_link_cont(daos_handle_t poh, const char *cont, const char *path)
 {

src/control/cmd/daos/container.go

@@ -251,6 +251,7 @@ type containerCreateCmd struct {
 	Mode            ConsModeFlag         `long:"mode" short:"M" description:"DFS consistency mode"`
 	ACLFile         string               `long:"acl-file" short:"A" description:"input file containing ACL"`
 	Group           ui.ACLPrincipalFlag  `long:"group" short:"g" description:"group who will own the container (group@[domain])"`
+	Attrs           ui.SetPropertiesFlag `long:"attrs" short:"a" description:"user-defined attributes (key:val[,key:val...])"`
 	Args            struct {
 		Label string `positional-arg-name:"label"`
 	} `positional-args:"yes"`
@@ -405,6 +406,25 @@ func (cmd *containerCreateCmd) contCreate() (string, error) {
 		contID = cmd.contUUID.String()
 	}
 
+	if len(cmd.Attrs.ParsedProps) != 0 {
+		attrs := make(attrList, 0, len(cmd.Attrs.ParsedProps))
+		for key, val := range cmd.Attrs.ParsedProps {
+			attrs = append(attrs, &attribute{
+				Name:  key,
+				Value: []byte(val),
+			})
+		}
+
+		if err := cmd.openContainer(C.DAOS_COO_RW); err != nil {
+			return "", errors.Wrapf(err, "failed to open new container %s", contID)
+		}
+		defer cmd.closeContainer()
+
+		if err := setDaosAttributes(cmd.cContHandle, contAttr, attrs); err != nil {
+			return "", errors.Wrapf(err, "failed to set user attributes on new container %s", contID)
+		}
+	}
+
 	cmd.Infof("Successfully created container %s", contID)
 	return contID, nil
 }
@@ -456,7 +476,33 @@ func (cmd *containerCreateCmd) contCreateUNS() (string, error) {
 	cPath := C.CString(cmd.Path)
 	defer freeString(cPath)
 
-	dunsErrno := C.duns_create_path(cmd.cPoolHandle, cPath, &dattr)
+	var dunsErrno C.int
+	attrs := cmd.Attrs.ParsedProps
+	if len(attrs) == 0 {
+		dunsErrno = C.duns_create_path(cmd.cPoolHandle, cPath, &dattr)
+	} else {
+		i := 0
+		attrNames := make([]*C.char, len(attrs))
+		valSizes := make([]C.size_t, len(attrs))
+		valBufs := make([]unsafe.Pointer, len(attrs))
+		for key, val := range attrs {
+			attrNames[i] = C.CString(key)
+			valSizes[i] = C.size_t(len(val))
+			valBufs[i] = C.malloc(valSizes[i])
+			valSlice := (*[1 << 30]byte)(valBufs[i])
+			copy(valSlice[:], val)
+			i++
+		}
+		defer func(nameSlice []*C.char, bufSlice []unsafe.Pointer) {
+			for i := 0; i < len(nameSlice); i++ {
+				freeString(nameSlice[i])
+				C.free(bufSlice[i])
+			}
+		}(attrNames, valBufs)
+
+		dunsErrno = C.duns_create_path_attr(cmd.cPoolHandle, cPath, C.int(len(attrs)), &attrNames[0], &valBufs[0], &valSizes[0], &dattr)
+	}
+
 	rc := C.daos_errno2der(dunsErrno)
 	if err := daosError(rc); err != nil {
 		return "", errors.Wrapf(err, "duns_create_path() failed")

src/include/daos_uns.h

@@ -161,6 +161,34 @@ struct duns_attr_t {
 int
 duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp);
 
+/**
+ * Create a special directory (POSIX) or file (HDF5) depending on the container type, and create a
+ * new DAOS container with a givren set of user attributes. The uuid of the container can be either
+ * passed in \a attrp->da_cuuid (deprecated) or generated internally and returned in \a da_cont.
+ * The extended attributes are set on the dir/file created that points to pool uuid, container
+ * uuid. This is to be used in a unified namespace solution to be able to map a path in the unified
+ * namespace to a location in the DAOS tier.  The container and pool can have labels, but the UNS
+ * stores the uuids only and so labels are ignored in \a attrp.
+ * The user is not required to call duns_destory_attrs on \a attrp as this call does not allocate
+ * any buffers in attrp.
+ *
+ * \param[in]	poh	Pool handle
+ * \param[in]	path	Valid path in an existing namespace.
+ * \param[in]	count	Number of container attributes
+ * \param[in]	names	Array of \a count null-terminated container attribute names.
+ * \param[in]	values	Array of \a count container attribute values
+ * \param[in]	sizes	Array of \a count elements containing the sizes of respective container
+ *			attribute values.
+ * \param[in,out]
+ *		attrp	Struct containing the attributes. The uuid of the
+ *			container created is returned in da_cuuid.
+ *
+ * \return		0 on Success. errno code on failure.
+ */
+int
+duns_create_path_attr(daos_handle_t poh, const char *path, int count, char const *const names[],
+		      void const *const values[], size_t const sizes[], struct duns_attr_t *attrp);
+
 /**
  * Retrieve the pool and container uuids from a path corresponding to a DAOS location. If this was a
  * path created with duns_create_path(), then this call would return the pool, container, and type

src/tests/ftest/dfuse/container_attrs.py

@@ -0,0 +1,113 @@
+"""
+  (C) Copyright 2020-2024 Intel Corporation.
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+"""
+
+import os
+import re
+
+from apricot import TestWithServers
+from dfuse_utils import get_dfuse, start_dfuse
+
+
+class DfuseContainerAttrs(TestWithServers):
+    """Check if the dfuse attributes of a container are properly managed.
+
+    :avocado: recursive
+    """
+
+    def setUp(self):
+        """Set up each test case."""
+        # obtain separate logs
+        self.update_log_file_names()
+
+        # Start the servers and agents
+        super().setUp()
+
+    def _check_attrs(self, dfuse, container_name):
+        """Check if the DFuse attributes of a container are loaded
+
+        Check in the log file of the dfuse instance if it contains the DFuse attributes of a given
+        container.  It also checks the value of the attributes found.
+
+        Args:
+            dfuse (Dfuse): DFuse instance to check
+            container_name (str): Name of the container
+        """
+        log_file = dfuse.get_log_file()
+        attrs = self.params.get("attrs", f"/run/{container_name}/*")
+        for attr in [attr.split(':') for attr in attrs.split(",")]:
+            match = None
+            attr_re = re.compile(r"^.+\ssetting\s+'" + attr[0] + r"'\s+is\s+(\d+)\s+seconds$")
+            for line in log_file:
+                match = attr_re.match(line)
+                if match:
+                    self.assertEqual(
+                        attr[1],
+                        match.group(1),
+                        f"Unexpected value for attribute {attr[0]}: "
+                        f"want={attr[1]}, got={match.group(1)}")
+                    break
+            self.assertIsNotNone(match, f"Setting of attribute {attr[0]} not found")
+
+    def test_dfuse_container_attrs(self):
+        """Jira ID: DAOS-14698.
+
+        Test Description:
+            Create a container with DFuse attributes
+            Mount a DFuse mount point
+            Check the output of the DFuse log
+        :avocado: tags=all,pr
+        :avocado: tags=vm
+        :avocado: tags=dfuse,container
+        :avocado: tags=DfuseContainerAttrs,test_dfuse_container_attrs
+        """
+        self.log.info("Creating DAOS pool")
+        pool = self.get_pool()
+
+        self.log_step("Creating DAOS container with Dfuse attributes")
+        container = self.get_container(pool, namespace="/run/container_01/*")
+
+        self.log_step("Mounting DFuse mount point")
+        dfuse = get_dfuse(self, self.hostlist_clients)
+        dfuse.env["D_LOG_FLUSH"] = "INFO"
+        start_dfuse(self, dfuse, pool, container)
+
+        self.log_step("Checking DFuse log file")
+        self._check_attrs(dfuse, "container_01")
+
+        self.log_step("Test passed")
+
+    def test_dfuse_subcontainer_attrs(self):
+        """Jira ID: DAOS-14698.
+
+        Test Description:
+            Create a container
+            Mount a DFuse mount point
+            Create a sub-container with DFuse attributes
+            Check the output of the DFuse log
+        :avocado: tags=all,pr
+        :avocado: tags=vm
+        :avocado: tags=dfuse,container
+        :avocado: tags=DfuseContainerAttrs,test_dfuse_subcontainer_attrs
+        """
+        self.log.info("Creating DAOS pool")
+        pool = self.get_pool()
+
+        self.log_step("Creating DAOS container")
+        container = self.get_container(pool, namespace="/run/container_02/*")
+
+        self.log_step("Mounting DFuse mount point")
+        dfuse = get_dfuse(self, self.hostlist_clients)
+        dfuse.env["D_LOG_FLUSH"] = "INFO"
+        start_dfuse(self, dfuse, pool, container)
+
+        self.log_step("Creating DAOS subcontainer with DFuse attributes")
+        sub_dir = os.path.join(dfuse.mount_dir.value, "foo")
+        container = self.get_container(pool, namespace="/run/container_03/*", path=sub_dir)
+
+        self.log_step("Checking DFuse log file")
+        self._check_attrs(dfuse, "container_03")
+
+        self.log_step("Test passed")

src/tests/ftest/dfuse/container_attrs.yaml

@@ -0,0 +1,30 @@
+hosts:
+  test_servers: 1
+  test_clients: 1
+
+timeout: 200
+
+server_config:
+  name: daos_server
+  engines_per_host: 1
+  engines:
+    0:
+      log_file: daos_server0.log
+      storage: auto
+
+pool:
+  scm_size: 80%
+
+container_01:
+  type: POSIX
+  attrs: dfuse-attr-time:666,dfuse-dentry-time:999
+  control_method: daos
+
+container_02:
+  type: POSIX
+  control_method: daos
+
+container_03:
+  type: POSIX
+  attrs: dfuse-attr-time:42,dfuse-dentry-time:404
+  control_method: daos

src/tests/ftest/util/daos_utils.py

@@ -75,7 +75,7 @@ def pool_autotest(self, pool):
 
     def container_create(self, pool, sys_name=None, path=None, cont_type=None,
                          oclass=None, dir_oclass=None, file_oclass=None, chunk_size=None,
-                         properties=None, acl_file=None, label=None):
+                         properties=None, acl_file=None, label=None, attrs=None):
         # pylint: disable=too-many-arguments
         """Create a container.
 
@@ -96,6 +96,8 @@ def container_create(self, pool, sys_name=None, path=None, cont_type=None,
                 pairs defining the container properties. Defaults to None
             acl_file (str, optional): ACL file. Defaults to None.
             label (str, optional): Container label. Defaults to None.
+            attrs (str, optional): String of comma-separated <name>:<value> pairs defining the
+                container user attributes.  Defaults to None.
 
         Returns:
             dict: the daos json command output converted to a python dictionary
@@ -110,10 +112,12 @@ def container_create(self, pool, sys_name=None, path=None, cont_type=None,
                 properties += ',rd_lvl:1'
         else:
             properties = 'rd_lvl:1'
+
         return self._get_json_result(
             ("container", "create"), pool=pool, sys_name=sys_name, path=path,
             type=cont_type, oclass=oclass, dir_oclass=dir_oclass, file_oclass=file_oclass,
-            chunk_size=chunk_size, properties=properties, acl_file=acl_file, label=label)
+            chunk_size=chunk_size, properties=properties, acl_file=acl_file, label=label,
+            attrs=attrs)
 
     def container_clone(self, src, dst):
         """Clone a container to a new container.

src/tests/ftest/util/daos_utils_base.py

@@ -300,6 +300,9 @@ def __init__(self):
                 #   --acl-file=PATH
                 #           input file containing ACL
                 self.acl_file = FormattedParameter("--acl-file={}", None)
+                #   --attrs=<name>:<value>[,<name>:<value>,...]
+                #           user-defined attributes
+                self.attrs = FormattedParameter("--attrs={}", None)
 
         class CreateSnapSubCommand(CommonContainerSubCommand):
             """Defines an object for the daos container create-snap command."""