object/search: Support numeric queries (#2733)

CHANGELOG.md

@@ -4,6 +4,8 @@ Changelog for NeoFS Node
 ## [Unreleased]
 
 ### Added
+- Support of numeric object search queries (#2733)
+- Support of `GT`, `GE`, `LT` and `LE` numeric comparison operators in CLI (#2733)
 
 ### Fixed
 - Access to `PUT` objects no longer grants `DELETE` rights (#2261)

cmd/neofs-cli/modules/object/search.go

@@ -85,6 +85,10 @@ var searchBinaryOpVocabulary = map[string]object.SearchMatchType{
 	"EQ":            object.MatchStringEqual,
 	"NE":            object.MatchStringNotEqual,
 	"COMMON_PREFIX": object.MatchCommonPrefix,
+	"GT":            object.MatchNumGT,
+	"GE":            object.MatchNumGE,
+	"LE":            object.MatchNumLE,
+	"LT":            object.MatchNumLT,
 }
 
 func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) {

go.mod

@@ -16,9 +16,9 @@ require (
 	github.com/nspcc-dev/hrw/v2 v2.0.0
 	github.com/nspcc-dev/locode-db v0.5.0
 	github.com/nspcc-dev/neo-go v0.105.1
-	github.com/nspcc-dev/neofs-api-go/v2 v2.14.0
+	github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240213170208-cfca09b5acbe
 	github.com/nspcc-dev/neofs-contract v0.19.1
-	github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240130073207-03ed6db7e1cd
+	github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240220193911-24254bf9aebe
 	github.com/nspcc-dev/tzhash v1.7.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/panjf2000/ants/v2 v2.8.2

go.sum

@@ -245,14 +245,14 @@ github.com/nspcc-dev/neo-go v0.105.1 h1:r0b2yIwLBi+ARBKU94gHL9oTFEB/XMJ0YlS2HN9Q
 github.com/nspcc-dev/neo-go v0.105.1/go.mod h1:GNh0cRALV/cuj+/xg2ZHDsrFbqcInqG7jjhqsLEnlNc=
 github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231127165613-b35f351f0ba0 h1:N+dMIBmteXjJpkH6UZ7HmNftuFxkqszfGLbhsEctnv0=
 github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231127165613-b35f351f0ba0/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag=
-github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 h1:jhuN8Ldqz7WApvUJRFY0bjRXE1R3iCkboMX5QVZhHVk=
-github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM=
+github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240213170208-cfca09b5acbe h1:Hoq88+PWS6tNnX4Y0jxE0C8wvxPI8UlVnCs2ZJDEy4Y=
+github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240213170208-cfca09b5acbe/go.mod h1:eaffSBIGhXUIMYvRBGXmlgQRLyyCWlzOft9jGYlqwrw=
 github.com/nspcc-dev/neofs-contract v0.19.1 h1:U1Uh+MlzfkalO0kRJ2pADZyHrmAOroC6KLFjdWnTNR0=
 github.com/nspcc-dev/neofs-contract v0.19.1/go.mod h1:ZOGouuwuHpgvYkx/LCGufGncIzEUhYEO18LL4cWEbyw=
 github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09fZo+KYi4=
 github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs=
-github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240130073207-03ed6db7e1cd h1:kRIn6i7BTa55ae4cH+UcqRfH//XC20mSC4E9WcWxkmM=
-github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240130073207-03ed6db7e1cd/go.mod h1:2PKUuH7kQaAmQ/USBgmiD/k08ssnSvayor6JAFhrC1c=
+github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240220193911-24254bf9aebe h1:VCJyY86/CSMTRjiXAPGGwRlss3FVGSGHZirCjQZzN2E=
+github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240220193911-24254bf9aebe/go.mod h1:icGhc6HFg+yKivBUoP7cut62SASuijDiWD5Txd6vWqY=
 github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
 github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
 github.com/nspcc-dev/tzhash v1.7.1 h1:6zmexLqdTF/ssbUAh7XJS7RxgKWaw28kdNpE/4UFdEU=

pkg/core/object/object.go

@@ -1,10 +1,15 @@
 package object
 
 import (
+	"errors"
+
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 )
 
+// ErrInvalidSearchQuery is returned when some object search query is invalid.
+var ErrInvalidSearchQuery = errors.New("invalid search query")
+
 // AddressOf returns the address of the object.
 func AddressOf(obj *object.Object) oid.Address {
 	var addr oid.Address

pkg/local_object_storage/engine/select.go

@@ -1,6 +1,9 @@
 package engine
 
 import (
+	"errors"
+
+	objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
@@ -64,6 +67,10 @@ func (e *StorageEngine) _select(prm SelectPrm) (SelectRes, error) {
 	e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
 		res, err := sh.Select(shPrm)
 		if err != nil {
+			if errors.Is(err, objectcore.ErrInvalidSearchQuery) {
+				outError = err
+				return true
+			}
 			e.reportShardError(sh, "could not select objects from shard", err)
 			return false
 		}

pkg/local_object_storage/metabase/select.go

@@ -4,9 +4,11 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"math/big"
 	"strconv"
 	"strings"
 
+	objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object"
 	"github.com/nspcc-dev/neofs-node/pkg/util"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
@@ -56,6 +58,12 @@ func (r SelectRes) AddressList() []oid.Address {
 }
 
 // Select returns list of addresses of objects that match search filters.
+//
+// Only creation epoch, payload size, user attributes and unknown system ones
+// are allowed with numeric operators. Values of numeric filters must be base-10
+// integers.
+//
+// Returns [object.ErrInvalidSearchQuery] if specified query is invalid.
 func (db *DB) Select(prm SelectPrm) (res SelectRes, err error) {
 	db.modeMtx.RLock()
 	defer db.modeMtx.RUnlock()
@@ -265,20 +273,75 @@ func (db *DB) selectFromFKBT(
 	f object.SearchFilter, // filter for operation and value
 	to map[string]int, // resulting cache
 	fNum int, // index of filter
-) { //
-	matchFunc, ok := db.matchers[f.Operation()]
-	if !ok {
-		db.log.Debug("missing matcher", zap.Uint32("operation", uint32(f.Operation())))
+) {
+	var nonNumMatcher matcher
+	op := f.Operation()
 
-		return
+	isNumOp := isNumericOp(op)
+	if !isNumOp {
+		var ok bool
+		nonNumMatcher, ok = db.matchers[op]
+		if !ok {
+			db.log.Debug("missing matcher", zap.Uint32("operation", uint32(op)))
+			return
+		}
 	}
 
 	fkbtRoot := tx.Bucket(name)
 	if fkbtRoot == nil {
 		return
 	}
 
-	err := matchFunc.matchBucket(fkbtRoot, f.Header(), f.Value(), func(k, _ []byte) error {
+	if isNumOp {
+		// TODO: big math takes less code but inefficient
+		filterNum, ok := new(big.Int).SetString(f.Value(), 10)
+		if !ok {
+			db.log.Debug("unexpected non-decimal numeric filter", zap.String("value", f.Value()))
+			return
+		}
+
+		var objNum big.Int
+
+		err := fkbtRoot.ForEach(func(objVal, _ []byte) error {
+			if len(objVal) == 0 {
+				return nil
+			}
+
+			_, ok := objNum.SetString(string(objVal), 10)
+			if !ok {
+				return nil
+			}
+
+			switch objNum.Cmp(filterNum) {
+			case -1:
+				ok = op == object.MatchNumLT || op == object.MatchNumLE
+			case 0:
+				ok = op == object.MatchNumLE || op == object.MatchNumGE
+			case 1:
+				ok = op == object.MatchNumGT || op == object.MatchNumGE
+			}
+			if !ok {
+				return nil
+			}
+
+			fkbtLeaf := fkbtRoot.Bucket(objVal)
+			if fkbtLeaf == nil {
+				return nil
+			}
+
+			return fkbtLeaf.ForEach(func(objAddr, _ []byte) error {
+				markAddressInCache(to, fNum, string(objAddr))
+				return nil
+			})
+		})
+		if err != nil {
+			db.log.Debug("error in FKBT selection", zap.String("error", err.Error()))
+		}
+
+		return
+	}
+
+	err := nonNumMatcher.matchBucket(fkbtRoot, f.Header(), f.Value(), func(k, _ []byte) error {
 		fkbtLeaf := fkbtRoot.Bucket(k)
 		if fkbtLeaf == nil {
 			return nil
@@ -473,6 +536,71 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f object.SearchFi
 	}
 
 	for i := range f {
+		op := f[i].Operation()
+		if isNumericOp(op) {
+			attr := f[i].Header()
+			if attr != object.FilterCreationEpoch && attr != object.FilterPayloadSize {
+				break
+			}
+
+			// filter by creation epoch or payload size, both uint64
+			filterVal := f[i].Value()
+			if len(filterVal) == 0 {
+				return false
+			}
+
+			if filterVal[0] == '-' {
+				if op == object.MatchNumLT || op == object.MatchNumLE {
+					return false
+				}
+				continue
+			}
+
+			if len(filterVal) > 20 { // max uint64 strlen
+				if op == object.MatchNumGT || op == object.MatchNumGE {
+					return false
+				}
+				continue
+			}
+
+			num, err := strconv.ParseUint(filterVal, 10, 64)
+			if err != nil {
+				if errors.Is(err, strconv.ErrRange) {
+					if op == object.MatchNumGT || op == object.MatchNumGE {
+						return false
+					}
+					continue
+				}
+				// has already been checked
+				db.log.Debug("unexpected failure to parse numeric filter uint value", zap.Error(err))
+				return false
+			}
+
+			var objVal uint64
+			if attr == object.FilterPayloadSize {
+				objVal = obj.PayloadSize()
+			} else {
+				objVal = obj.CreationEpoch()
+			}
+
+			switch {
+			case objVal > num:
+				if op == object.MatchNumLT || op == object.MatchNumLE {
+					return false
+				}
+			case objVal == num:
+				if op == object.MatchNumLT || op == object.MatchNumGT {
+					return false
+				}
+			case objVal < num:
+				if op == object.MatchNumGT || op == object.MatchNumGE {
+					return false
+				}
+			}
+
+			continue
+		}
+
 		matchFunc, ok := db.matchers[f[i].Operation()]
 		if !ok {
 			return false
@@ -514,7 +642,37 @@ func groupFilters(filters object.SearchFilters) (filterGroup, error) {
 	}
 
 	for i := range filters {
-		switch filters[i].Header() {
+		hdr := filters[i].Header()
+		if isNumericOp(filters[i].Operation()) {
+			switch hdr {
+			case
+				object.FilterVersion,
+				object.FilterID,
+				object.FilterContainerID,
+				object.FilterOwnerID,
+				object.FilterPayloadChecksum,
+				object.FilterType,
+				object.FilterPayloadHomomorphicHash,
+				object.FilterParentID,
+				object.FilterSplitID,
+				object.FilterRoot,
+				object.FilterPhysical:
+				// only object.FilterCreationEpoch and object.PayloadSize are numeric system
+				// object attributes now. Unsupported system attributes will lead to an empty
+				// results rather than a denial of service.
+				return res, fmt.Errorf("%w: invalid filter #%d: numeric filter with non-numeric system object attribute",
+					objectcore.ErrInvalidSearchQuery, i)
+			}
+
+			// TODO: big math takes less code but inefficient
+			_, ok := new(big.Int).SetString(filters[i].Value(), 10)
+			if !ok {
+				return res, fmt.Errorf("%w: invalid filter #%d: numeric filter with non-decimal value",
+					objectcore.ErrInvalidSearchQuery, i)
+			}
+		}
+
+		switch hdr {
 		case object.FilterContainerID: // support deprecated field
 			err := res.cnr.DecodeString(filters[i].Value())
 			if err != nil {
@@ -643,3 +801,7 @@ func filterExpired(tx *bbolt.Tx, epoch uint64, cID cid.ID, oIDs []oid.ID) ([]oid
 
 	return res, nil
 }
+
+func isNumericOp(op object.SearchMatchType) bool {
+	return op == object.MatchNumGT || op == object.MatchNumGE || op == object.MatchNumLT || op == object.MatchNumLE
+}