From ee604fa21435c47dbb5f7a419f48c38bac4bbd89 Mon Sep 17 00:00:00 2001 From: Andrey Butusov Date: Thu, 24 Oct 2024 19:06:10 +0300 Subject: [PATCH 1/2] config: update doc and examples to the correct version Signed-off-by: Andrey Butusov --- config/example/node.json | 13 ++-- config/example/node.yaml | 2 - config/mainnet/config.yml | 8 +- config/testnet/config.yml | 3 +- docs/storage-node-configuration.md | 117 ++++++++++++++++++++--------- 5 files changed, 89 insertions(+), 54 deletions(-) diff --git a/config/example/node.json b/config/example/node.json index 1bb1b8c759..331485fea7 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -35,8 +35,8 @@ "path": "/state" } }, - "grpc": { - "0": { + "grpc": [ + { "endpoint": "s01.neofs.devenv:8080", "conn_limit": 1, "tls": { @@ -45,17 +45,17 @@ "key": "/path/to/key" } }, - "1": { + { "endpoint": "s02.neofs.devenv:8080", "conn_limit": -1, "tls": { "enabled": false } }, - "2": { + { "endpoint": "s03.neofs.devenv:8080" } - }, + ], "tree": { "enabled": true, "cache_size": 15, @@ -98,8 +98,6 @@ }, "policer": { "head_timeout": "15s", - "cache_size": "1000001", - "cache_time": "31s", "replication_cooldown": "101ms", "object_batch_size": "11", "max_workers": "21" @@ -173,7 +171,6 @@ "writecache": { "enabled": true, "path": "tmp/1/cache", - "memcache_capacity": 2147483648, "small_object_size": 16384, "max_object_size": 134217728, "workers_number": 30, diff --git a/config/example/node.yaml b/config/example/node.yaml index 679e4d0454..85004e96ff 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -88,8 +88,6 @@ apiclient: policer: head_timeout: 15s # timeout for the Policer HEAD remote operation - cache_size: 1000001 # recently-handled objects cache size - cache_time: 31s # recently-handled objects cache expiration time replication_cooldown: 101ms # cooldown time b/w replication tasks submitting object_batch_size: 11 # replication's objects batch size max_workers: 21 # replication's worker pool's maximum size diff --git a/config/mainnet/config.yml b/config/mainnet/config.yml index d33570c4d4..c8c2ee20e9 100644 --- a/config/mainnet/config.yml +++ b/config/mainnet/config.yml @@ -10,14 +10,11 @@ node: attribute_2: User-Agent:NeoFS\/0.27 grpc: - num: 1 - 0: - endpoint: + - endpoint: tls: enabled: false storage: - shard_num: 1 shard: 0: metabase: @@ -48,10 +45,9 @@ prometheus: object: put: pool_size_remote: 100 - pool_size_local: 100 morph: - rpc_endpoint: + endpoints: - wss://rpc1.morph.fs.neo.org:40341/ws - wss://rpc2.morph.fs.neo.org:40341/ws - wss://rpc3.morph.fs.neo.org:40341/ws diff --git a/config/testnet/config.yml b/config/testnet/config.yml index 51b59b3fa6..70e18a7a76 100644 --- a/config/testnet/config.yml +++ b/config/testnet/config.yml @@ -2,7 +2,7 @@ logger: level: info morph: - rpc_endpoint: + endpoints: - wss://rpc01.morph.testnet.fs.neo.org:51331/ws - wss://rpc02.morph.testnet.fs.neo.org:51331/ws - wss://rpc03.morph.testnet.fs.neo.org:51331/ws @@ -28,7 +28,6 @@ prometheus: shutdown_timeout: 15s storage: - shard_num: 1 shard: 0: metabase: diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 0dfbe7a085..e0a11107f2 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -27,6 +27,7 @@ There are some custom types used for brevity: | `grpc` | [gRPC configuration](#grpc-section) | | `node` | [Node configuration](#node-section) | | `object` | [Object service configuration](#object-section) | +| `tree` | [Tree service configuration](#tree-section) | # `control` section @@ -42,7 +43,6 @@ control: |-------------------|----------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `authorized_keys` | `[]public key` | empty | List of public keys which are used to authorize requests to the control service. | | `grpc.endpoint` | `string` | empty | Address that control service listener binds to. | -| `grpc.conn_limit` | `int` | 0 | Number of accepted connections at a time, non-positive values keep connections unlimited. Connections that exceed limitation are accepted but not handled until some connection is closed. | # `grpc` section ```yaml @@ -60,10 +60,11 @@ grpc: Contains an array of gRPC endpoint configurations. The following table describes the format of each element. -| Parameter | Type | Default value | Description | -|---------------------------|-------------------------------|---------------|---------------------------------------------------------------------------| -| `endpoint` | `[]string` | empty | Address that service listener binds to. | -| `tls` | [TLS config](#tls-subsection) | | Address that control service listener binds to. | +| Parameter | Type | Default value | Description | +|-------------------|-------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------| +| `endpoint` | `string` | empty | Address that service listener binds to. | +| `conn_limit` | `int` | | Connection limits. Exceeding connection will not be declined, just blocked before active number decreases or client timeouts. | +| `tls` | [TLS config](#tls-subsection) | | Address that control service listener binds to. | ## `tls` subsection @@ -163,17 +164,18 @@ Contains configuration for each shard. Keys must be consecutive numbers starting `default` subsection has the same format and specifies defaults for missing values. The following table describes configuration for each shard. -| Parameter | Type | Default value | Description | -|-------------------------------------|---------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `compress` | `bool` | `false` | Flag to enable compression. | -| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | -| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | -| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | -| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | -| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | -| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | -| `small_object_size` | `size` | `1M` | Maximum size of an object stored in peapod. | -| `gc` | [GC config](#gc-subsection) | | GC configuration. | +| Parameter | Type | Default value | Description | +|-------------------------------------|----------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `compress` | `bool` | `false` | Flag to enable compression. | +| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | +| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | +| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | +| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | +| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | +| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | +| `small_object_size` | `size` | `1M` | Maximum size of an object stored in peapod. | +| `gc` | [GC config](#gc-subsection) | | GC configuration. | +| `pilorama` | [Pilorama Config](#pilorama-subsection) | | Pilorama configuration. | ### `blobstor` subsection @@ -191,11 +193,11 @@ blobstor: ``` #### Common options for sub-storages -| Parameter | Type | Default value | Description | -|-------------------------------------|-----------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0640` | Default permission for created files and directories. | -| `flush_interval` | `duration` | `10ms` | Time interval between batch writes to disk. | +| Parameter | Type | Default value | Description | +|-------------------------------------|------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0640` | Default permission for created files and directories. | +| `flush_interval` | `duration` | `10ms` | Time interval between batch writes to disk. | #### `fstree` type options | Parameter | Type | Default value | Description | @@ -258,15 +260,35 @@ writecache: workers_number: 30 ``` -| Parameter | Type | Default value | Description | -|----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------| -| `path` | `string` | | Path to the metabase file. | -| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | -| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | -| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | -| `workers_number` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | -| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | -| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| Parameter | Type | Default value | Description | +|---------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the writecache. | +| `path` | `string` | | Path to the metabase file. | +| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | +| `no_sync` | `bool` | `false` | Disable write synchronization, makes writes faster, but can lead to data loss. | +| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | +| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | +| `workers_number` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | +| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | +| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | + +### `pilorama` subsection + +```yaml +pilorama: + path: path/to/pilorama.db + max_batch_delay: 10ms + max_batch_size: 200 +``` + +| Parameter | Type | Default value | Description | +|-------------------|------------|---------------|-----------------------------------------------------------------------------------------| +| `path` | `string` | | Path to the pilorama database. If omitted, `pilorama.db` file is created blobstor.path. | +| `perm` | file mode | `0640` | Permissions to set for the database file. | +| `max_batch_size` | `int` | `1000` | Maximum amount of write operations to perform in a single transaction. | +| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| `no_sync` | `bool` | `false` | Disable write synchronization, makes writes faster, but can lead to data loss. | + # `node` section @@ -334,11 +356,12 @@ apiclient: stream_timeout: 20s reconnect_timeout: 30s ``` -| Parameter | Type | Default value | Description | -|-------------------|----------|---------------|-----------------------------------------------------------------------| -| dial_timeout | duration | `1m` | Timeout for dialing connections to other storage or inner ring nodes. | -| stream_timeout | duration | `15s` | Timeout for individual operations in a streaming RPC. | -| reconnect_timeout | duration | `30s` | Time to wait before reconnecting to a failed node. | +| Parameter | Type | Default value | Description | +|---------------------|----------|---------------|----------------------------------------------------------------------| +| `dial_timeout` | `duration` | `1m` | Timeout for dialing connections to other storage or inner ring nodes. | +| `stream_timeout` | `duration` | `15s` | Timeout for individual operations in a streaming RPC. | +| `reconnect_timeout` | `duration` | `30s` | Time to wait before reconnecting to a failed node. | +| `allow_external` | `bool` | `false` | Allow to fallback to addresses in `ExternalAddr` attribute. | # `policer` section @@ -386,4 +409,26 @@ object: | Parameter | Type | Default value | Description | |-----------------------------|-------|---------------|------------------------------------------------------------------------------------------------| | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | -| `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | \ No newline at end of file +| `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | + +# `tree` section +Contains tree-service related parameters. + +```yaml +tree: + enabled: true + cache_size: 15 + replication_worker_count: 32 + replication_channel_capacity: 32 + replication_timeout: 5s + sync_interval: 1h +``` + +| Parameter | Type | Default value | Description | +|--------------------------------|------------|---------------|----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `cache_size` | `int` | | Size for container cache. | +| `replication_worker_count` | `int` | | Number of replication workers. | +| `replication_channel_capacity` | `int` | | Capacity of replication channel. | +| `replication_timeout` | `duration` | | Timeout for replication process. | +| `sync_interval` | `duration` | | Interval for syncronization all trees. | From 652e230c3fc5d4ffcf1401cbd65fd063c05df555 Mon Sep 17 00:00:00 2001 From: Andrey Butusov Date: Fri, 25 Oct 2024 23:43:05 +0300 Subject: [PATCH 2/2] node, ir: validate config for unknown fields Add validation of ir and node configs by new structs with all possible fields. For node config add new option `WithValidate`, that is responsible for whether the config structure needs to be validated. Add tests. Closes #2663, #1950. Signed-off-by: Andrey Butusov --- CHANGELOG.md | 1 + cmd/internal/configvalidator/validate.go | 106 +++++++++ cmd/neofs-ir/defaults.go | 12 +- cmd/neofs-ir/defaults_test.go | 17 ++ cmd/neofs-ir/internal/validate/config.go | 213 +++++++++++++++++ cmd/neofs-ir/internal/validate/validate.go | 22 ++ .../internal/validate/validate_test.go | 139 +++++++++++ cmd/neofs-node/config/config.go | 8 + .../config/internal/validate/config.go | 181 +++++++++++++++ .../config/internal/validate/validate.go | 22 ++ .../config/internal/validate/validate_test.go | 218 ++++++++++++++++++ cmd/neofs-node/config/opts.go | 15 +- cmd/neofs-node/config/test/config.go | 1 + 13 files changed, 951 insertions(+), 4 deletions(-) create mode 100644 cmd/internal/configvalidator/validate.go create mode 100644 cmd/neofs-ir/defaults_test.go create mode 100644 cmd/neofs-ir/internal/validate/config.go create mode 100644 cmd/neofs-ir/internal/validate/validate.go create mode 100644 cmd/neofs-ir/internal/validate/validate_test.go create mode 100644 cmd/neofs-node/config/internal/validate/config.go create mode 100644 cmd/neofs-node/config/internal/validate/validate.go create mode 100644 cmd/neofs-node/config/internal/validate/validate_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 164598ca07..dc6ee77149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ attribute, which is used for container domain name in NNS contracts (#2954) - In inner ring config, default ports for TCP addresses are used if they are missing (#2969) - Metabase is refilled if an object exists in write-cache but is missing in metabase (#2977) - Pprof and metrics services stop at the end of SN's application lifecycle (#2976) +- Reject configuration with unknown fields (#2981) ### Removed - Support for node.key configuration (#2959) diff --git a/cmd/internal/configvalidator/validate.go b/cmd/internal/configvalidator/validate.go new file mode 100644 index 0000000000..cf01ca8f98 --- /dev/null +++ b/cmd/internal/configvalidator/validate.go @@ -0,0 +1,106 @@ +package configvalidator + +import ( + "errors" + "fmt" + "reflect" + "strings" +) + +// ErrUnknowField returns when an unknown field appears in the config. +var ErrUnknowField = errors.New("unknown field") + +// CheckForUnknownFields validates the config struct for unknown fields with the config map. +// If the field in the config is a map of a key and some value, need to use `mapstructure:,remain` and +// `prefix` tag with the key prefix, even it is empty string. +func CheckForUnknownFields(configMap map[string]any, config any) error { + return checkForUnknownFields(configMap, config, "") +} + +func checkForUnknownFields(configMap map[string]any, config any, currentPath string) error { + expectedFields := getFieldsFromStruct(config) + + for key, val := range configMap { + fullPath := key + if currentPath != "" { + fullPath = currentPath + "." + key + } + + _, other := expectedFields[",remain"] + _, exists := expectedFields[key] + if !exists && !other { + return fmt.Errorf("%w: %s", ErrUnknowField, fullPath) + } + + var fieldVal reflect.Value + if other && !exists { + fieldVal = reflect.ValueOf(config). + FieldByName(getStructFieldByTag(config, "prefix", key, strings.HasPrefix)) + } else { + fieldVal = reflect.ValueOf(config). + FieldByName(getStructFieldByTag(config, "mapstructure", key, func(a, b string) bool { + return a == b + })) + } + + switch fieldVal.Kind() { + case reflect.Slice: + if fieldVal.Len() == 0 { + return nil + } + fieldVal = fieldVal.Index(0) + case reflect.Map: + fieldVal = fieldVal.MapIndex(reflect.ValueOf(key)) + default: + } + + if !fieldVal.IsValid() { + return fmt.Errorf("%w: %s", ErrUnknowField, fullPath) + } + + nestedMap, okMap := val.(map[string]any) + nestedSlice, okSlice := val.([]any) + if (okMap || okSlice) && fieldVal.Kind() == reflect.Struct { + if okSlice { + for _, slice := range nestedSlice { + if nestedMap, okMap = slice.(map[string]any); okMap { + if err := checkForUnknownFields(nestedMap, fieldVal.Interface(), fullPath); err != nil { + return err + } + } + } + } else if err := checkForUnknownFields(nestedMap, fieldVal.Interface(), fullPath); err != nil { + return err + } + } else if okMap != (fieldVal.Kind() == reflect.Struct) { + return fmt.Errorf("%w: %s", ErrUnknowField, fullPath) + } + } + + return nil +} + +func getFieldsFromStruct(config any) map[string]struct{} { + fields := make(map[string]struct{}) + t := reflect.TypeOf(config) + for i := range t.NumField() { + field := t.Field(i) + if jsonTag := field.Tag.Get("mapstructure"); jsonTag != "" { + fields[jsonTag] = struct{}{} + } else { + fields[field.Name] = struct{}{} + } + } + return fields +} + +func getStructFieldByTag(config any, tagKey, tagValue string, comparison func(a, b string) bool) string { + t := reflect.TypeOf(config) + for i := range t.NumField() { + field := t.Field(i) + if jsonTag, ok := field.Tag.Lookup(tagKey); comparison(tagValue, jsonTag) && ok { + return field.Name + } + } + return "" +} diff --git a/cmd/neofs-ir/defaults.go b/cmd/neofs-ir/defaults.go index 41905a5f7d..8517c850cf 100644 --- a/cmd/neofs-ir/defaults.go +++ b/cmd/neofs-ir/defaults.go @@ -4,6 +4,7 @@ import ( "strings" "time" + "github.com/nspcc-dev/neofs-node/cmd/neofs-ir/internal/validate" "github.com/spf13/viper" ) @@ -29,9 +30,17 @@ func newConfig(path string) (*viper.Viper, error) { v.SetConfigType("yml") } err = v.ReadInConfig() + if err != nil { + return nil, err + } + } + + err = validate.ValidateStruct(v) + if err != nil { + return nil, err } - return v, err + return v, nil } func defaultConfiguration(cfg *viper.Viper) { @@ -60,7 +69,6 @@ func defaultConfiguration(cfg *viper.Viper) { cfg.SetDefault("wallet.address", "") // account address cfg.SetDefault("wallet.password", "") // password - cfg.SetDefault("timers.emit", "0") cfg.SetDefault("timers.stop_estimation.mul", 1) cfg.SetDefault("timers.stop_estimation.div", 4) cfg.SetDefault("timers.collect_basic_income.mul", 1) diff --git a/cmd/neofs-ir/defaults_test.go b/cmd/neofs-ir/defaults_test.go new file mode 100644 index 0000000000..a539641c05 --- /dev/null +++ b/cmd/neofs-ir/defaults_test.go @@ -0,0 +1,17 @@ +package main + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/cmd/neofs-ir/internal/validate" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestValidateDefaultConfig(t *testing.T) { + v := viper.New() + + defaultConfiguration(v) + + require.NoError(t, validate.ValidateStruct(v)) +} diff --git a/cmd/neofs-ir/internal/validate/config.go b/cmd/neofs-ir/internal/validate/config.go new file mode 100644 index 0000000000..8568b42af2 --- /dev/null +++ b/cmd/neofs-ir/internal/validate/config.go @@ -0,0 +1,213 @@ +package validate + +import "time" + +type validConfig struct { + Logger struct { + Level string `mapstructure:"level"` + } `mapstructure:"logger"` + + Wallet struct { + Path string `mapstructure:"path"` + Address string `mapstructure:"address"` + Password string `mapstructure:"password"` + } `mapstructure:"wallet"` + + WithoutMainnet bool `mapstructure:"without_mainnet"` + + Morph struct { + DialTimeout time.Duration `mapstructure:"dial_timeout"` + ReconnectionsNumber int `mapstructure:"reconnections_number"` + ReconnectionsDelay time.Duration `mapstructure:"reconnections_delay"` + Endpoints []string `mapstructure:"endpoints"` + Validators []string `mapstructure:"validators"` + Consensus struct { + Magic uint32 `mapstructure:"magic"` + Committee []string `mapstructure:"committee"` + + Storage struct { + Type string `mapstructure:"type"` + Path string `mapstructure:"path"` + } `mapstructure:"storage"` + + TimePerBlock time.Duration `mapstructure:"time_per_block"` + MaxTraceableBlocks uint32 `mapstructure:"max_traceable_blocks"` + SeedNodes []string `mapstructure:"seed_nodes"` + + Hardforks struct { + Name map[string]uint32 `mapstructure:",remain" prefix:""` + } `mapstructure:"hardforks"` + + ValidatorsHistory struct { + Height map[string]int `mapstructure:",remain" prefix:""` + } `mapstructure:"validators_history"` + + RPC struct { + Listen []string `mapstructure:"listen"` + TLS struct { + Enabled bool `mapstructure:"enabled"` + Listen []string `mapstructure:"listen"` + CertFile string `mapstructure:"cert_file"` + KeyFile string `mapstructure:"key_file"` + } `mapstructure:"tls"` + } `mapstructure:"rpc"` + + P2P struct { + DialTimeout time.Duration `mapstructure:"dial_timeout"` + ProtoTickInterval time.Duration `mapstructure:"proto_tick_interval"` + Listen []string `mapstructure:"listen"` + Peers struct { + Min int `mapstructure:"min"` + Max int `mapstructure:"max"` + Attempts int `mapstructure:"attempts"` + } `mapstructure:"peers"` + Ping struct { + Interval time.Duration `mapstructure:"interval"` + Timeout time.Duration `mapstructure:"timeout"` + } `mapstructure:"ping"` + } `mapstructure:"p2p"` + SetRolesInGenesis bool `mapstructure:"set_roles_in_genesis"` + } `mapstructure:"consensus"` + } `mapstructure:"morph"` + + FSChainAutodeploy bool `mapstructure:"fschain_autodeploy"` + + NNS struct { + SystemEmail string `mapstructure:"system_email"` + } `mapstructure:"nns"` + + Mainnet struct { + DialTimeout time.Duration `mapstructure:"dial_timeout"` + ReconnectionsNumber int `mapstructure:"reconnections_number"` + ReconnectionsDelay time.Duration `mapstructure:"reconnections_delay"` + Endpoints []string `mapstructure:"endpoints"` + } `mapstructure:"mainnet"` + + Control struct { + AuthorizedKeys []string `mapstructure:"authorized_keys"` + GRPC struct { + Endpoint string `mapstructure:"endpoint"` + } `mapstructure:"grpc"` + } `mapstructure:"control"` + + Governance struct { + Disable bool `mapstructure:"disable"` + } `mapstructure:"governance"` + + Node struct { + PersistentState struct { + Path string `mapstructure:"path"` + } `mapstructure:"persistent_state"` + } `mapstructure:"node"` + + Fee struct { + MainChain int64 `mapstructure:"main_chain"` + SideChain int64 `mapstructure:"side_chain"` + NamedContainerRegister int64 `mapstructure:"named_container_register"` + } `mapstructure:"fee"` + + Timers struct { + StopEstimation struct { + Mul int `mapstructure:"mul"` + Div int `mapstructure:"div"` + } `mapstructure:"stop_estimation"` + CollectBasicIncome struct { + Mul int `mapstructure:"mul"` + Div int `mapstructure:"div"` + } `mapstructure:"collect_basic_income"` + DistributeBasicIncome struct { + Mul int `mapstructure:"mul"` + Div int `mapstructure:"div"` + } `mapstructure:"distribute_basic_income"` + } `mapstructure:"timers"` + + Emit struct { + Storage struct { + Amount int64 `mapstructure:"amount"` + } `mapstructure:"storage"` + Mint struct { + Value int64 `mapstructure:"value"` + CacheSize int `mapstructure:"cache_size"` + Threshold int `mapstructure:"threshold"` + } `mapstructure:"mint"` + Gas struct { + BalanceThreshold int64 `mapstructure:"balance_threshold"` + } `mapstructure:"gas"` + } `mapstructure:"emit"` + + Workers struct { + Alphabet int `mapstructure:"alphabet"` + Balance int `mapstructure:"balance"` + Container int `mapstructure:"container"` + NeoFS int `mapstructure:"neofs"` + Netmap int `mapstructure:"netmap"` + Reputation int `mapstructure:"reputation"` + } `mapstructure:"workers"` + + Audit struct { + Timeout struct { + Get time.Duration `mapstructure:"get"` + Head time.Duration `mapstructure:"head"` + RangeHash time.Duration `mapstructure:"rangehash"` + Search time.Duration `mapstructure:"search"` + } `mapstructure:"timeout"` + Task struct { + ExecPoolSize int `mapstructure:"exec_pool_size"` + QueueCapacity int `mapstructure:"queue_capacity"` + } `mapstructure:"task"` + PDP struct { + PairsPoolSize int `mapstructure:"pairs_pool_size"` + MaxSleepInterval time.Duration `mapstructure:"max_sleep_interval"` + } `mapstructure:"pdp"` + POR struct { + PoolSize int `mapstructure:"pool_size"` + } `mapstructure:"por"` + } `mapstructure:"audit"` + + Indexer struct { + CacheTimeout time.Duration `mapstructure:"cache_timeout"` + } `mapstructure:"indexer"` + + NetmapCleaner struct { + Enabled bool `mapstructure:"enabled"` + Threshold int `mapstructure:"threshold"` + } `mapstructure:"netmap_cleaner"` + + Contracts struct { + NeoFS string `mapstructure:"neofs"` + Processing string `mapstructure:"processing"` + Audit string `mapstructure:"audit"` + Balance string `mapstructure:"balance"` + Container string `mapstructure:"container"` + NeoFSID string `mapstructure:"neofsid"` + Netmap string `mapstructure:"netmap"` + Proxy string `mapstructure:"proxy"` + Reputation string `mapstructure:"reputation"` + Alphabet struct { + AZ string `mapstructure:"az"` + Buky string `mapstructure:"buky"` + Vedi string `mapstructure:"vedi"` + Glagoli string `mapstructure:"glagoli"` + Dobro string `mapstructure:"dobro"` + Yest string `mapstructure:"yest"` + Zhivete string `mapstructure:"zhivete"` + } `mapstructure:"alphabet"` + } `mapstructure:"contracts"` + + Pprof struct { + Enabled bool `mapstructure:"enabled"` + Address string `mapstructure:"address"` + ShutdownTimeout time.Duration `mapstructure:"shutdown_timeout"` + } `mapstructure:"pprof"` + + Prometheus struct { + Enabled bool `mapstructure:"enabled"` + Address string `mapstructure:"address"` + ShutdownTimeout time.Duration `mapstructure:"shutdown_timeout"` + } `mapstructure:"prometheus"` + + Settlement struct { + BasicIncomeRate int64 `mapstructure:"basic_income_rate"` + AuditFee int64 `mapstructure:"audit_fee"` + } `mapstructure:"settlement"` +} diff --git a/cmd/neofs-ir/internal/validate/validate.go b/cmd/neofs-ir/internal/validate/validate.go new file mode 100644 index 0000000000..ce88e4ded8 --- /dev/null +++ b/cmd/neofs-ir/internal/validate/validate.go @@ -0,0 +1,22 @@ +package validate + +import ( + "fmt" + + "github.com/nspcc-dev/neofs-node/cmd/internal/configvalidator" + "github.com/spf13/viper" +) + +// ValidateStruct validates the viper config structure. +func ValidateStruct(v *viper.Viper) error { + var cfg validConfig + if err := v.Unmarshal(&cfg); err != nil { + return fmt.Errorf("unable to decode config: %w", err) + } + + if err := configvalidator.CheckForUnknownFields(v.AllSettings(), cfg); err != nil { + return err + } + + return nil +} diff --git a/cmd/neofs-ir/internal/validate/validate_test.go b/cmd/neofs-ir/internal/validate/validate_test.go new file mode 100644 index 0000000000..cfada1fc0a --- /dev/null +++ b/cmd/neofs-ir/internal/validate/validate_test.go @@ -0,0 +1,139 @@ +package validate + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestCheckForUnknownFields(t *testing.T) { + tests := []struct { + name string + config string + wantErr bool + }{ + { + name: "with all right fields", + config: ` +morph: + dial_timeout: 1m + reconnections_number: 5 + reconnections_delay: 5s + endpoints: + - wss://sidechain1.fs.neo.org:30333/ws + - wss://sidechain2.fs.neo.org:30333/ws + validators: + - 0283120f4c8c1fc1d792af5063d2def9da5fddc90bc1384de7fcfdda33c3860170 + consensus: + magic: 15405 + committee: + - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 + - 02103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e + - 03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699 + - 02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62 + storage: + type: boltdb + path: ./db/morph.bolt + time_per_block: 1s + max_traceable_blocks: 11520 + seed_nodes: + - node2 + - node3:20333 + hardforks: + name: 1730000 + validators_history: + 0: 4 + 4: 1 + 12: 4 + rpc: + listen: + - localhost + - localhost:30334 + tls: + enabled: false + listen: + - localhost:30335 + - localhost:30336 + cert_file: serv.crt + key_file: serv.key + p2p: + dial_timeout: 1m + proto_tick_interval: 2s + listen: + - localhost + - localhost:20334 + peers: + min: 1 + max: 5 + attempts: 20 + ping: + interval: 30s + timeout: 90s + set_roles_in_genesis: true +`, + wantErr: false, + }, + { + name: "unknown morph.consensus.timeout", + config: ` +morph: + consensus: + p2p: + ping: + interval: 30s + timeout: 90s + set_roles_in_genesis: true +`, + wantErr: true, + }, + { + name: "morph.consensus.storage.type expected type string", + config: ` +morph: + consensus: + storage: + type: + path: ./db/morph.bolt +`, + wantErr: true, + }, + { + name: "unknown field morph.attr", + config: ` +morph: + dial_timeout: 1m + reconnections_number: 5 + attr: 123 +`, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := viper.New() + v.SetConfigType("yaml") + require.NoError(t, v.ReadConfig(strings.NewReader(tt.config))) + + err := ValidateStruct(v) + fmt.Println(err) + if (err != nil) != tt.wantErr { + t.Errorf("CheckForUnknownFields() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestCheckForUnknownFieldsExample(t *testing.T) { + const exampleConfigPrefix = "../../../../config/" + + path := filepath.Join(exampleConfigPrefix, "example/ir.yaml") + v := viper.New() + v.SetConfigFile(path) + + require.NoError(t, v.ReadInConfig()) + require.NoError(t, ValidateStruct(v)) +} diff --git a/cmd/neofs-node/config/config.go b/cmd/neofs-node/config/config.go index c68bed9fcd..c8ce47e6de 100644 --- a/cmd/neofs-node/config/config.go +++ b/cmd/neofs-node/config/config.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/internal" + "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/internal/validate" "github.com/spf13/viper" ) @@ -54,6 +55,13 @@ func New(_ Prm, opts ...Option) *Config { } } + if o.validate { + err := validate.ValidateStruct(v) + if err != nil { + panic(err) + } + } + return &Config{ v: v, opts: *o, diff --git a/cmd/neofs-node/config/internal/validate/config.go b/cmd/neofs-node/config/internal/validate/config.go new file mode 100644 index 0000000000..7fe047cff9 --- /dev/null +++ b/cmd/neofs-node/config/internal/validate/config.go @@ -0,0 +1,181 @@ +package validate + +import ( + "time" +) + +type valideConfig struct { + Logger struct { + Level string `mapstructure:"level"` + } `mapstructure:"logger"` + + Pprof struct { + Enabled bool `mapstructure:"enabled"` + Address string `mapstructure:"address"` + ShutdownTimeout time.Duration `mapstructure:"shutdown_timeout"` + } `mapstructure:"pprof"` + + Prometheus struct { + Enabled bool `mapstructure:"enabled"` + Address string `mapstructure:"address"` + ShutdownTimeout time.Duration `mapstructure:"shutdown_timeout"` + } `mapstructure:"prometheus"` + + Node struct { + Wallet struct { + Path string `mapstructure:"path"` + Address string `mapstructure:"address"` + Password string `mapstructure:"password"` + } `mapstructure:"wallet"` + + Addresses []string `mapstructure:"addresses"` + Relay bool `mapstructure:"relay"` + + PersistentSessions struct { + Path string `mapstructure:"path"` + } `mapstructure:"persistent_sessions"` + + PersistentState struct { + Path string `mapstructure:"path"` + } `mapstructure:"persistent_state"` + + Attribute map[string]string `mapstructure:",remain" prefix:"attribute_"` + } `mapstructure:"node"` + + GRPC []struct { + Endpoint string `mapstructure:"endpoint"` + ConnLimit int `mapstructure:"conn_limit"` + + TLS struct { + Enabled bool `mapstructure:"enabled"` + Certificate string `mapstructure:"certificate"` + Key string `mapstructure:"key"` + } `mapstructure:"tls"` + } `mapstructure:"grpc"` + + Tree struct { + Enabled bool `mapstructure:"enabled"` + CacheSize int `mapstructure:"cache_size"` + ReplicationWorkerCount int `mapstructure:"replication_worker_count"` + ReplicationChannelCap int `mapstructure:"replication_channel_capacity"` + ReplicationTimeout time.Duration `mapstructure:"replication_timeout"` + SyncInterval time.Duration `mapstructure:"sync_interval"` + } `mapstructure:"tree"` + + Control struct { + AuthorizedKeys []string `mapstructure:"authorized_keys"` + + GRPC struct { + Endpoint string `mapstructure:"endpoint"` + } `mapstructure:"grpc"` + } `mapstructure:"control"` + + Contracts struct { + Balance string `mapstructure:"balance"` + Container string `mapstructure:"container"` + Netmap string `mapstructure:"netmap"` + Reputation string `mapstructure:"reputation"` + Proxy string `mapstructure:"proxy"` + } `mapstructure:"contracts"` + + Morph struct { + DialTimeout time.Duration `mapstructure:"dial_timeout"` + CacheTTL time.Duration `mapstructure:"cache_ttl"` + ReconnectionsNumber int `mapstructure:"reconnections_number"` + ReconnectionsDelay time.Duration `mapstructure:"reconnections_delay"` + Endpoints []string `mapstructure:"endpoints"` + } `mapstructure:"morph"` + + APIClient struct { + DialTimeout time.Duration `mapstructure:"dial_timeout"` + StreamTimeout time.Duration `mapstructure:"stream_timeout"` + AllowExternal bool `mapstructure:"allow_external"` + ReconnectTimeout time.Duration `mapstructure:"reconnect_timeout"` + } `mapstructure:"apiclient"` + + Policer struct { + HeadTimeout time.Duration `mapstructure:"head_timeout"` + ReplicationCooldown time.Duration `mapstructure:"replication_cooldown"` + ObjectBatchSize int `mapstructure:"object_batch_size"` + MaxWorkers int `mapstructure:"max_workers"` + } `mapstructure:"policer"` + + Replicator struct { + PutTimeout time.Duration `mapstructure:"put_timeout"` + PoolSize int `mapstructure:"pool_size"` + } `mapstructure:"replicator"` + + Object struct { + Delete struct { + TombstoneLifetime int `mapstructure:"tombstone_lifetime"` + } `mapstructure:"delete"` + + Put struct { + PoolSizeRemote int `mapstructure:"pool_size_remote"` + } `mapstructure:"put"` + } `mapstructure:"object"` + + Storage struct { + ShardPoolSize int `mapstructure:"shard_pool_size"` + ShardROErrorThreshold int `mapstructure:"shard_ro_error_threshold"` + IgnoreUninitedShards bool `mapstructure:"ignore_uninited_shards"` + Shard struct { + Default shardDetails `mapstructure:"default"` + ShardList map[string]shardDetails `mapstructure:",remain" prefix:""` + } `mapstructure:"shard"` + } `mapstructure:"storage"` +} + +type shardDetails struct { + Mode string `mapstructure:"mode"` + ResyncMetabase bool `mapstructure:"resync_metabase"` + + WriteCache struct { + Enabled bool `mapstructure:"enabled"` + Path string `mapstructure:"path"` + Capacity string `mapstructure:"capacity"` + NoSync bool `mapstructure:"no_sync"` + SmallObjectSize string `mapstructure:"small_object_size"` + MaxObjectSize string `mapstructure:"max_object_size"` + WorkersNumber int `mapstructure:"workers_number"` + MaxBatchDelay time.Duration `mapstructure:"max_batch_delay"` + MaxBatchSize int `mapstructure:"max_batch_size"` + } `mapstructure:"writecache"` + + Metabase struct { + Path string `mapstructure:"path"` + Perm string `mapstructure:"perm"` + MaxBatchSize int `mapstructure:"max_batch_size"` + MaxBatchDelay time.Duration `mapstructure:"max_batch_delay"` + } `mapstructure:"metabase"` + + Compress bool `mapstructure:"compress"` + CompressionExcludeContentTypes []string `mapstructure:"compression_exclude_content_types"` + + Pilorama struct { + Path string `mapstructure:"path"` + Perm string `mapstructure:"perm"` + MaxBatchDelay time.Duration `mapstructure:"max_batch_delay"` + MaxBatchSize int `mapstructure:"max_batch_size"` + NoSync bool `mapstructure:"no_sync"` + } `mapstructure:"pilorama"` + + Blobstor []struct { + Type string `mapstructure:"type"` + Path string `mapstructure:"path"` + Perm string `mapstructure:"perm"` + FlushInterval time.Duration `mapstructure:"flush_interval"` + Depth int `mapstructure:"depth"` + NoSync bool `mapstructure:"no_sync"` + CombinedCountLimit int `mapstructure:"combined_count_limit"` + CombinedSizeLimit string `mapstructure:"combined_size_limit"` + CombinedSizeThreshold string `mapstructure:"combined_size_threshold"` + } `mapstructure:"blobstor"` + + GC struct { + RemoverBatchSize int `mapstructure:"remover_batch_size"` + RemoverSleepInterval time.Duration `mapstructure:"remover_sleep_interval"` + } `mapstructure:"gc"` + + SmallObjectSize string `mapstructure:"small_object_size"` +} diff --git a/cmd/neofs-node/config/internal/validate/validate.go b/cmd/neofs-node/config/internal/validate/validate.go new file mode 100644 index 0000000000..e5736ec4a7 --- /dev/null +++ b/cmd/neofs-node/config/internal/validate/validate.go @@ -0,0 +1,22 @@ +package validate + +import ( + "fmt" + + "github.com/nspcc-dev/neofs-node/cmd/internal/configvalidator" + "github.com/spf13/viper" +) + +// ValidateStruct validates the viper config structure. +func ValidateStruct(v *viper.Viper) error { + var cfg valideConfig + if err := v.Unmarshal(&cfg); err != nil { + return fmt.Errorf("unable to decode config: %w", err) + } + + if err := configvalidator.CheckForUnknownFields(v.AllSettings(), cfg); err != nil { + return err + } + + return nil +} diff --git a/cmd/neofs-node/config/internal/validate/validate_test.go b/cmd/neofs-node/config/internal/validate/validate_test.go new file mode 100644 index 0000000000..272dedf015 --- /dev/null +++ b/cmd/neofs-node/config/internal/validate/validate_test.go @@ -0,0 +1,218 @@ +package validate + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestCheckForUnknownFields(t *testing.T) { + tests := []struct { + name string + config string + wantErr bool + }{ + { + name: "with all right fields", + config: ` +node: + wallet: + path: "./wallet.json" + address: "NcpJzXcSDrh5CCizf4K9Ro6w4t59J5LKzz" + password: "password" + addresses: + - s01.neofs.devenv:8080 + - /dns4/s02.neofs.devenv/tcp/8081 + - grpc://127.0.0.1:8082 + - grpcs://localhost:8083 + attribute_0: "Price:11" + attribute_1: UN-LOCODE:RU MSK + attribute_2: VerifiedNodesDomain:nodes.some-org.neofs + relay: true + persistent_sessions: + path: /sessions + persistent_state: + path: /state +`, + wantErr: false, + }, + { + name: "unknown node.password", + config: ` +node: + wallet: + path: "./wallet.json" + address: "NcpJzXcSDrh5CCizf4K9Ro6w4t59J5LKzz" + password: "password" + addresses: + - s01.neofs.devenv:8080 + - /dns4/s02.neofs.devenv/tcp/8081 + - grpc://127.0.0.1:8082 + - grpcs://localhost:8083 + attribute_0: "Price:11" + attribute_1: UN-LOCODE:RU MSK + attribute_2: VerifiedNodesDomain:nodes.some-org.neofs + relay: true + persistent_sessions: + path: /sessions + persistent_state: + path: /state +`, + wantErr: true, + }, + { + name: "node.wallet.address expected type string", + config: ` +node: + wallet: + path: "./wallet.json" + address: + password: "password" + addresses: s01.neofs.devenv:8080 + attribute_0: "Price:11" + attribute_1: UN-LOCODE:RU MSK + attribute_2: VerifiedNodesDomain:nodes.some-org.neofs + relay: true + persistent_sessions: + path: /sessions + persistent_state: + path: /state +`, + wantErr: true, + }, + { + name: "unknown field node.attr", + config: ` +node: + wallet: + path: "./wallet.json" + address: "NcpJzXcSDrh5CCizf4K9Ro6w4t59J5LKzz" + password: "password" + addresses: s01.neofs.devenv:8080 + attribute_0: "Price:11" + attribute_1: UN-LOCODE:RU MSK + attribute_2: VerifiedNodesDomain:nodes.some-org.neofs + attr: attr + relay: true + persistent_sessions: + path: /sessions + persistent_state: + path: /state +`, + wantErr: true, + }, + { + name: "grpc right", + config: ` +grpc: + - endpoint: s01.neofs.devenv:8080 + conn_limit: 1 + tls: + enabled: true + certificate: /path/to/cert + key: /path/to/key + + - endpoint: s02.neofs.devenv:8080 + conn_limit: -1 + tls: + enabled: false + - endpoint: s03.neofs.devenv:8080 +`, + wantErr: false, + }, + { + name: "unknown field grpc.key", + config: ` +grpc: + - endpoint: s01.neofs.devenv:8080 + conn_limit: 1 + tls: + enabled: true + certificate: /path/to/cert + key: /path/to/key + + - endpoint: s02.neofs.devenv:8080 + conn_limit: -1 + tls: + enabled: false + - endpoint: s03.neofs.devenv:8080 +`, + wantErr: true, + }, + { + name: "unknown field grpc.unknown", + config: ` +grpc: + - endpoint: s01.neofs.devenv:8080 + conn_limit: 1 + tls: + enabled: true + certificate: /path/to/cert + key: /path/to/key + + - endpoint: s02.neofs.devenv:8080 + conn_limit: -1 + tls: + enabled: false + - endpoint: s03.neofs.devenv:8080 + - unknown: field +`, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := viper.New() + v.SetConfigType("yaml") + require.NoError(t, v.ReadConfig(strings.NewReader(tt.config))) + + err := ValidateStruct(v) + fmt.Println(err) + if (err != nil) != tt.wantErr { + t.Errorf("CheckForUnknownFields() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestCheckForUnknownFieldsExamples(t *testing.T) { + const exampleConfigPrefix = "../../../../../config/" + t.Run("example json", func(t *testing.T) { + path := filepath.Join(exampleConfigPrefix, "example/node.json") + v := viper.New() + v.SetConfigFile(path) + + require.NoError(t, v.ReadInConfig()) + require.NoError(t, ValidateStruct(v)) + }) + + t.Run("example yaml", func(t *testing.T) { + path := filepath.Join(exampleConfigPrefix, "example/node.yaml") + v := viper.New() + v.SetConfigFile(path) + + require.NoError(t, v.ReadInConfig()) + require.NoError(t, ValidateStruct(v)) + }) + + t.Run("mainnet", func(t *testing.T) { + p := filepath.Join(exampleConfigPrefix, "mainnet/config.yml") + v := viper.New() + v.SetConfigFile(p) + + require.NoError(t, v.ReadInConfig()) + require.NoError(t, ValidateStruct(v)) + }) + t.Run("testnet", func(t *testing.T) { + p := filepath.Join(exampleConfigPrefix, "testnet/config.yml") + v := viper.New() + v.SetConfigFile(p) + + require.NoError(t, v.ReadInConfig()) + require.NoError(t, ValidateStruct(v)) + }) +} diff --git a/cmd/neofs-node/config/opts.go b/cmd/neofs-node/config/opts.go index f7f7685f71..a20319732e 100644 --- a/cmd/neofs-node/config/opts.go +++ b/cmd/neofs-node/config/opts.go @@ -1,11 +1,14 @@ package config type opts struct { - path string + path string + validate bool } func defaultOpts() *opts { - return new(opts) + return &opts{ + validate: true, + } } // Option allows to set an optional parameter of the Config. @@ -18,3 +21,11 @@ func WithConfigFile(path string) Option { o.path = path } } + +// WithValidate returns an option that is responsible +// for whether the config structure needs to be validated. +func WithValidate(validate bool) Option { + return func(o *opts) { + o.validate = validate + } +} diff --git a/cmd/neofs-node/config/test/config.go b/cmd/neofs-node/config/test/config.go index f7921ce599..5528df5dad 100644 --- a/cmd/neofs-node/config/test/config.go +++ b/cmd/neofs-node/config/test/config.go @@ -15,6 +15,7 @@ func fromFile(path string) *config.Config { return config.New(p, config.WithConfigFile(path), + config.WithValidate(false), ) }