From 510073784335e8d8ec8f8e4cc988bc2aad176c8e Mon Sep 17 00:00:00 2001 From: Simon Hornby Date: Fri, 8 Sep 2023 09:50:06 +0200 Subject: [PATCH] feat: add hot reloading and an optional, simpler file format to offline mode (#242) --- README.md | 40 +++-- examples/simple-bootstrap.json | 13 ++ server/src/builder.rs | 27 ++-- server/src/cli.rs | 3 + server/src/client_api.rs | 1 + server/src/frontend_api.rs | 1 + server/src/lib.rs | 1 + server/src/main.rs | 12 ++ server/src/offline/mod.rs | 2 + server/src/offline/offline_hotload.rs | 218 ++++++++++++++++++++++++++ 10 files changed, 297 insertions(+), 21 deletions(-) create mode 100644 examples/simple-bootstrap.json create mode 100644 server/src/offline/mod.rs create mode 100644 server/src/offline/offline_hotload.rs diff --git a/README.md b/README.md index 223df2bd..ac0daf58 100644 --- a/README.md +++ b/README.md @@ -135,10 +135,10 @@ Edge mode also supports dynamic tokens, meaning that Edge doesn't need a token t Even though Edge supports dynamic tokens, you still have the option of providing a token through the command line argument or environment variable. This way, since Edge already knows about your token at start up, it will sync your features for that token and should be ready for your requests right away (_warm up / hot start_). ### Front-end tokens -[Front-end tokens](https://docs.getunleash.io/reference/api-tokens-and-client-keys#front-end-tokens) can also be used with `/api/frontend` and `/api/proxy` endpoints, however they are not allowed to fetch features upstream. -In order to use these tokens correctly and make sure they return the correct information, it's important that the features they are allowed to access are already present in that Edge node's features cache. -The easiest way to ensure this is by passing in at least one client token as one of the command line arguments, -ensuring it has access to the same features as the front-end token you'll be using. +[Front-end tokens](https://docs.getunleash.io/reference/api-tokens-and-client-keys#front-end-tokens) can also be used with `/api/frontend` and `/api/proxy` endpoints, however they are not allowed to fetch features upstream. +In order to use these tokens correctly and make sure they return the correct information, it's important that the features they are allowed to access are already present in that Edge node's features cache. +The easiest way to ensure this is by passing in at least one client token as one of the command line arguments, +ensuring it has access to the same features as the front-end token you'll be using. If you're using a frontend token that doesn't have data in the node's feature cache, you will receive an HTTP Status code: 511 Network Authentication Required along with a body of which project and environment you will need to add a client token for. #### Enterprise @@ -210,15 +210,31 @@ graph LR B-->|Fetch toggles| C[Features dump] ``` -Offline mode should be used when you don't have a connection to an upstream node, such as your Unleash instance itself or another Edge instance. It can also be used when you need to have full control of both the data your clients will get and which tokens can be used to access it. +Offline mode is useful when there is no connection to an upstream node, such as your Unleash instance or another Edge instance, or as a tool to make working with Unleash easier during development. -Since this mode does not connect to an upstream node, it needs a downloaded JSON dump of a result from a query against an Unleash server on the [/api/client/features](https://docs.getunleash.io/reference/api/unleash/get-client-feature) endpoint as well as a comma-separated list of tokens that should be allowed to access the server. +To use offline mode, you'll need a features file. The easiest way to get one is to download a JSON dump of a result from a query against an Unleash server on the [/api/client/features](https://docs.getunleash.io/reference/api/unleash/get-client-feature) endpoint. You can also use a hand rolled, human readable JSON version of the features file. Edge will automatically convert it to the API format when it starts up. Here's an example: -If your token follows the Unleash API token format `[project]:[environment].`, Edge will filter the features dump to match the project contained in the token. +``` json +{ + "featureOne": { + "enabled": true, + "variant": "variantOne" + }, + "featureTwo": { + "enabled": false, + "variant": "variantTwo" + }, + "featureThree": { + "enabled": true + } +} +``` + +The simplified JSON format should be an object with a key for each feature. You can force the result of `is_enabled` in your SDK by setting the enabled property, likewise can also force the result of `get_variant` by specifying the name of the variant you want. This format is primarily for development. -If you'd rather use a simple token like `secret-123`, any query against `/api/client/features` will receive the dump passed in on the command line. +When using offline mode you must specify one or more tokens at startup. These tokens will let your SDKs access Edge. Tokens following the Unleash API format [project]:[environment]. allow Edge to recognize the project and environment specified in the token, returning only the relevant features to the calling SDK. On the other hand, for tokens not adhering to this format, Edge will return all features if there is an exact match with any of the startup tokens. -When using offline mode, you can think of these tokens as [proxy client keys](https://docs.getunleash.io/reference/api-tokens-and-client-keys#proxy-client-keys). +To make local development easier, you can specify a reload interval in seconds; this will cause Edge to reload the features file from disk every X seconds. This can be useful for local development. Since offline mode does not connect to an upstream node, it does not support metrics or dynamic tokens. @@ -229,8 +245,10 @@ $ ./unleash-edge offline --help Usage: unleash-edge offline [OPTIONS] Options: - -b, --bootstrap-file [env: BOOTSTRAP_FILE=] - -t, --tokens [env: TOKENS=] + -b, --bootstrap-file [env: BOOTSTRAP_FILE=] + -t, --tokens [env: TOKENS=] + -r, --reload-interval [env: RELOAD_INTERVAL=] + ``` ## [Metrics](https://docs.getunleash.io/reference/api/unleash/metrics) diff --git a/examples/simple-bootstrap.json b/examples/simple-bootstrap.json new file mode 100644 index 00000000..17a1b09b --- /dev/null +++ b/examples/simple-bootstrap.json @@ -0,0 +1,13 @@ +{ + "featureOne": { + "enabled": true, + "variant": "variantOne" + }, + "featureTwo": { + "enabled": false, + "variant": "variantTwo" + }, + "featureThree": { + "enabled": true + } +} diff --git a/server/src/builder.rs b/server/src/builder.rs index 4288d62e..58c861f0 100644 --- a/server/src/builder.rs +++ b/server/src/builder.rs @@ -2,9 +2,11 @@ use chrono::Duration; use dashmap::DashMap; use reqwest::Url; use std::fs::File; +use std::io::Read; use std::sync::Arc; use std::{io::BufReader, str::FromStr}; +use crate::offline::offline_hotload::{load_bootstrap, load_offline_engine_cache}; use crate::persistence::file::FilePersister; use crate::persistence::redis::RedisPersister; use crate::persistence::EdgePersistence; @@ -84,13 +86,13 @@ pub(crate) fn build_offline_mode( for edge_token in edge_tokens { token_cache.insert(edge_token.token.clone(), edge_token.clone()); - features_cache.insert( - crate::tokens::cache_key(&edge_token), + + load_offline_engine_cache( + &edge_token, + features_cache.clone(), + engine_cache.clone(), client_features.clone(), ); - let mut engine_state = EngineState::default(); - engine_state.take_state(client_features.clone()); - engine_cache.insert(crate::tokens::cache_key(&edge_token), engine_state); } Ok((token_cache, features_cache, engine_cache)) } @@ -98,11 +100,16 @@ pub(crate) fn build_offline_mode( fn build_offline(offline_args: OfflineArgs) -> EdgeResult { if let Some(bootstrap) = offline_args.bootstrap_file { let file = File::open(bootstrap.clone()).map_err(|_| EdgeError::NoFeaturesFile)?; - let reader = BufReader::new(file); - let client_features: ClientFeatures = serde_json::from_reader(reader).map_err(|e| { - let path = format!("{}", bootstrap.clone().display()); - EdgeError::InvalidBackupFile(path, e.to_string()) - })?; + + let mut reader = BufReader::new(file); + let mut content = String::new(); + + reader + .read_to_string(&mut content) + .map_err(|_| EdgeError::NoFeaturesFile)?; + + let client_features = load_bootstrap(&bootstrap)?; + build_offline_mode(client_features, offline_args.tokens) } else { Err(EdgeError::NoFeaturesFile) diff --git a/server/src/cli.rs b/server/src/cli.rs index 7607303c..34db2a7a 100644 --- a/server/src/cli.rs +++ b/server/src/cli.rs @@ -181,6 +181,9 @@ pub struct OfflineArgs { /// Tokens that should be allowed to connect to Edge. Supports a comma separated list or multiple instances of the `--tokens` argument #[clap(short, long, env, value_delimiter = ',')] pub tokens: Vec, + /// The interval in seconds between reloading the bootstrap file. Disabled if unset or 0 + #[clap(short, long, env, default_value_t = 0)] + pub reload_interval: u64, } #[derive(Args, Debug, Clone)] diff --git a/server/src/client_api.rs b/server/src/client_api.rs index 28884471..c7f82e50 100644 --- a/server/src/client_api.rs +++ b/server/src/client_api.rs @@ -628,6 +628,7 @@ mod tests { .app_data(Data::new(crate::cli::EdgeMode::Offline(OfflineArgs { bootstrap_file: Some(PathBuf::from("../examples/features.json")), tokens: vec!["secret_123".into()], + reload_interval: 0, }))) .service(web::scope("/api/client").service(get_features)), ) diff --git a/server/src/frontend_api.rs b/server/src/frontend_api.rs index fea0319d..b514e3a4 100644 --- a/server/src/frontend_api.rs +++ b/server/src/frontend_api.rs @@ -994,6 +994,7 @@ mod tests { .app_data(Data::new(EdgeMode::Offline(OfflineArgs { bootstrap_file: None, tokens: vec!["secret-123".into()], + reload_interval: 0, }))) .service(web::scope("/api/frontend").service(super::get_frontend_all_features)), ) diff --git a/server/src/lib.rs b/server/src/lib.rs index 794c6cfd..a951868e 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -15,6 +15,7 @@ pub mod http; pub mod internal_backstage; pub mod metrics; pub mod middleware; +pub mod offline; #[cfg(not(tarpaulin_include))] pub mod openapi; pub mod persistence; diff --git a/server/src/main.rs b/server/src/main.rs index d148d44b..36711da7 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -16,6 +16,7 @@ use unleash_edge::builder::build_caches_and_refreshers; use unleash_edge::cli::{CliArgs, EdgeMode}; use unleash_edge::metrics::client_metrics::MetricsCache; use unleash_edge::middleware::request_tracing::RequestTracing; +use unleash_edge::offline::offline_hotload; use unleash_edge::persistence::{persist_data, EdgePersistence}; use unleash_edge::types::{EdgeToken, TokenRefresh, TokenValidationStatus}; use unleash_edge::{admin_api, cli, client_api, frontend_api, health_checker, openapi}; @@ -57,6 +58,7 @@ async fn main() -> Result<(), anyhow::Error> { let token_validator_schedule = token_validator.clone(); let lazy_feature_cache = features_cache.clone(); let lazy_token_cache = token_cache.clone(); + let lazy_engine_cache = engine_cache.clone(); let metrics_cache = Arc::new(MetricsCache::default()); let metrics_cache_clone = metrics_cache.clone(); @@ -154,6 +156,16 @@ async fn main() -> Result<(), anyhow::Error> { } } } + cli::EdgeMode::Offline(offline_args) if offline_args.reload_interval > 0 => { + tokio::select! { + _ = offline_hotload::start_hotload_loop(lazy_feature_cache, lazy_engine_cache, offline_args) => { + tracing::info!("Hotloader unexpectedly shut down."); + }, + _ = server.run() => { + tracing::info!("Actix is shutting down. No pending tasks."); + }, + } + } _ => tokio::select! { _ = server.run() => { tracing::info!("Actix is shutting down. Persisting data"); diff --git a/server/src/offline/mod.rs b/server/src/offline/mod.rs new file mode 100644 index 00000000..852f4b28 --- /dev/null +++ b/server/src/offline/mod.rs @@ -0,0 +1,2 @@ +#[cfg(not(tarpaulin_include))] +pub mod offline_hotload; diff --git a/server/src/offline/offline_hotload.rs b/server/src/offline/offline_hotload.rs new file mode 100644 index 00000000..2047dfe5 --- /dev/null +++ b/server/src/offline/offline_hotload.rs @@ -0,0 +1,218 @@ +use std::{ + collections::HashMap, + fs::File, + io::{BufReader, Read}, + path::Path, + str::FromStr, + sync::Arc, + time::Duration, +}; + +use dashmap::DashMap; +use serde::Deserialize; +use unleash_types::client_features::{ + ClientFeature, ClientFeatures, Strategy, Variant, WeightType, +}; +use unleash_yggdrasil::EngineState; + +use crate::{cli::OfflineArgs, error::EdgeError, types::EdgeToken}; + +pub async fn start_hotload_loop( + features_cache: Arc>, + engine_cache: Arc>, + offline_args: OfflineArgs, +) { + let known_tokens = offline_args.tokens; + let bootstrap_path = offline_args.bootstrap_file; + + loop { + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(offline_args.reload_interval)) => { + let bootstrap = bootstrap_path.as_ref().map(|bootstrap_path|load_bootstrap(bootstrap_path)); + match bootstrap { + Some(Ok(bootstrap)) => { + let edge_tokens: Vec = known_tokens + .iter() + .map(|token| EdgeToken::from_str(token).unwrap_or_else(|_| EdgeToken::offline_token(token))) + .collect(); + + for edge_token in edge_tokens { + load_offline_engine_cache(&edge_token, features_cache.clone(), engine_cache.clone(), bootstrap.clone()); + } + }, + Some(Err(e)) => { + tracing::error!("Error loading bootstrap file: {:?}", e); + } + None => { + tracing::debug!("No bootstrap file provided"); + } + }; + } + } + } +} + +pub(crate) fn load_offline_engine_cache( + edge_token: &EdgeToken, + features_cache: Arc>, + engine_cache: Arc>, + client_features: ClientFeatures, +) { + features_cache.insert( + crate::tokens::cache_key(edge_token), + client_features.clone(), + ); + let mut engine_state = EngineState::default(); + engine_state.take_state(client_features); + engine_cache.insert(crate::tokens::cache_key(edge_token), engine_state); +} + +#[derive(Deserialize)] +struct SimpleFeature { + enabled: bool, + variant: Option, +} + +fn make_simple_bootstrap(simple_bootstrap: HashMap) -> ClientFeatures { + let features = simple_bootstrap + .iter() + .map(|(feature_name, simple_feat)| { + let variants = simple_feat.variant.as_ref().map(|variant_name| { + vec![Variant { + name: variant_name.clone(), + weight: 1000, + weight_type: Some(WeightType::Fix), + stickiness: Some("default".into()), + payload: None, + overrides: None, + }] + }); + + ClientFeature { + name: feature_name.clone(), + enabled: simple_feat.enabled, + variants, + strategies: Some(vec![Strategy { + name: "default".into(), + parameters: Some(HashMap::new()), + sort_order: None, + segments: None, + constraints: Some(vec![]), + variants: Some(vec![]), + }]), + project: Some("default".into()), + ..Default::default() + } + }) + .collect(); + ClientFeatures { + version: 2, + features, + segments: None, + query: None, + } +} + +pub(crate) fn load_bootstrap(bootstrap_path: &Path) -> Result { + let file = File::open(bootstrap_path).map_err(|_| EdgeError::NoFeaturesFile)?; + + let mut reader = BufReader::new(file); + let mut content = String::new(); + + reader + .read_to_string(&mut content) + .map_err(|_| EdgeError::NoFeaturesFile)?; + + parse_bootstrap(content).map_err(|e| { + let path = format!("{}", bootstrap_path.to_path_buf().display()); + EdgeError::InvalidBackupFile(path, e.to_string()) + }) +} + +fn parse_bootstrap(content: String) -> Result { + let client_features: Result = + serde_json::from_str::>(&content) + .map(make_simple_bootstrap) + .or_else(|_| serde_json::from_str(&content)); + + client_features +} + +#[cfg(test)] +mod tests { + use super::parse_bootstrap; + + #[test] + fn loads_simple_bootstrap_format() { + let simple_bootstrap = r#" + { + "feature1": { + "enabled": true, + "variant": "variant1" + } + }"#; + parse_bootstrap(simple_bootstrap.to_string()).unwrap(); + } + + #[test] + fn simple_bootstrap_parses_to_client_features_correctly() { + let simple_bootstrap = r#" + { + "feature1": { + "enabled": true, + "variant": "variant1" + } + }"#; + let client_features = parse_bootstrap(simple_bootstrap.to_string()).unwrap(); + assert_eq!(client_features.features.len(), 1); + assert_eq!(client_features.features[0].name, "feature1"); + assert!(client_features.features[0].enabled); + assert_eq!( + client_features.features[0].variants.as_ref().unwrap()[0].name, + "variant1" + ); + } + + #[test] + fn simple_bootstrap_does_not_require_variants() { + let simple_bootstrap = r#" + { + "feature1": { + "enabled": true + } + }"#; + parse_bootstrap(simple_bootstrap.to_string()).unwrap(); + } + + #[test] + fn falls_back_to_standard_unleash_format() { + let simple_bootstrap = r#" + { + "version": 2, + "features": [ + { + "strategies": [ + { + "name": "default", + "constraints": [], + "parameters": {} + } + ], + "impressionData": false, + "enabled": true, + "name": "custom.constraint", + "description": "", + "project": "default", + "stale": false, + "type": "release", + "variants": [] + } + ], + "query": { + "environment": "development", + "inlineSegmentConstraints": true + } + }"#; + parse_bootstrap(simple_bootstrap.to_string()).unwrap(); + } +}