Add a config.strategy to enable overwrite of properties from env vars

README.md

@@ -675,6 +675,24 @@ value just disappear if the substitution is not found:
  // this array could have one or two elements
  path = [ "a", ${?OPTIONAL_A} ]
 
+By setting the JVM property `-Dconfig.override_with_env_vars=true`
+it is possible to override any configuration value using environment
+variables even if an explicit substitution is not specified.
+
+The environment variable value will override any pre-existing value
+and also any value provided as Java property.
+
+With this option enabled only environment variables starting with
+`CONFIG_FORCE_` are considered, and the name is mangled as follows:
+
+ - the prefix `CONFIG_FORCE_` is stripped
+ - single underscore(`_`) is converted into a dot(`.`)
+ - double underscore(`__`) is converted into a dash(`-`)
+ - triple underscore(`___`) is converted into a single underscore(`_`)
+
+i.e. The environment variable `CONFIG_FORCE_a_b__c___d` set the
+configuration key `a.b-c_d`
+
 ### Concatenation
 
 Values _on the same line_ are concatenated (for strings and

build.sbt

@@ -85,7 +85,17 @@ lazy val configLib = Project("config", file("config"))
  Test/ run / fork := true
 
  //env vars for tests
- Test / envVars ++= Map("testList.0" -> "0", "testList.1" -> "1")
+ Test / envVars ++= Map("testList.0" -> "0",
+ "testList.1" -> "1",
+ "CONFIG_FORCE_b" -> "5",
+ "CONFIG_FORCE_testList_0" -> "10",
+ "CONFIG_FORCE_testList_1" -> "11",
+ "CONFIG_FORCE_42___a" -> "1",
+ "CONFIG_FORCE_a_b_c" -> "2",
+ "CONFIG_FORCE_a__c" -> "3",
+ "CONFIG_FORCE_a___c" -> "4",
+ "CONFIG_FORCE_akka_version" -> "foo",
+ "CONFIG_FORCE_akka_event__handler__dispatcher_max__pool__size" -> "10")
 
  OsgiKeys.exportPackage := Seq("com.typesafe.config", "com.typesafe.config.impl")
  publish := sys.error("use publishSigned instead of plain publish")

config/checkstyle-config.xml

@@ -1,8 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE module PUBLIC
- "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
- "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
-
+ "-//Checkstyle//DTD Check Configuration 1.3//EN"
+ "https://checkstyle.org/dtds/configuration_1_3.dtd">
 <module name="Checker">
  <module name="SuppressionFilter">
  <property name="file" value="config/checkstyle-suppressions.xml"/>

config/checkstyle-suppressions.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE suppressions PUBLIC "-//Puppy Crawl//DTD Suppressions 1.1//EN"
- "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
+<!DOCTYPE suppressions PUBLIC
+ "-//Checkstyle//DTD Suppressions 1.1//EN"
+ "https://checkstyle.org/dtds/suppressions_1_1.dtd">
 <suppressions>
  <!-- don't care about javadoc coverage of methods in impl -->
  <suppress checks="JavadocMethod"

config/src/main/java/com/typesafe/config/ConfigFactory.java

@@ -36,6 +36,7 @@
  */
 public final class ConfigFactory {
  private static final String STRATEGY_PROPERTY_NAME = "config.strategy";
+ private static final String OVERRIDE_WITH_ENV_PROPERTY_NAME = "config.override_with_env_vars";
 
  private ConfigFactory() {
  }
@@ -383,7 +384,11 @@ public static Config defaultReference(ClassLoader loader) {
  * @return the default override configuration
  */
  public static Config defaultOverrides() {
- return systemProperties();
+ if (getOverrideWithEnv()) {
+ return systemEnvironmentOverrides().withFallback(systemProperties());
+ } else {
+ return systemProperties();
+ }
  }
 
  /**
@@ -394,7 +399,7 @@ public static Config defaultOverrides() {
  * @return the default override configuration
  */
  public static Config defaultOverrides(ClassLoader loader) {
- return systemProperties();
+ return defaultOverrides();
  }
 
  /**
@@ -497,6 +502,7 @@ public static void invalidateCaches() {
  // all caches
  ConfigImpl.reloadSystemPropertiesConfig();
  ConfigImpl.reloadEnvVariablesConfig();
+ ConfigImpl.reloadEnvVariablesOverridesConfig();
  }
 
  /**
@@ -549,6 +555,50 @@ public static Config systemProperties() {
  return ConfigImpl.systemPropertiesAsConfig();
  }
 
+ /**
+ * Gets a <code>Config</code> containing the system's environment variables
+ * used to override configuration keys.
+ * Environment variables taken in considerations are starting with
+ * {@code CONFIG_FORCE_}
+ * 
+ * <p>
+ * Environment variables are mangled in the following way after stripping the prefix "CONFIG_FORCE_":
+ * <table border="1">
+ * <tr>
+ * <th bgcolor="silver">Env Var</th>
+ * <th bgcolor="silver">Config</th>
+ * </tr>
+ * <tr>
+ * <td>_&nbsp;&nbsp;&nbsp;[1 underscore]</td>
+ * <td>. [dot]</td>
+ * </tr>
+ * <tr>
+ * <td>__&nbsp;&nbsp;[2 underscore]</td>
+ * <td>- [dash]</td>
+ * </tr>
+ * <tr>
+ * <td>___&nbsp;[3 underscore]</td>
+ * <td>_ [underscore]</td>
+ * </tr>
+ * </table>
+ * 
+ * <p>
+ * A variable like: {@code CONFIG_FORCE_a_b__c___d}
+ * is translated to a config key: {@code a.b-c_d}
+ * 
+ * <p>
+ * This method can return a global immutable singleton, so it's preferred
+ * over parsing system properties yourself.
+ * <p>
+ * {@link #defaultOverrides} will include the system environment variables as
+ * overrides if `config.override_with_env_vars` is set to `true`.
+ *
+ * @return system environment variable overrides parsed into a <code>Config</code>
+ */
+ public static Config systemEnvironmentOverrides() {
+ return ConfigImpl.envVariablesOverridesAsConfig();
+ }
+
  /**
  * Gets a <code>Config</code> containing the system's environment variables.
  * This method can return a global immutable singleton.
@@ -1063,4 +1113,10 @@ private static ConfigLoadingStrategy getConfigLoadingStrategy() {
  return new DefaultConfigLoadingStrategy();
  }
  }
+
+ private static Boolean getOverrideWithEnv() {
+ String overrideWithEnv = System.getProperties().getProperty(OVERRIDE_WITH_ENV_PROPERTY_NAME);
+
+ return Boolean.parseBoolean(overrideWithEnv);
+ }
 }

config/src/main/java/com/typesafe/config/impl/ConfigImpl.java

@@ -32,6 +32,7 @@
  * For use only by the {@link com.typesafe.config} package.
  */
 public class ConfigImpl {
+ private static final String ENV_VAR_OVERRIDE_PREFIX = "CONFIG_FORCE_";
 
  private static class LoaderCache {
  private Config currentSystemProperties;
@@ -360,6 +361,43 @@ public static void reloadEnvVariablesConfig() {
  EnvVariablesHolder.envVariables = loadEnvVariables();
  }
 
+
+
+ private static AbstractConfigObject loadEnvVariablesOverrides() {
+ Map<String, String> env = new HashMap(System.getenv());
+ Map<String, String> result = new HashMap(System.getenv());
+
+ for (String key : env.keySet()) {
+ if (key.startsWith(ENV_VAR_OVERRIDE_PREFIX)) {
+ result.put(ConfigImplUtil.envVariableAsProperty(key, ENV_VAR_OVERRIDE_PREFIX), env.get(key));
+ }
+ }
+
+ return PropertiesParser.fromStringMap(newSimpleOrigin("env variables overrides"), result);
+ }
+
+ private static class EnvVariablesOverridesHolder {
+ static volatile AbstractConfigObject envVariables = loadEnvVariablesOverrides();
+ }
+
+ static AbstractConfigObject envVariablesOverridesAsConfigObject() {
+ try {
+ return EnvVariablesOverridesHolder.envVariables;
+ } catch (ExceptionInInitializerError e) {
+ throw ConfigImplUtil.extractInitializerError(e);
+ }
+ }
+
+ public static Config envVariablesOverridesAsConfig() {
+ return envVariablesOverridesAsConfigObject().toConfig();
+ }
+
+ public static void reloadEnvVariablesOverridesConfig() {
+ // ConfigFactory.invalidateCaches() relies on this having the side
+ // effect that it drops all caches
+ EnvVariablesOverridesHolder.envVariables = loadEnvVariablesOverrides();
+ }
+
  public static Config defaultReference(final ClassLoader loader) {
  return computeCachedConfig(loader, "defaultReference", new Callable<Config>() {
  @Override

config/src/main/java/com/typesafe/config/impl/ConfigImplUtil.java

@@ -235,6 +235,52 @@ static String toCamelCase(String originalName) {
  return nameBuilder.toString();
  }
 
+ private static char underscoreMappings(int num) {
+ // Rationale on name mangling:
+ //
+ // Most shells (e.g. bash, sh, etc.) doesn't support any character other
+ // than alphanumeric and `_` in environment variables names.
+ // In HOCON the default separator is `.` so it is directly translated to a
+ // single `_` for convenience; `-` and `_` are less often present in config
+ // keys but they have to be representable and the only possible mapping is
+ // `_` repeated.
+ switch (num) {
+ case 1: return '.';
+ case 2: return '-';
+ case 3: return '_';
+ default: return 0;
+ }
+ }
+
+ static String envVariableAsProperty(String variable, String prefix) throws ConfigException {
+ StringBuilder builder = new StringBuilder();
+
+ String strippedPrefix = variable.substring(prefix.length(), variable.length());
+
+ int underscores = 0;
+ for (char c : strippedPrefix.toCharArray()) {
+ if (c == '_') {
+ underscores++;
+ } else {
+ if (underscores > 0 && underscores < 4) {
+ builder.append(underscoreMappings(underscores));
+ } else if (underscores > 3) {
+ throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores.");
+ }
+ underscores = 0;
+ builder.append(c);
+ }
+ }
+
+ if (underscores > 0 && underscores < 4) {
+ builder.append(underscoreMappings(underscores));
+ } else if (underscores > 3) {
+ throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores.");
+ }
+
+ return builder.toString();
+ }
+
  /**
  * Guess configuration syntax from given filename.
  *

config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala

@@ -1090,6 +1090,53 @@ class ConfigTest extends TestUtils {
  assertEquals(10, resolved.getInt("bar.nested.a.q"))
  }
 
+ @Test
+ def testEnvVariablesNameMangling() {
+ assertEquals("a", ConfigImplUtil.envVariableAsProperty("prefix_a", "prefix_"))
+ assertEquals("a.b", ConfigImplUtil.envVariableAsProperty("prefix_a_b", "prefix_"))
+ assertEquals("a.b.c", ConfigImplUtil.envVariableAsProperty("prefix_a_b_c", "prefix_"))
+ assertEquals("a.b-c-d", ConfigImplUtil.envVariableAsProperty("prefix_a_b__c__d", "prefix_"))
+ assertEquals("a.b_c_d", ConfigImplUtil.envVariableAsProperty("prefix_a_b___c___d", "prefix_"))
+
+ intercept[ConfigException.BadPath] {
+ ConfigImplUtil.envVariableAsProperty("prefix_____", "prefix_")
+ }
+ intercept[ConfigException.BadPath] {
+ ConfigImplUtil.envVariableAsProperty("prefix_a_b___c____d", "prefix_")
+ }
+ }
+
+ @Test
+ def testLoadWithEnvSubstitutions() {
+ System.setProperty("config.override_with_env_vars", "true")
+
+ try {
+ val loader02 = new TestClassLoader(this.getClass().getClassLoader(),
+ Map("reference.conf" -> resourceFile("test02.conf").toURI.toURL()))
+
+ val loader04 = new TestClassLoader(this.getClass().getClassLoader(),
+ Map("reference.conf" -> resourceFile("test04.conf").toURI.toURL()))
+
+ val conf02 = withContextClassLoader(loader02) {
+ ConfigFactory.load()
+ }
+
+ val conf04 = withContextClassLoader(loader04) {
+ ConfigFactory.load()
+ }
+
+ assertEquals(1, conf02.getInt("42_a"))
+ assertEquals(2, conf02.getInt("a.b.c"))
+ assertEquals(3, conf02.getInt("a-c"))
+ assertEquals(4, conf02.getInt("a_c"))
+
+ assertEquals("foo", conf04.getString("akka.version"))
+ assertEquals(10, conf04.getInt("akka.event-handler-dispatcher.max-pool-size"))
+ } finally {
+ System.clearProperty("config.override_with_env_vars")
+ }
+ }
+
  @Test
  def renderRoundTrip() {
  val allBooleans = true :: false :: Nil

config/src/test/scala/com/typesafe/config/impl/PublicApiTest.scala

@@ -653,6 +653,26 @@ class PublicApiTest extends TestUtils {
  }
  }
 
+ @Test
+ def loadEnvironmentVariablesOverridesIfConfigured(): Unit = {
+ assertEquals("config.override_with_env_vars is not set", null, System.getProperty("config.override_with_env_vars"))
+
+ System.setProperty("config.override_with_env_vars", "true")
+
+ try {
+ val loaderB2 = new TestClassLoader(this.getClass().getClassLoader(),
+ Map("reference.conf" -> resourceFile("b_2.conf").toURI.toURL()))
+
+ val configB2 = withContextClassLoader(loaderB2) {
+ ConfigFactory.load()
+ }
+
+ assertEquals(5, configB2.getInt("b"))
+ } finally {
+ System.clearProperty("config.override_with_env_vars")
+ }
+ }
+
  @Test
  def usesContextClassLoaderForApplicationConf() {
  val loaderA1 = new TestClassLoader(this.getClass().getClassLoader(),
@@ -1145,4 +1165,4 @@ object TestStrategy {
  private var invocations = 0
  def getIncovations() = invocations
  def increment() = invocations += 1
-}
+}

project/build.properties

@@ -1 +1 @@
-sbt.version=1.2.7
+sbt.version=1.2.8