diff --git a/larky/src/main/java/com/verygood/security/larky/modules/DecoratorConfig.java b/larky/src/main/java/com/verygood/security/larky/modules/DecoratorConfig.java new file mode 100644 index 000000000..621083485 --- /dev/null +++ b/larky/src/main/java/com/verygood/security/larky/modules/DecoratorConfig.java @@ -0,0 +1,125 @@ +package com.verygood.security.larky.modules; + +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +@Getter +@AllArgsConstructor +@Builder +public class DecoratorConfig { + + public static class InvalidDecoratorConfigException extends RuntimeException { + + public InvalidDecoratorConfigException(String message) { + super(message); + } + } + + @Getter + @AllArgsConstructor + @Builder + public static class NonLuhnValidTransformPattern { + + private final String search; + private final String replace; + } + + @Getter + @AllArgsConstructor + @Builder + public static class NonLuhnValidPattern { + + private final String validatePattern; + private final List transformPatterns; + } + + private final String searchPattern; + private final String replacePattern; + private final NonLuhnValidPattern nonLuhnValidPattern; + + + public static DecoratorConfig fromObject(Object decoratorConfig) { + if (!(decoratorConfig instanceof Map)) { + return null; + } + Map map = (Map) decoratorConfig; + + DecoratorConfigBuilder decoratorConfigBuilder = DecoratorConfig.builder(); + + decoratorConfigBuilder.searchPattern(getString(map, "searchPattern")); + decoratorConfigBuilder.replacePattern(getString(map, "replacePattern")); + + Map nonLuhnValidPattern = getMap(map, "nonLuhnValidPattern"); + if (nonLuhnValidPattern != null) { + NonLuhnValidPattern.NonLuhnValidPatternBuilder nonLuhnValidPatternBuilder = NonLuhnValidPattern.builder(); + nonLuhnValidPatternBuilder.validatePattern(getString(nonLuhnValidPattern, "validatePattern")); + ImmutableList.Builder transformPatterns = ImmutableList.builder(); + for (Object transformPattern : getList(nonLuhnValidPattern, "transformPatterns")) { + NonLuhnValidTransformPattern.NonLuhnValidTransformPatternBuilder transformPatternBuilder = NonLuhnValidTransformPattern.builder(); + Map transformPatternMap = toMap(transformPattern); + transformPatternBuilder.search(getString(transformPatternMap, "search")); + transformPatternBuilder.replace(getString(transformPatternMap, "replace")); + transformPatterns.add(transformPatternBuilder.build()); + } + nonLuhnValidPatternBuilder.transformPatterns(transformPatterns.build()); + decoratorConfigBuilder.nonLuhnValidPattern(nonLuhnValidPatternBuilder.build()); + } + return decoratorConfigBuilder.build(); + } + + private static Map toMap(Object obj) { + if (obj == null) { + return null; + } + if (!(obj instanceof Map)) { + throw new InvalidDecoratorConfigException( + String.format("'%s' must be dict", obj) + ); + } + return (Map) obj; + } + + private static String getString(Map map, String field) { + if (!map.containsKey(field)) { + return null; + } + Object value = map.get(field); + if (!(value instanceof String)) { + throw new InvalidDecoratorConfigException( + String.format("'%s' field must be string", field) + ); + } + return (String) value; + } + + private static Map getMap(Map map, String field) { + if (!map.containsKey(field)) { + return null; + } + Object value = map.get(field); + if (!(value instanceof Map)) { + throw new InvalidDecoratorConfigException( + String.format("'%s' field must be dict", field) + ); + } + return (Map) value; + } + + private static List getList(Map map, String field) { + if (!map.containsKey(field)) { + return Collections.emptyList(); + } + Object value = map.get(field); + if (!(value instanceof List)) { + throw new InvalidDecoratorConfigException( + String.format("'%s' field must be array", field) + ); + } + return (List) value; + } +} diff --git a/larky/src/main/java/com/verygood/security/larky/modules/VaultModule.java b/larky/src/main/java/com/verygood/security/larky/modules/VaultModule.java index c97e7c16f..61848696a 100644 --- a/larky/src/main/java/com/verygood/security/larky/modules/VaultModule.java +++ b/larky/src/main/java/com/verygood/security/larky/modules/VaultModule.java @@ -1,9 +1,13 @@ package com.verygood.security.larky.modules; import com.google.common.collect.ImmutableList; +import com.verygood.security.larky.modules.DecoratorConfig.InvalidDecoratorConfigException; +import com.verygood.security.larky.modules.vgs.vault.NoopVault; import com.verygood.security.larky.modules.vgs.vault.defaults.DefaultVault; import com.verygood.security.larky.modules.vgs.vault.spi.LarkyVault; -import com.verygood.security.larky.modules.vgs.vault.NoopVault; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; import net.starlark.java.annot.Param; import net.starlark.java.annot.ParamType; import net.starlark.java.annot.StarlarkBuiltin; @@ -12,9 +16,6 @@ import net.starlark.java.eval.NoneType; import net.starlark.java.eval.Starlark; -import java.util.List; -import java.util.ServiceLoader; - @StarlarkBuiltin( name = "vault", @@ -107,13 +108,23 @@ public VaultModule() { allowedTypes = { @ParamType(type = List.class) }), + @Param( + name = "decorator_config", + doc = "alias decorator config", + named = true, + defaultValue = "None", + allowedTypes = { + @ParamType(type = NoneType.class), + @ParamType(type = Map.class), + }), }) @Override - public Object redact(Object value, Object storage, Object format, List tags) throws EvalException { + public Object redact(Object value, Object storage, Object format, List tags, Object decoratorConfig) throws EvalException { validateStorage(storage); validateFormat(format); + validateDecoratorConfig(decoratorConfig); - return vault.redact(value, storage, format, tags); + return vault.redact(value, storage, format, tags, decoratorConfig); } @StarlarkMethod( @@ -174,4 +185,23 @@ private void validateFormat(Object format) throws EvalException { } } + private void validateDecoratorConfig(Object decoratorConfig) throws EvalException { + if (decoratorConfig != null && !(decoratorConfig instanceof NoneType) && !(decoratorConfig instanceof Map)) { + throw Starlark.errorf(String.format( + "Decorator config of type %s is not supported in VAULT, expecting Map", + decoratorConfig.getClass().getName() + )); + } else { + try { + DecoratorConfig.fromObject(decoratorConfig); + } catch (InvalidDecoratorConfigException e) { + throw Starlark.errorf(String.format( + "Decorator config '%s' is invalid. %s", + decoratorConfig, e.getMessage() + )); + } + } + } + + } diff --git a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/NoopVault.java b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/NoopVault.java index 1f6aa843d..88fb7c313 100644 --- a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/NoopVault.java +++ b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/NoopVault.java @@ -1,6 +1,7 @@ package com.verygood.security.larky.modules.vgs.vault; import com.verygood.security.larky.modules.vgs.vault.spi.LarkyVault; +import java.util.Map; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Starlark; @@ -8,7 +9,7 @@ public class NoopVault implements LarkyVault { @Override - public Object redact(Object value, Object storage, Object format, List tags) throws EvalException { + public Object redact(Object value, Object storage, Object format, List tags, Object decoratorConfig) throws EvalException { throw Starlark.errorf("vault.redact operation must be overridden"); } diff --git a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/AliasDecorator.java b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/AliasDecorator.java new file mode 100644 index 000000000..5b85e75cc --- /dev/null +++ b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/AliasDecorator.java @@ -0,0 +1,42 @@ +package com.verygood.security.larky.modules.vgs.vault.defaults; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import net.starlark.java.eval.EvalException; + +public class AliasDecorator implements TokenizeFunction { + + private final TokenizeFunction tokenizeFunction; + private final Pattern searchPattern; + private final String replacePattern; + + + public AliasDecorator( + TokenizeFunction tokenizeFunction, + String searchPattern, + String replacePattern) { + this.tokenizeFunction = tokenizeFunction; + this.searchPattern = Pattern.compile(searchPattern); + this.replacePattern = replacePattern; + } + + @Override + public String tokenize(String toTokenize) throws EvalException { + + final Matcher matcher = searchPattern.matcher(toTokenize); + + if (!matcher.find()) { + // Fallback to generic + return new UUIDAliasGenerator().generate(toTokenize); + + } + return tokenize(matcher, replacePattern); + } + + private String tokenize(Matcher matcher, String replacePattern) throws EvalException { + final String tokenGroup = matcher.group("token"); + String tokenized = tokenizeFunction.tokenize(tokenGroup); + String preFormatted = replacePattern.replace("${token}", "%s"); + return matcher.replaceFirst(String.format(preFormatted, tokenized)); + } +} diff --git a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/DefaultVault.java b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/DefaultVault.java index b699426e7..e81b4e38f 100644 --- a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/DefaultVault.java +++ b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/DefaultVault.java @@ -1,15 +1,15 @@ package com.verygood.security.larky.modules.vgs.vault.defaults; import com.google.common.collect.ImmutableMap; +import com.verygood.security.larky.modules.DecoratorConfig; import com.verygood.security.larky.modules.vgs.vault.spi.LarkyVault; - -import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.NoneType; -import net.starlark.java.eval.Starlark; - import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.NoneType; +import net.starlark.java.eval.Starlark; public class DefaultVault implements LarkyVault { @@ -40,13 +40,32 @@ public class DefaultVault implements LarkyVault { .build(); @Override - public Object redact(Object value, Object storage, Object format, List tags) throws EvalException { + public Object redact(Object value, Object storage, Object format, List tags, Object decoratorConfig) throws EvalException { String sValue = getValue(value); - String alias = getAliasGenerator(format).generate(sValue); + AliasGenerator generator = getAliasGenerator(format); + Optional aliasDecorator = resolveAliasDecorator(decoratorConfig, generator); + + String alias = aliasDecorator.isPresent() + ? aliasDecorator.get().tokenize(sValue) + : generator.generate(sValue); getStorage(storage).put(alias, value); return alias; } + private Optional resolveAliasDecorator(Object decoratorConfig, AliasGenerator generator) { + DecoratorConfig config = DecoratorConfig.fromObject(decoratorConfig); + if (config != null && config.getSearchPattern() != null && config.getReplacePattern() != null) { + AliasDecorator decorator = new AliasDecorator( + generator::generate, + config.getSearchPattern(), + config.getReplacePattern() + ); + return Optional.of(decorator); + } else { + return Optional.empty(); + } + } + @Override public Object reveal(Object value, Object storage) throws EvalException { String sValue = getValue(value); diff --git a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/TokenizeFunction.java b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/TokenizeFunction.java new file mode 100644 index 000000000..f8306f369 --- /dev/null +++ b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/defaults/TokenizeFunction.java @@ -0,0 +1,9 @@ +package com.verygood.security.larky.modules.vgs.vault.defaults; + +import net.starlark.java.eval.EvalException; + +@FunctionalInterface +public interface TokenizeFunction { + + String tokenize(String value) throws EvalException; +} diff --git a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/spi/LarkyVault.java b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/spi/LarkyVault.java index 3fa1bdab6..d41392856 100644 --- a/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/spi/LarkyVault.java +++ b/larky/src/main/java/com/verygood/security/larky/modules/vgs/vault/spi/LarkyVault.java @@ -7,7 +7,7 @@ public interface LarkyVault extends StarlarkValue { - Object redact(Object value, Object storage, Object format, List tags) throws EvalException; + Object redact(Object value, Object storage, Object format, List tags, Object decoratorConfig) throws EvalException; Object reveal(Object value, Object storage) throws EvalException; diff --git a/larky/src/test/java/com/verygood/security/larky/modules/vgs/vault/VaultModuleSPITest.java b/larky/src/test/java/com/verygood/security/larky/modules/vgs/vault/VaultModuleSPITest.java index 0fcc1ae03..9232c1144 100644 --- a/larky/src/test/java/com/verygood/security/larky/modules/vgs/vault/VaultModuleSPITest.java +++ b/larky/src/test/java/com/verygood/security/larky/modules/vgs/vault/VaultModuleSPITest.java @@ -46,7 +46,7 @@ public void testNoopModule_exception() throws Exception { // Assert Exceptions Assertions.assertThrows(EvalException.class, () -> { - vault.redact("fail", Starlark.NONE, Starlark.NONE, null); + vault.redact("fail", Starlark.NONE, Starlark.NONE, null, null); }, "vault.redact operation must be overridden" ); @@ -68,7 +68,7 @@ public void testDefaultModule_ok() throws Exception { // Invoke Vault String secret = "4111111111111111"; - String alias = (String) vault.redact(secret, Starlark.NONE, Starlark.NONE, null); + String alias = (String) vault.redact(secret, Starlark.NONE, Starlark.NONE, null, null); String result = (String) vault.reveal(alias, Starlark.NONE); // Assert OK @@ -85,7 +85,7 @@ public void testSPIModule_single_ok() throws Exception { // Invoke Vault String secret = "4111111111111111"; - String alias = (String) vault.redact(secret, Starlark.NONE, Starlark.NONE, null); + String alias = (String) vault.redact(secret, Starlark.NONE, Starlark.NONE, null, null); String result = (String) vault.reveal(alias, Starlark.NONE); // Assert OK diff --git a/larky/src/test/resources/vgs_tests/vault/test_default_vault.star b/larky/src/test/resources/vgs_tests/vault/test_default_vault.star index 195f27454..4597ccc3e 100644 --- a/larky/src/test/resources/vgs_tests/vault/test_default_vault.star +++ b/larky/src/test/resources/vgs_tests/vault/test_default_vault.star @@ -127,6 +127,115 @@ def _luhn_mod10(digits): return sum%10 +def _test_alias_decorator(): + secret = "1111111111" + alias = vault.redact(secret, decorator_config={ + "searchPattern": "(?[0-9]{6})(?[0-9]{4})", + "replacePattern": "98${token}${lastFour}", + "nonLuhnValidPattern": { + "validatePattern": "[0-9]{6}(?[0-9]{6})[0-9]{4}", + "transformPatterns": [ + { + "search": "(?[0-9]{6})(?[0-9]{4})", + "replace": "98${token}${lastFour}", + } + ] + } + }) + + asserts.assert_that(alias).is_not_equal_to(secret) + asserts.assert_true(alias.startswith("98")) + asserts.assert_true(alias.endswith("1111")) + +def _test_alias_decorator_empty(): + card_number = "4111111111111111" + redacted_card_number = vault.redact(card_number, decorator_config={}) + + asserts.assert_that(redacted_card_number[:4]).is_equal_to('tok_') + +def _test_invalid_alias_decorator_invalid_config_format(): + asserts.assert_fails(lambda : vault.redact("whatever", decorator_config="invalid"), + """in call to redact\\(\\), parameter 'decorator_config' got value of type 'string', want 'NoneType or Map'""" + ) + +def _test_invalid_alias_decorator_invalid_config_search_pattern(): + asserts.assert_fails(lambda : vault.redact("1111111111", decorator_config={ + "searchPattern": {}, + "replacePattern": "98${token}${lastFour}" + }), + """Decorator config '{"searchPattern": {}, "replacePattern": "98\\${token}\\${lastFour}"}' is invalid\\. 'searchPattern' field must be string""" + ) + +def _test_invalid_alias_decorator_invalid_config_replace_pattern(): + asserts.assert_fails(lambda : vault.redact("1111111111", decorator_config={ + "searchPattern": "(?[0-9]{6})(?[0-9]{4})", + "replacePattern": ["98${token}${lastFour}"] + }), + """Decorator config '{"searchPattern": "\\(\\?\\[0-9]\\{6}\\)\\(\\?\\[0-9\\]\\{4}\\)", "replacePattern": \\["98\\${token}\\${lastFour}"]}' is invalid\\. 'replacePattern' field must be string""" + ) + +def _test_invalid_alias_decorator_invalid_config_non_luhn_valid_format(): + asserts.assert_fails(lambda : vault.redact("1111111111", decorator_config={ + "nonLuhnValidPattern": "[0-9]+" + }), + """Decorator config '{"nonLuhnValidPattern": "\\[0-9\\]\\+"}' is invalid\\. 'nonLuhnValidPattern' field must be dict""" + ) + +def _test_invalid_alias_decorator_invalid_config_non_luhn_valid_validate_pattern(): + asserts.assert_fails(lambda : vault.redact("1111111111", decorator_config={ + "nonLuhnValidPattern": { + "validatePattern": [], + "transformPatterns": [ + { + "search": "[0-9]{10}", + "replace": "98${token}", + } + ] + } + }), + """Decorator config '{"nonLuhnValidPattern": {"validatePattern": \\[\\], "transformPatterns": \\[{"search": "\\[0-9\\]\\{10}", "replace": "98\\${token}"}\\]}}' is invalid\\. 'validatePattern' field must be string""" + ) + +def _test_invalid_alias_decorator_invalid_config_non_luhn_valid_transform_patterns(): + asserts.assert_fails(lambda : vault.redact("1111111111", decorator_config={ + "nonLuhnValidPattern": { + "validatePattern": "[0-9]{10}", + "transformPatterns": "[0-9]{10}" + } + }), + """Decorator config '{"nonLuhnValidPattern": {"validatePattern": "\\[0-9\\]\\{10}", "transformPatterns": "\\[0-9\\]\\{10\\}"}}' is invalid\\. 'transformPatterns' field must be array""" + ) + +def _test_invalid_alias_decorator_invalid_config_non_luhn_valid_search(): + asserts.assert_fails(lambda : vault.redact("1111111111", decorator_config={ + "nonLuhnValidPattern": { + "validatePattern": "[0-9]{10}", + "transformPatterns": [ + { + "search": {}, + "replace": "98${token}", + } + ] + } + }), + """Decorator config '{"nonLuhnValidPattern": {"validatePattern": "\\[0-9\\]\\{10}", "transformPatterns": \\[{"search": \\{}, "replace": "98\\${token}"}\\]}}' is invalid\\. 'search' field must be string""" + ) + +def _test_invalid_alias_decorator_invalid_config_non_luhn_valid_replace(): + asserts.assert_fails(lambda : vault.redact("1111111111", decorator_config={ + "nonLuhnValidPattern": { + "validatePattern": "[0-9]{10}", + "transformPatterns": [ + { + "search": "[0-9]{10}", + "replace": ["98${token}", "99${token}"], + } + ] + } + }), + """Decorator config '{"nonLuhnValidPattern": {"validatePattern": "\\[0-9\\]\\{10}", "transformPatterns": \\[{"search": "\\[0-9\\]\\{10}", "replace": \\["98\\${token}", "99\\${token}"\\]}\\]}}' is invalid\\. 'replace' field must be string""" + ) + def _suite(): _suite = unittest.TestSuite() @@ -151,6 +260,18 @@ def _suite(): _suite.addTest(unittest.FunctionTestCase(_test_valid_format_preserving)) _suite.addTest(unittest.FunctionTestCase(_test_invalid_format_preserving)) + # Alias Decorator Tests + _suite.addTest(unittest.FunctionTestCase(_test_alias_decorator)) + _suite.addTest(unittest.FunctionTestCase(_test_alias_decorator_empty)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_format)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_search_pattern)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_replace_pattern)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_non_luhn_valid_format)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_non_luhn_valid_validate_pattern)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_non_luhn_valid_transform_patterns)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_non_luhn_valid_search)) + _suite.addTest(unittest.FunctionTestCase(_test_invalid_alias_decorator_invalid_config_non_luhn_valid_replace)) + return _suite