diff --git a/adr/0002-support-object-values-masking.md b/adr/0002-support-object-values-masking.md deleted file mode 100644 index e69de29b..00000000 diff --git a/src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark.java b/src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark.java index a1e48a1f..b8d95da2 100644 --- a/src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark.java +++ b/src/jmh/java/dev/blaauwendraad/masker/json/JsonMaskerBenchmark.java @@ -1,9 +1,17 @@ package dev.blaauwendraad.masker.json; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import dev.blaauwendraad.masker.json.config.JsonMaskingConfig; -import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.Warmup; import randomgen.json.RandomJsonGenerator; import randomgen.json.RandomJsonGeneratorConfig; @@ -13,6 +21,9 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static randomgen.json.JsonStringCharacters.getPrintableAsciiCharacters; @Warmup(iterations = 1, time = 3) @Fork(value = 1) @@ -23,11 +34,11 @@ public class JsonMaskerBenchmark { @org.openjdk.jmh.annotations.State(Scope.Thread) public static class State { - @Param({"1kb", "128kb", "2mb"}) + @Param({ "1kb", "128kb", "2mb" }) String jsonSize; - @Param({"0.01", "0.1"}) + @Param({ "0.01", "0.1" }) double maskedKeyProbability; - @Param({"none", "8"}) + @Param({ "none", "8" }) String obfuscationLength; private String jsonString; private byte[] jsonBytes; @@ -39,6 +50,11 @@ public synchronized void setup() { Set keysToBeMasked = getTargetKeys(); RandomJsonGeneratorConfig config = RandomJsonGeneratorConfig.builder() + .setAllowedCharacters( + getPrintableAsciiCharacters().stream() + .filter(c -> c != '"') + .collect(Collectors.toSet()) + ) .setTargetKeys(keysToBeMasked) .setTargetKeyPercentage(maskedKeyProbability) .setTargetJsonSizeBytes(BenchmarkUtils.parseSize(jsonSize)) @@ -49,7 +65,9 @@ public synchronized void setup() { jsonMasker = JsonMasker.getMasker( JsonMaskingConfig.custom(keysToBeMasked, JsonMaskingConfig.TargetKeyMode.MASK) - .obfuscationLength(Objects.equals(obfuscationLength, "none") ? -1 : Integer.parseInt(obfuscationLength)) + .obfuscationLength(Objects.equals(obfuscationLength, "none") + ? -1 + : Integer.parseInt(obfuscationLength)) .build() ); objectMapper = new ObjectMapper(); @@ -75,7 +93,12 @@ public int baselineCountBytes(State state) { @Benchmark public String jacksonString(State state) throws IOException { - return ParseAndMaskUtil.mask(state.jsonString, state.getTargetKeys(), JsonMaskingConfig.TargetKeyMode.MASK, state.objectMapper).toString(); + return ParseAndMaskUtil.mask( + state.jsonString, + state.getTargetKeys(), + JsonMaskingConfig.TargetKeyMode.MASK, + state.objectMapper + ).toString(); } @Benchmark diff --git a/src/main/java/dev/blaauwendraad/masker/json/KeyContainsMasker.java b/src/main/java/dev/blaauwendraad/masker/json/KeyContainsMasker.java index e3a3e2d7..ff861ce6 100644 --- a/src/main/java/dev/blaauwendraad/masker/json/KeyContainsMasker.java +++ b/src/main/java/dev/blaauwendraad/masker/json/KeyContainsMasker.java @@ -6,7 +6,6 @@ import dev.blaauwendraad.masker.json.util.FixedLengthTargetValueMaskUtil; import dev.blaauwendraad.masker.json.util.Utf8Util; -import java.nio.charset.StandardCharsets; import java.util.Set; import static dev.blaauwendraad.masker.json.util.AsciiCharacter.isDoubleQuote; @@ -115,9 +114,11 @@ public byte[] mask(byte[] input) { * or object. Now let's verify the found JSON key is a target key. */ int keyLength = closingQuoteIndex - openingQuoteIndex - 1; // minus one for the quote - byte[] keyBytesBuffer = new byte[keyLength]; - System.arraycopy(maskingState.getMessage(), openingQuoteIndex + 1, keyBytesBuffer, 0, keyLength); - String key = new String(keyBytesBuffer, StandardCharsets.UTF_8); + String key = new String( + maskingState.getMessage(), + openingQuoteIndex + 1 /* plus one for the opening quote */, + keyLength + ); if (!maskingConfig.caseSensitiveTargetKeys()) { key = key.toLowerCase(); } @@ -333,9 +334,11 @@ private static void maskObjectValueInPlace(MaskingState maskingState, JsonMaskin } int closingQuoteIndex = maskingState.currentIndex(); int keyLength = closingQuoteIndex - openingQuoteIndex - 1; // minus one for the quote - byte[] keyBytesBuffer = new byte[keyLength]; - System.arraycopy(maskingState.getMessage(), openingQuoteIndex + 1, keyBytesBuffer, 0, keyLength); - String key = new String(keyBytesBuffer, StandardCharsets.UTF_8); + String key = new String( + maskingState.getMessage(), + openingQuoteIndex + 1 + /* plus one for the opening quote */, keyLength + ); if (!maskingConfig.caseSensitiveTargetKeys()) { key = key.toLowerCase(); } diff --git a/src/main/java/dev/blaauwendraad/masker/json/util/AsciiCharacter.java b/src/main/java/dev/blaauwendraad/masker/json/util/AsciiCharacter.java index 6f49e6fe..bb5e7689 100644 --- a/src/main/java/dev/blaauwendraad/masker/json/util/AsciiCharacter.java +++ b/src/main/java/dev/blaauwendraad/masker/json/util/AsciiCharacter.java @@ -123,7 +123,7 @@ public static boolean isSquareBracketClose(byte inputByte) { } /** - * Tests if the given byte corresponds to an opening curly bracket '{@literal {}' in ASCII encoding. + * Tests if the given byte corresponds to an opening curly bracket '{@literal {}}' in ASCII encoding. * * @param inputByte the input byte * @return true if the byte corresponds to an opening curly diff --git a/src/main/java/dev/blaauwendraad/masker/json/util/FixedLengthTargetValueMaskUtil.java b/src/main/java/dev/blaauwendraad/masker/json/util/FixedLengthTargetValueMaskUtil.java index 38d56506..5a577621 100644 --- a/src/main/java/dev/blaauwendraad/masker/json/util/FixedLengthTargetValueMaskUtil.java +++ b/src/main/java/dev/blaauwendraad/masker/json/util/FixedLengthTargetValueMaskUtil.java @@ -18,8 +18,6 @@ private FixedLengthTargetValueMaskUtil() { * @param fixedMaskLength the length of the fixed-length mask byte string. * @param targetValueLength the length of the target value slice. * @param maskByte the byte used for each byte in the mask - * @return a new array corresponding to the input bytes array with the target value replaced with a fixed length - * mask */ public static void replaceTargetValueWithFixedLengthMask( MaskingState maskingState, diff --git a/src/test/java/randomgen/json/RandomJsonGenerator.java b/src/test/java/randomgen/json/RandomJsonGenerator.java index e5fee2df..30fc5bbd 100644 --- a/src/test/java/randomgen/json/RandomJsonGenerator.java +++ b/src/test/java/randomgen/json/RandomJsonGenerator.java @@ -1,7 +1,14 @@ package randomgen.json; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.*; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BigIntegerNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.NumericNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import java.math.BigInteger; import java.nio.charset.StandardCharsets; @@ -12,7 +19,12 @@ public class RandomJsonGenerator { private final ThreadLocalRandom random = ThreadLocalRandom.current(); private enum NodeType { - arrayNode, objectNode, booleanNode, nullNode, stringNode, numberNode + arrayNode, + objectNode, + booleanNode, + nullNode, + stringNode, + numberNode } public RandomJsonGenerator(RandomJsonGeneratorConfig config) { @@ -31,7 +43,8 @@ private JsonNode createRandomJsonNode(Context context, int depth) { } else if (config.hasTargetSize() && depth == 0) { // always start with an object root if generating json of certain size nodeType = NodeType.objectNode; - } else if ((depth < config.getMaxNodeDepth() / 3) && (nodeType != NodeType.objectNode && nodeType != NodeType.arrayNode)) { + } else if ((depth < config.getMaxNodeDepth() / 3) && (nodeType != NodeType.objectNode + && nodeType != NodeType.arrayNode)) { // forcefully override chance to 50% to create an object if only <33% of max node depth is reached if (random.nextBoolean()) { nodeType = NodeType.objectNode; @@ -120,7 +133,7 @@ private String getRandomString() { } private Character getRandomCharacter() { - Character[] characters = config.getStringCharacters().toArray(new Character[0]); + Character[] characters = config.getAllowedCharacters().toArray(new Character[0]); int randomIndex = random.nextInt(characters.length - 1); return characters[randomIndex]; } diff --git a/src/test/java/randomgen/json/RandomJsonGeneratorConfig.java b/src/test/java/randomgen/json/RandomJsonGeneratorConfig.java index 1481b375..b736a0c0 100644 --- a/src/test/java/randomgen/json/RandomJsonGeneratorConfig.java +++ b/src/test/java/randomgen/json/RandomJsonGeneratorConfig.java @@ -20,7 +20,7 @@ public class RandomJsonGeneratorConfig { private final int maxObjectKeys; private final int maxNodeDepth; private final double targetKeyPercentage; // percentage of object keys which are target keys - private final Set stringCharacters; + private final Set allowedCharacters; private final Set targetKeys; private final int targetJsonSizeBytes; @@ -34,7 +34,7 @@ public RandomJsonGeneratorConfig( int maxObjectKeys, int maxNodeDepth, double targetKeyPercentage, - Set stringCharacters, + Set allowedCharacters, Set targetKeys, int targetJsonSizeBytes ) { @@ -46,7 +46,7 @@ public RandomJsonGeneratorConfig( this.maxObjectKeys = maxObjectKeys; this.maxNodeDepth = maxNodeDepth; this.targetKeyPercentage = targetKeyPercentage; - this.stringCharacters = stringCharacters; + this.allowedCharacters = allowedCharacters; this.targetKeys = targetKeys; this.maxStringLength = maxStringLength; this.targetJsonSizeBytes = targetJsonSizeBytes; @@ -108,8 +108,8 @@ public double getTargetKeyPercentage() { return targetKeyPercentage; } - public Set getStringCharacters() { - return stringCharacters; + public Set getAllowedCharacters() { + return allowedCharacters; } public int getTargetJsonSizeBytes() { @@ -130,11 +130,11 @@ public static class Builder { private int maxObjectKeys = 5; private int maxNodeDepth = 10; private double targetKeyPercentage = 0.2; - private Set allowedCharacters = mergeCharSets(mergeCharSets( + private Set allowedCharacters = mergeCharSets( getPrintableAsciiCharacters(), getUnicodeControlCharacters(), getRandomPrintableUnicodeCharacters() - )); + ); private Set targetKeys = defaultTargetKeys; private int targetJsonSizeBytes = -1; // no target, random size depending on other constraints @@ -183,8 +183,8 @@ public Builder setTargetKeyPercentage(double targetKeyPercentage) { return this; } - public Builder setStringCharacters(Set stringCharacters) { - this.allowedCharacters = stringCharacters; + public Builder setAllowedCharacters(Set allowedCharacters) { + this.allowedCharacters = allowedCharacters; return this; } diff --git a/src/test/java/randomgen/json/RandomJsonGeneratorTest.java b/src/test/java/randomgen/json/RandomJsonGeneratorTest.java index 9026c0c9..6d088382 100644 --- a/src/test/java/randomgen/json/RandomJsonGeneratorTest.java +++ b/src/test/java/randomgen/json/RandomJsonGeneratorTest.java @@ -1,14 +1,17 @@ package randomgen.json; import com.fasterxml.jackson.databind.JsonNode; +import dev.blaauwendraad.masker.json.JsonMasker; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.nio.charset.StandardCharsets; -import java.text.DecimalFormat; -import java.util.DoubleSummaryStatistics; +import java.util.Set; + +import static randomgen.json.JsonStringCharacters.getPrintableAsciiCharacters; public class RandomJsonGeneratorTest { @ParameterizedTest @@ -30,14 +33,21 @@ void testGeneratesJsonForAGivenSize(int numberOfTests) { // we risk having more than 1% difference due to requirement to return a valid json int targetJsonSizeBytes = (i + 5) * 1024; RandomJsonGenerator randomJsonGenerator = - new RandomJsonGenerator(RandomJsonGeneratorConfig.builder().setTargetJsonSizeBytes(targetJsonSizeBytes).createConfig()); + new RandomJsonGenerator(RandomJsonGeneratorConfig.builder() + .setTargetJsonSizeBytes(targetJsonSizeBytes) + .createConfig()); JsonNode randomJsonNode = randomJsonGenerator.createRandomJsonNode(); int actualSizeBytes = randomJsonNode.toString().getBytes(StandardCharsets.UTF_8).length; double allowedDifference = targetJsonSizeBytes * 0.01; - Assertions.assertEquals(targetJsonSizeBytes, actualSizeBytes, allowedDifference, () -> "Expected json to be of size " + targetJsonSizeBytes + " (±1%), got: " + actualSizeBytes); + Assertions.assertEquals( + targetJsonSizeBytes, + actualSizeBytes, + allowedDifference, + () -> "Expected json to be of size " + targetJsonSizeBytes + " (±1%), got: " + actualSizeBytes + ); } } @@ -45,27 +55,65 @@ void testGeneratesJsonForAGivenSize(int numberOfTests) { void testGenerate1MbJson() { int targetJsonSizeBytes = 1024 * 1024; RandomJsonGenerator randomJsonGenerator = - new RandomJsonGenerator(RandomJsonGeneratorConfig.builder().setTargetJsonSizeBytes(targetJsonSizeBytes).createConfig()); + new RandomJsonGenerator(RandomJsonGeneratorConfig.builder() + .setTargetJsonSizeBytes(targetJsonSizeBytes) + .createConfig()); JsonNode randomJsonNode = randomJsonGenerator.createRandomJsonNode(); int actualSizeBytes = randomJsonNode.toString().getBytes(StandardCharsets.UTF_8).length; double allowedDifference = targetJsonSizeBytes * 0.001; - Assertions.assertEquals(targetJsonSizeBytes, actualSizeBytes, allowedDifference, () -> "Expected json to be of size " + targetJsonSizeBytes + " (±0.1%), got: " + actualSizeBytes); + Assertions.assertEquals( + targetJsonSizeBytes, + actualSizeBytes, + allowedDifference, + () -> "Expected json to be of size " + targetJsonSizeBytes + " (±0.1%), got: " + actualSizeBytes + ); } @Test void testGenerate10MbJson() { int targetJsonSizeBytes = 10 * 1024 * 1024; RandomJsonGenerator randomJsonGenerator = - new RandomJsonGenerator(RandomJsonGeneratorConfig.builder().setTargetJsonSizeBytes(targetJsonSizeBytes).createConfig()); + new RandomJsonGenerator(RandomJsonGeneratorConfig.builder() + .setTargetJsonSizeBytes(targetJsonSizeBytes) + .createConfig()); JsonNode randomJsonNode = randomJsonGenerator.createRandomJsonNode(); int actualSizeBytes = randomJsonNode.toString().getBytes(StandardCharsets.UTF_8).length; double allowedDifference = targetJsonSizeBytes * 0.001; - Assertions.assertEquals(targetJsonSizeBytes, actualSizeBytes, allowedDifference, () -> "Expected json to be of size " + targetJsonSizeBytes + " (±0.1%), got: " + actualSizeBytes); + Assertions.assertEquals( + targetJsonSizeBytes, + actualSizeBytes, + allowedDifference, + () -> "Expected json to be of size " + targetJsonSizeBytes + " (±0.1%), got: " + actualSizeBytes + ); + } + + @Test + @Disabled + void profileMasker() { + int targetJsonSizeBytes = 1024 * 1024; + RandomJsonGenerator randomJsonGenerator = + new RandomJsonGenerator(RandomJsonGeneratorConfig.builder() + .setAllowedCharacters(getPrintableAsciiCharacters()) + .setTargetJsonSizeBytes(targetJsonSizeBytes) + .createConfig()); + + JsonNode randomJsonNode = randomJsonGenerator.createRandomJsonNode(); + + var json = randomJsonNode.toString(); + + JsonMasker jsonMasker = JsonMasker.getMasker(Set.of("targetKey1", "targetKey2", "targetKey3", "targetKey4")); + + int size = 0; + for (int i = 0; i < 500; i++) { + size += jsonMasker.mask(json).length(); + } + + System.out.println(size); } }