Skip to content

Commit

Permalink
My latest random stuff
Browse files Browse the repository at this point in the history
  • Loading branch information
breus committed Dec 17, 2023
1 parent f74a687 commit 3319f14
Show file tree
Hide file tree
Showing 8 changed files with 276 additions and 39 deletions.
6 changes: 3 additions & 3 deletions adr/0001-support-array-masking.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ Supporting arrays of mixed types (number + strings + arrays + objects) possesses
challenge: Up to now, the json-masker did not have to keep track of current nesting level. I.e.: it did not matter where
we found `"maskMe": "a"`, since all such instances are masked.

To bring support for masking of strings and numbers inside arrays, we will have to know exactly which field is being
processed right now, to correctly mask a JSON like this:
To bring support for masking of strings and numbers inside arrays, we will have to know the depth of the field which is
being processed right now, to correctly mask a JSON like this:

```json
{
Expand Down Expand Up @@ -47,7 +47,7 @@ We could simply not support arrays with mixed types, considering that those are
in most strongly typed languages such as Java).
\
However, given that the purpose of this library is to provide high-performance with as correct semantics as
possible, we only have the following options that would be obvious for the user:
possible for any JSON, we only have the following options that would be obvious for the user:

1. Mask all mixed values of an array (Solution 1)
2. Fail if a maskable array contains anything different than string/number
Expand Down
Empty file.
18 changes: 8 additions & 10 deletions src/main/java/dev/blaauwendraad/masker/json/KeyContainsMasker.java
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ public byte[] mask(byte[] input) {
}
boolean keyMatched = targetKeys.contains(key);
if (allowMode && keyMatched) {
skipValue(maskingState); // the value belongs to a JSON key which is explicitly allowed, so skip it
skipAllValues(maskingState); // the value belongs to a JSON key which is explicitly allowed, so skip it
continue;
}
if (!allowMode && !keyMatched) {
Expand Down Expand Up @@ -187,8 +187,7 @@ private boolean isStartOfMaskableValue(MaskingState maskingState) {
(maskingConfig.isArrayMaskingEnabled() && AsciiJsonUtil.isArrayStart(maskingState.byteAtCurrentIndex()))
|| (maskingConfig.isNumberMaskingEnabled()
&& AsciiJsonUtil.isFirstNumberChar(maskingState.byteAtCurrentIndex())) || (
maskingConfig.isObjectValuesMaskingEnabled()
&& AsciiJsonUtil.isObjectStart(maskingState.byteAtCurrentIndex()));
AsciiJsonUtil.isObjectStart(maskingState.byteAtCurrentIndex()));
}

/**
Expand Down Expand Up @@ -299,12 +298,11 @@ private static void maskArrayValueInPlace(MaskingState maskingState, JsonMasking
} else if (AsciiJsonUtil.isFirstNumberChar(maskingState.byteAtCurrentIndex())
&& maskingConfig.isNumberMaskingEnabled()) {
maskNumberValueInPlace(maskingState, maskingConfig);
} else if (AsciiJsonUtil.isObjectStart(maskingState.byteAtCurrentIndex())
&& maskingConfig.isObjectValuesMaskingEnabled()) {
} else if (AsciiJsonUtil.isObjectStart(maskingState.byteAtCurrentIndex())) {
maskObjectValueInPlace(maskingState, maskingConfig);
} else {
// non-maskable values
skipValue(maskingState);
skipAllValues(maskingState);
}
skipWhitespaceCharacters(maskingState);
if (AsciiCharacter.isComma(maskingState.byteAtCurrentIndex())) {
Expand All @@ -315,6 +313,7 @@ private static void maskArrayValueInPlace(MaskingState maskingState, JsonMasking
}

/**
* Masks all values (depending on the {@link JsonMaskingConfig} in the object.
* @param maskingState the current masking state
* @param maskingConfig the masking configuration
*/
Expand Down Expand Up @@ -361,8 +360,7 @@ private static void maskObjectValueInPlace(MaskingState maskingState, JsonMaskin
} else if (AsciiJsonUtil.isFirstNumberChar(maskingState.byteAtCurrentIndex())
&& maskingConfig.isNumberMaskingEnabled()) {
maskNumberValueInPlace(maskingState, maskingConfig);
} else if (AsciiJsonUtil.isObjectStart(maskingState.byteAtCurrentIndex())
&& maskingConfig.isObjectValuesMaskingEnabled()) {
} else if (AsciiJsonUtil.isObjectStart(maskingState.byteAtCurrentIndex())) {
maskObjectValueInPlace(maskingState, maskingConfig);
} else {
while (!AsciiCharacter.isComma(maskingState.byteAtCurrentIndex())
Expand All @@ -372,7 +370,7 @@ private static void maskObjectValueInPlace(MaskingState maskingState, JsonMaskin
}
}
} else {
skipValue(maskingState);
skipAllValues(maskingState);
}
skipWhitespaceCharacters(maskingState);
if (AsciiCharacter.isComma(maskingState.byteAtCurrentIndex())) {
Expand Down Expand Up @@ -438,7 +436,7 @@ private static void maskNumberValueInPlace(MaskingState maskingState, JsonMaskin
* Note: in case the value is an object or array, it skips the entire object and array and all the included elements
* in it (e.g. nested arrays, objects, etc.)
*/
private static void skipValue(MaskingState maskingState) {
private static void skipAllValues(MaskingState maskingState) {
if (AsciiCharacter.isLowercaseN(maskingState.byteAtCurrentIndex())
|| AsciiCharacter.isLowercaseT(maskingState.byteAtCurrentIndex())) { // null and true
maskingState.setCurrentIndex(maskingState.currentIndex() + 4);
Expand Down
26 changes: 0 additions & 26 deletions src/test/java/dev/blaauwendraad/masker/json/FuzzingTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,32 +13,6 @@
final class FuzzingTest {
private static final int SECONDS_FOR_EACH_TEST_TO_RUN = 10;

@ParameterizedTest
@ValueSource(ints = { SECONDS_FOR_EACH_TEST_TO_RUN })
// duration in seconds the tests runs for
void fuzzTestNoFailuresKeyContainsAlgorithm(int secondsToRunTest) {
long startTime = System.currentTimeMillis();
int randomTestExecuted = 0;
while (System.currentTimeMillis() < startTime + 10 * 1000) {
Set<String> targetKeys = Set.of("targetKey1", "targetKey2");
KeyContainsMasker keyContainsMasker = new KeyContainsMasker(JsonMaskingConfig.getDefault(targetKeys));
RandomJsonGenerator randomJsonGenerator =
new RandomJsonGenerator(RandomJsonGeneratorConfig.builder().createConfig());
JsonNode randomJsonNode = randomJsonGenerator.createRandomJsonNode();
Assertions.assertDoesNotThrow(
() -> keyContainsMasker.mask(randomJsonNode.toPrettyString()),
randomJsonNode.toPrettyString()
);
randomTestExecuted++;
}
System.out.printf(
"Executed %d randomly generated test scenarios in %d seconds%n",
randomTestExecuted,
secondsToRunTest
);
}

@ParameterizedTest
@ValueSource(ints = { SECONDS_FOR_EACH_TEST_TO_RUN })
// duration in seconds the tests runs for
void fuzzing_NoArrayNoObjectValueMasking(int secondsToRunTest) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ public static List<JsonMaskerTestInstance> getJsonMaskerTestInstancesFromFile(
if (caseSensitiveTargetKeys != null && caseSensitiveTargetKeys.booleanValue()) {
configBuilder.caseSensitiveTargetKeys();
}
// JsonNode objectValueMasking = maskerConfig.findValue("objectValueMasking");
// if (objectValueMasking != null && !objectValueMasking.asBoolean()) {
// configBuilder.disableObjectValueMasking();
// }
}
JsonMaskingConfig maskingConfig = configBuilder.build();
var input = jsonNode.get("input").toString();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package dev.blaauwendraad.masker.json;

import com.fasterxml.jackson.databind.JsonNode;
import dev.blaauwendraad.masker.json.config.JsonMaskingConfig;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import randomgen.json.RandomJsonGenerator;
import randomgen.json.RandomJsonGeneratorConfig;

import javax.annotation.Nonnull;
import javax.annotation.ParametersAreNonnullByDefault;
import java.time.Duration;
import java.time.Instant;
import java.util.Set;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;

/**
* This class contains fuzzing tests which are meant to spot infinite loops and program failures for all combination
* of {@link JsonMasker} and {@link JsonMaskingConfig}.
* <p>
* For each {@link JsonMaskingConfig}, random JSON inputs are generated against which the masker runs and the only thing
* that is tested it doesn't cause an exception or gets stuck in a loop.
*/
@ParametersAreNonnullByDefault
final class NoFailingExecutionFuzzingTest {
private static final Duration DEFAULT_TEST_INSTANCE_DURATION = Duration.ofSeconds(3);

@ParameterizedTest
@MethodSource("failureFuzzingConfigurations")
// duration in seconds the tests runs for
void defaultJsonMasker(JsonMaskingConfig jsonMaskingConfig, Duration durationToRunEachTest) throws InterruptedException {
Instant startTime = Instant.now();
AtomicInteger randomTestExecuted = new AtomicInteger();
AtomicReference<String> lastExecutedJson = new AtomicReference<>();
ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(1, 1, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
threadPoolExecutor.execute(() -> {
while (Instant.ofEpochMilli(System.currentTimeMillis()).isBefore(startTime.plus(durationToRunEachTest))) {
KeyContainsMasker keyContainsMasker = new KeyContainsMasker(jsonMaskingConfig);
RandomJsonGenerator randomJsonGenerator =
new RandomJsonGenerator(RandomJsonGeneratorConfig.builder().setMaxArraySize(3).setMaxNodeDepth(3).setMaxObjectKeys(2).createConfig());
JsonNode randomJsonNode = randomJsonGenerator.createRandomJsonNode();
String jsonString = randomJsonNode.toPrettyString();
lastExecutedJson.set(jsonString);
Assertions.assertDoesNotThrow(
() -> keyContainsMasker.mask(jsonString),
randomJsonNode.toPrettyString()
);
randomTestExecuted.incrementAndGet();
}
});
threadPoolExecutor.awaitTermination(4, TimeUnit.SECONDS);
System.out.printf(
"Executed %d randomly generated test scenarios in %d seconds%n",
randomTestExecuted.get(),
durationToRunEachTest.toSeconds()
);
// This is created to see on which input the program gets stuck in a loop
System.out.printf("Last executed JSON:\n%s", lastExecutedJson.toString());
}


@Nonnull
private static Stream<Arguments> failureFuzzingConfigurations() {
Set<String> targetKeys = Set.of("targetKey1", "targetKey2");
return Stream.of(
Arguments.of(
JsonMaskingConfig.getDefault(targetKeys), DEFAULT_TEST_INSTANCE_DURATION
),
Arguments.of(
JsonMaskingConfig.custom(targetKeys, JsonMaskingConfig.TargetKeyMode.MASK)
.caseSensitiveTargetKeys().build(), DEFAULT_TEST_INSTANCE_DURATION
),
Arguments.of(
JsonMaskingConfig.custom(targetKeys, JsonMaskingConfig.TargetKeyMode.MASK)
.disableArrayValueMasking().build(), DEFAULT_TEST_INSTANCE_DURATION
),
Arguments.of(
JsonMaskingConfig.custom(targetKeys, JsonMaskingConfig.TargetKeyMode.MASK)
.build(), DEFAULT_TEST_INSTANCE_DURATION
)
);
}


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package dev.blaauwendraad.masker.json;

import dev.blaauwendraad.masker.json.config.JsonMaskerAlgorithmType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;

import javax.annotation.ParametersAreNonnullByDefault;
import java.io.IOException;
import java.util.Set;
import java.util.stream.Stream;

@ParametersAreNonnullByDefault
final class NoObjectValueMaskingTest {
@ParameterizedTest
@MethodSource("noObjectMaskingFile")
void multiTargetKey(JsonMaskerTestInstance testInstance) {
Assertions.assertEquals(testInstance.expectedOutput(), testInstance.jsonMasker().mask(testInstance.input()));
}

private static Stream<JsonMaskerTestInstance> noObjectMaskingFile() throws IOException {
return JsonMaskerTestUtil.getJsonMaskerTestInstancesFromFile("test-no-object-value-masking.json", Set.of(
JsonMaskerAlgorithmType.values())).stream();
}

}
144 changes: 144 additions & 0 deletions src/test/resources/test-no-object-value-masking.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
[
{
"maskerConfig": {
"objectValueMasking": false
},
"targetKeys": [
"maskMe",
"andMaskMe"
],
"input": {
"someKey": {
"maskMe": [
null,
true,
123,
"yes",
"no"
],
"dontMaskMe": {
"andMaskMe": "hello"
}
}
},
"expectedOutput": {
"someKey": {
"maskMe": [
null,
true,
123,
"***",
"**"
],
"dontMaskMe": {
"andMaskMe": "*****"
}
}
}
},
{
"maskerConfig": {
"objectValueMasking": false
},
"targetKeys": [
"targetKey1",
"targetKey2"
],
"input": {
"targetKey1": {
"targetKey2": "\u001C"
},
"targetKey2": "W"
},
"expectedOutput": {
"targetKey1": {
"targetKey2": "*"
},
"targetKey2": "*"
}
},
{
"maskerConfig": {
"objectValueMasking": false
},
"targetKeys": [
"targetKey1",
"targetKey2"
],
"input": {
"targetKey1": {
"targetKey2": "hey",
"targetKey3": "\u001C"
},
"targetKey2": true,
"targetKey4": null
},
"expectedOutput": {
"targetKey1": {
"targetKey2": "***",
"targetKey3": "\u001C"
},
"targetKey2": true,
"targetKey4": null
}
},
{
"maskerConfig": {
"objectValueMasking": false
},
"targetKeys": [
"targetKey1",
"targetKey2"
],
"input": {
"targetKey1": {
"targetKey2": [
{},
null,
{
"targetKey1": "mask"
}
],
"targetKey3": "\u001C"
},
"targetKey2": true,
"targetKey4": null
},
"expectedOutput": {
"targetKey1": {
"targetKey2": [
{},
null,
{
"targetKey1": "****"
}
],
"targetKey3": "\u001C"
},
"targetKey2": true,
"targetKey4": null
}
},
{
"maskerConfig": {
"objectValueMasking": false
},
"targetKeys": [
"targetKey1"
],
"input": {
"targetKey1": [
{
"pbNGs還": {}
}
]
},
"expectedOutput": {
"targetKey1": [
{
"pbNGs還": {}
}
]
}
}
]

0 comments on commit 3319f14

Please sign in to comment.