Skip to content

Commit

Permalink
add test case
Browse files Browse the repository at this point in the history
  • Loading branch information
jimeng committed Oct 7, 2024
1 parent ebe3d00 commit 6153921
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 33 deletions.
2 changes: 2 additions & 0 deletions src/main/java/org/simdjson/BitIndexes.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.simdjson;

import java.util.Arrays;

public class BitIndexes {

private final int[] indexes;
Expand Down
58 changes: 25 additions & 33 deletions src/main/java/org/simdjson/SimdJsonParser2.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ static class JsonNode {
private JsonNode ptr;
private byte[] buffer;
private final int targetParseNum;
// every time json string is processed, currentVersion will be incremented by 1
private long currentVersion = 0;
// pruning, when alreadyProcessedCols == NUM
private long alreadyProcessedCols = 0;
Expand Down Expand Up @@ -84,22 +85,31 @@ public String[] parse(byte[] buffer, int len) {
return getResult();
}

private void parseElement(String fieldName) {
if (fieldName == null) {
int start = bitIndexes.advance();
int realEnd = bitIndexes.advance();
while (realEnd > start) {
if (buffer[--realEnd] == '"') {
break;
}
}
fieldName = new String(buffer, start + 1, realEnd - start - 1);
private String parseField() {
int start = bitIndexes.advance();
int next = bitIndexes.peek();
String field = new String(buffer, start, next - start).trim();
if ("null".equalsIgnoreCase(field)) {
return null;
}
if (!ptr.getChildren().containsKey(fieldName)) {
// field type is string or type is decimal
if (field.startsWith("\"")) {
field = field.substring(1, field.length() - 1);
}
return field;
}

private void parseElement(String expectFieldName) {
// if expectFieldName is null, parent is map, else is list
if (expectFieldName == null) {
expectFieldName = parseField();
bitIndexes.advance(); // skip :
}
if (!ptr.getChildren().containsKey(expectFieldName)) {
skip(false);
return;
}
ptr = ptr.getChildren().get(fieldName);
ptr = ptr.getChildren().get(expectFieldName);
switch (buffer[bitIndexes.peek()]) {
case '{' -> {
parseMap();
Expand All @@ -125,7 +135,7 @@ private void parseMap() {
}
ptr.setStart(bitIndexes.peek());
bitIndexes.advance();
while (bitIndexes.hasNext() && buffer[bitIndexes.peek()] != '}' && alreadyProcessedCols < targetParseNum) {
while (bitIndexes.hasNext() && buffer[bitIndexes.peek()] != '}' && alreadyProcessedCols <= targetParseNum) {
parseElement(null);
if (buffer[bitIndexes.peek()] == ',') {
bitIndexes.advance();
Expand All @@ -150,7 +160,7 @@ private void parseList() {
ptr.setStart(bitIndexes.peek());
bitIndexes.advance();
int i = 0;
while (bitIndexes.hasNext() && buffer[bitIndexes.peek()] != ']' && alreadyProcessedCols < targetParseNum) {
while (bitIndexes.hasNext() && buffer[bitIndexes.peek()] != ']' && alreadyProcessedCols <= targetParseNum) {
parseElement("" + i);
if (buffer[bitIndexes.peek()] == ',') {
bitIndexes.advance();
Expand Down Expand Up @@ -198,26 +208,8 @@ private String skip(boolean retainValue) {
bitIndexes.advance();
return retainValue ? new String(buffer, start, end - start + 1) : null;
}
case '"' -> {
bitIndexes.advance();
int realEnd = bitIndexes.peek();
while (realEnd > start) {
if (buffer[--realEnd] == '"') {
break;
}
}
return retainValue ? new String(buffer, start + 1, realEnd - start - 1) : null;
}
default -> {
bitIndexes.advance();
int realEnd = bitIndexes.peek();
while (realEnd >= start) {
--realEnd;
if (buffer[realEnd] >= '0' && buffer[realEnd] <= '9') {
break;
}
}
return retainValue ? new String(buffer, start, realEnd - start + 1) : null;
return parseField();
}
}
}
Expand Down
33 changes: 33 additions & 0 deletions src/test/java/org/simdjson/JsonMultiValueParsingTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package org.simdjson;

import static org.simdjson.testutils.SimdJsonAssertions.assertThat;
import static org.simdjson.testutils.TestUtils.toUtf8;

import org.junit.jupiter.api.Test;

public class JsonMultiValueParsingTest {
@Test
public void test() {
byte[] json = toUtf8("{\"field1\":{\"field2\":\"value2\",\"field3\":3},\"field4\":[\"value4\",\"value5\"],\"field5\":null}");
SimdJsonParser2 parser = new SimdJsonParser2("field1.field2", "field1.field3", "field4", "field4.0", "field5");
String[] result = parser.parse(json, json.length);
assertThat(result[0]).isEqualTo("value2");
assertThat(result[1]).isEqualTo("3");
assertThat(result[2]).isEqualTo("[\"value4\",\"value5\"]");
assertThat(result[3]).isEqualTo("value4");
assertThat(result[4]).isEqualTo(null);
}

@Test
public void testNonAsciiCharacters() {
byte[] json = toUtf8("{\"ąćśńźż\": 1, \"\\u20A9\\u0E3F\": 2, \"αβγ\": 3, \"😀abc😀\": 4}");
SimdJsonParser2 parser = new SimdJsonParser2("ąćśńźż", "\\u20A9\\u0E3F", "αβγ", "😀abc😀");
// when
String[] result = parser.parse(json, json.length);
// then
assertThat(result[0]).isEqualTo("1");
assertThat(result[1]).isEqualTo("2");
assertThat(result[2]).isEqualTo("3");
assertThat(result[3]).isEqualTo("4");
}
}

0 comments on commit 6153921

Please sign in to comment.