-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[test] Clean up integration test schema setup (#633)
This PR does NOT change any logic in main code. It tries to become the first step to refactor & cleanup test code and make it easier to add integration tests & unit tests in the future. The change in this PR includes: (1) Use avsc file to replace loosely structured schema string in test classes (2) Create a simple input file schema builder to construct Avro input schema. (3) Clean up some data generator method to make it easier to use. Note for reviewer: The new changes lies in: (1) PushInputSchemaBuilder.java (New simple test schema builder) (2) TestWriteUtils.java (Parse schema from .avsc files, change some data gen methods) (3) Move some schema strings into venice-test-common/src/integrationTest/resources folder.
- Loading branch information
1 parent
5684066
commit 90f4631
Showing
70 changed files
with
909 additions
and
828 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 8 additions & 10 deletions
18
...venice-common/src/test/java/com/linkedin/venice/etl/ETLValueSchemaTransformationTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,33 +1,31 @@ | ||
package com.linkedin.venice.etl; | ||
|
||
import static com.linkedin.venice.utils.TestWriteUtils.ETL_UNION_VALUE_SCHEMA_STRING_WITHOUT_NULL; | ||
import static com.linkedin.venice.utils.TestWriteUtils.ETL_UNION_VALUE_SCHEMA_STRING_WITH_NULL; | ||
import static com.linkedin.venice.utils.TestWriteUtils.ETL_VALUE_SCHEMA_STRING; | ||
import static com.linkedin.venice.utils.TestWriteUtils.ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA; | ||
import static com.linkedin.venice.utils.TestWriteUtils.ETL_UNION_VALUE_WITH_NULL_SCHEMA; | ||
import static com.linkedin.venice.utils.TestWriteUtils.ETL_VALUE_SCHEMA; | ||
|
||
import org.apache.avro.Schema; | ||
import org.testng.Assert; | ||
import org.testng.annotations.Test; | ||
|
||
|
||
public class ETLValueSchemaTransformationTest { | ||
@Test | ||
public void testRecordSchemaBecomesUnionWithNull() { | ||
Schema valueSchema = Schema.parse(ETL_VALUE_SCHEMA_STRING); | ||
ETLValueSchemaTransformation transformation = ETLValueSchemaTransformation.fromSchema(valueSchema); | ||
ETLValueSchemaTransformation transformation = ETLValueSchemaTransformation.fromSchema(ETL_VALUE_SCHEMA); | ||
Assert.assertEquals(transformation, ETLValueSchemaTransformation.UNIONIZE_WITH_NULL); | ||
} | ||
|
||
@Test | ||
public void testUnionSchemaWithoutNullAddsNull() { | ||
Schema valueSchema = Schema.parse(ETL_UNION_VALUE_SCHEMA_STRING_WITHOUT_NULL); | ||
ETLValueSchemaTransformation transformation = ETLValueSchemaTransformation.fromSchema(valueSchema); | ||
ETLValueSchemaTransformation transformation = | ||
ETLValueSchemaTransformation.fromSchema(ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA); | ||
Assert.assertEquals(transformation, ETLValueSchemaTransformation.ADD_NULL_TO_UNION); | ||
} | ||
|
||
@Test | ||
public void testUnionSchemaWithNullStaysUnchanged() { | ||
Schema valueSchema = Schema.parse(ETL_UNION_VALUE_SCHEMA_STRING_WITH_NULL); | ||
ETLValueSchemaTransformation transformation = ETLValueSchemaTransformation.fromSchema(valueSchema); | ||
ETLValueSchemaTransformation transformation = | ||
ETLValueSchemaTransformation.fromSchema(ETL_UNION_VALUE_WITH_NULL_SCHEMA); | ||
Assert.assertEquals(transformation, ETLValueSchemaTransformation.NONE); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.