Search in sources :

Example 36 with TransformContext

use of io.cdap.cdap.etl.api.TransformContext in project hydrator-plugins by cdapio.

the class JSONParserTest method testInvalidJsonPathForNonNullableSchema.

@Test
public void testInvalidJsonPathForNonNullableSchema() throws Exception {
    final String[] jsonPaths = { "expensive:$.expensive", "bicycle_color:$.store.bicycle.color", "bicycle_price:$.store.bicycle.price", "window:$.store.window" };
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    JSONParser.Config config = new JSONParser.Config("body", Joiner.on(",").join(jsonPaths), OUTPUT4.toString());
    Transform<StructuredRecord, StructuredRecord> transform = new JSONParser(config);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(INPUT1);
    transform.configurePipeline(mockPipelineConfigurer);
    TransformContext context = new MockTransformContext();
    transform.initialize(context);
    transform.transform(StructuredRecord.builder(INPUT1).set("body", json).build(), emitter);
    Assert.assertEquals(0, emitter.getEmitted().size());
}
Also used : MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockPipelineConfigurer(io.cdap.cdap.etl.mock.common.MockPipelineConfigurer) TransformContext(io.cdap.cdap.etl.api.TransformContext) MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockEmitter(io.cdap.cdap.etl.mock.common.MockEmitter) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) Test(org.junit.Test)

Example 37 with TransformContext

use of io.cdap.cdap.etl.api.TransformContext in project hydrator-plugins by cdapio.

the class UnionSplitterTest method testSplitOnField.

@Test
public void testSplitOnField() throws Exception {
    Schema rec1Schema = Schema.recordOf("rec1", Schema.Field.of("x", Schema.of(Schema.Type.INT)), Schema.Field.of("y", Schema.of(Schema.Type.INT)));
    Schema rec2Schema = Schema.recordOf("rec2", Schema.Field.of("y", Schema.of(Schema.Type.INT)), Schema.Field.of("z", Schema.of(Schema.Type.INT)));
    StructuredRecord rec1 = StructuredRecord.builder(rec1Schema).set("x", 0).set("y", 1).build();
    StructuredRecord rec2 = StructuredRecord.builder(rec2Schema).set("y", 2).set("z", 3).build();
    Schema inputSchema = Schema.recordOf("union", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.unionOf(Schema.of(Schema.Type.NULL), Schema.of(Schema.Type.BOOLEAN), Schema.of(Schema.Type.INT), Schema.of(Schema.Type.LONG), Schema.of(Schema.Type.FLOAT), Schema.of(Schema.Type.DOUBLE), Schema.of(Schema.Type.STRING), rec1Schema, rec2Schema)));
    Schema nullSchema = Schema.recordOf("union.null", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.NULL)));
    Schema boolSchema = Schema.recordOf("union.boolean", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.BOOLEAN)));
    Schema intSchema = Schema.recordOf("union.int", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.INT)));
    Schema longSchema = Schema.recordOf("union.long", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.LONG)));
    Schema floatSchema = Schema.recordOf("union.float", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.FLOAT)));
    Schema doubleSchema = Schema.recordOf("union.double", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.DOUBLE)));
    Schema stringSchema = Schema.recordOf("union.string", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.STRING)));
    Schema withRec1Schema = Schema.recordOf("union.rec1", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", rec1Schema));
    Schema withRec2Schema = Schema.recordOf("union.rec2", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", rec2Schema));
    UnionSplitter unionSplitter = new UnionSplitter(new UnionSplitter.Conf("b", true));
    TransformContext context = new MockTransformContext();
    unionSplitter.initialize(context);
    MockMultiOutputEmitter<StructuredRecord> mockEmitter = new MockMultiOutputEmitter<>();
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", null).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", true).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", new byte[] { 5 }).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5L).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5.5f).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5.5d).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", "5").build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", rec1).build(), mockEmitter);
    unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", rec2).build(), mockEmitter);
    Map<String, List<StructuredRecord>> expected = new HashMap<>();
    expected.put("null", ImmutableList.of(StructuredRecord.builder(nullSchema).set("a", 0L).set("b", null).build()));
    expected.put("boolean", ImmutableList.of(StructuredRecord.builder(boolSchema).set("a", 0L).set("b", true).build()));
    expected.put("int", ImmutableList.of(StructuredRecord.builder(intSchema).set("a", 0L).set("b", 5).build()));
    expected.put("long", ImmutableList.of(StructuredRecord.builder(longSchema).set("a", 0L).set("b", 5L).build()));
    expected.put("float", ImmutableList.of(StructuredRecord.builder(floatSchema).set("a", 0L).set("b", 5.5f).build()));
    expected.put("double", ImmutableList.of(StructuredRecord.builder(doubleSchema).set("a", 0L).set("b", 5.5d).build()));
    expected.put("string", ImmutableList.of(StructuredRecord.builder(stringSchema).set("a", 0L).set("b", "5").build()));
    expected.put("rec1", ImmutableList.of(StructuredRecord.builder(withRec1Schema).set("a", 0L).set("b", rec1).build()));
    expected.put("rec2", ImmutableList.of(StructuredRecord.builder(withRec2Schema).set("a", 0L).set("b", rec2).build()));
    Map<String, List<Object>> actual = mockEmitter.getEmitted();
    Assert.assertEquals(expected, actual);
    Assert.assertEquals(0, context.getFailureCollector().getValidationFailures().size());
}
Also used : MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) HashMap(java.util.HashMap) Schema(io.cdap.cdap.api.data.schema.Schema) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) TransformContext(io.cdap.cdap.etl.api.TransformContext) MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockMultiOutputEmitter(io.cdap.cdap.etl.mock.common.MockMultiOutputEmitter) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Test(org.junit.Test)

Aggregations

TransformContext (io.cdap.cdap.etl.api.TransformContext)37 StructuredRecord (io.cdap.cdap.api.data.format.StructuredRecord)35 MockTransformContext (io.cdap.cdap.etl.mock.transform.MockTransformContext)35 Test (org.junit.Test)34 MockEmitter (io.cdap.cdap.etl.mock.common.MockEmitter)30 Schema (io.cdap.cdap.api.data.schema.Schema)15 ValidationException (io.cdap.cdap.etl.api.validation.ValidationException)4 Base32 (org.apache.commons.codec.binary.Base32)4 Base64 (org.apache.commons.codec.binary.Base64)4 Cause (io.cdap.cdap.etl.api.validation.ValidationFailure.Cause)3 MockPipelineConfigurer (io.cdap.cdap.etl.mock.common.MockPipelineConfigurer)3 StageMetrics (io.cdap.cdap.etl.api.StageMetrics)2 DefaultStageMetrics (io.cdap.cdap.etl.common.DefaultStageMetrics)2 NoopStageStatisticsCollector (io.cdap.cdap.etl.common.NoopStageStatisticsCollector)2 StageStatisticsCollector (io.cdap.cdap.etl.common.StageStatisticsCollector)2 TrackedMultiOutputTransform (io.cdap.cdap.etl.common.TrackedMultiOutputTransform)2 Hex (org.apache.commons.codec.binary.Hex)2 ImmutableList (com.google.common.collect.ImmutableList)1 MockMultiOutputEmitter (io.cdap.cdap.etl.mock.common.MockMultiOutputEmitter)1 HashMap (java.util.HashMap)1