use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.
the class XMLParserTest method testEmitErrors.
@Test
public void testEmitErrors() throws Exception {
XMLParser.Config config = new XMLParser.Config("body", "UTF-8", "title:/book/title,author:/book/author," + "year:/book/year", "title:string,author:int,year:int", "Write to error dataset");
Transform<StructuredRecord, StructuredRecord> transform = new XMLParser(config);
transform.initialize(new MockTransformContext());
StructuredRecord inputRecord = StructuredRecord.builder(INPUT).set("offset", 1).set("body", "<book category=\"COOKING\"><title lang=\"en\">Everyday Italian</title>" + "<author>Giada De Laurentiis</author><year>2005</year><price>30.00</price></book>").build();
MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
transform.transform(inputRecord, emitter);
Assert.assertEquals(0, emitter.getEmitted().size());
Assert.assertEquals(1, emitter.getErrors().size());
InvalidEntry<StructuredRecord> invalidEntry = emitter.getErrors().get(0);
Assert.assertEquals(31, invalidEntry.getErrorCode());
Assert.assertEquals("offset", 1, invalidEntry.getInvalidRecord().<Integer>get("offset").intValue());
Assert.assertEquals("body", "<book category=\"COOKING\"><title lang=\"en\">Everyday Italian</title><author>Giada " + "De Laurentiis</author><year>2005</year><price>30.00</price>" + "</book>", invalidEntry.getInvalidRecord().get("body"));
}
use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.
the class NullFieldSplitterTest method testSameSchemaSplit.
@Test
public void testSameSchemaSplit() throws Exception {
Schema schema = Schema.recordOf("test", Schema.Field.of("x", Schema.nullableOf(Schema.of(Schema.Type.LONG))), Schema.Field.of("y", Schema.nullableOf(Schema.of(Schema.Type.INT))), Schema.Field.of("z", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
NullFieldSplitter nullFieldSplitter = new NullFieldSplitter(new NullFieldSplitter.Conf("z", false));
nullFieldSplitter.initialize(new MockTransformContext());
MockMultiOutputEmitter<StructuredRecord> mockEmitter = new MockMultiOutputEmitter<>();
// test non-null
nullFieldSplitter.transform(StructuredRecord.builder(schema).set("x", 0L).set("z", "abc").build(), mockEmitter);
Assert.assertEquals(ImmutableMap.of(NullFieldSplitter.NON_NULL_PORT, ImmutableList.of(StructuredRecord.builder(schema).set("x", 0L).set("z", "abc").build())), mockEmitter.getEmitted());
// test null
mockEmitter.clear();
nullFieldSplitter.transform(StructuredRecord.builder(schema).set("y", 5).build(), mockEmitter);
Assert.assertEquals(ImmutableMap.of(NullFieldSplitter.NULL_PORT, ImmutableList.of(StructuredRecord.builder(schema).set("y", 5).build())), mockEmitter.getEmitted());
}
use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.
the class NullFieldSplitterTest method testModifySchemaSplit.
@Test
public void testModifySchemaSplit() throws Exception {
Schema nullSchema = Schema.recordOf("test", Schema.Field.of("x", Schema.nullableOf(Schema.of(Schema.Type.LONG))), Schema.Field.of("y", Schema.nullableOf(Schema.of(Schema.Type.INT))), Schema.Field.of("z", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
Schema nonNullSchema = Schema.recordOf("test.nonnull", Schema.Field.of("x", Schema.nullableOf(Schema.of(Schema.Type.LONG))), Schema.Field.of("y", Schema.nullableOf(Schema.of(Schema.Type.INT))), Schema.Field.of("z", Schema.of(Schema.Type.STRING)));
NullFieldSplitter nullFieldSplitter = new NullFieldSplitter(new NullFieldSplitter.Conf("z", true));
nullFieldSplitter.initialize(new MockTransformContext());
MockMultiOutputEmitter<StructuredRecord> mockEmitter = new MockMultiOutputEmitter<>();
// test non-null
nullFieldSplitter.transform(StructuredRecord.builder(nullSchema).set("x", 0L).set("z", "abc").build(), mockEmitter);
Assert.assertEquals(ImmutableMap.of(NullFieldSplitter.NON_NULL_PORT, ImmutableList.of(StructuredRecord.builder(nonNullSchema).set("x", 0L).set("z", "abc").build())), mockEmitter.getEmitted());
// test null
mockEmitter.clear();
nullFieldSplitter.transform(StructuredRecord.builder(nullSchema).set("y", 5).build(), mockEmitter);
Assert.assertEquals(ImmutableMap.of(NullFieldSplitter.NULL_PORT, ImmutableList.of(StructuredRecord.builder(nullSchema).set("y", 5).build())), mockEmitter.getEmitted());
}
use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.
the class UnionSplitterTest method testSplitOnField.
@Test
public void testSplitOnField() throws Exception {
Schema rec1Schema = Schema.recordOf("rec1", Schema.Field.of("x", Schema.of(Schema.Type.INT)), Schema.Field.of("y", Schema.of(Schema.Type.INT)));
Schema rec2Schema = Schema.recordOf("rec2", Schema.Field.of("y", Schema.of(Schema.Type.INT)), Schema.Field.of("z", Schema.of(Schema.Type.INT)));
StructuredRecord rec1 = StructuredRecord.builder(rec1Schema).set("x", 0).set("y", 1).build();
StructuredRecord rec2 = StructuredRecord.builder(rec2Schema).set("y", 2).set("z", 3).build();
Schema inputSchema = Schema.recordOf("union", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.unionOf(Schema.of(Schema.Type.NULL), Schema.of(Schema.Type.BOOLEAN), Schema.of(Schema.Type.INT), Schema.of(Schema.Type.LONG), Schema.of(Schema.Type.FLOAT), Schema.of(Schema.Type.DOUBLE), Schema.of(Schema.Type.STRING), rec1Schema, rec2Schema)));
Schema nullSchema = Schema.recordOf("union.null", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.NULL)));
Schema boolSchema = Schema.recordOf("union.boolean", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.BOOLEAN)));
Schema intSchema = Schema.recordOf("union.int", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.INT)));
Schema longSchema = Schema.recordOf("union.long", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.LONG)));
Schema floatSchema = Schema.recordOf("union.float", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.FLOAT)));
Schema doubleSchema = Schema.recordOf("union.double", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.DOUBLE)));
Schema stringSchema = Schema.recordOf("union.string", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", Schema.of(Schema.Type.STRING)));
Schema withRec1Schema = Schema.recordOf("union.rec1", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", rec1Schema));
Schema withRec2Schema = Schema.recordOf("union.rec2", Schema.Field.of("a", Schema.of(Schema.Type.LONG)), Schema.Field.of("b", rec2Schema));
UnionSplitter unionSplitter = new UnionSplitter(new UnionSplitter.Conf("b", true));
TransformContext context = new MockTransformContext();
unionSplitter.initialize(context);
MockMultiOutputEmitter<StructuredRecord> mockEmitter = new MockMultiOutputEmitter<>();
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", null).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", true).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", new byte[] { 5 }).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5L).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5.5f).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", 5.5d).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", "5").build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", rec1).build(), mockEmitter);
unionSplitter.transform(StructuredRecord.builder(inputSchema).set("a", 0L).set("b", rec2).build(), mockEmitter);
Map<String, List<StructuredRecord>> expected = new HashMap<>();
expected.put("null", ImmutableList.of(StructuredRecord.builder(nullSchema).set("a", 0L).set("b", null).build()));
expected.put("boolean", ImmutableList.of(StructuredRecord.builder(boolSchema).set("a", 0L).set("b", true).build()));
expected.put("int", ImmutableList.of(StructuredRecord.builder(intSchema).set("a", 0L).set("b", 5).build()));
expected.put("long", ImmutableList.of(StructuredRecord.builder(longSchema).set("a", 0L).set("b", 5L).build()));
expected.put("float", ImmutableList.of(StructuredRecord.builder(floatSchema).set("a", 0L).set("b", 5.5f).build()));
expected.put("double", ImmutableList.of(StructuredRecord.builder(doubleSchema).set("a", 0L).set("b", 5.5d).build()));
expected.put("string", ImmutableList.of(StructuredRecord.builder(stringSchema).set("a", 0L).set("b", "5").build()));
expected.put("rec1", ImmutableList.of(StructuredRecord.builder(withRec1Schema).set("a", 0L).set("b", rec1).build()));
expected.put("rec2", ImmutableList.of(StructuredRecord.builder(withRec2Schema).set("a", 0L).set("b", rec2).build()));
Map<String, List<Object>> actual = mockEmitter.getEmitted();
Assert.assertEquals(expected, actual);
Assert.assertEquals(0, context.getFailureCollector().getValidationFailures().size());
}
Aggregations