Search in sources :

Example 1 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class XMLParserTest method testExitOnError.

@Test
public void testExitOnError() throws Exception {
    XMLParser.Config config = new XMLParser.Config("body", "UTF-8", "title:/book/title,author:/book/author," + "year:/book/year", "title:string,author:int,year:int", "Exit on error");
    try {
        Transform<StructuredRecord, StructuredRecord> transform = new XMLParser(config);
        transform.initialize(new MockTransformContext());
        StructuredRecord inputRecord = StructuredRecord.builder(INPUT).set("offset", 1).set("body", "<book category=\"COOKING\"><title lang=\"en\">Everyday Italian</title>" + "<author>Giada De Laurentiis</author><year>2005</year><price>30.00</price></book>").build();
        MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
        transform.transform(inputRecord, emitter);
        Assert.fail();
    } catch (IllegalStateException e) {
        LOG.error("Test passes. Exiting on exception.", e);
    }
}
Also used : MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockEmitter(io.cdap.cdap.etl.mock.common.MockEmitter) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) Test(org.junit.Test)

Example 2 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class XMLParserTest method testXpathArray.

@Test
public void testXpathArray() throws Exception {
    Schema schema = Schema.recordOf("record", Schema.Field.of("category", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
    XMLParser.Config config = new XMLParser.Config("body", "UTF-8", "category://book/@category,title://book/title", "category:string,title:string", "Exit on error");
    Transform<StructuredRecord, StructuredRecord> transform = new XMLParser(config);
    transform.initialize(new MockTransformContext());
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    StructuredRecord inputRecord = StructuredRecord.builder(INPUT).set("offset", 1).set("body", "<bookstore><book category=\"cooking\"><title lang=\"en\">Everyday Italian</title>" + "<author>Giada De Laurentiis</author><year>2005</year><price>30.00</price></book>" + "<book category=\"children\"><title lang=\"en\">Harry Potter</title><author>J K. Rowling</author>" + "<year>2005</year><price>29.99</price></book></bookstore>").build();
    transform.transform(inputRecord, emitter);
    List<StructuredRecord> expected = ImmutableList.of(StructuredRecord.builder(schema).set("category", "cooking").set("title", "Everyday Italian").build());
    Assert.assertEquals(expected, emitter.getEmitted());
}
Also used : MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockEmitter(io.cdap.cdap.etl.mock.common.MockEmitter) Schema(io.cdap.cdap.api.data.schema.Schema) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) Test(org.junit.Test)

Example 3 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class XMLParserTest method testXpathWithMultipleElements.

@Test
public void testXpathWithMultipleElements() throws Exception {
    Schema schema = Schema.recordOf("record", Schema.Field.of("category", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("price", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), Schema.Field.of("year", Schema.nullableOf(Schema.of(Schema.Type.INT))), Schema.Field.of("subcategory", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
    XMLParser.Config config = new XMLParser.Config("body", "UTF-16 (Unicode with byte-order mark)", "category://book/@category,title://book/title,year:/bookstore/book[price>35.00]/year," + "price:/bookstore/book[price>35.00]/price,subcategory://book/subcategory", "category:string,title:string,price:double,year:int,subcategory:string", "Write to error dataset");
    Transform<StructuredRecord, StructuredRecord> transform = new XMLParser(config);
    transform.initialize(new MockTransformContext());
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    StructuredRecord inputRecord = StructuredRecord.builder(INPUT).set("offset", 1).set("body", "<bookstore><book category=\"cooking\"><subcategory><type>Continental</type></subcategory>" + "<title lang=\"en\">Everyday Italian</title><author>Giada De Laurentiis</author><year>2005</year>" + "<price>30.00</price></book></bookstore>").build();
    transform.transform(inputRecord, emitter);
    List<StructuredRecord> expected = ImmutableList.of(StructuredRecord.builder(schema).set("category", "cooking").set("title", "Everyday Italian").set("subcategory", "<subcategory><type>Continental</type></subcategory>").build());
    Assert.assertEquals(expected, emitter.getEmitted());
    emitter.clear();
    inputRecord = StructuredRecord.builder(INPUT).set("offset", 2).set("body", "<bookstore><book category=\"children\"><subcategory><type>Series</type></subcategory>" + "<title lang=\"en\">Harry Potter</title><author>J K. Rowling</author><year>2005</year><price>49.99</price>" + "</book></bookstore>").build();
    transform.transform(inputRecord, emitter);
    expected = ImmutableList.of(StructuredRecord.builder(schema).set("category", "children").set("title", "Harry Potter").set("price", 49.99d).set("year", 2005).set("subcategory", "<subcategory><type>Series</type></subcategory>").build());
    Assert.assertEquals(expected, emitter.getEmitted());
    emitter.clear();
    inputRecord = StructuredRecord.builder(INPUT).set("offset", 3).set("body", "<bookstore><book category=\"web\"><subcategory><type>Basics</type></subcategory>" + "<title lang=\"en\">Learning XML</title><author>Erik T. Ray</author><year>2003</year><price>39.95</price>" + "</book></bookstore>").build();
    transform.transform(inputRecord, emitter);
    expected = ImmutableList.of(StructuredRecord.builder(schema).set("category", "web").set("title", "Learning XML").set("price", 39.95d).set("year", 2003).set("subcategory", "<subcategory><type>Basics</type></subcategory>").build());
    Assert.assertEquals(expected, emitter.getEmitted());
    emitter.clear();
}
Also used : MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockEmitter(io.cdap.cdap.etl.mock.common.MockEmitter) Schema(io.cdap.cdap.api.data.schema.Schema) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) Test(org.junit.Test)

Example 4 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class XMLParserTest method testXMLParserWithSimpleXPath.

@Test
public void testXMLParserWithSimpleXPath() throws Exception {
    Schema schema = Schema.recordOf("record", Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("author", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("year", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
    XMLParser.Config config = new XMLParser.Config("body", "UTF-8", "title:/book/title,author:/book/author,year:/book/year", "title:string,author:string,year:string", "Write to error dataset");
    Transform<StructuredRecord, StructuredRecord> transform = new XMLParser(config);
    transform.initialize(new MockTransformContext());
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    StructuredRecord inputRecord = StructuredRecord.builder(INPUT).set("offset", 1).set("body", "<book category=\"COOKING\"><title lang=\"en\">Everyday Italian</title>" + "<author>Giada De Laurentiis</author><year>2005</year><price>30.00</price></book>").build();
    transform.transform(inputRecord, emitter);
    List<StructuredRecord> expected = ImmutableList.of(StructuredRecord.builder(schema).set("title", "Everyday Italian").set("author", "Giada De Laurentiis").set("year", "2005").build());
    Assert.assertEquals(expected, emitter.getEmitted());
}
Also used : MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockEmitter(io.cdap.cdap.etl.mock.common.MockEmitter) Schema(io.cdap.cdap.api.data.schema.Schema) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) Test(org.junit.Test)

Example 5 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class CompressorTest method testZipCompress.

@Test
public void testZipCompress() throws Exception {
    Transform<StructuredRecord, StructuredRecord> transform = new Compressor(new Compressor.Config("a:ZIP", OUTPUT.toString()));
    MockTransformContext context = new MockTransformContext();
    transform.initialize(context);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(StructuredRecord.builder(INPUT).set("a", "This is a test for testing zip compression").set("b", "2").set("c", "3").set("d", "4").set("e", "5").build(), emitter);
    byte[] expected = compressZIP("This is a test for testing zip compression".getBytes());
    byte[] actual = emitter.getEmitted().get(0).get("a");
    Assert.assertEquals(2, emitter.getEmitted().get(0).getSchema().getFields().size());
    Assert.assertArrayEquals(expected, actual);
}
Also used : MockTransformContext(io.cdap.cdap.etl.mock.transform.MockTransformContext) MockEmitter(io.cdap.cdap.etl.mock.common.MockEmitter) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) Test(org.junit.Test)

Aggregations

StructuredRecord (io.cdap.cdap.api.data.format.StructuredRecord)59 MockTransformContext (io.cdap.cdap.etl.mock.transform.MockTransformContext)59 Test (org.junit.Test)58 MockEmitter (io.cdap.cdap.etl.mock.common.MockEmitter)52 TransformContext (io.cdap.cdap.etl.api.TransformContext)35 Schema (io.cdap.cdap.api.data.schema.Schema)27 LookupConfig (io.cdap.cdap.etl.api.LookupConfig)7 LookupTableConfig (io.cdap.cdap.etl.api.LookupTableConfig)7 ValidationException (io.cdap.cdap.etl.api.validation.ValidationException)4 Base32 (org.apache.commons.codec.binary.Base32)4 Base64 (org.apache.commons.codec.binary.Base64)4 Cause (io.cdap.cdap.etl.api.validation.ValidationFailure.Cause)3 MockMultiOutputEmitter (io.cdap.cdap.etl.mock.common.MockMultiOutputEmitter)3 MockPipelineConfigurer (io.cdap.cdap.etl.mock.common.MockPipelineConfigurer)3 HashMap (java.util.HashMap)2 Hex (org.apache.commons.codec.binary.Hex)2 ImmutableList (com.google.common.collect.ImmutableList)1 MockLookupProvider (io.cdap.cdap.etl.mock.common.MockLookupProvider)1 List (java.util.List)1