Examples with MockEmitter io.cdap.cdap.etl.mock.common.MockEmitter used on opensource projects

Example 46 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class LogParserTransformTest method testS3LogTransform.

@Test
public void testS3LogTransform() throws Exception {
    StructuredRecord botRecord = StructuredRecord.builder(STRING_SCHEMA).set("body", "13a9f69e4a00effd6b4b891dcbcabef632ef9a9da7c localhost " + "[22/Jan/2015:11:03:21 +0000] 122.122.111.11 - 6006CA0AE4 REST.GET.OBJECT " + "ubuntu/this/is/some/folder " + "\"GET /my/uri.gif releases/dists/precise/releases/i18n/Translation-en HTTP/1.1\" " + "403 AccessDenied 231 - 10 - \"-\" \"Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.  17)\" -").build();
    StructuredRecord browserRecord = StructuredRecord.builder(STRING_SCHEMA).set("body", "13a9f69e4a00effd6b4b891dcef632ef9afe38cc8b0 localhost " + "[31/Jan/2015:21:57:57 +0000] 133.133.133.133 - 0E94306589 REST.GET.OBJECT " + "downloads/this/is/another/folder/with/a/file/file.zip " + "\"GET /my/uri.jpg HTTP/1.1\" 304 - - 195750039 198 - " + "\"-\" \"Mozilla/5.0 Gecko/20100115 Firefox/3.6\" -").build();
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    S3_TRANSFORM.transform(botRecord, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    Assert.assertEquals("/my/uri.gif", output.get("uri"));
    Assert.assertEquals("122.122.111.11", output.get("ip"));
    Assert.assertEquals("unknown", output.get("browser"));
    Assert.assertEquals("", output.get("device"));
    Assert.assertEquals(403, output.<Integer>get("httpStatus").intValue());
    Assert.assertEquals(1421924601000L, output.<Long>get("ts").longValue());
    S3_TRANSFORM.transform(browserRecord, emitter);
    output = emitter.getEmitted().get(1);
    Assert.assertEquals("/my/uri.jpg", output.get("uri"));
    Assert.assertEquals("133.133.133.133", output.get("ip"));
    Assert.assertEquals("Firefox", output.get("browser"));
    Assert.assertEquals("Personal computer", output.get("device"));
    Assert.assertEquals(304, output.<Integer>get("httpStatus").intValue());
    Assert.assertEquals(1422741477000L, output.<Long>get("ts").longValue());
}

Example 47 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class CSVParserTest method testNullableFields.

@Test
public void testNullableFields() throws Exception {
    Schema schema = Schema.recordOf("nullables", Schema.Field.of("int", Schema.nullableOf(Schema.of(Schema.Type.INT))), Schema.Field.of("long", Schema.nullableOf(Schema.of(Schema.Type.LONG))), Schema.Field.of("float", Schema.nullableOf(Schema.of(Schema.Type.FLOAT))), Schema.Field.of("double", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), Schema.Field.of("bool", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))), Schema.Field.of("string", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
    CSVParser.Config config = new CSVParser.Config("DEFAULT", null, "body", schema.toString());
    Transform<StructuredRecord, StructuredRecord> transform = new CSVParser(config);
    transform.initialize(null);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(StructuredRecord.builder(INPUT1).set("body", "1,2,3,4,true,abc").build(), emitter);
    StructuredRecord expected = StructuredRecord.builder(schema).set("int", 1).set("long", 2L).set("float", 3f).set("double", 4d).set("bool", true).set("string", "abc").build();
    Assert.assertEquals(1, emitter.getEmitted().size());
    Assert.assertEquals(expected, emitter.getEmitted().get(0));
    emitter.clear();
    transform.transform(StructuredRecord.builder(INPUT1).set("body", ",,,,,").build(), emitter);
    expected = StructuredRecord.builder(schema).set("int", null).set("long", null).set("float", null).set("double", null).set("bool", null).set("string", "").build();
    Assert.assertEquals(1, emitter.getEmitted().size());
    Assert.assertEquals(expected, emitter.getEmitted().get(0));
}

Example 48 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class CSVParserTest method testPassThrough.

@Test
public void testPassThrough() throws Exception {
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    CSVParser.Config config = new CSVParser.Config("DEFAULT", null, "body", OUTPUT3.toString());
    Transform<StructuredRecord, StructuredRecord> transform = new CSVParser(config);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(INPUT2);
    transform.configurePipeline(mockPipelineConfigurer);
    transform.initialize(null);
    transform.transform(StructuredRecord.builder(INPUT2).set("body", "10,stringA,3,4.32,true").set("offset", 10).build(), emitter);
    Assert.assertEquals(10L, emitter.getEmitted().get(0).<Long>get("a").longValue());
    Assert.assertEquals("stringA", emitter.getEmitted().get(0).get("b"));
    Assert.assertEquals(3, emitter.getEmitted().get(0).<Integer>get("c").intValue());
    Assert.assertEquals(4.32d, emitter.getEmitted().get(0).get("d"), 0.0001d);
    Assert.assertEquals(true, emitter.getEmitted().get(0).get("e"));
    // Pass through from input.
    Assert.assertEquals(10, emitter.getEmitted().get(0).<Integer>get("offset").intValue());
}

Example 49 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class CSVParserTest method testEmitErrors.

@Test
public void testEmitErrors() throws Exception {
    CSVParser.Config config = new CSVParser.Config("DEFAULT", null, "body", OUTPUT5.toString());
    Transform<StructuredRecord, StructuredRecord> transform = new CSVParser(config);
    transform.initialize(new MockTransformContext());
    StructuredRecord inputRecord = StructuredRecord.builder(INPUT2).set("offset", 1).set("body", "0,\"020\"1,\"BS:12345  ORDER:111\"4").build();
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(inputRecord, emitter);
    Assert.assertEquals(0, emitter.getEmitted().size());
    Assert.assertEquals(1, emitter.getErrors().size());
    InvalidEntry<StructuredRecord> invalidEntry = emitter.getErrors().get(0);
    Assert.assertEquals(31, invalidEntry.getErrorCode());
    Assert.assertEquals("offset", 1, invalidEntry.getInvalidRecord().<Integer>get("offset").intValue());
    Assert.assertEquals("body", "0,\"020\"1,\"BS:12345  ORDER:111\"4", invalidEntry.getInvalidRecord().get("body"));
}

Example 50 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class CompressorTest method testGZIPCompress.

@Test
public void testGZIPCompress() throws Exception {
    Transform<StructuredRecord, StructuredRecord> transform = new Compressor(new Compressor.Config("a:GZIP", OUTPUT.toString()));
    MockTransformContext context = new MockTransformContext();
    transform.initialize(context);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(StructuredRecord.builder(INPUT).set("a", "This is a test for testing gzip compression").set("b", "2").set("c", "3").set("d", "4").set("e", "5").build(), emitter);
    byte[] expected = compressGZIP("This is a test for testing gzip compression".getBytes());
    byte[] actual = emitter.getEmitted().get(0).get("a");
    Assert.assertEquals(2, emitter.getEmitted().get(0).getSchema().getFields().size());
    Assert.assertArrayEquals(expected, actual);
}