Examples with MockEmitter io.cdap.cdap.etl.mock.common.MockEmitter used on opensource projects

Example 16 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class DecompressorTest method testZipCompress.

@Test
public void testZipCompress() throws Exception {
    String decompressTester = "This is a test for testing zip compression";
    Transform<StructuredRecord, StructuredRecord> transform = new Decompressor(new Decompressor.Config("a:ZIP", OUTPUT.toString()));
    MockTransformContext context = new MockTransformContext();
    transform.initialize(context);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    byte[] compressed = zip(decompressTester.getBytes());
    transform.transform(StructuredRecord.builder(INPUT).set("a", compressed).set("b", "2").set("c", "3").set("d", "4").set("e", "5").build(), emitter);
    String actual = emitter.getEmitted().get(0).get("a");
    Assert.assertEquals(2, emitter.getEmitted().get(0).getSchema().getFields().size());
    Assert.assertEquals(decompressTester, actual);
}

Example 17 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class DecompressorTest method testSnappyCompress.

@Test
public void testSnappyCompress() throws Exception {
    String decompressTester = "This is a test for testing snappy compression";
    Transform<StructuredRecord, StructuredRecord> transform = new Decompressor(new Decompressor.Config("a:SNAPPY", OUTPUT.toString()));
    MockTransformContext context = new MockTransformContext();
    transform.initialize(context);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    byte[] compressed = Snappy.compress(decompressTester.getBytes());
    transform.transform(StructuredRecord.builder(INPUT).set("a", compressed).set("b", "2").set("c", "3").set("d", "4").set("e", "5").build(), emitter);
    String actual = emitter.getEmitted().get(0).get("a");
    Assert.assertEquals(2, emitter.getEmitted().get(0).getSchema().getFields().size());
    Assert.assertEquals(decompressTester, actual);
}

Example 18 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class JavaScriptTransformTest method testSimple.

@Test
public void testSimple() throws Exception {
    JavaScriptTransform.Config config = new JavaScriptTransform.Config("function transform(x, emitter, context) { x.intField = x.intField * 1024; emitter.emit(x); }", null, null);
    Transform<StructuredRecord, StructuredRecord> transform = new JavaScriptTransform(config);
    transform.initialize(new MockTransformContext());
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(RECORD1, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    // check record1
    Assert.assertEquals(SCHEMA, output.getSchema());
    Assert.assertTrue(output.get("booleanField"));
    Assert.assertEquals(28 * 1024, output.<Integer>get("intField").intValue());
    Assert.assertEquals(99L, output.<Long>get("longField").longValue());
    Assert.assertTrue(Math.abs(2.71f - (Float) output.get("floatField")) < 0.000001);
    Assert.assertTrue(Math.abs(3.14 - (Double) output.get("doubleField")) < 0.000001);
    Assert.assertArrayEquals(Bytes.toBytes("foo"), (byte[]) output.get("bytesField"));
    Assert.assertEquals("bar", output.get("stringField"));
    Assert.assertEquals("baz", output.get("nullableField"));
    Assert.assertEquals("hello", output.get("unionField"));
    Map<String, Integer> expectedMapField = ImmutableMap.of("foo", 13, "bar", 17);
    List<String> expectedListField = ImmutableList.of("foo", "bar", "baz");
    Assert.assertEquals(expectedMapField, output.get("mapField"));
    Assert.assertEquals(expectedListField, output.get("arrayField"));
    emitter.clear();
    // check record2
    transform.transform(RECORD2, emitter);
    output = emitter.getEmitted().get(0);
    Assert.assertEquals(SCHEMA, output.getSchema());
    Assert.assertFalse((Boolean) output.get("booleanField"));
    Assert.assertEquals(-28 * 1024, output.<Integer>get("intField").intValue());
    Assert.assertEquals(-99L, output.<Long>get("longField").longValue());
    Assert.assertTrue(Math.abs(-2.71f - (Float) output.get("floatField")) < 0.000001);
    Assert.assertTrue(Math.abs(-3.14 - (Double) output.get("doubleField")) < 0.000001);
    Assert.assertArrayEquals(Bytes.toBytes("hello"), (byte[]) output.get("bytesField"));
    Assert.assertEquals("world", output.get("stringField"));
    Assert.assertNull(output.get("nullableField"));
    Assert.assertEquals(3, output.<Integer>get("unionField").intValue());
    expectedMapField = ImmutableMap.of();
    expectedListField = ImmutableList.of();
    Assert.assertEquals(expectedMapField, output.get("mapField"));
    Assert.assertEquals(expectedListField, output.get("arrayField"));
}

Example 19 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class JavaScriptTransformTest method testEmitErrors.

@Test
public void testEmitErrors() throws Exception {
    Schema inputSchema = Schema.recordOf("smallerSchema", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    Schema outputSchema = Schema.recordOf("smallerSchema", Schema.Field.of("x", Schema.of(Schema.Type.INT)), Schema.Field.of("y", Schema.of(Schema.Type.INT)));
    JavaScriptTransform.Config config = new JavaScriptTransform.Config("function transform(input, emitter, context) { " + "emitter.emit({\"x\":input['x'], \"y\":input['x']+5});" + "emitter.emitError({\"errorCode\":31, \"errorMsg\":\"error!\", \"invalidRecord\": input});" + " }", outputSchema.toString(), null);
    JavaScriptTransform transform = new JavaScriptTransform(config);
    transform.initialize(new MockTransformContext());
    transform.setErrorSchema(inputSchema);
    StructuredRecord inputRecord = StructuredRecord.builder(inputSchema).set("x", 25).build();
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(inputRecord, emitter);
    Assert.assertEquals(1, emitter.getEmitted().size());
    Assert.assertEquals(1, emitter.getErrors().size());
    InvalidEntry<StructuredRecord> invalidEntry = emitter.getErrors().get(0);
    Assert.assertEquals(31, invalidEntry.getErrorCode());
    Assert.assertEquals("error!", invalidEntry.getErrorMsg());
    Assert.assertEquals(25, invalidEntry.getInvalidRecord().<Integer>get("x").intValue());
    Assert.assertNull(invalidEntry.getInvalidRecord().get("y"));
}

Example 20 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class JavaScriptTransformTest method testArguments.

@Test
public void testArguments() throws Exception {
    Schema schema = Schema.recordOf("x", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    JavaScriptTransform.Config config = new JavaScriptTransform.Config("function transform(input, emitter, context) { " + "var multiplier = context.getArguments().get('multiplier');" + "emitter.emit({\"x\": input.x * multiplier});" + " }", schema.toString(), null);
    JavaScriptTransform transform = new JavaScriptTransform(config);
    MockTransformContext transformContext = new MockTransformContext("stage", ImmutableMap.of("multiplier", "5"));
    transform.initialize(transformContext);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(StructuredRecord.builder(schema).set("x", 2).build(), emitter);
    Assert.assertEquals(ImmutableList.of(StructuredRecord.builder(schema).set("x", 10).build()), emitter.getEmitted());
}