Examples with MockEmitter io.cdap.cdap.etl.mock.common.MockEmitter used on opensource projects

Example 41 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class ProjectionTransformTest method testConvertToFloat.

@Test
public void testConvertToFloat() throws Exception {
    ProjectionTransform.ProjectionTransformConfig config = new ProjectionTransform.ProjectionTransformConfig(null, null, "intField:float,longField:float", null);
    Transform<StructuredRecord, StructuredRecord> transform = new ProjectionTransform(config);
    TransformContext transformContext = new MockTransformContext();
    transform.initialize(transformContext);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(SIMPLE_TYPES_RECORD, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    Schema expectedSchema = Schema.recordOf("record.projected", Schema.Field.of("booleanField", Schema.of(Schema.Type.BOOLEAN)), Schema.Field.of("intField", Schema.of(Schema.Type.FLOAT)), Schema.Field.of("longField", Schema.of(Schema.Type.FLOAT)), Schema.Field.of("floatField", Schema.of(Schema.Type.FLOAT)), Schema.Field.of("doubleField", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("bytesField", Schema.of(Schema.Type.BYTES)), Schema.Field.of("stringField", Schema.of(Schema.Type.STRING)));
    Assert.assertEquals(expectedSchema, output.getSchema());
    Assert.assertTrue((Boolean) output.get("booleanField"));
    Assert.assertEquals(28f, output.get("intField"), 0.0001f);
    Assert.assertEquals(99f, output.get("longField"), 0.0001f);
    Assert.assertTrue(Math.abs(2.71f - (Float) output.get("floatField")) < 0.000001);
    Assert.assertTrue(Math.abs(3.14 - (Double) output.get("doubleField")) < 0.000001);
    Assert.assertArrayEquals(Bytes.toBytes("foo"), (byte[]) output.get("bytesField"));
    Assert.assertEquals("bar", output.get("stringField"));
}

Example 42 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class ProjectionTransformTest method testDropFields.

@Test
public void testDropFields() throws Exception {
    Schema schema = Schema.recordOf("three", Schema.Field.of("x", Schema.of(Schema.Type.INT)), Schema.Field.of("y", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("z", Schema.arrayOf(Schema.of(Schema.Type.INT))));
    StructuredRecord input = StructuredRecord.builder(schema).set("x", 1).set("y", 3.14).set("z", new int[] { 1, 2, 3 }).build();
    ProjectionTransform.ProjectionTransformConfig config = new ProjectionTransform.ProjectionTransformConfig("y, z", null, null, null);
    Transform<StructuredRecord, StructuredRecord> transform = new ProjectionTransform(config);
    TransformContext transformContext = new MockTransformContext();
    transform.initialize(transformContext);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(input, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    Schema expectedSchema = Schema.recordOf("three.projected", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    Assert.assertEquals(expectedSchema, output.getSchema());
    Assert.assertEquals(1, output.<Integer>get("x").intValue());
}

Example 43 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class ProjectionTransformTest method testConvertToLong.

@Test
public void testConvertToLong() throws Exception {
    ProjectionTransform.ProjectionTransformConfig config = new ProjectionTransform.ProjectionTransformConfig(null, null, "intField:long,floatField:long,doubleField:long", null);
    Transform<StructuredRecord, StructuredRecord> transform = new ProjectionTransform(config);
    TransformContext transformContext = new MockTransformContext();
    transform.initialize(transformContext);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(SIMPLE_TYPES_RECORD, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    Schema expectedSchema = Schema.recordOf("record.projected", Schema.Field.of("booleanField", Schema.of(Schema.Type.BOOLEAN)), Schema.Field.of("intField", Schema.of(Schema.Type.LONG)), Schema.Field.of("longField", Schema.of(Schema.Type.LONG)), Schema.Field.of("floatField", Schema.of(Schema.Type.LONG)), Schema.Field.of("doubleField", Schema.of(Schema.Type.LONG)), Schema.Field.of("bytesField", Schema.of(Schema.Type.BYTES)), Schema.Field.of("stringField", Schema.of(Schema.Type.STRING)));
    Assert.assertEquals(expectedSchema, output.getSchema());
    Assert.assertTrue((Boolean) output.get("booleanField"));
    Assert.assertEquals(28L, output.<Long>get("intField").longValue());
    Assert.assertEquals(99L, output.<Long>get("longField").longValue());
    Assert.assertEquals(3, output.<Long>get("floatField").longValue());
    Assert.assertEquals(3, output.<Long>get("doubleField").longValue());
    Assert.assertArrayEquals(Bytes.toBytes("foo"), (byte[]) output.get("bytesField"));
    Assert.assertEquals("bar", output.get("stringField"));
}

Example 44 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class JavaScriptTransformTest method testComplex.

@Test
public void testComplex() throws Exception {
    Schema inner2Schema = Schema.recordOf("inner2", Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("val", Schema.of(Schema.Type.DOUBLE)));
    Schema inner1Schema = Schema.recordOf("inner1", Schema.Field.of("list", Schema.arrayOf(Schema.mapOf(Schema.of(Schema.Type.STRING), inner2Schema))));
    Schema schema = Schema.recordOf("complex", Schema.Field.of("num", Schema.of(Schema.Type.INT)), Schema.Field.of("inner1", inner1Schema));
    /*
    {
      "complex": {
        "num": 8,
        "inner1": {
          "list": [
            "map": {
              "p": {
                "name": "pi",
                "val": 3.14
              },
              "e": {
                "name": "e",
                "val": 2.71
              }
            }
          ]
        }
      }
    }
    */
    StructuredRecord pi = StructuredRecord.builder(inner2Schema).set("name", "pi").set("val", 3.14).build();
    StructuredRecord e = StructuredRecord.builder(inner2Schema).set("name", "e").set("val", 2.71).build();
    StructuredRecord inner1 = StructuredRecord.builder(inner1Schema).set("list", Lists.newArrayList(ImmutableMap.of("p", pi, "e", e))).build();
    StructuredRecord input = StructuredRecord.builder(schema).set("num", 8).set("inner1", inner1).build();
    Schema outputSchema = Schema.recordOf("output", Schema.Field.of("x", Schema.of(Schema.Type.DOUBLE)));
    JavaScriptTransform.Config config = new JavaScriptTransform.Config("function transform(input, emitter, context) {\n" + "  var pi = input.inner1.list[0].p;\n" + "  var e = input.inner1.list[0].e;\n" + "  var val = power(pi.val, 3) + power(e.val, 2);\n" + "  print(pi); print(e);\n print(context);\n" + "  context.getMetrics().count(\"script.transform.count\", 1);\n" + "  context.getMetrics().pipelineCount(\"script.transform.count\", 1);\n" + "  context.getLogger().info(\"Log test from Script Transform\");\n" + "  emitter.emit({ 'x':val });\n" + "}" + "function power(x, y) { \n" + "  var ans = 1; \n" + "  for (i = 0; i < y; i++) { \n" + "    ans = ans * x;\n" + "  }\n" + "  return ans;\n" + "}", outputSchema.toString(), null);
    Transform<StructuredRecord, StructuredRecord> transform = new JavaScriptTransform(config);
    MockTransformContext mockContext = new MockTransformContext("transform.1");
    transform.initialize(mockContext);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(input, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    Assert.assertEquals(outputSchema, output.getSchema());
    Assert.assertTrue(Math.abs(2.71 * 2.71 + 3.14 * 3.14 * 3.14 - (Double) output.get("x")) < 0.000001);
    Assert.assertEquals(1, mockContext.getMockMetrics().getCount("script.transform.count"));
    Assert.assertEquals(1, mockContext.getMockMetrics().getPipelineCount("transform.1.script.transform.count"));
}

Example 45 with MockEmitter

use of io.cdap.cdap.etl.mock.common.MockEmitter in project hydrator-plugins by cdapio.

the class LogParserTransformTest method testErrorDatasetForInvalidS3Log.

@Test
public void testErrorDatasetForInvalidS3Log() throws Exception {
    StructuredRecord botRecord = StructuredRecord.builder(STRING_SCHEMA).set("body", "13a9f69e4a00effd6b4b891dcbcabef632ef9a9da7c - 6006CA0AE4 REST.GET.OBJECT " + "ubuntu/this/is/some/folder " + "\"GET /my/uri.gif releases/dists/precise/releases/i18n/Translation-en HTTP/1.1\" " + "403 AccessDenied 231 - 10 - \"-\" \"Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.  17)\" -").build();
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    S3_TRANSFORM.transform(botRecord, emitter);
    Assert.assertEquals(0, emitter.getEmitted().size());
    Assert.assertEquals(1, emitter.getErrors().size());
    InvalidEntry<StructuredRecord> invalidEntry = emitter.getErrors().get(0);
    Assert.assertEquals(31, invalidEntry.getErrorCode());
    Assert.assertEquals("Error Message", "Couldn't parse log, because the log did not match the S3 format.", invalidEntry.getErrorMsg());
    Assert.assertEquals("Error Record", botRecord, invalidEntry.getInvalidRecord());
}