Examples with MockTransformContext io.cdap.cdap.etl.mock.transform.MockTransformContext used on opensource projects

Example 41 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class ProjectionTransformTest method testConvertToLong.

@Test
public void testConvertToLong() throws Exception {
    ProjectionTransform.ProjectionTransformConfig config = new ProjectionTransform.ProjectionTransformConfig(null, null, "intField:long,floatField:long,doubleField:long", null);
    Transform<StructuredRecord, StructuredRecord> transform = new ProjectionTransform(config);
    TransformContext transformContext = new MockTransformContext();
    transform.initialize(transformContext);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(SIMPLE_TYPES_RECORD, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    Schema expectedSchema = Schema.recordOf("record.projected", Schema.Field.of("booleanField", Schema.of(Schema.Type.BOOLEAN)), Schema.Field.of("intField", Schema.of(Schema.Type.LONG)), Schema.Field.of("longField", Schema.of(Schema.Type.LONG)), Schema.Field.of("floatField", Schema.of(Schema.Type.LONG)), Schema.Field.of("doubleField", Schema.of(Schema.Type.LONG)), Schema.Field.of("bytesField", Schema.of(Schema.Type.BYTES)), Schema.Field.of("stringField", Schema.of(Schema.Type.STRING)));
    Assert.assertEquals(expectedSchema, output.getSchema());
    Assert.assertTrue((Boolean) output.get("booleanField"));
    Assert.assertEquals(28L, output.<Long>get("intField").longValue());
    Assert.assertEquals(99L, output.<Long>get("longField").longValue());
    Assert.assertEquals(3, output.<Long>get("floatField").longValue());
    Assert.assertEquals(3, output.<Long>get("doubleField").longValue());
    Assert.assertArrayEquals(Bytes.toBytes("foo"), (byte[]) output.get("bytesField"));
    Assert.assertEquals("bar", output.get("stringField"));
}

Example 42 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class JavaScriptTransformTest method testComplex.

@Test
public void testComplex() throws Exception {
    Schema inner2Schema = Schema.recordOf("inner2", Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("val", Schema.of(Schema.Type.DOUBLE)));
    Schema inner1Schema = Schema.recordOf("inner1", Schema.Field.of("list", Schema.arrayOf(Schema.mapOf(Schema.of(Schema.Type.STRING), inner2Schema))));
    Schema schema = Schema.recordOf("complex", Schema.Field.of("num", Schema.of(Schema.Type.INT)), Schema.Field.of("inner1", inner1Schema));
    /*
    {
      "complex": {
        "num": 8,
        "inner1": {
          "list": [
            "map": {
              "p": {
                "name": "pi",
                "val": 3.14
              },
              "e": {
                "name": "e",
                "val": 2.71
              }
            }
          ]
        }
      }
    }
    */
    StructuredRecord pi = StructuredRecord.builder(inner2Schema).set("name", "pi").set("val", 3.14).build();
    StructuredRecord e = StructuredRecord.builder(inner2Schema).set("name", "e").set("val", 2.71).build();
    StructuredRecord inner1 = StructuredRecord.builder(inner1Schema).set("list", Lists.newArrayList(ImmutableMap.of("p", pi, "e", e))).build();
    StructuredRecord input = StructuredRecord.builder(schema).set("num", 8).set("inner1", inner1).build();
    Schema outputSchema = Schema.recordOf("output", Schema.Field.of("x", Schema.of(Schema.Type.DOUBLE)));
    JavaScriptTransform.Config config = new JavaScriptTransform.Config("function transform(input, emitter, context) {\n" + "  var pi = input.inner1.list[0].p;\n" + "  var e = input.inner1.list[0].e;\n" + "  var val = power(pi.val, 3) + power(e.val, 2);\n" + "  print(pi); print(e);\n print(context);\n" + "  context.getMetrics().count(\"script.transform.count\", 1);\n" + "  context.getMetrics().pipelineCount(\"script.transform.count\", 1);\n" + "  context.getLogger().info(\"Log test from Script Transform\");\n" + "  emitter.emit({ 'x':val });\n" + "}" + "function power(x, y) { \n" + "  var ans = 1; \n" + "  for (i = 0; i < y; i++) { \n" + "    ans = ans * x;\n" + "  }\n" + "  return ans;\n" + "}", outputSchema.toString(), null);
    Transform<StructuredRecord, StructuredRecord> transform = new JavaScriptTransform(config);
    MockTransformContext mockContext = new MockTransformContext("transform.1");
    transform.initialize(mockContext);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(input, emitter);
    StructuredRecord output = emitter.getEmitted().get(0);
    Assert.assertEquals(outputSchema, output.getSchema());
    Assert.assertTrue(Math.abs(2.71 * 2.71 + 3.14 * 3.14 * 3.14 - (Double) output.get("x")) < 0.000001);
    Assert.assertEquals(1, mockContext.getMockMetrics().getCount("script.transform.count"));
    Assert.assertEquals(1, mockContext.getMockMetrics().getPipelineCount("transform.1.script.transform.count"));
}

Example 43 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class CSVParserTest method testEmitErrors.

@Test
public void testEmitErrors() throws Exception {
    CSVParser.Config config = new CSVParser.Config("DEFAULT", null, "body", OUTPUT5.toString());
    Transform<StructuredRecord, StructuredRecord> transform = new CSVParser(config);
    transform.initialize(new MockTransformContext());
    StructuredRecord inputRecord = StructuredRecord.builder(INPUT2).set("offset", 1).set("body", "0,\"020\"1,\"BS:12345  ORDER:111\"4").build();
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(inputRecord, emitter);
    Assert.assertEquals(0, emitter.getEmitted().size());
    Assert.assertEquals(1, emitter.getErrors().size());
    InvalidEntry<StructuredRecord> invalidEntry = emitter.getErrors().get(0);
    Assert.assertEquals(31, invalidEntry.getErrorCode());
    Assert.assertEquals("offset", 1, invalidEntry.getInvalidRecord().<Integer>get("offset").intValue());
    Assert.assertEquals("body", "0,\"020\"1,\"BS:12345  ORDER:111\"4", invalidEntry.getInvalidRecord().get("body"));
}

Example 44 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class CompressorTest method testGZIPCompress.

@Test
public void testGZIPCompress() throws Exception {
    Transform<StructuredRecord, StructuredRecord> transform = new Compressor(new Compressor.Config("a:GZIP", OUTPUT.toString()));
    MockTransformContext context = new MockTransformContext();
    transform.initialize(context);
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    transform.transform(StructuredRecord.builder(INPUT).set("a", "This is a test for testing gzip compression").set("b", "2").set("c", "3").set("d", "4").set("e", "5").build(), emitter);
    byte[] expected = compressGZIP("This is a test for testing gzip compression".getBytes());
    byte[] actual = emitter.getEmitted().get(0).get("a");
    Assert.assertEquals(2, emitter.getEmitted().get(0).getSchema().getFields().size());
    Assert.assertArrayEquals(expected, actual);
}

Example 45 with MockTransformContext

use of io.cdap.cdap.etl.mock.transform.MockTransformContext in project hydrator-plugins by cdapio.

the class DecoderTest method testBase32Decoder.

@Test
public void testBase32Decoder() throws Exception {
    String test = "This is a test for testing base32 decoding";
    Transform<StructuredRecord, StructuredRecord> encoder = new Encoder(new Encoder.Config("a:BASE32", OUTPUT.toString()));
    TransformContext encoderContext = new MockTransformContext();
    encoder.initialize(encoderContext);
    MockEmitter<StructuredRecord> emitterEncoded = new MockEmitter<>();
    encoder.transform(StructuredRecord.builder(INPUT).set("a", test).set("b", "2").set("c", "3").set("d", "4").set("e", "5").build(), emitterEncoded);
    Base32 base32 = new Base32();
    byte[] expected = base32.encode(test.getBytes("UTF-8"));
    byte[] actual = emitterEncoded.getEmitted().get(0).get("a");
    Assert.assertEquals(2, emitterEncoded.getEmitted().get(0).getSchema().getFields().size());
    Assert.assertArrayEquals(expected, actual);
    Transform<StructuredRecord, StructuredRecord> decoder = new Decoder(new Decoder.Config("a:BASE32", OUTPUTSTR.toString()));
    MockTransformContext context = new MockTransformContext();
    decoder.initialize(context);
    MockEmitter<StructuredRecord> emitterDecoded = new MockEmitter<>();
    decoder.transform(emitterEncoded.getEmitted().get(0), emitterDecoded);
    Assert.assertEquals(2, emitterDecoded.getEmitted().get(0).getSchema().getFields().size());
    Assert.assertEquals(test, emitterDecoded.getEmitted().get(0).get("a"));
    Assert.assertEquals(0, encoderContext.getFailureCollector().getValidationFailures().size());
}