Examples with MockPipelineConfigurer io.cdap.cdap.etl.mock.common.MockPipelineConfigurer used on opensource projects

Example 66 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class JSONFormatterTest method testSchemaValidation.

@Test
public void testSchemaValidation() throws Exception {
    JSONFormatter.Config config = new JSONFormatter.Config(OUTPUT1.toString());
    Transform<StructuredRecord, StructuredRecord> transform = new JSONFormatter(config);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(INPUT1);
    transform.configurePipeline(mockPipelineConfigurer);
    Assert.assertEquals(OUTPUT1, mockPipelineConfigurer.getOutputSchema());
}

Example 67 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class JSONParserTest method testInvalidJsonPathForNonNullableSchema.

@Test
public void testInvalidJsonPathForNonNullableSchema() throws Exception {
    final String[] jsonPaths = { "expensive:$.expensive", "bicycle_color:$.store.bicycle.color", "bicycle_price:$.store.bicycle.price", "window:$.store.window" };
    MockEmitter<StructuredRecord> emitter = new MockEmitter<>();
    JSONParser.Config config = new JSONParser.Config("body", Joiner.on(",").join(jsonPaths), OUTPUT4.toString());
    Transform<StructuredRecord, StructuredRecord> transform = new JSONParser(config);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(INPUT1);
    transform.configurePipeline(mockPipelineConfigurer);
    TransformContext context = new MockTransformContext();
    transform.initialize(context);
    transform.transform(StructuredRecord.builder(INPUT1).set("body", json).build(), emitter);
    Assert.assertEquals(0, emitter.getEmitted().size());
}

Example 68 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class NormalizeTest method testInvalidOutputSchema.

@Test
public void testInvalidOutputSchema() throws Exception {
    // schema with no ID field
    Schema outputSchema = Schema.recordOf("outputSchema", Schema.Field.of(DATE, Schema.of(Schema.Type.STRING)), Schema.Field.of(ATTRIBUTE_TYPE, Schema.of(Schema.Type.STRING)), Schema.Field.of(ATTRIBUTE_VALUE, Schema.of(Schema.Type.STRING)));
    Normalize.NormalizeConfig config = new Normalize.NormalizeConfig(validFieldMapping, validFieldNormalizing, outputSchema.toString());
    MockPipelineConfigurer configurer = new MockPipelineConfigurer(INPUT_SCHEMA);
    new Normalize(config).configurePipeline(configurer);
    FailureCollector collector = configurer.getStageConfigurer().getFailureCollector();
    Assert.assertEquals(1, collector.getValidationFailures().size());
    Assert.assertEquals(1, collector.getValidationFailures().get(0).getCauses().size());
    ValidationFailure.Cause expectedCause = new ValidationFailure.Cause();
    expectedCause.addAttribute(CauseAttributes.STAGE_CONFIG, Normalize.NormalizeConfig.FIELD_MAPPING);
    expectedCause.addAttribute(CauseAttributes.CONFIG_ELEMENT, "CustomerId:Id");
    Assert.assertEquals(expectedCause, collector.getValidationFailures().get(0).getCauses().get(0));
}

Example 69 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class GroupByTestRun method testGroupByCollectSet.

@Test
public void testGroupByCollectSet() {
    /*
        <ts, user, item, price> --> group by item, uniqueUsers:CollectSet(user) --> item table
     */
    Schema purchaseSchema = Schema.recordOf("purchase", Schema.Field.of("ts", Schema.of(Schema.Type.LONG)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("item", Schema.of(Schema.Type.STRING)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)));
    Schema aggSchema = Schema.recordOf("purchase.agg", Schema.Field.of("item", Schema.of(Schema.Type.STRING)), Schema.Field.of("uniqueUsers", Schema.arrayOf(Schema.of(Schema.Type.STRING))));
    GroupByConfig groupByConfig = new GroupByConfig("item", "uniqueUsers:CollectSet(user)");
    GroupByAggregator groupByAggregator = new GroupByAggregator(groupByConfig);
    MockPipelineConfigurer mockConfigurer = new MockPipelineConfigurer(purchaseSchema, Collections.emptyMap());
    groupByAggregator.configurePipeline(mockConfigurer);
    Assert.assertEquals(aggSchema, mockConfigurer.getOutputSchema());
}

Example 70 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class GroupByTestRun method testGroupByConditionValidation.

@Test
public void testGroupByConditionValidation() {
    Schema purchaseSchema = Schema.recordOf("purchase", Schema.Field.of("ts", Schema.of(Schema.Type.LONG)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("item", Schema.of(Schema.Type.STRING)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)));
    Schema aggSchema = Schema.recordOf("purchase.agg", Schema.Field.of("item", Schema.of(Schema.Type.STRING)), Schema.Field.of("maxPrice", Schema.of(Schema.Type.DOUBLE)));
    GroupByConfig groupByConfig = new GroupByConfig("item", "maxPrice:MaxIf(price):condition(price<5)");
    GroupByAggregator groupByAggregator = new GroupByAggregator(groupByConfig);
    MockPipelineConfigurer mockConfigurer = new MockPipelineConfigurer(purchaseSchema, Collections.emptyMap());
    groupByAggregator.configurePipeline(mockConfigurer);
    Assert.assertEquals(aggSchema, mockConfigurer.getOutputSchema());
}