Examples with MockPipelineConfigurer io.cdap.cdap.etl.mock.common.MockPipelineConfigurer used on opensource projects

Example 21 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class GroupByTestRun method testCountDistinctValidation.

@Test
public void testCountDistinctValidation() {
    String groupByField = "state";
    Schema customerSchema = Schema.recordOf("customers", Schema.Field.of("string", Schema.of(Schema.Type.STRING)), Schema.Field.of("int", Schema.of(Schema.Type.INT)), Schema.Field.of("long", Schema.of(Schema.Type.LONG)), Schema.Field.of("float", Schema.of(Schema.Type.FLOAT)), Schema.Field.of("double", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("boolean", Schema.of(Schema.Type.BOOLEAN)), Schema.Field.of("array", Schema.arrayOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("record", Schema.recordOf("record")), Schema.Field.of("map", Schema.mapOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.STRING))), Schema.Field.of("enum", Schema.enumWith("A", "B")), Schema.Field.of("union", Schema.unionOf(Schema.of(Schema.Type.STRING), Schema.of(Schema.Type.BOOLEAN))), Schema.Field.of("bytes", Schema.of(Schema.Type.BYTES)), Schema.Field.of(groupByField, Schema.of(Schema.Type.STRING)));
    MockPipelineConfigurer configurer = new MockPipelineConfigurer(customerSchema, Collections.emptyMap());
    validateCountDistinct(configurer, "string", groupByField);
    validateCountDistinct(configurer, "int", groupByField);
    validateCountDistinct(configurer, "long", groupByField);
    validateCountDistinct(configurer, "boolean", groupByField);
    assertCountDistinctFailure(configurer, "float", groupByField, "CountDistinct for floats should fail validation");
    assertCountDistinctFailure(configurer, "double", groupByField, "CountDistinct for doubles should fail validation");
    assertCountDistinctFailure(configurer, "array", groupByField, "CountDistinct for arrays should fail validation");
    assertCountDistinctFailure(configurer, "record", groupByField, "CountDistinct for records should fail validation");
    assertCountDistinctFailure(configurer, "map", groupByField, "CountDistinct for maps should fail validation");
    assertCountDistinctFailure(configurer, "enum", groupByField, "CountDistinct for enums should fail validation");
    assertCountDistinctFailure(configurer, "union", groupByField, "CountDistinct for unions should fail validation");
    assertCountDistinctFailure(configurer, "bytes", groupByField, "CountDistinct for bytes should fail validation");
}

Example 22 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class GroupByTestRun method testGroupByConditionValidationFailure.

@Test
public void testGroupByConditionValidationFailure() {
    Schema purchaseSchema = Schema.recordOf("purchase", Schema.Field.of("ts", Schema.of(Schema.Type.LONG)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("item", Schema.of(Schema.Type.STRING)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)));
    GroupByConfig groupByConfig = new GroupByConfig("item", "maxPrice:MaxIf(price):condition(test<5)");
    GroupByAggregator groupByAggregator = new GroupByAggregator(groupByConfig);
    MockPipelineConfigurer mockConfigurer = new MockPipelineConfigurer(purchaseSchema, Collections.emptyMap());
    try {
        groupByAggregator.configurePipeline(mockConfigurer);
        Assert.fail("Condition with wrong input field that does not exists in schema should fail validation");
    } catch (ValidationException e) {
        Assert.assertEquals(1, e.getFailures().size());
        Assert.assertEquals("test", e.getFailures().get(0).getCauses().get(0).getAttribute("configElement"));
    }
}

Example 23 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class RowDenormalizerConfigTest method testDenormalizerWithWrongKeyField.

@Test(expected = ValidationException.class)
public void testDenormalizerWithWrongKeyField() throws Exception {
    Schema inputSchema = Schema.recordOf("record", Schema.Field.of("KeyField", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("NameField", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("ValueField", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
    MockPipelineConfigurer configurer = new MockPipelineConfigurer(inputSchema, Collections.emptyMap());
    RowDenormalizerConfig config = new RowDenormalizerConfig("WrongKeyField", "NameField", "ValueField", "Firstname," + "Lastname,Address", "Lastname:lname,Address:addr");
    RowDenormalizerAggregator aggregator = new RowDenormalizerAggregator(config);
    aggregator.configurePipeline(configurer);
    configurer.getStageConfigurer().getFailureCollector().getOrThrowException();
}

Example 24 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class KVTableSinkTest method testTableSinkWithInvalidFieldType.

@Test(expected = IllegalArgumentException.class)
public void testTableSinkWithInvalidFieldType() {
    Schema inputSchema = Schema.recordOf("purchase", // only string and bytes are supported, this is invalid type
    Schema.Field.of("rowKey", Schema.of(Schema.Type.LONG)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)));
    KVTableSink.KVTableConfig kvTableConfig = new KVTableSink.KVTableConfig("purchases", "rowKey", "user");
    KVTableSink kvTableSink = new KVTableSink(kvTableConfig);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(inputSchema);
    kvTableSink.configurePipeline(mockPipelineConfigurer);
}

Example 25 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class KVTableSinkTest method testkvTableConfigure.

@Test
public void testkvTableConfigure() {
    Schema inputSchema = Schema.recordOf("purchase", Schema.Field.of("rowKey", Schema.of(Schema.Type.BYTES)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)));
    KVTableSink.KVTableConfig kvTableConfig = new KVTableSink.KVTableConfig("purchases", "rowKey", "user");
    KVTableSink kvTableSink = new KVTableSink(kvTableConfig);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(inputSchema);
    kvTableSink.configurePipeline(mockPipelineConfigurer);
}