Examples with MockPipelineConfigurer io.cdap.cdap.etl.mock.common.MockPipelineConfigurer used on opensource projects

Example 26 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class KVTableSinkTest method testTableSinkWithNullableLongTypeForValue.

@Test(expected = IllegalArgumentException.class)
public void testTableSinkWithNullableLongTypeForValue() {
    Schema inputSchema = Schema.recordOf("purchase", // only string and bytes are supported, this is invalid type
    Schema.Field.of("user", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("rowKey", Schema.of(Schema.Type.LONG)));
    KVTableSink.KVTableConfig kvTableConfig = new KVTableSink.KVTableConfig("purchases", "rowKey", "user");
    KVTableSink kvTableSink = new KVTableSink(kvTableConfig);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(inputSchema);
    kvTableSink.configurePipeline(mockPipelineConfigurer);
}

Example 27 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class KVTableSinkTest method testTableSinkWithMissingKeyField.

@Test(expected = IllegalArgumentException.class)
public void testTableSinkWithMissingKeyField() {
    Schema inputSchema = Schema.recordOf("purchase", Schema.Field.of("uniqueId", Schema.of(Schema.Type.STRING)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)));
    KVTableSink.KVTableConfig kvTableConfig = new KVTableSink.KVTableConfig("purchases", "rowKey", "user");
    KVTableSink kvTableSink = new KVTableSink(kvTableConfig);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(inputSchema);
    kvTableSink.configurePipeline(mockPipelineConfigurer);
}

Example 28 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class TableSinkTest method testTableSink.

@Test
public void testTableSink() {
    Schema outputSchema = Schema.recordOf("purchase", Schema.Field.of("rowkey", Schema.of(Schema.Type.STRING)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("count", Schema.of(Schema.Type.INT)));
    Schema inputSchema = Schema.recordOf("purchase", Schema.Field.of("rowkey", Schema.of(Schema.Type.STRING)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("count", Schema.of(Schema.Type.INT)));
    TableSinkConfig tableSinkConfig = new TableSinkConfig("tableSink", "rowkey", outputSchema.toString());
    TableSink tableSink = new TableSink(tableSinkConfig);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(inputSchema);
    tableSink.configurePipeline(mockPipelineConfigurer);
    Assert.assertEquals(outputSchema, mockPipelineConfigurer.getOutputSchema());
}

Example 29 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class TableSinkTest method testTableSinkWithMissingRowKeyField.

@Test
public void testTableSinkWithMissingRowKeyField() {
    Schema outputSchema = Schema.recordOf("purchase", Schema.Field.of("rowkey", Schema.of(Schema.Type.STRING)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)));
    Schema inputSchema = Schema.recordOf("purchase", Schema.Field.of("userid", Schema.of(Schema.Type.STRING)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)));
    TableSinkConfig tableSinkConfig = new TableSinkConfig("tableSink", "rowkey", outputSchema.toString());
    TableSink tableSink = new TableSink(tableSinkConfig);
    MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(inputSchema);
    tableSink.configurePipeline(mockPipelineConfigurer);
    FailureCollector collector = mockPipelineConfigurer.getStageConfigurer().getFailureCollector();
    Assert.assertEquals(1, collector.getValidationFailures().size());
}

Example 30 with MockPipelineConfigurer

use of io.cdap.cdap.etl.mock.common.MockPipelineConfigurer in project hydrator-plugins by cdapio.

the class JavaScriptTransformTest method testSchemaValidation.

@Test
public void testSchemaValidation() throws Exception {
    Schema outputSchema = Schema.recordOf("smallerSchema", Schema.Field.of("x", Schema.of(Schema.Type.INT)), Schema.Field.of("y", Schema.of(Schema.Type.LONG)));
    JavaScriptTransform.Config config = new JavaScriptTransform.Config("function transform(input, emitter, context) { emitter.emit({ 'x':input.intField, 'y':input.longField }); }", outputSchema.toString(), null);
    Schema inputSchema = Schema.recordOf("biggerSchema", Schema.Field.of("x", Schema.of(Schema.Type.INT)), Schema.Field.of("y", Schema.of(Schema.Type.LONG)), Schema.Field.of("z", Schema.of(Schema.Type.DOUBLE)));
    MockPipelineConfigurer configurer = new MockPipelineConfigurer(inputSchema, Collections.emptyMap());
    new JavaScriptTransform(config).configurePipeline(configurer);
    Assert.assertEquals(outputSchema, configurer.getOutputSchema());
    // if schame is null in config, then input schema is set as output schema
    config = new JavaScriptTransform.Config("function transform(input, emitter, context) { emitter.emit({ 'x':input.intField, 'y':input.longField }); }", null, null);
    new JavaScriptTransform(config).configurePipeline(configurer);
    Assert.assertEquals(inputSchema, configurer.getOutputSchema());
}