use of co.cask.cdap.etl.proto.v2.ETLStage in project cdap by caskdata.
the class PipelineTest method testStringCaseTransform.
@Test
public void testStringCaseTransform() throws Exception {
String inputName = "transformTestInput";
String outputName = "transformTestOutput";
// create the pipeline config
ETLStage source = new ETLStage("source", MockSource.getPlugin(inputName));
ETLStage sink = new ETLStage("sink", MockSink.getPlugin(outputName));
Map<String, String> transformProperties = new HashMap<>();
transformProperties.put("lowerFields", "first");
transformProperties.put("upperFields", "last");
ETLStage transform = new ETLStage("transform", new ETLPlugin(StringCaseTransform.NAME, Transform.PLUGIN_TYPE, transformProperties, null));
ETLBatchConfig pipelineConfig = ETLBatchConfig.builder("* * * * *").addStage(source).addStage(sink).addStage(transform).addConnection(source.getName(), transform.getName()).addConnection(transform.getName(), sink.getName()).build();
// create the pipeline
ApplicationId pipelineId = NamespaceId.DEFAULT.app("transformTestPipeline");
ApplicationManager appManager = deployApplication(pipelineId, new AppRequest<>(APP_ARTIFACT, pipelineConfig));
// write the input
Schema schema = Schema.recordOf("name", Schema.Field.of("first", Schema.of(Schema.Type.STRING)), Schema.Field.of("last", Schema.of(Schema.Type.STRING)));
DataSetManager<Table> inputManager = getDataset(inputName);
List<StructuredRecord> inputRecords = new ArrayList<>();
inputRecords.add(StructuredRecord.builder(schema).set("first", "Samuel").set("last", "Jackson").build());
MockSource.writeInput(inputManager, inputRecords);
WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME);
workflowManager.start();
workflowManager.waitForFinish(4, TimeUnit.MINUTES);
DataSetManager<Table> outputManager = getDataset(outputName);
List<StructuredRecord> outputRecords = MockSink.readOutput(outputManager);
List<StructuredRecord> expected = new ArrayList<>();
expected.add(StructuredRecord.builder(schema).set("first", "samuel").set("last", "JACKSON").build());
Assert.assertEquals(expected, outputRecords);
}
use of co.cask.cdap.etl.proto.v2.ETLStage in project cdap by caskdata.
the class BatchPipelineSpecGenerator method generateSpec.
@Override
public BatchPipelineSpec generateSpec(ETLBatchConfig config) {
BatchPipelineSpec.Builder specBuilder = BatchPipelineSpec.builder();
for (ETLStage endingAction : config.getPostActions()) {
String name = endingAction.getName();
DefaultPipelineConfigurer<T> pipelineConfigurer = new DefaultPipelineConfigurer<>(configurer, name, engine);
PluginSpec pluginSpec = configurePlugin(endingAction.getName(), endingAction.getPlugin(), pipelineConfigurer);
specBuilder.addAction(new ActionSpec(name, pluginSpec));
}
configureStages(config, specBuilder);
return specBuilder.build();
}
Aggregations