use of co.cask.cdap.etl.proto.v2.ETLStage in project cdap by caskdata.
the class PipelineSpecGenerator method configureStage.
/**
* Configures a stage and returns the spec for it.
*
* @param stage the user provided configuration for the stage
* @param validatedPipeline the validated pipeline config
* @param pluginConfigurer configurer used to configure the stage
* @return the spec for the stage
*/
private ConfiguredStage configureStage(ETLStage stage, ValidatedPipeline validatedPipeline, DefaultPipelineConfigurer<T> pluginConfigurer) {
String stageName = stage.getName();
ETLPlugin stagePlugin = stage.getPlugin();
PluginSpec pluginSpec = configurePlugin(stageName, stagePlugin, pluginConfigurer);
DefaultStageConfigurer stageConfigurer = pluginConfigurer.getStageConfigurer();
Map<String, StageSpec.Port> outputSchemas = new HashMap<>();
Map<String, Schema> inputSchemas = stageConfigurer.getInputSchemas();
if (pluginSpec.getType().equals(SplitterTransform.PLUGIN_TYPE)) {
Map<String, Schema> outputPortSchemas = stageConfigurer.getOutputPortSchemas();
for (Map.Entry<String, String> outputEntry : validatedPipeline.getOutputPorts(stageName).entrySet()) {
String outputStage = outputEntry.getKey();
String outputPort = outputEntry.getValue();
if (outputPort == null) {
throw new IllegalArgumentException(String.format("Connection from Splitter '%s' to '%s' must specify a port.", stageName, outputStage));
}
outputSchemas.put(outputStage, new StageSpec.Port(outputPort, outputPortSchemas.get(outputPort)));
}
} else {
Schema outputSchema = stageConfigurer.getOutputSchema();
// all the same
if (Condition.PLUGIN_TYPE.equals(pluginSpec.getType())) {
outputSchema = null;
for (Schema schema : inputSchemas.values()) {
if (schema != null) {
// todo: fix this cleanly and fully
if (outputSchema != null && !outputSchema.equals(schema)) {
throw new IllegalArgumentException("Cannot have different input schemas going into stage " + stageName);
}
outputSchema = schema;
}
}
}
for (String outputStage : validatedPipeline.getOutputs(stageName)) {
outputSchemas.put(outputStage, new StageSpec.Port(null, outputSchema));
}
}
StageSpec stageSpec = StageSpec.builder(stageName, pluginSpec).addInputSchemas(inputSchemas).addOutputPortSchemas(outputSchemas).setErrorSchema(stageConfigurer.getErrorSchema()).setProcessTimingEnabled(validatedPipeline.isProcessTimingEnabled()).setStageLoggingEnabled(validatedPipeline.isStageLoggingEnabled()).build();
return new ConfiguredStage(stageSpec, pluginConfigurer.getPipelineProperties());
}
use of co.cask.cdap.etl.proto.v2.ETLStage in project cdap by caskdata.
the class PipelineSpecGenerator method validateConfig.
/**
* Validate that this is a valid pipeline. A valid pipeline has the following properties:
*
* All stages in the pipeline have a unique name.
* Source stages have at least one output and no inputs.
* Sink stages have at least one input and no outputs.
* There are no cycles in the pipeline.
* All inputs into a stage have the same schema.
* ErrorTransforms only have BatchSource, Transform, or BatchAggregator as input stages.
* AlertPublishers have at least one input and no outputs and don't have SparkSink or BatchSink as input.
*
* Returns the stages in the order they should be configured to ensure that all input stages are configured
* before their output.
*
* @param config the user provided configuration
* @return the order to configure the stages in
* @throws IllegalArgumentException if the pipeline is invalid
*/
private ValidatedPipeline validateConfig(ETLConfig config) {
config.validate();
if (config.getStages().isEmpty()) {
throw new IllegalArgumentException("A pipeline must contain at least one stage.");
}
Set<String> actionStages = new HashSet<>();
Set<String> conditionStages = new HashSet<>();
Map<String, String> stageTypes = new HashMap<>();
// check stage name uniqueness
Set<String> stageNames = new HashSet<>();
for (ETLStage stage : config.getStages()) {
if (!stageNames.add(stage.getName())) {
throw new IllegalArgumentException(String.format("Invalid pipeline. Multiple stages are named %s. Please ensure all stage names are unique", stage.getName()));
}
// if stage is Action stage, add it to the Action stage set
if (isAction(stage.getPlugin().getType())) {
actionStages.add(stage.getName());
}
// if the stage is condition add it to the Condition stage set
if (stage.getPlugin().getType().equals(Condition.PLUGIN_TYPE)) {
conditionStages.add(stage.getName());
}
stageTypes.put(stage.getName(), stage.getPlugin().getType());
}
// check that the from and to are names of actual stages
// also check that conditions have at most 2 outgoing connections each label with true or
// false but not both
Map<String, Boolean> conditionBranch = new HashMap<>();
for (Connection connection : config.getConnections()) {
if (!stageNames.contains(connection.getFrom())) {
throw new IllegalArgumentException(String.format("Invalid connection %s. %s is not a stage.", connection, connection.getFrom()));
}
if (!stageNames.contains(connection.getTo())) {
throw new IllegalArgumentException(String.format("Invalid connection %s. %s is not a stage.", connection, connection.getTo()));
}
if (conditionStages.contains(connection.getFrom())) {
if (connection.getCondition() == null) {
String msg = String.format("For condition stage %s, the connection %s is not marked with either " + "'true' or 'false'.", connection.getFrom(), connection);
throw new IllegalArgumentException(msg);
}
// check if connection from the condition node is marked as true or false multiple times
if (conditionBranch.containsKey(connection.getFrom()) && connection.getCondition().equals(conditionBranch.get(connection.getFrom()))) {
String msg = String.format("For condition stage '%s', more than one outgoing connections are marked as %s.", connection.getFrom(), connection.getCondition());
throw new IllegalArgumentException(msg);
}
conditionBranch.put(connection.getFrom(), connection.getCondition());
}
}
List<ETLStage> traversalOrder = new ArrayList<>(stageNames.size());
// can only have empty connections if the pipeline consists of a single action.
if (config.getConnections().isEmpty()) {
if (actionStages.size() == 1 && stageNames.size() == 1) {
traversalOrder.add(config.getStages().iterator().next());
return new ValidatedPipeline(traversalOrder, config);
} else {
throw new IllegalArgumentException("Invalid pipeline. There are no connections between stages. " + "This is only allowed if the pipeline consists of a single action plugin.");
}
}
Dag dag = new Dag(config.getConnections());
Set<String> controlStages = Sets.union(actionStages, conditionStages);
Map<String, ETLStage> stages = new HashMap<>();
for (ETLStage stage : config.getStages()) {
String stageName = stage.getName();
Set<String> stageInputs = dag.getNodeInputs(stageName);
Set<String> stageOutputs = dag.getNodeOutputs(stageName);
String stageType = stage.getPlugin().getType();
boolean isSource = isSource(stageType);
boolean isSink = isSink(stageType);
// check source plugins are sources in the dag
if (isSource) {
if (!stageInputs.isEmpty() && !controlStages.containsAll(stageInputs)) {
throw new IllegalArgumentException(String.format("%s %s has incoming connections from %s. %s stages cannot have any incoming connections.", stageType, stageName, stageType, Joiner.on(',').join(stageInputs)));
}
} else if (isSink) {
if (!stageOutputs.isEmpty() && !controlStages.containsAll(stageOutputs)) {
throw new IllegalArgumentException(String.format("%s %s has outgoing connections to %s. %s stages cannot have any outgoing connections.", stageType, stageName, stageType, Joiner.on(',').join(stageOutputs)));
}
} else if (ErrorTransform.PLUGIN_TYPE.equals(stageType)) {
for (String inputStage : stageInputs) {
String inputType = stageTypes.get(inputStage);
if (!VALID_ERROR_INPUTS.contains(inputType)) {
throw new IllegalArgumentException(String.format("ErrorTransform %s cannot have stage %s of type %s as input. Only %s stages can emit errors.", stageName, inputStage, inputType, Joiner.on(',').join(VALID_ERROR_INPUTS)));
}
}
}
boolean isAction = isAction(stageType);
if (!isAction && !stageType.equals(Condition.PLUGIN_TYPE) && !isSource && stageInputs.isEmpty()) {
throw new IllegalArgumentException(String.format("Stage %s is unreachable, it has no incoming connections.", stageName));
}
if (!isAction && !isSink && stageOutputs.isEmpty()) {
throw new IllegalArgumentException(String.format("Stage %s is a dead end, it has no outgoing connections.", stageName));
}
stages.put(stageName, stage);
}
validateConditionBranches(conditionStages, dag);
for (String stageName : dag.getTopologicalOrder()) {
traversalOrder.add(stages.get(stageName));
}
return new ValidatedPipeline(traversalOrder, config);
}
use of co.cask.cdap.etl.proto.v2.ETLStage in project cdap by caskdata.
the class DataPipelineTest method testMacroEvaluationActionPipeline.
public void testMacroEvaluationActionPipeline(Engine engine) throws Exception {
ETLStage action1 = new ETLStage("action1", MockAction.getPlugin("actionTable", "action1.row", "action1.column", "${value}"));
ETLBatchConfig etlConfig = co.cask.cdap.etl.proto.v2.ETLBatchConfig.builder("* * * * *").addStage(action1).setEngine(engine).build();
// set runtime arguments for macro substitution
Map<String, String> runtimeArguments = ImmutableMap.of("value", "macroValue");
AppRequest<co.cask.cdap.etl.proto.v2.ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
ApplicationId appId = NamespaceId.DEFAULT.app("macroActionTest-" + engine);
ApplicationManager appManager = deployApplication(appId, appRequest);
WorkflowManager manager = appManager.getWorkflowManager(SmartWorkflow.NAME);
manager.setRuntimeArgs(runtimeArguments);
manager.start(ImmutableMap.of("logical.start.time", "0"));
manager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
DataSetManager<Table> actionTableDS = getDataset("actionTable");
Assert.assertEquals("macroValue", MockAction.readOutput(actionTableDS, "action1.row", "action1.column"));
}
use of co.cask.cdap.etl.proto.v2.ETLStage in project cdap by caskdata.
the class DataPipelineTest method testKVTableLookup.
@Test
public void testKVTableLookup() throws Exception {
addDatasetInstance(KeyValueTable.class.getName(), "ageTable");
DataSetManager<KeyValueTable> lookupTable = getDataset("ageTable");
lookupTable.get().write("samuel".getBytes(Charsets.UTF_8), "12".getBytes(Charsets.UTF_8));
lookupTable.get().write("bob".getBytes(Charsets.UTF_8), "36".getBytes(Charsets.UTF_8));
lookupTable.get().write("jane".getBytes(Charsets.UTF_8), "25".getBytes(Charsets.UTF_8));
lookupTable.flush();
ETLBatchConfig etlConfig = ETLBatchConfig.builder("* * * * *").addStage(new ETLStage("source", MockSource.getPlugin("inputTable"))).addStage(new ETLStage("transform", LookupTransform.getPlugin("person", "age", "ageTable"))).addStage(new ETLStage("sink", MockSink.getPlugin("outputTable"))).addConnection("source", "transform").addConnection("transform", "sink").build();
ApplicationId appId = NamespaceId.DEFAULT.app("testKVTableLookup");
AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
ApplicationManager appManager = deployApplication(appId, appRequest);
// set up input data
Schema inputSchema = Schema.recordOf("person", Schema.Field.of("person", Schema.of(Schema.Type.STRING)));
StructuredRecord recordSamuel = StructuredRecord.builder(inputSchema).set("person", "samuel").build();
StructuredRecord recordBob = StructuredRecord.builder(inputSchema).set("person", "bob").build();
StructuredRecord recordJane = StructuredRecord.builder(inputSchema).set("person", "jane").build();
DataSetManager<Table> inputTable = getDataset("inputTable");
MockSource.writeInput(inputTable, ImmutableList.of(recordSamuel, recordBob, recordJane));
WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME).start();
workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
Schema schema = Schema.recordOf("person", Schema.Field.of("person", Schema.of(Schema.Type.STRING)), Schema.Field.of("age", Schema.of(Schema.Type.STRING)));
Set<StructuredRecord> expected = new HashSet<>();
expected.add(StructuredRecord.builder(schema).set("person", "samuel").set("age", "12").build());
expected.add(StructuredRecord.builder(schema).set("person", "bob").set("age", "36").build());
expected.add(StructuredRecord.builder(schema).set("person", "jane").set("age", "25").build());
DataSetManager<Table> outputTable = getDataset("outputTable");
Set<StructuredRecord> actual = new HashSet<>(MockSink.readOutput(outputTable));
Assert.assertEquals(expected, actual);
validateMetric(3, appId, "source.records.out");
validateMetric(3, appId, "sink.records.in");
deleteDatasetInstance(NamespaceId.DEFAULT.dataset("inputTable"));
deleteDatasetInstance(NamespaceId.DEFAULT.dataset("outputTable"));
}
use of co.cask.cdap.etl.proto.v2.ETLStage in project cdap by caskdata.
the class DataPipelineTest method testSplitterToConnector.
private void testSplitterToConnector(Engine engine) throws Exception {
Schema schema = Schema.recordOf("user", Schema.Field.of("id", Schema.of(Schema.Type.LONG)), Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING))), Schema.Field.of("email", Schema.nullableOf(Schema.of(Schema.Type.STRING))));
StructuredRecord user0 = StructuredRecord.builder(schema).set("id", 0L).build();
StructuredRecord user1 = StructuredRecord.builder(schema).set("id", 1L).set("email", "one@example.com").build();
StructuredRecord user2 = StructuredRecord.builder(schema).set("id", 2L).set("name", "two").build();
StructuredRecord user3 = StructuredRecord.builder(schema).set("id", 3L).set("name", "three").set("email", "three@example.com").build();
String sourceName = "splitconSource" + engine.name();
String sink1Name = "splitconSink1" + engine.name();
String sink2Name = "splitconSink2" + engine.name();
/*
*
* |null --> sink1
* |null--> identity-agg --> splitter2 --|
* source --> splitter1--| |non-null --|
* | |--> sink2
* |non-null-----------------------------------------|
*/
ETLBatchConfig config = ETLBatchConfig.builder("* * * * *").setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("splitter1", NullFieldSplitterTransform.getPlugin("name"))).addStage(new ETLStage("splitter2", NullFieldSplitterTransform.getPlugin("email"))).addStage(new ETLStage("identity", IdentityAggregator.getPlugin())).addStage(new ETLStage("sink1", MockSink.getPlugin(sink1Name))).addStage(new ETLStage("sink2", MockSink.getPlugin(sink2Name))).addConnection("source", "splitter1").addConnection("splitter1", "identity", "null").addConnection("splitter1", "sink2", "non-null").addConnection("identity", "splitter2").addConnection("splitter2", "sink1", "null").addConnection("splitter2", "sink2", "non-null").build();
AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
ApplicationId appId = NamespaceId.DEFAULT.app("SplitConTest-" + engine);
ApplicationManager appManager = deployApplication(appId, appRequest);
// write input data
DataSetManager<Table> inputManager = getDataset(sourceName);
MockSource.writeInput(inputManager, ImmutableList.of(user0, user1, user2, user3));
// run pipeline
WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME);
workflowManager.start();
workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
// check output
// sink1 should only have records where both name and email are null (user0)
DataSetManager<Table> sinkManager = getDataset(sink1Name);
Set<StructuredRecord> expected = ImmutableSet.of(user0);
Set<StructuredRecord> actual = Sets.newHashSet(MockSink.readOutput(sinkManager));
Assert.assertEquals(expected, actual);
// sink2 should have anything with a non-null name or non-null email
sinkManager = getDataset(sink2Name);
expected = ImmutableSet.of(user1, user2, user3);
actual = Sets.newHashSet(MockSink.readOutput(sinkManager));
Assert.assertEquals(expected, actual);
validateMetric(4, appId, "source.records.out");
validateMetric(4, appId, "splitter1.records.in");
validateMetric(2, appId, "splitter1.records.out.null");
validateMetric(2, appId, "splitter1.records.out.non-null");
validateMetric(2, appId, "identity.records.in");
validateMetric(2, appId, "identity.records.out");
validateMetric(2, appId, "splitter2.records.in");
validateMetric(1, appId, "splitter2.records.out.null");
validateMetric(1, appId, "splitter2.records.out.non-null");
validateMetric(1, appId, "sink1.records.in");
validateMetric(3, appId, "sink2.records.in");
}
Aggregations