Search in sources :

Example 6 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.

the class ValidationUtils method validate.

/**
 * Validate plugin based on the {@link StageValidationRequest}
 *
 * @param validationRequest {@link StageValidationRequest} with plugin properties
 * @param pluginConfigurer  {@link PluginConfigurer} for using the plugin
 * @param macroFn           {@link Function} for evaluating macros
 * @return {@link StageValidationResponse} in json format
 */
public static StageValidationResponse validate(String namespace, StageValidationRequest validationRequest, PluginConfigurer pluginConfigurer, Function<Map<String, String>, Map<String, String>> macroFn, FeatureFlagsProvider featureFlagsProvider) {
    ETLStage stageConfig = validationRequest.getStage();
    ValidatingConfigurer validatingConfigurer = new ValidatingConfigurer(pluginConfigurer, featureFlagsProvider);
    // Batch or Streaming doesn't matter for a single stage.
    PipelineSpecGenerator<ETLBatchConfig, BatchPipelineSpec> pipelineSpecGenerator = new BatchPipelineSpecGenerator(namespace, validatingConfigurer, null, Collections.emptySet(), Collections.emptySet(), Engine.SPARK, featureFlagsProvider);
    DefaultStageConfigurer stageConfigurer = new DefaultStageConfigurer(stageConfig.getName());
    for (StageSchema stageSchema : validationRequest.getInputSchemas()) {
        stageConfigurer.addInputSchema(stageSchema.getStage(), stageSchema.getSchema());
        stageConfigurer.addInputStage(stageSchema.getStage());
    }
    DefaultPipelineConfigurer pipelineConfigurer = new DefaultPipelineConfigurer(validatingConfigurer, stageConfig.getName(), Engine.SPARK, stageConfigurer, featureFlagsProvider);
    // evaluate macros
    Map<String, String> evaluatedProperties = macroFn.apply(stageConfig.getPlugin().getProperties());
    ETLPlugin originalConfig = stageConfig.getPlugin();
    ETLPlugin evaluatedConfig = new ETLPlugin(originalConfig.getName(), originalConfig.getType(), evaluatedProperties, originalConfig.getArtifactConfig());
    try {
        StageSpec spec = pipelineSpecGenerator.configureStage(stageConfig.getName(), evaluatedConfig, pipelineConfigurer).build();
        return new StageValidationResponse(spec);
    } catch (ValidationException e) {
        return new StageValidationResponse(e.getFailures());
    }
}
Also used : ValidationException(io.cdap.cdap.etl.api.validation.ValidationException) BatchPipelineSpecGenerator(io.cdap.cdap.etl.batch.BatchPipelineSpecGenerator) ETLPlugin(io.cdap.cdap.etl.proto.v2.ETLPlugin) DefaultStageConfigurer(io.cdap.cdap.etl.common.DefaultStageConfigurer) ETLBatchConfig(io.cdap.cdap.etl.proto.v2.ETLBatchConfig) BatchPipelineSpec(io.cdap.cdap.etl.batch.BatchPipelineSpec) StageSchema(io.cdap.cdap.etl.proto.v2.validation.StageSchema) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) ValidatingConfigurer(io.cdap.cdap.etl.validation.ValidatingConfigurer) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) DefaultPipelineConfigurer(io.cdap.cdap.etl.common.DefaultPipelineConfigurer) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse)

Example 7 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.

the class DataPipelineServiceTest method testValidateStageSingleInvalidConfigProperty.

@Test
public void testValidateStageSingleInvalidConfigProperty() throws Exception {
    // StringValueFilterTransform will be configured to filter records where field x has value 'y'
    // it will be invalid because the type of field x will be an int instead of the required string
    String stageName = "tx";
    Map<String, String> properties = new HashMap<>();
    properties.put("field", "x");
    properties.put("value", "y");
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(StringValueFilterTransform.NAME, Transform.PLUGIN_TYPE, properties));
    Schema inputSchema = Schema.recordOf("x", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    StageValidationRequest requestBody = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), false);
    StageValidationResponse actual = sendRequest(requestBody);
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    // the stage will add 2 causes for invalid input field failure. One is related to input field and the other is
    // related to config property.
    Assert.assertEquals(2, failure.getCauses().size());
    Assert.assertEquals("field", failure.getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(STAGE));
    Assert.assertEquals("x", failure.getCauses().get(1).getAttribute(CauseAttributes.INPUT_SCHEMA_FIELD));
    Assert.assertEquals("input", failure.getCauses().get(1).getAttribute(CauseAttributes.INPUT_STAGE));
    Assert.assertEquals(stageName, failure.getCauses().get(1).getAttribute(STAGE));
}
Also used : StageSchema(io.cdap.cdap.etl.proto.v2.validation.StageSchema) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) HashMap(java.util.HashMap) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) StageSchema(io.cdap.cdap.etl.proto.v2.validation.StageSchema) Schema(io.cdap.cdap.api.data.schema.Schema) ETLPlugin(io.cdap.cdap.etl.proto.v2.ETLPlugin) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse) ValidationFailure(io.cdap.cdap.etl.api.validation.ValidationFailure) Test(org.junit.Test)

Example 8 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.

the class DataPipelineServiceTest method testValidationFailureWithNPE.

@Test
public void testValidationFailureWithNPE() throws Exception {
    String stageName = "npe";
    ETLStage stage = new ETLStage(stageName, NullErrorTransform.getPlugin());
    StageValidationResponse actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), false));
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(STAGE));
    Assert.assertNotNull(failure.getCauses().get(0).getAttribute(CauseAttributes.STACKTRACE));
}
Also used : StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse) ValidationFailure(io.cdap.cdap.etl.api.validation.ValidationFailure) Test(org.junit.Test)

Example 9 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.

the class DataPipelineServiceTest method testValidateStageMissingRequiredProperty.

// tests that plugins that cannot be instantiated due to missing required properties are captured
@Test
public void testValidateStageMissingRequiredProperty() throws Exception {
    String stageName = "tx";
    // string filter requires the field name and the value
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(StringValueFilterTransform.NAME, Transform.PLUGIN_TYPE, Collections.emptyMap()));
    StageValidationResponse actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), false));
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(2, actual.getFailures().size());
    Set<String> properties = new HashSet<>();
    properties.add(actual.getFailures().get(0).getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    properties.add(actual.getFailures().get(1).getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    Set<String> expected = new HashSet<>();
    expected.add("field");
    expected.add("value");
    Assert.assertEquals(expected, properties);
}
Also used : StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) ETLPlugin(io.cdap.cdap.etl.proto.v2.ETLPlugin) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 10 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.

the class DataPipelineServiceTest method testValidateMultiInputInvalidInputField.

@Test
public void testValidateMultiInputInvalidInputField() throws Exception {
    // StringValueFilterTransform will be configured to filter records where field x has value 'y'
    // it will be invalid because the type of field x will be an int instead of the required string
    String stageName = "tx";
    Map<String, String> properties = new HashMap<>();
    properties.put("field", "x");
    properties.put("value", "y");
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(StringValueFilterTransform.NAME, Transform.PLUGIN_TYPE, properties));
    Schema inputSchema = Schema.recordOf("x", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    StageValidationRequest requestBody = new StageValidationRequest(stage, ImmutableList.of(new StageSchema("input1", inputSchema), new StageSchema("input2", inputSchema)), false);
    StageValidationResponse actual = sendRequest(requestBody);
    List<String> expectedInputs = ImmutableList.of("input1", "input2");
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    // the stage will add 3 causes. Two are related to input field and one is related to config property.
    Assert.assertEquals(3, failure.getCauses().size());
    Assert.assertEquals("field", failure.getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(STAGE));
    Assert.assertEquals(stageName, failure.getCauses().get(1).getAttribute(STAGE));
    Assert.assertEquals(stageName, failure.getCauses().get(2).getAttribute(STAGE));
    Assert.assertTrue(expectedInputs.contains(failure.getCauses().get(1).getAttribute(CauseAttributes.INPUT_STAGE)));
    Assert.assertTrue(expectedInputs.contains(failure.getCauses().get(2).getAttribute(CauseAttributes.INPUT_STAGE)));
}
Also used : StageSchema(io.cdap.cdap.etl.proto.v2.validation.StageSchema) StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) HashMap(java.util.HashMap) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) StageSchema(io.cdap.cdap.etl.proto.v2.validation.StageSchema) Schema(io.cdap.cdap.api.data.schema.Schema) ETLPlugin(io.cdap.cdap.etl.proto.v2.ETLPlugin) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse) ValidationFailure(io.cdap.cdap.etl.api.validation.ValidationFailure) Test(org.junit.Test)

Aggregations

StageValidationRequest (io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest)34 StageValidationResponse (io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse)34 ETLStage (io.cdap.cdap.etl.proto.v2.ETLStage)32 Test (org.junit.Test)30 Schema (io.cdap.cdap.api.data.schema.Schema)20 ETLPlugin (io.cdap.cdap.etl.proto.v2.ETLPlugin)20 StageSchema (io.cdap.cdap.etl.proto.v2.validation.StageSchema)20 ValidationFailure (io.cdap.cdap.etl.api.validation.ValidationFailure)18 HashMap (java.util.HashMap)12 StageSpec (io.cdap.cdap.etl.proto.v2.spec.StageSpec)6 ImmutableMap (com.google.common.collect.ImmutableMap)4 Gson (com.google.gson.Gson)4 GsonBuilder (com.google.gson.GsonBuilder)4 JsonSyntaxException (com.google.gson.JsonSyntaxException)4 ArtifactScope (io.cdap.cdap.api.artifact.ArtifactScope)4 MacroEvaluator (io.cdap.cdap.api.macro.MacroEvaluator)4 MacroParserOptions (io.cdap.cdap.api.macro.MacroParserOptions)4 BasicArguments (io.cdap.cdap.etl.common.BasicArguments)4 ConnectionMacroEvaluator (io.cdap.cdap.etl.common.ConnectionMacroEvaluator)4 DefaultMacroEvaluator (io.cdap.cdap.etl.common.DefaultMacroEvaluator)4