Examples with StageValidationRequest io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest used on opensource projects

Example 16 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by cdapio.

the class RemoteValidationTask method run.

@Override
public void run(RunnableTaskContext context) throws Exception {
    SystemAppTaskContext systemAppContext = context.getRunnableTaskSystemAppContext();
    RemoteValidationRequest remoteValidationRequest = GSON.fromJson(context.getParam(), RemoteValidationRequest.class);
    String namespace = remoteValidationRequest.getNamespace();
    String originalRequest = remoteValidationRequest.getRequest();
    StageValidationRequest validationRequest;
    try {
        validationRequest = GSON.fromJson(originalRequest, StageValidationRequest.class);
        validationRequest.validate();
    } catch (JsonSyntaxException e) {
        throw new IllegalArgumentException(String.format("Unable to decode request body %s", originalRequest), e);
    } catch (IllegalArgumentException e) {
        throw new IllegalArgumentException("Invalid stage config", e);
    }
    Map<String, String> arguments = Collections.emptyMap();
    // this option.
    if (validationRequest.getResolveMacrosFromPreferences()) {
        try {
            arguments = systemAppContext.getPreferencesForNamespace(namespace, true);
        } catch (IllegalArgumentException iae) {
            // If this is the case, we return a 404 error.
            throw new IllegalArgumentException(String.format(NAMESPACE_DOES_NOT_EXIST, namespace), iae);
        }
    }
    Map<String, MacroEvaluator> evaluators = ImmutableMap.of(SecureStoreMacroEvaluator.FUNCTION_NAME, new SecureStoreMacroEvaluator(namespace, systemAppContext), OAuthMacroEvaluator.FUNCTION_NAME, new OAuthMacroEvaluator(systemAppContext), ConnectionMacroEvaluator.FUNCTION_NAME, new ConnectionMacroEvaluator(namespace, systemAppContext));
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(arguments), evaluators, DefaultMacroEvaluator.MAP_FUNCTIONS);
    MacroParserOptions macroParserOptions = MacroParserOptions.builder().skipInvalidMacros().setEscaping(false).setFunctionWhitelist(evaluators.keySet()).build();
    Function<Map<String, String>, Map<String, String>> macroFn = macroProperties -> systemAppContext.evaluateMacros(namespace, macroProperties, macroEvaluator, macroParserOptions);
    PluginConfigurer pluginConfigurer = systemAppContext.createPluginConfigurer(namespace);
    StageValidationResponse validationResponse = ValidationUtils.validate(namespace, validationRequest, pluginConfigurer, macroFn, systemAppContext);
    // If the validation success and if it only involves system artifacts, then we don't need to restart task runner
    if (validationResponse.getFailures().isEmpty()) {
        StageSpec spec = validationResponse.getSpec();
        if (spec != null) {
            context.setTerminateOnComplete(!ArtifactScope.SYSTEM.equals(spec.getPlugin().getArtifact().getScope()));
        }
    }
    context.writeResult(GSON.toJson(validationResponse).getBytes());
}

Example 17 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by cdapio.

the class ValidationUtils method validate.

/**
 * Validate plugin based on the {@link StageValidationRequest}
 *
 * @param validationRequest {@link StageValidationRequest} with plugin properties
 * @param pluginConfigurer  {@link PluginConfigurer} for using the plugin
 * @param macroFn           {@link Function} for evaluating macros
 * @return {@link StageValidationResponse} in json format
 */
public static StageValidationResponse validate(String namespace, StageValidationRequest validationRequest, PluginConfigurer pluginConfigurer, Function<Map<String, String>, Map<String, String>> macroFn, FeatureFlagsProvider featureFlagsProvider) {
    ETLStage stageConfig = validationRequest.getStage();
    ValidatingConfigurer validatingConfigurer = new ValidatingConfigurer(pluginConfigurer, featureFlagsProvider);
    // Batch or Streaming doesn't matter for a single stage.
    PipelineSpecGenerator<ETLBatchConfig, BatchPipelineSpec> pipelineSpecGenerator = new BatchPipelineSpecGenerator(namespace, validatingConfigurer, null, Collections.emptySet(), Collections.emptySet(), Engine.SPARK, featureFlagsProvider);
    DefaultStageConfigurer stageConfigurer = new DefaultStageConfigurer(stageConfig.getName());
    for (StageSchema stageSchema : validationRequest.getInputSchemas()) {
        stageConfigurer.addInputSchema(stageSchema.getStage(), stageSchema.getSchema());
        stageConfigurer.addInputStage(stageSchema.getStage());
    }
    DefaultPipelineConfigurer pipelineConfigurer = new DefaultPipelineConfigurer(validatingConfigurer, stageConfig.getName(), Engine.SPARK, stageConfigurer, featureFlagsProvider);
    // evaluate macros
    Map<String, String> evaluatedProperties = macroFn.apply(stageConfig.getPlugin().getProperties());
    ETLPlugin originalConfig = stageConfig.getPlugin();
    ETLPlugin evaluatedConfig = new ETLPlugin(originalConfig.getName(), originalConfig.getType(), evaluatedProperties, originalConfig.getArtifactConfig());
    try {
        StageSpec spec = pipelineSpecGenerator.configureStage(stageConfig.getName(), evaluatedConfig, pipelineConfigurer).build();
        return new StageValidationResponse(spec);
    } catch (ValidationException e) {
        return new StageValidationResponse(e.getFailures());
    }
}

Example 18 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by cdapio.

the class DataPipelineServiceTest method testValidateStageMultipleErrors.

// test that multiple exceptions set in an InvalidStageException are captured as failures
@Test
public void testValidateStageMultipleErrors() throws Exception {
    // configure an invalid regex and a set the source and destination to the same value,
    // which should generate 2 errors
    String stageName = "stg";
    Map<String, String> properties = new HashMap<>();
    properties.put("filterRegex", "[");
    properties.put("sourceFileset", "files");
    properties.put("destinationFileset", "files");
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(FileMoveAction.NAME, Action.PLUGIN_TYPE, properties));
    StageValidationRequest request = new StageValidationRequest(stage, Collections.emptyList(), false);
    StageValidationResponse actual = sendRequest(request);
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(2, actual.getFailures().size());
    ValidationFailure failure1 = actual.getFailures().get(0);
    Assert.assertEquals("filterRegex", failure1.getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    Assert.assertEquals(stageName, failure1.getCauses().get(0).getAttribute(STAGE));
    // failure 2 should have 2 causes one for each config property
    ValidationFailure failure2 = actual.getFailures().get(1);
    Assert.assertEquals(2, failure2.getCauses().size());
}

Example 19 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by cdapio.

the class DataPipelineServiceTest method testValidationFailureForAggregator.

@Test
public void testValidationFailureForAggregator() throws Exception {
    String stageName = "ag";
    ETLStage stage = new ETLStage(stageName, DistinctAggregator.getPlugin("id,name"));
    // input schema does not contain name field
    Schema inputSchema = Schema.recordOf("id", Schema.Field.of("id", Schema.of(Schema.Type.STRING)));
    StageValidationRequest requestBody = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), false);
    StageValidationResponse actual = sendRequest(requestBody);
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(STAGE));
    Assert.assertEquals("fields", failure.getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    Assert.assertEquals("name", failure.getCauses().get(0).getAttribute(CauseAttributes.CONFIG_ELEMENT));
}

Example 20 with StageValidationRequest

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by cdapio.

the class DataPipelineServiceTest method testValidateStagePluginNotFound.

@Test
public void testValidateStagePluginNotFound() throws Exception {
    String name = MockSource.NAME;
    String type = BatchSource.PLUGIN_TYPE;
    ArtifactSelectorConfig requestedArtifact = new ArtifactSelectorConfig(ArtifactScope.USER.name(), batchMocksArtifactId.getArtifact() + "-ghost", batchMocksArtifactId.getVersion());
    String stageName = "src";
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(name, type, Collections.emptyMap(), requestedArtifact));
    StageValidationResponse actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), false));
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_ID));
    Assert.assertEquals(type, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_TYPE));
    Assert.assertEquals(name, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_NAME));
    Assert.assertEquals(requestedArtifact.getName(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_NAME));
    Assert.assertEquals(requestedArtifact.getScope(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_SCOPE));
    Assert.assertEquals(requestedArtifact.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_VERSION));
    Assert.assertEquals(batchMocksArtifactId.getArtifact(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_NAME));
    Assert.assertEquals(ArtifactScope.SYSTEM.name(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_SCOPE));
    Assert.assertEquals(batchMocksArtifactId.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_VERSION));
}