Examples with StageValidationResponse io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse used on opensource projects

Example 11 with StageValidationResponse

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse in project cdap by caskdata.

the class ValidationUtilsTest method testValidateException.

@Test
public void testValidateException() {
    Schema testSchema = Schema.recordOf("a", Schema.Field.of("a", Schema.of(Schema.Type.STRING)));
    StageSchema testStageSchema = new StageSchema("source", testSchema);
    ETLStage etlStage = new ETLStage("source", MockSource.getPlugin(null));
    StageValidationRequest validationRequest = new StageValidationRequest(etlStage, Collections.singletonList(testStageSchema), false);
    StageValidationResponse stageValidationResponse = ValidationUtils.validate(NamespaceId.DEFAULT.getNamespace(), validationRequest, getPluginConfigurer(MockSource.PLUGIN_CLASS), properties -> properties, MOCK_FEATURE_FLAGS_PROVIDER);
    Assert.assertEquals(1, stageValidationResponse.getFailures().size());
}

Example 12 with StageValidationResponse

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse in project cdap by caskdata.

the class ValidationUtilsTest method testMacroSubstitution.

@Test
public void testMacroSubstitution() {
    Schema testSchema = Schema.recordOf("a", Schema.Field.of("a", Schema.of(Schema.Type.STRING)));
    StageSchema testStageSchema = new StageSchema("source", testSchema);
    ETLStage etlStage = new ETLStage("source", MockSource.getPlugin("@{tName}"));
    StageValidationRequest validationRequest = new StageValidationRequest(etlStage, Collections.singletonList(testStageSchema), false);
    String testtable = "testtable";
    StageValidationResponse stageValidationResponse = ValidationUtils.validate(NamespaceId.DEFAULT.getNamespace(), validationRequest, getPluginConfigurer(MockSource.PLUGIN_CLASS), properties -> {
        Map<String, String> propertiesCopy = new HashMap<>(properties);
        if (propertiesCopy.getOrDefault("tableName", "").equals("@{tName}")) {
            propertiesCopy.put("tableName", testtable);
        }
        return propertiesCopy;
    }, MOCK_FEATURE_FLAGS_PROVIDER);
    Assert.assertTrue(stageValidationResponse.getFailures().isEmpty());
    Assert.assertEquals(testtable, stageValidationResponse.getSpec().getPlugin().getProperties().get("tableName"));
}

Example 13 with StageValidationResponse

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse in project cdap by caskdata.

the class RemoteValidationTask method run.

@Override
public void run(RunnableTaskContext context) throws Exception {
    SystemAppTaskContext systemAppContext = context.getRunnableTaskSystemAppContext();
    RemoteValidationRequest remoteValidationRequest = GSON.fromJson(context.getParam(), RemoteValidationRequest.class);
    String namespace = remoteValidationRequest.getNamespace();
    String originalRequest = remoteValidationRequest.getRequest();
    StageValidationRequest validationRequest;
    try {
        validationRequest = GSON.fromJson(originalRequest, StageValidationRequest.class);
        validationRequest.validate();
    } catch (JsonSyntaxException e) {
        throw new IllegalArgumentException(String.format("Unable to decode request body %s", originalRequest), e);
    } catch (IllegalArgumentException e) {
        throw new IllegalArgumentException("Invalid stage config", e);
    }
    Map<String, String> arguments = Collections.emptyMap();
    // this option.
    if (validationRequest.getResolveMacrosFromPreferences()) {
        try {
            arguments = systemAppContext.getPreferencesForNamespace(namespace, true);
        } catch (IllegalArgumentException iae) {
            // If this is the case, we return a 404 error.
            throw new IllegalArgumentException(String.format(NAMESPACE_DOES_NOT_EXIST, namespace), iae);
        }
    }
    Map<String, MacroEvaluator> evaluators = ImmutableMap.of(SecureStoreMacroEvaluator.FUNCTION_NAME, new SecureStoreMacroEvaluator(namespace, systemAppContext), OAuthMacroEvaluator.FUNCTION_NAME, new OAuthMacroEvaluator(systemAppContext), ConnectionMacroEvaluator.FUNCTION_NAME, new ConnectionMacroEvaluator(namespace, systemAppContext));
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(arguments), evaluators, DefaultMacroEvaluator.MAP_FUNCTIONS);
    MacroParserOptions macroParserOptions = MacroParserOptions.builder().skipInvalidMacros().setEscaping(false).setFunctionWhitelist(evaluators.keySet()).build();
    Function<Map<String, String>, Map<String, String>> macroFn = macroProperties -> systemAppContext.evaluateMacros(namespace, macroProperties, macroEvaluator, macroParserOptions);
    PluginConfigurer pluginConfigurer = systemAppContext.createPluginConfigurer(namespace);
    StageValidationResponse validationResponse = ValidationUtils.validate(namespace, validationRequest, pluginConfigurer, macroFn, systemAppContext);
    // If the validation success and if it only involves system artifacts, then we don't need to restart task runner
    if (validationResponse.getFailures().isEmpty()) {
        StageSpec spec = validationResponse.getSpec();
        if (spec != null) {
            context.setTerminateOnComplete(!ArtifactScope.SYSTEM.equals(spec.getPlugin().getArtifact().getScope()));
        }
    }
    context.writeResult(GSON.toJson(validationResponse).getBytes());
}

Example 14 with StageValidationResponse

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse in project cdap by caskdata.

the class DataPipelineServiceTest method testValidateStageMultipleErrors.

// test that multiple exceptions set in an InvalidStageException are captured as failures
@Test
public void testValidateStageMultipleErrors() throws Exception {
    // configure an invalid regex and a set the source and destination to the same value,
    // which should generate 2 errors
    String stageName = "stg";
    Map<String, String> properties = new HashMap<>();
    properties.put("filterRegex", "[");
    properties.put("sourceFileset", "files");
    properties.put("destinationFileset", "files");
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(FileMoveAction.NAME, Action.PLUGIN_TYPE, properties));
    StageValidationRequest request = new StageValidationRequest(stage, Collections.emptyList(), false);
    StageValidationResponse actual = sendRequest(request);
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(2, actual.getFailures().size());
    ValidationFailure failure1 = actual.getFailures().get(0);
    Assert.assertEquals("filterRegex", failure1.getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    Assert.assertEquals(stageName, failure1.getCauses().get(0).getAttribute(STAGE));
    // failure 2 should have 2 causes one for each config property
    ValidationFailure failure2 = actual.getFailures().get(1);
    Assert.assertEquals(2, failure2.getCauses().size());
}

Example 15 with StageValidationResponse

use of io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse in project cdap by caskdata.

the class DataPipelineServiceTest method testSecureMacroSubstitution.

@Test
public void testSecureMacroSubstitution() throws Exception {
    // StringValueFilterTransform checks that the field exists in the input schema
    String stageName = "tx";
    Map<String, String> properties = new HashMap<>();
    properties.put("field", "${secure(field)}");
    properties.put("value", "y");
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(StringValueFilterTransform.NAME, Transform.PLUGIN_TYPE, properties));
    Schema inputSchema = Schema.recordOf("x", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    // this call happens when no value for 'field' is in the secure store, which should not result in
    // any failures because the macro could not be enabled so the check is skipped
    StageValidationRequest requestBody = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), false);
    StageValidationResponse actual = sendRequest(requestBody);
    Assert.assertTrue(actual.getFailures().isEmpty());
    // now set the value of 'field' to be 'z', which is not in the input schema
    // this call should result in a failure
    getSecureStoreManager().put("default", "field", "z", "desc", Collections.emptyMap());
    actual = sendRequest(requestBody);
    Assert.assertNull(actual.getSpec());
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    // the stage will add 2 causes for invalid input field failure. One is related to input field and the other is
    // related to config property.
    Assert.assertEquals(1, failure.getCauses().size());
    Assert.assertEquals("field", failure.getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(STAGE));
}