use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.
the class DataPipelineServiceTest method testConnectionMacroSubstitution.
@Test
public void testConnectionMacroSubstitution() throws Exception {
String stageName = "tx";
// test using no connection macro, it should behave as a normal plugin
Map<String, String> properties = Collections.singletonMap("tableName", "test");
ETLStage stage = new ETLStage(stageName, new ETLPlugin(MockSource.NAME, BatchSource.PLUGIN_TYPE, properties));
StageValidationRequest requestBody = new StageValidationRequest(stage, Collections.emptyList(), false);
StageValidationResponse actual = sendRequest(requestBody);
Assert.assertTrue(actual.getFailures().isEmpty());
// use a connection macro, the validation should also pass
properties = Collections.singletonMap("connectionConfig", "${conn(testconn)}");
addConnection("testconn", new ConnectionCreationRequest("", new PluginInfo("test", "dummy", null, Collections.singletonMap("tableName", "newtest"), new ArtifactSelectorConfig())));
stage = new ETLStage(stageName + "conn", new ETLPlugin(MockSource.NAME, BatchSource.PLUGIN_TYPE, properties));
requestBody = new StageValidationRequest(stage, Collections.emptyList(), false);
actual = sendRequest(requestBody);
Assert.assertTrue(actual.getFailures().isEmpty());
// test the properties can still be correctly set if the connection property get evaluated to a json object
addConnection("testconn", new ConnectionCreationRequest("", new PluginInfo("test", "dummy", null, ImmutableMap.of("tableName", "aaa", "key1", "${badval}"), new ArtifactSelectorConfig())));
getPreferencesService().setProperties(NamespaceId.DEFAULT, Collections.singletonMap("badval", "{\"a\" : 1}"));
// test it can still pass validation
actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), true));
Assert.assertTrue(actual.getFailures().isEmpty());
}
use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.
the class DataPipelineServiceTest method testValidateStagePluginNotFound.
@Test
public void testValidateStagePluginNotFound() throws Exception {
String name = MockSource.NAME;
String type = BatchSource.PLUGIN_TYPE;
ArtifactSelectorConfig requestedArtifact = new ArtifactSelectorConfig(ArtifactScope.USER.name(), batchMocksArtifactId.getArtifact() + "-ghost", batchMocksArtifactId.getVersion());
String stageName = "src";
ETLStage stage = new ETLStage(stageName, new ETLPlugin(name, type, Collections.emptyMap(), requestedArtifact));
StageValidationResponse actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), false));
Assert.assertEquals(1, actual.getFailures().size());
ValidationFailure failure = actual.getFailures().iterator().next();
Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_ID));
Assert.assertEquals(type, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_TYPE));
Assert.assertEquals(name, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_NAME));
Assert.assertEquals(requestedArtifact.getName(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_NAME));
Assert.assertEquals(requestedArtifact.getScope(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_SCOPE));
Assert.assertEquals(requestedArtifact.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_VERSION));
Assert.assertEquals(batchMocksArtifactId.getArtifact(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_NAME));
Assert.assertEquals(ArtifactScope.SYSTEM.name(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_SCOPE));
Assert.assertEquals(batchMocksArtifactId.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_VERSION));
}
use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.
the class DataPipelineServiceTest method testMacroResolutionFromProperties.
@Test
public void testMacroResolutionFromProperties() throws Exception {
// StringValueFilterTransform checks that the field exists in the input schema
String stageName = "tx";
Map<String, String> properties = new HashMap<>();
properties.put("field", "x");
properties.put("value", "${someProperty}");
ETLStage stage = new ETLStage(stageName, new ETLPlugin(StringValueFilterTransform.NAME, Transform.PLUGIN_TYPE, properties));
Schema inputSchema = Schema.recordOf("x", Schema.Field.of("x", Schema.of(Schema.Type.STRING)));
// Set the preference value in the store
getPreferencesService().setProperties(NamespaceId.DEFAULT, Collections.singletonMap("someProperty", "someValue"));
// This call should include the resolved value for Field
StageValidationRequest requestBody1 = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), true);
StageValidationResponse actual1 = sendRequest(requestBody1);
Assert.assertTrue(actual1.getFailures().isEmpty());
Assert.assertNotNull(actual1.getSpec().getPlugin());
Assert.assertEquals("someValue", actual1.getSpec().getPlugin().getProperties().get("value"));
// This call should NOT include the resolved value for Field
StageValidationRequest requestBody2 = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), false);
StageValidationResponse actual2 = sendRequest(requestBody2);
Assert.assertTrue(actual2.getFailures().isEmpty());
Assert.assertNotNull(actual2.getSpec().getPlugin());
Assert.assertEquals("${someProperty}", actual2.getSpec().getPlugin().getProperties().get("value"));
}
use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.
the class DataPipelineServiceTest method testValidationFailureForJoiner.
@Test
public void testValidationFailureForJoiner() throws Exception {
String stageName = "joiner";
// join key field t2_cust_name does not exist
ETLStage stage = new ETLStage(stageName, MockJoiner.getPlugin("t1.customer_id=t2.cust_id&" + "t1.customer_name=t2.t2_cust_name", "t1,t2", ""));
StageSchema inputSchema1 = new StageSchema("t1", Schema.recordOf("id", Schema.Field.of("customer_id", Schema.of(Schema.Type.STRING)), Schema.Field.of("customer_name", Schema.of(Schema.Type.STRING))));
// t1.customer_id type string does not match t2.cust_id type int
StageSchema inputSchema2 = new StageSchema("t2", Schema.recordOf("id", Schema.Field.of("cust_id", Schema.of(Schema.Type.INT)), Schema.Field.of("cust_name", Schema.of(Schema.Type.STRING))));
StageValidationRequest requestBody = new StageValidationRequest(stage, ImmutableList.of(inputSchema1, inputSchema2), false);
StageValidationResponse actual = sendRequest(requestBody);
Assert.assertNull(actual.getSpec());
Assert.assertEquals(2, actual.getFailures().size());
ValidationFailure fieldDoesNotExist = actual.getFailures().get(0);
Assert.assertEquals(stageName, fieldDoesNotExist.getCauses().get(0).getAttribute(STAGE));
Assert.assertEquals("t1.customer_id=t2.cust_id", fieldDoesNotExist.getCauses().get(0).getAttribute(CauseAttributes.CONFIG_ELEMENT));
ValidationFailure typeMismatch = actual.getFailures().get(1);
Assert.assertEquals(stageName, typeMismatch.getCauses().get(0).getAttribute(STAGE));
Assert.assertEquals("t1.customer_name=t2.t2_cust_name", typeMismatch.getCauses().get(0).getAttribute(CauseAttributes.CONFIG_ELEMENT));
}
use of io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest in project cdap by caskdata.
the class DataPipelineServiceTest method testValidationFailureForAggregator.
@Test
public void testValidationFailureForAggregator() throws Exception {
String stageName = "ag";
ETLStage stage = new ETLStage(stageName, DistinctAggregator.getPlugin("id,name"));
// input schema does not contain name field
Schema inputSchema = Schema.recordOf("id", Schema.Field.of("id", Schema.of(Schema.Type.STRING)));
StageValidationRequest requestBody = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), false);
StageValidationResponse actual = sendRequest(requestBody);
Assert.assertNull(actual.getSpec());
Assert.assertEquals(1, actual.getFailures().size());
ValidationFailure failure = actual.getFailures().iterator().next();
Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(STAGE));
Assert.assertEquals("fields", failure.getCauses().get(0).getAttribute(CauseAttributes.STAGE_CONFIG));
Assert.assertEquals("name", failure.getCauses().get(0).getAttribute(CauseAttributes.CONFIG_ELEMENT));
}
Aggregations