Search in sources :

Example 26 with ArtifactSelectorConfig

use of io.cdap.cdap.etl.proto.ArtifactSelectorConfig in project cdap by cdapio.

the class DataPipelineServiceTest method testValidateStagePluginNotFound.

@Test
public void testValidateStagePluginNotFound() throws Exception {
    String name = MockSource.NAME;
    String type = BatchSource.PLUGIN_TYPE;
    ArtifactSelectorConfig requestedArtifact = new ArtifactSelectorConfig(ArtifactScope.USER.name(), batchMocksArtifactId.getArtifact() + "-ghost", batchMocksArtifactId.getVersion());
    String stageName = "src";
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(name, type, Collections.emptyMap(), requestedArtifact));
    StageValidationResponse actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), false));
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_ID));
    Assert.assertEquals(type, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_TYPE));
    Assert.assertEquals(name, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_NAME));
    Assert.assertEquals(requestedArtifact.getName(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_NAME));
    Assert.assertEquals(requestedArtifact.getScope(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_SCOPE));
    Assert.assertEquals(requestedArtifact.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_VERSION));
    Assert.assertEquals(batchMocksArtifactId.getArtifact(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_NAME));
    Assert.assertEquals(ArtifactScope.SYSTEM.name(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_SCOPE));
    Assert.assertEquals(batchMocksArtifactId.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_VERSION));
}
Also used : StageValidationRequest(io.cdap.cdap.etl.proto.v2.validation.StageValidationRequest) ArtifactSelectorConfig(io.cdap.cdap.etl.proto.ArtifactSelectorConfig) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) ETLPlugin(io.cdap.cdap.etl.proto.v2.ETLPlugin) StageValidationResponse(io.cdap.cdap.etl.proto.v2.validation.StageValidationResponse) ValidationFailure(io.cdap.cdap.etl.api.validation.ValidationFailure) Test(org.junit.Test)

Example 27 with ArtifactSelectorConfig

use of io.cdap.cdap.etl.proto.ArtifactSelectorConfig in project cdap by cdapio.

the class ConnectionStoreTest method testNotFound.

@Test
public void testNotFound() throws Exception {
    ConnectionId connectionId = new ConnectionId(new NamespaceSummary("default", "", 0L), "nonexisting");
    // get non-existing connection
    try {
        connectionStore.getConnection(connectionId);
        Assert.fail();
    } catch (ConnectionNotFoundException e) {
    // expected
    }
    try {
        connectionStore.deleteConnection(connectionId);
        Assert.fail();
    } catch (ConnectionNotFoundException e) {
    // expected
    }
    // put a connection in the store
    NamespaceSummary oldNamespace = new NamespaceSummary("default", "", 0L);
    ConnectionId id = new ConnectionId(oldNamespace, "myconn");
    Connection connection = new Connection("myconn", "GCS", "GCS connection", false, false, 0L, 0L, new PluginInfo("GCS", "connector", "Google Cloud Platform", ImmutableMap.of("project", "abc"), new ArtifactSelectorConfig("SYSTEM", "google-cloud", "1.0.0")));
    connectionStore.saveConnection(id, connection, false);
    // get the connection with a namespace with a higher generation id should fail
    try {
        connectionStore.getConnection(new ConnectionId(new NamespaceSummary("default", "", 1L), "myconn"));
        Assert.fail();
    } catch (ConnectionNotFoundException e) {
    // expected
    }
}
Also used : ConnectionId(io.cdap.cdap.etl.proto.connection.ConnectionId) ConnectionNotFoundException(io.cdap.cdap.etl.proto.connection.ConnectionNotFoundException) ArtifactSelectorConfig(io.cdap.cdap.etl.proto.ArtifactSelectorConfig) Connection(io.cdap.cdap.etl.proto.connection.Connection) NamespaceSummary(io.cdap.cdap.api.NamespaceSummary) PluginInfo(io.cdap.cdap.etl.proto.connection.PluginInfo) Test(org.junit.Test)

Example 28 with ArtifactSelectorConfig

use of io.cdap.cdap.etl.proto.ArtifactSelectorConfig in project cdap by caskdata.

the class ETLBatchConfigTest method testUpgrade.

@Test
public void testUpgrade() throws Exception {
    final ArtifactSelectorConfig artifact = new ArtifactSelectorConfig("SYSTEM", "universal", "1.0.0");
    ETLStage source = new ETLStage("source", new Plugin("DataGenerator", ImmutableMap.of("p1", "v1"), artifact), null);
    io.cdap.cdap.etl.proto.v2.ETLStage sourceNew = from(source, BatchSource.PLUGIN_TYPE);
    ETLStage transform1 = new ETLStage("transform1", new Plugin("Script", ImmutableMap.of("script", "something"), null));
    io.cdap.cdap.etl.proto.v2.ETLStage transform1New = from(transform1, Transform.PLUGIN_TYPE);
    ETLStage transform2 = new ETLStage("transform2", new Plugin("Script", null, null));
    io.cdap.cdap.etl.proto.v2.ETLStage transform2New = from(transform2, Transform.PLUGIN_TYPE);
    ETLStage transform3 = new ETLStage("transform3", new Plugin("Validator", ImmutableMap.of("p1", "v1", "p2", "v2")), null);
    io.cdap.cdap.etl.proto.v2.ETLStage transform3New = from(transform3, Transform.PLUGIN_TYPE);
    ETLStage sink1 = new ETLStage("sink1", new Plugin("Table", ImmutableMap.of("rowkey", "xyz"), artifact), null);
    io.cdap.cdap.etl.proto.v2.ETLStage sink1New = from(sink1, BatchSink.PLUGIN_TYPE);
    ETLStage sink2 = new ETLStage("sink2", new Plugin("HDFS", ImmutableMap.of("name", "abc"), artifact), null);
    io.cdap.cdap.etl.proto.v2.ETLStage sink2New = from(sink2, BatchSink.PLUGIN_TYPE);
    Set<Connection> connections = new HashSet<>();
    connections.add(new Connection(sourceNew.getName(), transform1New.getName()));
    connections.add(new Connection(transform1New.getName(), transform2New.getName()));
    connections.add(new Connection(transform2New.getName(), transform3New.getName()));
    connections.add(new Connection(transform3New.getName(), sink1New.getName()));
    connections.add(new Connection(transform3New.getName(), sink2New.getName()));
    String schedule = "*/5 * * * *";
    Resources resources = new Resources(1024, 1);
    ETLBatchConfig config = ETLBatchConfig.builder(schedule).setSource(source).addSink(sink1).addSink(sink2).addTransform(transform1).addTransform(transform2).addTransform(transform3).addConnections(connections).setResources(resources).setDriverResources(resources).build();
    io.cdap.cdap.etl.proto.v2.ETLBatchConfig configNew = io.cdap.cdap.etl.proto.v2.ETLBatchConfig.builder(schedule).addStage(sourceNew).addStage(sink1New).addStage(sink2New).addStage(transform1New).addStage(transform2New).addStage(transform3New).addConnections(connections).setResources(resources).setDriverResources(resources).build();
    Assert.assertEquals(configNew, config.upgrade(new UpgradeContext() {

        @Nullable
        @Override
        public ArtifactSelectorConfig getPluginArtifact(String pluginType, String pluginName) {
            return null;
        }
    }));
}
Also used : ArtifactSelectorConfig(io.cdap.cdap.etl.proto.ArtifactSelectorConfig) Connection(io.cdap.cdap.etl.proto.Connection) UpgradeContext(io.cdap.cdap.etl.proto.UpgradeContext) Resources(io.cdap.cdap.api.Resources) ETLPlugin(io.cdap.cdap.etl.proto.v2.ETLPlugin) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 29 with ArtifactSelectorConfig

use of io.cdap.cdap.etl.proto.ArtifactSelectorConfig in project cdap by caskdata.

the class ETLBatchConfigTest method testUpgrade.

@Test
public void testUpgrade() throws Exception {
    final ArtifactSelectorConfig artifact = new ArtifactSelectorConfig("SYSTEM", "universal", "1.0.0");
    ETLStage source = new ETLStage("DataGenerator", ImmutableMap.of("p1", "v1"), null);
    io.cdap.cdap.etl.proto.v1.ETLStage sourceNew = new io.cdap.cdap.etl.proto.v1.ETLStage("DataGenerator.1", new Plugin(source.getName(), source.getProperties(), artifact), source.getErrorDatasetName());
    ETLStage transform1 = new ETLStage("Script", ImmutableMap.of("script", "something"), null);
    io.cdap.cdap.etl.proto.v1.ETLStage transform1New = new io.cdap.cdap.etl.proto.v1.ETLStage("Script.2", new Plugin(transform1.getName(), transform1.getProperties(), artifact), transform1.getErrorDatasetName());
    ETLStage transform2 = new ETLStage("Script", null, null);
    io.cdap.cdap.etl.proto.v1.ETLStage transform2New = new io.cdap.cdap.etl.proto.v1.ETLStage("Script.3", new Plugin(transform2.getName(), transform2.getProperties(), artifact), transform2.getErrorDatasetName());
    ETLStage transform3 = new ETLStage("Validator", ImmutableMap.of("p1", "v1", "p2", "v2"), "errorDS");
    io.cdap.cdap.etl.proto.v1.ETLStage transform3New = new io.cdap.cdap.etl.proto.v1.ETLStage("Validator.4", new Plugin(transform3.getName(), transform3.getProperties(), artifact), transform3.getErrorDatasetName());
    ETLStage sink1 = new ETLStage("Table", ImmutableMap.of("rowkey", "xyz"), null);
    io.cdap.cdap.etl.proto.v1.ETLStage sink1New = new io.cdap.cdap.etl.proto.v1.ETLStage("Table.5", new Plugin(sink1.getName(), sink1.getProperties(), artifact), sink1.getErrorDatasetName());
    ETLStage sink2 = new ETLStage("HDFS", ImmutableMap.of("name", "abc"), null);
    io.cdap.cdap.etl.proto.v1.ETLStage sink2New = new io.cdap.cdap.etl.proto.v1.ETLStage("HDFS.6", new Plugin(sink2.getName(), sink2.getProperties(), artifact), sink2.getErrorDatasetName());
    ETLStage action = new ETLStage("Email", ImmutableMap.of("email", "slj@example.com"), null);
    io.cdap.cdap.etl.proto.v1.ETLStage actionNew = new io.cdap.cdap.etl.proto.v1.ETLStage("Email.1", new Plugin(action.getName(), action.getProperties(), artifact), action.getErrorDatasetName());
    List<Connection> connections = new ArrayList<>();
    connections.add(new Connection(sourceNew.getName(), transform1New.getName()));
    connections.add(new Connection(transform1New.getName(), transform2New.getName()));
    connections.add(new Connection(transform2New.getName(), transform3New.getName()));
    connections.add(new Connection(transform3New.getName(), sink1New.getName()));
    connections.add(new Connection(transform3New.getName(), sink2New.getName()));
    String schedule = "*/5 * * * *";
    Resources resources = new Resources(1024, 1);
    ETLBatchConfig config = new ETLBatchConfig(schedule, source, ImmutableList.of(sink1, sink2), ImmutableList.of(transform1, transform2, transform3), resources, ImmutableList.of(action));
    io.cdap.cdap.etl.proto.v1.ETLBatchConfig configNew = io.cdap.cdap.etl.proto.v1.ETLBatchConfig.builder(schedule).setSource(sourceNew).addSink(sink1New).addSink(sink2New).addTransform(transform1New).addTransform(transform2New).addTransform(transform3New).addConnections(connections).setResources(resources).setDriverResources(resources).addAction(actionNew).build();
    Assert.assertEquals(configNew, config.upgrade(new UpgradeContext() {

        @Nullable
        @Override
        public ArtifactSelectorConfig getPluginArtifact(String pluginType, String pluginName) {
            return new ArtifactSelectorConfig(ArtifactScope.SYSTEM.name(), "universal", "1.0.0");
        }
    }));
}
Also used : ArtifactSelectorConfig(io.cdap.cdap.etl.proto.ArtifactSelectorConfig) Connection(io.cdap.cdap.etl.proto.Connection) ArrayList(java.util.ArrayList) UpgradeContext(io.cdap.cdap.etl.proto.UpgradeContext) Resources(io.cdap.cdap.api.Resources) Plugin(io.cdap.cdap.etl.proto.v1.Plugin) Test(org.junit.Test)

Example 30 with ArtifactSelectorConfig

use of io.cdap.cdap.etl.proto.ArtifactSelectorConfig in project cdap by caskdata.

the class ETLStage method upgradeStage.

io.cdap.cdap.etl.proto.v1.ETLStage upgradeStage(String name, String pluginType, UpgradeContext upgradeContext) {
    ArtifactSelectorConfig artifactSelectorConfig = upgradeContext.getPluginArtifact(pluginType, this.name);
    Plugin plugin = new Plugin(this.name, getProperties(), artifactSelectorConfig);
    return new io.cdap.cdap.etl.proto.v1.ETLStage(name, plugin, this.errorDatasetName);
}
Also used : ArtifactSelectorConfig(io.cdap.cdap.etl.proto.ArtifactSelectorConfig) Plugin(io.cdap.cdap.etl.proto.v1.Plugin)

Aggregations

ArtifactSelectorConfig (io.cdap.cdap.etl.proto.ArtifactSelectorConfig)65 Test (org.junit.Test)45 PluginInfo (io.cdap.cdap.etl.proto.connection.PluginInfo)20 StructuredRecord (io.cdap.cdap.api.data.format.StructuredRecord)19 Schema (io.cdap.cdap.api.data.schema.Schema)19 ETLStage (io.cdap.cdap.etl.proto.v2.ETLStage)19 File (java.io.File)18 ETLPlugin (io.cdap.cdap.etl.proto.v2.ETLPlugin)17 Table (io.cdap.cdap.api.dataset.table.Table)15 ETLBatchConfig (io.cdap.cdap.etl.proto.v2.ETLBatchConfig)15 ApplicationManager (io.cdap.cdap.test.ApplicationManager)15 HashSet (java.util.HashSet)15 ProgramRunStatus (io.cdap.cdap.proto.ProgramRunStatus)14 AppRequest (io.cdap.cdap.proto.artifact.AppRequest)14 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)14 WorkflowManager (io.cdap.cdap.test.WorkflowManager)14 ArrayList (java.util.ArrayList)14 TimeoutException (java.util.concurrent.TimeoutException)14 HashMap (java.util.HashMap)13 ArtifactSummary (io.cdap.cdap.api.artifact.ArtifactSummary)12