Examples with ETLPlugin io.cdap.cdap.etl.proto.v2.ETLPlugin used on opensource projects

Example 6 with ETLPlugin

use of io.cdap.cdap.etl.proto.v2.ETLPlugin in project cdap by caskdata.

the class DataPipelineTest method testSimpleUpgradePipelinesWithSnapshotArtifact.

/* Tests upgrade for a deployed application. Also tests that SNAPSHOT artifacts are being considered for upgrade.
     1. Deploy an application with older application artifact (1.0.0) and older filter plugin version (1.0.0).
     2. Add new versions of application artifact (0.0.9, 1.1.0, 1.2.0) and filter plugin artifacts (1.0.5, 1.1.0).
     3. Also deploy a snapshot version of app artifact 1.3.0-SNAPSHOT and plugin artifact 1.1.1-SNAPSHOT bind to it.
     3. Upgrade the older deployed application.
     4. Verify that after upgrading, application artifact and filter plugin artifact is upgraded to use latest version
        in its config and it uses snapshot versions for both.
   */
@Test
public void testSimpleUpgradePipelinesWithSnapshotArtifact() throws Exception {
    ArtifactSelectorConfig currentArtifactSelector = new ArtifactSelectorConfig(ArtifactScope.USER.name(), "test-plugins", "1.0.0");
    Engine engine = Engine.MAPREDUCE;
    String sourceName = "testSource" + engine.name();
    String sinkName = "testSink" + engine.name();
    ETLBatchConfig etlConfig = ETLBatchConfig.builder().setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("filter", PluggableFilterTransform.getPlugin(ValueFilter.NAME, ValueFilter.getProperties("${field}", "${value}"), currentArtifactSelector))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addConnection("source", "filter").addConnection("filter", "sink").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app("sparkProgramTest");
    // Deploy app with artifact version 1.0.0.
    ApplicationManager appManager = deployApplication(appId, appRequest);
    ApplicationDetail oldAppDetail = getAppDetail(appId);
    ETLBatchConfig oldBatchConfig = GSON.fromJson(oldAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> oldStageMap = oldBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Upgrade application with allowSnapshot set to true.
    appManager.upgrade(Collections.emptySet(), true);
    ApplicationDetail upgradedAppDetail = getAppDetail(appId);
    ETLBatchConfig newBatchConfig = GSON.fromJson(upgradedAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> newStageMap = newBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Compare stages that should be same after upgrade.
    Assert.assertEquals(oldStageMap.get("source"), newStageMap.get("source"));
    Assert.assertEquals(oldStageMap.get("sink"), newStageMap.get("sink"));
    // Verify that after upgrade, application upgrades artifact version to latest version available.
    Assert.assertEquals(UPGRADE_APP_ARTIFACT_ID_3_SNAPSHOT.getVersion(), upgradedAppDetail.getArtifact().getVersion());
    // Check if the filter stage, for which version should be upgraded to desired version in SYSTEM scope.
    ETLPlugin upgradedPlugin = newStageMap.get("filter").getPlugin();
    Assert.assertEquals(upgradedPlugin.getArtifactConfig().getVersion(), "1.1.1-SNAPSHOT");
    Assert.assertEquals(ArtifactScope.valueOf(upgradedPlugin.getArtifactConfig().getScope().toUpperCase()), ArtifactScope.USER);
}

Example 7 with ETLPlugin

use of io.cdap.cdap.etl.proto.v2.ETLPlugin in project cdap by caskdata.

the class AutoJoinerTest method testAutoJoinWithMacros.

private void testAutoJoinWithMacros(Engine engine, List<String> required, Schema expectedSchema, Set<StructuredRecord> expectedRecords, boolean excludeUsers, boolean excludePurchases) throws Exception {
    /*
         users ------|
                     |--> join --> sink
         purchases --|

         joinOn: users.region = purchases.region and users.user_id = purchases.user_id
     */
    String userInput = UUID.randomUUID().toString();
    String purchaseInput = UUID.randomUUID().toString();
    String output = UUID.randomUUID().toString();
    Map<String, String> joinerProps = new HashMap<>();
    joinerProps.put(MockAutoJoiner.Conf.STAGES, "${stages}");
    joinerProps.put(MockAutoJoiner.Conf.KEY, "${key}");
    joinerProps.put(MockAutoJoiner.Conf.REQUIRED, "${required}");
    joinerProps.put(MockAutoJoiner.Conf.SELECT, "${select}");
    if (engine == Engine.SPARK || (required.size() < 2 && engine == Engine.MAPREDUCE)) {
        joinerProps.put(MockAutoJoiner.Conf.SCHEMA, "${schema}");
    }
    ETLBatchConfig config = ETLBatchConfig.builder().addStage(new ETLStage("users", MockSource.getPlugin(userInput))).addStage(new ETLStage("purchases", MockSource.getPlugin(purchaseInput))).addStage(new ETLStage("join", new ETLPlugin(MockAutoJoiner.NAME, BatchJoiner.PLUGIN_TYPE, joinerProps))).addStage(new ETLStage("sink", MockSink.getPlugin(output))).addConnection("users", "join").addConnection("purchases", "join").addConnection("join", "sink").setEngine(engine).build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
    ApplicationId appId = NamespaceId.DEFAULT.app(UUID.randomUUID().toString());
    ApplicationManager appManager = deployApplication(appId, appRequest);
    // write input data
    if (!excludeUsers) {
        List<StructuredRecord> userData = Arrays.asList(USER_ALICE, USER_ALYCE, USER_BOB);
        DataSetManager<Table> inputManager = getDataset(userInput);
        MockSource.writeInput(inputManager, userData);
    }
    if (!excludePurchases) {
        List<StructuredRecord> purchaseData = new ArrayList<>();
        purchaseData.add(StructuredRecord.builder(PURCHASE_SCHEMA).set("region", "us").set("user_id", 0).set("purchase_id", 123).build());
        purchaseData.add(StructuredRecord.builder(PURCHASE_SCHEMA).set("region", "us").set("user_id", 2).set("purchase_id", 456).build());
        DataSetManager<Table> inputManager = getDataset(purchaseInput);
        MockSource.writeInput(inputManager, purchaseData);
    }
    WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME);
    List<JoinField> selectedFields = new ArrayList<>();
    selectedFields.add(new JoinField("purchases", "region"));
    selectedFields.add(new JoinField("purchases", "purchase_id"));
    selectedFields.add(new JoinField("purchases", "user_id"));
    selectedFields.add(new JoinField("users", "name"));
    Map<String, String> joinerProperties = MockAutoJoiner.getProperties(Arrays.asList("purchases", "users"), Arrays.asList("region", "user_id"), required, Collections.emptyList(), selectedFields, true);
    Map<String, String> runtimeArgs = new HashMap<>();
    runtimeArgs.put("stages", joinerProperties.get(MockAutoJoiner.Conf.STAGES));
    runtimeArgs.put("key", joinerProperties.get(MockAutoJoiner.Conf.KEY));
    runtimeArgs.put("required", joinerProperties.get(MockAutoJoiner.Conf.REQUIRED));
    runtimeArgs.put("select", joinerProperties.get(MockAutoJoiner.Conf.SELECT));
    runtimeArgs.put("schema", expectedSchema.toString());
    workflowManager.startAndWaitForGoodRun(runtimeArgs, ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    DataSetManager<Table> outputManager = getDataset(output);
    List<StructuredRecord> outputRecords = MockSink.readOutput(outputManager);
    Assert.assertEquals(expectedRecords, new HashSet<>(outputRecords));
}

Example 8 with ETLPlugin

use of io.cdap.cdap.etl.proto.v2.ETLPlugin in project cdap by caskdata.

the class DataPipelineServiceTest method testConnectionMacroSubstitution.

@Test
public void testConnectionMacroSubstitution() throws Exception {
    String stageName = "tx";
    // test using no connection macro, it should behave as a normal plugin
    Map<String, String> properties = Collections.singletonMap("tableName", "test");
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(MockSource.NAME, BatchSource.PLUGIN_TYPE, properties));
    StageValidationRequest requestBody = new StageValidationRequest(stage, Collections.emptyList(), false);
    StageValidationResponse actual = sendRequest(requestBody);
    Assert.assertTrue(actual.getFailures().isEmpty());
    // use a connection macro, the validation should also pass
    properties = Collections.singletonMap("connectionConfig", "${conn(testconn)}");
    addConnection("testconn", new ConnectionCreationRequest("", new PluginInfo("test", "dummy", null, Collections.singletonMap("tableName", "newtest"), new ArtifactSelectorConfig())));
    stage = new ETLStage(stageName + "conn", new ETLPlugin(MockSource.NAME, BatchSource.PLUGIN_TYPE, properties));
    requestBody = new StageValidationRequest(stage, Collections.emptyList(), false);
    actual = sendRequest(requestBody);
    Assert.assertTrue(actual.getFailures().isEmpty());
    // test the properties can still be correctly set if the connection property get evaluated to a json object
    addConnection("testconn", new ConnectionCreationRequest("", new PluginInfo("test", "dummy", null, ImmutableMap.of("tableName", "aaa", "key1", "${badval}"), new ArtifactSelectorConfig())));
    getPreferencesService().setProperties(NamespaceId.DEFAULT, Collections.singletonMap("badval", "{\"a\" : 1}"));
    // test it can still pass validation
    actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), true));
    Assert.assertTrue(actual.getFailures().isEmpty());
}

Example 9 with ETLPlugin

use of io.cdap.cdap.etl.proto.v2.ETLPlugin in project cdap by caskdata.

the class DataPipelineServiceTest method testValidateStagePluginNotFound.

@Test
public void testValidateStagePluginNotFound() throws Exception {
    String name = MockSource.NAME;
    String type = BatchSource.PLUGIN_TYPE;
    ArtifactSelectorConfig requestedArtifact = new ArtifactSelectorConfig(ArtifactScope.USER.name(), batchMocksArtifactId.getArtifact() + "-ghost", batchMocksArtifactId.getVersion());
    String stageName = "src";
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(name, type, Collections.emptyMap(), requestedArtifact));
    StageValidationResponse actual = sendRequest(new StageValidationRequest(stage, Collections.emptyList(), false));
    Assert.assertEquals(1, actual.getFailures().size());
    ValidationFailure failure = actual.getFailures().iterator().next();
    Assert.assertEquals(stageName, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_ID));
    Assert.assertEquals(type, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_TYPE));
    Assert.assertEquals(name, failure.getCauses().get(0).getAttribute(CauseAttributes.PLUGIN_NAME));
    Assert.assertEquals(requestedArtifact.getName(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_NAME));
    Assert.assertEquals(requestedArtifact.getScope(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_SCOPE));
    Assert.assertEquals(requestedArtifact.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.REQUESTED_ARTIFACT_VERSION));
    Assert.assertEquals(batchMocksArtifactId.getArtifact(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_NAME));
    Assert.assertEquals(ArtifactScope.SYSTEM.name(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_SCOPE));
    Assert.assertEquals(batchMocksArtifactId.getVersion(), failure.getCauses().get(0).getAttribute(CauseAttributes.SUGGESTED_ARTIFACT_VERSION));
}

Example 10 with ETLPlugin

use of io.cdap.cdap.etl.proto.v2.ETLPlugin in project cdap by caskdata.

the class DataPipelineServiceTest method testMacroResolutionFromProperties.

@Test
public void testMacroResolutionFromProperties() throws Exception {
    // StringValueFilterTransform checks that the field exists in the input schema
    String stageName = "tx";
    Map<String, String> properties = new HashMap<>();
    properties.put("field", "x");
    properties.put("value", "${someProperty}");
    ETLStage stage = new ETLStage(stageName, new ETLPlugin(StringValueFilterTransform.NAME, Transform.PLUGIN_TYPE, properties));
    Schema inputSchema = Schema.recordOf("x", Schema.Field.of("x", Schema.of(Schema.Type.STRING)));
    // Set the preference value in the store
    getPreferencesService().setProperties(NamespaceId.DEFAULT, Collections.singletonMap("someProperty", "someValue"));
    // This call should include the resolved value for Field
    StageValidationRequest requestBody1 = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), true);
    StageValidationResponse actual1 = sendRequest(requestBody1);
    Assert.assertTrue(actual1.getFailures().isEmpty());
    Assert.assertNotNull(actual1.getSpec().getPlugin());
    Assert.assertEquals("someValue", actual1.getSpec().getPlugin().getProperties().get("value"));
    // This call should NOT include the resolved value for Field
    StageValidationRequest requestBody2 = new StageValidationRequest(stage, Collections.singletonList(new StageSchema("input", inputSchema)), false);
    StageValidationResponse actual2 = sendRequest(requestBody2);
    Assert.assertTrue(actual2.getFailures().isEmpty());
    Assert.assertNotNull(actual2.getSpec().getPlugin());
    Assert.assertEquals("${someProperty}", actual2.getSpec().getPlugin().getProperties().get("value"));
}