Examples with ApplicationDetail io.cdap.cdap.proto.ApplicationDetail used on opensource projects

Example 21 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class DataPipelineTest method testUpgradePipelinesWithNoChangeInPluginRange.

/* Tests upgrade for a deployed application with a plugin using plugin range.
   1. Deploy an application with older application artifact (1.0.0) and filter plugin version with range
      [1.0.0-2.0.0) to make sure latest version of plugin should be included in it.
   2. Add new versions of application artifacts (0.0.9, 1.1.0, 1.2.0) and filter plugin artifacts (1.0.5, 1.1.0) in
      SYSTEM scope (in test class setup).
   3. Upgrade the older deployed application.
   4. Verify that after upgrading, application artifact uses latest version in its config.
      But plugin range is not updated as latest version of plugin is still included in the range.
  */
@Test
public void testUpgradePipelinesWithNoChangeInPluginRange() throws Exception {
    ArtifactSelectorConfig currentArtifactSelector = new ArtifactSelectorConfig(ArtifactScope.USER.name(), "test-plugins", "[1.0.0,2.0.0)");
    Engine engine = Engine.MAPREDUCE;
    String sourceName = "testSource" + engine.name();
    String sinkName = "testSink" + engine.name();
    ETLBatchConfig etlConfig = ETLBatchConfig.builder().setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("filter", PluggableFilterTransform.getPlugin(ValueFilter.NAME, ValueFilter.getProperties("${field}", "${value}"), currentArtifactSelector))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addConnection("source", "filter").addConnection("filter", "sink").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app("sparkProgramTest");
    // Deploy app with artifact version 1.0.0.
    ApplicationManager appManager = deployApplication(appId, appRequest);
    ApplicationDetail oldAppDetail = getAppDetail(appId);
    ETLBatchConfig oldBatchConfig = GSON.fromJson(oldAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> oldStageMap = oldBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Upgrade application with consider snapshot enabled to consider highest version of plugin available which is
    // 1.1.1-SNAPSHOT.
    appManager.upgrade(Collections.emptySet(), true);
    ApplicationDetail upgradedAppDetail = getAppDetail(appId);
    ETLBatchConfig newBatchConfig = GSON.fromJson(upgradedAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> newStageMap = newBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Compare stages that should be same after upgrade.
    Assert.assertEquals(oldStageMap.get("source"), newStageMap.get("source"));
    Assert.assertEquals(oldStageMap.get("sink"), newStageMap.get("sink"));
    // The filter stage is not changed as latest plugin is in the range.
    Assert.assertEquals(oldStageMap.get("filter"), newStageMap.get("filter"));
    // Verify that after upgrade, application upgrades artifact version to latest version available.
    Assert.assertEquals(upgradedAppDetail.getArtifact().getVersion(), UPGRADE_APP_ARTIFACT_ID_3_SNAPSHOT.getVersion());
}

Example 22 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class DataPipelineTest method testSimpleUpgradePipelines.

/* Tests upgrade for a deployed application.
     1. Deploy an application with older application artifact (1.0.0) and older filter plugin version (1.0.0).
     2. Add new versions of application artifacts (0.0.9, 1.1.0, 1.2.0) and filter plugin artifacts (1.0.5, 1.1.0) in
        SYSTEM scope (in test class setup).
     3. Upgrade the older deployed application.
     4. Verify that after upgrading, application artifact and filter plugin artifact is upgraded to use latest version
        in its config.
   */
@Test
public void testSimpleUpgradePipelines() throws Exception {
    ArtifactSelectorConfig currentArtifactSelector = new ArtifactSelectorConfig(ArtifactScope.USER.name(), "test-plugins", "1.0.0");
    Engine engine = Engine.MAPREDUCE;
    String sourceName = "testSource" + engine.name();
    String sinkName = "testSink" + engine.name();
    ETLBatchConfig etlConfig = ETLBatchConfig.builder().setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("filter", PluggableFilterTransform.getPlugin(ValueFilter.NAME, ValueFilter.getProperties("${field}", "${value}"), currentArtifactSelector))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addConnection("source", "filter").addConnection("filter", "sink").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app("sparkProgramTest");
    // Deploy app with artifact version 1.0.0.
    ApplicationManager appManager = deployApplication(appId, appRequest);
    ApplicationDetail oldAppDetail = getAppDetail(appId);
    ETLBatchConfig oldBatchConfig = GSON.fromJson(oldAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> oldStageMap = oldBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Upgrade application.
    appManager.upgrade();
    ApplicationDetail upgradedAppDetail = getAppDetail(appId);
    ETLBatchConfig newBatchConfig = GSON.fromJson(upgradedAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> newStageMap = newBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Compare stages that should be same after upgrade.
    Assert.assertEquals(oldStageMap.get("source"), newStageMap.get("source"));
    Assert.assertEquals(oldStageMap.get("sink"), newStageMap.get("sink"));
    // Verify that after upgrade, application upgrades artifact version to latest version available.
    Assert.assertEquals(UPGRADE_APP_ARTIFACT_ID_2.getVersion(), upgradedAppDetail.getArtifact().getVersion());
    // Check if the filter stage, for which version should be upgraded to desired version in SYSTEM scope.
    ETLPlugin upgradedPlugin = newStageMap.get("filter").getPlugin();
    Assert.assertEquals(upgradedPlugin.getArtifactConfig().getVersion(), "1.1.0");
    Assert.assertEquals(ArtifactScope.valueOf(upgradedPlugin.getArtifactConfig().getScope().toUpperCase()), ArtifactScope.SYSTEM);
}

Example 23 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by caskdata.

the class ApplicationLifecycleService method getAppDetails.

/**
 * Gets details for a set of given applications.
 *
 * @param appIds the set of application id to get details
 * @return a {@link Map} from the application id to the corresponding detail. There will be no entry for applications
 * that don't exist.
 * @throws Exception if failed to get details.
 */
public Map<ApplicationId, ApplicationDetail> getAppDetails(Collection<ApplicationId> appIds) throws Exception {
    Set<? extends EntityId> visibleIds = accessEnforcer.isVisible(new HashSet<>(appIds), authenticationContext.getPrincipal());
    Set<ApplicationId> filterIds = appIds.stream().filter(visibleIds::contains).collect(Collectors.toSet());
    Map<ApplicationId, ApplicationSpecification> appSpecs = store.getApplications(filterIds);
    Map<ApplicationId, String> principals = ownerAdmin.getOwnerPrincipals(filterIds);
    Map<ApplicationId, ApplicationDetail> result = new HashMap<>();
    for (Map.Entry<ApplicationId, ApplicationSpecification> entry : appSpecs.entrySet()) {
        ApplicationId appId = entry.getKey();
        result.put(appId, enforceApplicationDetailAccess(appId, ApplicationDetail.fromSpec(entry.getValue(), principals.get(appId))));
    }
    return result;
}

Example 24 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by caskdata.

the class ApplicationLifecycleService method processApplications.

private void processApplications(List<Map.Entry<ApplicationId, ApplicationSpecification>> list, Consumer<ApplicationDetail> consumer) {
    Set<ApplicationId> appIds = list.stream().map(Map.Entry::getKey).collect(Collectors.toSet());
    Set<? extends EntityId> visible = accessEnforcer.isVisible(appIds, authenticationContext.getPrincipal());
    list.removeIf(entry -> !visible.contains(entry.getKey()));
    appIds.removeIf(id -> !visible.contains(id));
    try {
        Map<ApplicationId, String> owners = ownerAdmin.getOwnerPrincipals(appIds);
        for (Map.Entry<ApplicationId, ApplicationSpecification> entry : list) {
            ApplicationDetail applicationDetail = ApplicationDetail.fromSpec(entry.getValue(), owners.get(entry.getKey()));
            try {
                capabilityReader.checkAllEnabled(entry.getValue());
            } catch (CapabilityNotAvailableException ex) {
                LOG.debug("Application {} is ignored due to exception.", applicationDetail.getName(), ex);
                continue;
            }
            consumer.accept(applicationDetail);
        }
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}

Example 25 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by caskdata.

the class ApplicationClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    ApplicationId app = NamespaceId.DEFAULT.app(FakeApp.NAME);
    Assert.assertEquals(0, appClient.list(NamespaceId.DEFAULT).size());
    // deploy app
    LOG.info("Deploying app");
    appClient.deploy(NamespaceId.DEFAULT, createAppJarFile(FakeApp.class, FakeApp.NAME, "1.0.0-SNAPSHOT"));
    appClient.waitForDeployed(app, 30, TimeUnit.SECONDS);
    Assert.assertEquals(1, appClient.list(NamespaceId.DEFAULT).size());
    try {
        // check program list
        LOG.info("Checking program list for app");
        Map<ProgramType, List<ProgramRecord>> programs = appClient.listProgramsByType(app);
        verifyProgramNames(FakeApp.MAPREDUCES, programs.get(ProgramType.MAPREDUCE));
        verifyProgramNames(FakeApp.WORKFLOWS, programs.get(ProgramType.WORKFLOW));
        verifyProgramNames(FakeApp.SERVICES, programs.get(ProgramType.SERVICE));
        verifyProgramNames(FakeApp.MAPREDUCES, appClient.listPrograms(app, ProgramType.MAPREDUCE));
        verifyProgramNames(FakeApp.WORKFLOWS, appClient.listPrograms(app, ProgramType.WORKFLOW));
        verifyProgramNames(FakeApp.SERVICES, appClient.listPrograms(app, ProgramType.SERVICE));
        verifyProgramNames(FakeApp.MAPREDUCES, appClient.listAllPrograms(NamespaceId.DEFAULT, ProgramType.MAPREDUCE));
        verifyProgramNames(FakeApp.WORKFLOWS, appClient.listAllPrograms(NamespaceId.DEFAULT, ProgramType.WORKFLOW));
        verifyProgramNames(FakeApp.SERVICES, appClient.listAllPrograms(NamespaceId.DEFAULT, ProgramType.SERVICE));
        verifyProgramRecords(FakeApp.ALL_PROGRAMS, appClient.listAllPrograms(NamespaceId.DEFAULT));
        ApplicationDetail appDetail = appClient.get(app);
        ArtifactSummary expected = new ArtifactSummary(FakeApp.NAME, "1.0.0-SNAPSHOT");
        Assert.assertEquals(expected, appDetail.getArtifact());
    } finally {
        // delete app
        LOG.info("Deleting app");
        appClient.delete(app);
        appClient.waitForDeleted(app, 30, TimeUnit.SECONDS);
        Assert.assertEquals(0, appClient.list(NamespaceId.DEFAULT).size());
    }
}