Examples with ApplicationDetail io.cdap.cdap.proto.ApplicationDetail used on opensource projects

Example 16 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class UpgradeTool method upgrade.

private boolean upgrade(final ApplicationId appId) throws Exception {
    final ApplicationDetail appDetail = appClient.get(appId);
    if (!upgrader.shouldUpgrade(appDetail.getArtifact())) {
        LOG.debug("Skipping app {}.", appId);
        return false;
    }
    Upgrader.UpgradeAction action;
    if (dryrun) {
        action = new Upgrader.UpgradeAction() {

            @Override
            public boolean upgrade(AppRequest<? extends ETLConfig> appRequest) throws Exception {
                LOG.info("Writing dryrun app request for pipeline: {}", appId);
                String filename = String.format("%s-%s.json", appId.getNamespace(), appId.getApplication());
                File outputFile = new File(outputDir, filename);
                try (BufferedWriter writer = Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8)) {
                    writer.write(GSON.toJson(appRequest));
                }
                return true;
            }
        };
    } else {
        action = new Upgrader.UpgradeAction() {

            @Override
            public boolean upgrade(AppRequest<? extends ETLConfig> appRequest) {
                LOG.info("Updating pipeline: {}", appId);
                try {
                    appClient.update(appId, appRequest);
                    return true;
                } catch (Exception e) {
                    LOG.error("Error updating pipeline {}.", appId, e);
                    if (errorDir != null) {
                        File errorFile = new File(errorDir, String.format("%s-%s.json", appId.getParent(), appId.getEntityName()));
                        LOG.error("Writing config for pipeline {} to {} for further manual investigation.", appId, errorFile.getAbsolutePath());
                        try (OutputStreamWriter outputStreamWriter = new OutputStreamWriter(new FileOutputStream(errorFile))) {
                            outputStreamWriter.write(GSON.toJson(appRequest));
                        } catch (IOException e1) {
                            LOG.error("Error writing config out for manual investigation.", e1);
                        }
                    }
                    return false;
                }
            }
        };
    }
    return upgrader.upgrade(appDetail.getArtifact(), appDetail.getConfiguration(), action);
}

Example 17 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class SupportBundlePipelineInfoTask method collect.

@Override
public void collect() throws IOException, NotFoundException {
    for (NamespaceId namespaceId : namespaces) {
        Iterable<ApplicationDetail> appDetails;
        if (requestApplication == null) {
            appDetails = remoteApplicationDetailFetcher.list(namespaceId.getNamespace());
        } else {
            try {
                appDetails = Collections.singletonList(remoteApplicationDetailFetcher.get(new ApplicationId(namespaceId.getNamespace(), requestApplication)));
            } catch (NotFoundException e) {
                LOG.debug("Failed to find application {} ", requestApplication, e);
                continue;
            }
        }
        for (ApplicationDetail appDetail : appDetails) {
            String application = appDetail.getName();
            ApplicationId applicationId = new ApplicationId(namespaceId.getNamespace(), application);
            File appFolderPath = new File(basePath, appDetail.getName());
            DirUtils.mkdirs(appFolderPath);
            try (FileWriter file = new FileWriter(new File(appFolderPath, appDetail.getName() + ".json"))) {
                GSON.toJson(appDetail, file);
            }
            ProgramId programId = new ProgramId(namespaceId.getNamespace(), appDetail.getName(), programType, programName);
            Iterable<RunRecord> runRecordList;
            if (runId != null) {
                ProgramRunId programRunId = new ProgramRunId(namespaceId.getNamespace(), appDetail.getName(), programType, programName, runId);
                RunRecordDetail runRecordDetail = remoteProgramRunRecordFetcher.getRunRecordMeta(programRunId);
                runRecordList = Collections.singletonList(runRecordDetail);
            } else {
                runRecordList = getRunRecords(programId);
            }
            SupportBundleRuntimeInfoTask supportBundleRuntimeInfoTask = new SupportBundleRuntimeInfoTask(appFolderPath, namespaceId, applicationId, programType, programId, remoteMetricsSystemClient, runRecordList, appDetail);
            SupportBundlePipelineRunLogTask supportBundlePipelineRunLogTask = new SupportBundlePipelineRunLogTask(appFolderPath, programId, remoteLogsFetcher, runRecordList);
            String runtimeInfoClassName = supportBundleRuntimeInfoTask.getClass().getName();
            String runtimeInfoTaskName = uuid.concat(": ").concat(runtimeInfoClassName).concat(": ").concat(appDetail.getName());
            supportBundleJob.executeTask(supportBundleRuntimeInfoTask, basePath.getPath(), runtimeInfoTaskName, runtimeInfoTaskName);
            String runtimeLogClassName = supportBundlePipelineRunLogTask.getClass().getName();
            String runtimeLogTaskName = uuid.concat(": ").concat(runtimeLogClassName).concat(": ").concat(appDetail.getName());
            supportBundleJob.executeTask(supportBundlePipelineRunLogTask, basePath.getPath(), runtimeLogTaskName, runtimeLogClassName);
        }
    }
}

Example 18 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class AppLifecycleHttpHandler method getApplicationDetails.

/**
 * Gets {@link ApplicationDetail} for a set of applications. It expects a post body as a array of object, with each
 * object specifying the applciation id and an optional version. E.g.
 *
 * <pre>
 * {@code
 * [
 *   {"appId":"XYZ", "version":"1.2.3"},
 *   {"appId":"ABC"},
 *   {"appId":"FOO", "version":"2.3.4"},
 * ]
 * }
 * </pre>
 * The response will be an array of {@link BatchApplicationDetail} object, which either indicates a success (200) or
 * failure for each of the requested application in the same order as the request.
 */
@POST
@Path("/appdetail")
public void getApplicationDetails(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespace) throws Exception {
    List<ApplicationId> appIds = decodeAndValidateBatchApplication(validateNamespace(namespace), request);
    Map<ApplicationId, ApplicationDetail> details = applicationLifecycleService.getAppDetails(appIds);
    List<BatchApplicationDetail> result = new ArrayList<>();
    for (ApplicationId appId : appIds) {
        ApplicationDetail detail = details.get(appId);
        if (detail == null) {
            result.add(new BatchApplicationDetail(new NotFoundException(appId)));
        } else {
            result.add(new BatchApplicationDetail(detail));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result));
}

Example 19 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class ApplicationLifecycleService method processApplications.

private void processApplications(List<Map.Entry<ApplicationId, ApplicationSpecification>> list, Consumer<ApplicationDetail> consumer) {
    Set<ApplicationId> appIds = list.stream().map(Map.Entry::getKey).collect(Collectors.toSet());
    Set<? extends EntityId> visible = accessEnforcer.isVisible(appIds, authenticationContext.getPrincipal());
    list.removeIf(entry -> !visible.contains(entry.getKey()));
    appIds.removeIf(id -> !visible.contains(id));
    try {
        Map<ApplicationId, String> owners = ownerAdmin.getOwnerPrincipals(appIds);
        for (Map.Entry<ApplicationId, ApplicationSpecification> entry : list) {
            ApplicationDetail applicationDetail = ApplicationDetail.fromSpec(entry.getValue(), owners.get(entry.getKey()));
            try {
                capabilityReader.checkAllEnabled(entry.getValue());
            } catch (CapabilityNotAvailableException ex) {
                LOG.debug("Application {} is ignored due to exception.", applicationDetail.getName(), ex);
                continue;
            }
            consumer.accept(applicationDetail);
        }
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}

Example 20 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class DataPipelineTest method testSimpleUpgradePipelinesWithSnapshotArtifact.

/* Tests upgrade for a deployed application. Also tests that SNAPSHOT artifacts are being considered for upgrade.
     1. Deploy an application with older application artifact (1.0.0) and older filter plugin version (1.0.0).
     2. Add new versions of application artifact (0.0.9, 1.1.0, 1.2.0) and filter plugin artifacts (1.0.5, 1.1.0).
     3. Also deploy a snapshot version of app artifact 1.3.0-SNAPSHOT and plugin artifact 1.1.1-SNAPSHOT bind to it.
     3. Upgrade the older deployed application.
     4. Verify that after upgrading, application artifact and filter plugin artifact is upgraded to use latest version
        in its config and it uses snapshot versions for both.
   */
@Test
public void testSimpleUpgradePipelinesWithSnapshotArtifact() throws Exception {
    ArtifactSelectorConfig currentArtifactSelector = new ArtifactSelectorConfig(ArtifactScope.USER.name(), "test-plugins", "1.0.0");
    Engine engine = Engine.MAPREDUCE;
    String sourceName = "testSource" + engine.name();
    String sinkName = "testSink" + engine.name();
    ETLBatchConfig etlConfig = ETLBatchConfig.builder().setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("filter", PluggableFilterTransform.getPlugin(ValueFilter.NAME, ValueFilter.getProperties("${field}", "${value}"), currentArtifactSelector))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addConnection("source", "filter").addConnection("filter", "sink").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app("sparkProgramTest");
    // Deploy app with artifact version 1.0.0.
    ApplicationManager appManager = deployApplication(appId, appRequest);
    ApplicationDetail oldAppDetail = getAppDetail(appId);
    ETLBatchConfig oldBatchConfig = GSON.fromJson(oldAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> oldStageMap = oldBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Upgrade application with allowSnapshot set to true.
    appManager.upgrade(Collections.emptySet(), true);
    ApplicationDetail upgradedAppDetail = getAppDetail(appId);
    ETLBatchConfig newBatchConfig = GSON.fromJson(upgradedAppDetail.getConfiguration(), ETLBatchConfig.class);
    Map<String, ETLStage> newStageMap = newBatchConfig.getStages().stream().collect(Collectors.toMap(ETLStage::getName, e -> e));
    // Compare stages that should be same after upgrade.
    Assert.assertEquals(oldStageMap.get("source"), newStageMap.get("source"));
    Assert.assertEquals(oldStageMap.get("sink"), newStageMap.get("sink"));
    // Verify that after upgrade, application upgrades artifact version to latest version available.
    Assert.assertEquals(UPGRADE_APP_ARTIFACT_ID_3_SNAPSHOT.getVersion(), upgradedAppDetail.getArtifact().getVersion());
    // Check if the filter stage, for which version should be upgraded to desired version in SYSTEM scope.
    ETLPlugin upgradedPlugin = newStageMap.get("filter").getPlugin();
    Assert.assertEquals(upgradedPlugin.getArtifactConfig().getVersion(), "1.1.1-SNAPSHOT");
    Assert.assertEquals(ArtifactScope.valueOf(upgradedPlugin.getArtifactConfig().getScope().toUpperCase()), ArtifactScope.USER);
}