Search in sources :

Example 11 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by caskdata.

the class AppLifecycleHttpHandlerTest method testListAndGet.

@Test
public void testListAndGet() throws Exception {
    // deploy without name to testnamespace1
    deploy(AllProgramsApp.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    // deploy with name to testnamespace2
    String ns2AppName = AllProgramsApp.NAME + "2";
    Id.Namespace ns2 = Id.Namespace.from(TEST_NAMESPACE2);
    Id.Artifact ns2ArtifactId = Id.Artifact.from(ns2, AllProgramsApp.class.getSimpleName(), "1.0.0-SNAPSHOT");
    HttpResponse response = addAppArtifact(ns2ArtifactId, AllProgramsApp.class);
    Assert.assertEquals(200, response.getResponseCode());
    Id.Application appId = Id.Application.from(ns2, ns2AppName);
    response = deploy(appId, new AppRequest<>(ArtifactSummary.from(ns2ArtifactId.toArtifactId())));
    Assert.assertEquals(200, response.getResponseCode());
    // deploy with name and version to testnamespace2
    ApplicationId app1 = new ApplicationId(TEST_NAMESPACE2, ns2AppName, VERSION1);
    response = deploy(app1, new AppRequest<>(ArtifactSummary.from(ns2ArtifactId.toArtifactId())));
    Assert.assertEquals(200, response.getResponseCode());
    // verify testnamespace1 has 1 app
    List<JsonObject> apps = getAppList(TEST_NAMESPACE1);
    Assert.assertEquals(1, apps.size());
    // verify testnamespace2 has 2 app
    apps = getAppList(TEST_NAMESPACE2);
    Assert.assertEquals(2, apps.size());
    // get and verify app details in testnamespace1
    JsonObject result = getAppDetails(TEST_NAMESPACE1, AllProgramsApp.NAME);
    ApplicationSpecification spec = Specifications.from(new AllProgramsApp());
    Assert.assertEquals(AllProgramsApp.NAME, result.get("name").getAsString());
    Assert.assertEquals(AllProgramsApp.DESC, result.get("description").getAsString());
    // Validate the datasets
    JsonArray datasets = result.get("datasets").getAsJsonArray();
    Assert.assertEquals(spec.getDatasets().size(), datasets.size());
    Assert.assertTrue(StreamSupport.stream(datasets.spliterator(), false).map(JsonObject.class::cast).map(obj -> obj.get("name").getAsString()).allMatch(dataset -> spec.getDatasets().containsKey(dataset)));
    // Validate the programs
    JsonArray programs = result.get("programs").getAsJsonArray();
    int totalPrograms = Arrays.stream(io.cdap.cdap.api.app.ProgramType.values()).mapToInt(type -> spec.getProgramsByType(type).size()).reduce(0, (l, r) -> l + r);
    Assert.assertEquals(totalPrograms, programs.size());
    Assert.assertTrue(StreamSupport.stream(programs.spliterator(), false).map(JsonObject.class::cast).allMatch(obj -> {
        String type = obj.get("type").getAsString().toUpperCase();
        io.cdap.cdap.api.app.ProgramType programType = io.cdap.cdap.api.app.ProgramType.valueOf(type);
        return spec.getProgramsByType(programType).contains(obj.get("name").getAsString());
    }));
    // get and verify app details in testnamespace2
    List<BatchApplicationDetail> appDetails = getAppDetails(TEST_NAMESPACE2, Arrays.asList(ImmutablePair.of(ns2AppName, null), ImmutablePair.of(ns2AppName, VERSION1)));
    Assert.assertEquals(2, appDetails.size());
    Assert.assertTrue(appDetails.stream().allMatch(d -> d.getStatusCode() == 200));
    ApplicationDetail appDetail = appDetails.get(0).getDetail();
    Assert.assertNotNull(appDetail);
    Assert.assertEquals(ns2AppName, appDetail.getName());
    Assert.assertEquals(ApplicationId.DEFAULT_VERSION, appDetail.getAppVersion());
    appDetail = appDetails.get(1).getDetail();
    Assert.assertNotNull(appDetail);
    Assert.assertEquals(ns2AppName, appDetail.getName());
    Assert.assertEquals(VERSION1, appDetail.getAppVersion());
    // delete app in testnamespace1
    response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1));
    Assert.assertEquals(200, response.getResponseCode());
    // delete app in testnamespace2
    response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2));
    Assert.assertEquals(200, response.getResponseCode());
    deleteArtifact(ns2ArtifactId, 200);
    // verify testnamespace2 has 0 app
    apps = getAppList(TEST_NAMESPACE2);
    Assert.assertEquals(0, apps.size());
}
Also used : JsonObject(com.google.gson.JsonObject) ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) Arrays(java.util.Arrays) TypeToken(com.google.gson.reflect.TypeToken) ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) AppLifecycleHttpHandler(io.cdap.cdap.gateway.handlers.AppLifecycleHttpHandler) UGIProvider(io.cdap.cdap.security.impersonation.UGIProvider) HttpResponse(io.cdap.common.http.HttpResponse) AppWithSchedule(io.cdap.cdap.AppWithSchedule) ArtifactSummary(io.cdap.cdap.api.artifact.ArtifactSummary) AuthenticationContext(io.cdap.cdap.security.spi.authentication.AuthenticationContext) ConfigTestApp(io.cdap.cdap.ConfigTestApp) PreferencesService(io.cdap.cdap.config.PreferencesService) AppWithDataset(io.cdap.cdap.AppWithDataset) BatchApplicationDetail(io.cdap.cdap.proto.BatchApplicationDetail) ImmutableSet(com.google.common.collect.ImmutableSet) MetadataSubscriberService(io.cdap.cdap.metadata.MetadataSubscriberService) ImmutableMap(com.google.common.collect.ImmutableMap) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) MessagingService(io.cdap.cdap.messaging.MessagingService) Set(java.util.Set) ApplicationLifecycleService(io.cdap.cdap.internal.app.services.ApplicationLifecycleService) MetadataServiceClient(io.cdap.cdap.data2.metadata.writer.MetadataServiceClient) AppDeploymentInfo(io.cdap.cdap.internal.app.deploy.pipeline.AppDeploymentInfo) Id(io.cdap.cdap.common.id.Id) JsonArray(com.google.gson.JsonArray) List(java.util.List) MetricsSystemClient(io.cdap.cdap.api.metrics.MetricsSystemClient) Constants(io.cdap.cdap.common.conf.Constants) AppWithNoServices(io.cdap.cdap.AppWithNoServices) ProfileId(io.cdap.cdap.proto.id.ProfileId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) NotFoundException(io.cdap.cdap.common.NotFoundException) Singleton(com.google.inject.Singleton) BeforeClass(org.junit.BeforeClass) AppFabricTestBase(io.cdap.cdap.internal.app.services.http.AppFabricTestBase) AccessEnforcer(io.cdap.cdap.security.spi.authorization.AccessEnforcer) ApplicationWithPrograms(io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms) ProgramType(io.cdap.cdap.proto.ProgramType) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) NamespaceNotFoundException(io.cdap.cdap.common.NamespaceNotFoundException) ManagerFactory(io.cdap.cdap.app.deploy.ManagerFactory) Profile(io.cdap.cdap.proto.profile.Profile) StreamSupport(java.util.stream.StreamSupport) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) CurrentUGIProvider(io.cdap.cdap.security.impersonation.CurrentUGIProvider) AppWithDatasetDuplicate(io.cdap.cdap.AppWithDatasetDuplicate) Before(org.junit.Before) UsageRegistry(io.cdap.cdap.data2.registry.UsageRegistry) Scheduler(io.cdap.cdap.scheduler.Scheduler) ProgramId(io.cdap.cdap.proto.id.ProgramId) Config(io.cdap.cdap.api.Config) Impersonator(io.cdap.cdap.security.impersonation.Impersonator) Test(org.junit.Test) Scopes(com.google.inject.Scopes) Store(io.cdap.cdap.app.store.Store) Mockito(org.mockito.Mockito) Provides(com.google.inject.Provides) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) CapabilityReader(io.cdap.cdap.internal.capability.CapabilityReader) OwnerAdmin(io.cdap.cdap.security.impersonation.OwnerAdmin) AllProgramsApp(io.cdap.cdap.AllProgramsApp) AppRequest(io.cdap.cdap.proto.artifact.AppRequest) Assert(org.junit.Assert) AbstractModule(com.google.inject.AbstractModule) Specifications(io.cdap.cdap.internal.app.deploy.Specifications) HttpResponseCodes(org.jboss.resteasy.util.HttpResponseCodes) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) HttpResponse(io.cdap.common.http.HttpResponse) JsonObject(com.google.gson.JsonObject) AllProgramsApp(io.cdap.cdap.AllProgramsApp) AppRequest(io.cdap.cdap.proto.artifact.AppRequest) JsonArray(com.google.gson.JsonArray) ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) BatchApplicationDetail(io.cdap.cdap.proto.BatchApplicationDetail) BatchApplicationDetail(io.cdap.cdap.proto.BatchApplicationDetail) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) ProfileId(io.cdap.cdap.proto.id.ProfileId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) ProgramId(io.cdap.cdap.proto.id.ProgramId) ProgramType(io.cdap.cdap.proto.ProgramType) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 12 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by caskdata.

the class UpgradeTool method upgrade.

private boolean upgrade(final ApplicationId appId) throws Exception {
    final ApplicationDetail appDetail = appClient.get(appId);
    if (!upgrader.shouldUpgrade(appDetail.getArtifact())) {
        LOG.debug("Skipping app {}.", appId);
        return false;
    }
    Upgrader.UpgradeAction action;
    if (dryrun) {
        action = new Upgrader.UpgradeAction() {

            @Override
            public boolean upgrade(AppRequest<? extends ETLConfig> appRequest) throws Exception {
                LOG.info("Writing dryrun app request for pipeline: {}", appId);
                String filename = String.format("%s-%s.json", appId.getNamespace(), appId.getApplication());
                File outputFile = new File(outputDir, filename);
                try (BufferedWriter writer = Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8)) {
                    writer.write(GSON.toJson(appRequest));
                }
                return true;
            }
        };
    } else {
        action = new Upgrader.UpgradeAction() {

            @Override
            public boolean upgrade(AppRequest<? extends ETLConfig> appRequest) {
                LOG.info("Updating pipeline: {}", appId);
                try {
                    appClient.update(appId, appRequest);
                    return true;
                } catch (Exception e) {
                    LOG.error("Error updating pipeline {}.", appId, e);
                    if (errorDir != null) {
                        File errorFile = new File(errorDir, String.format("%s-%s.json", appId.getParent(), appId.getEntityName()));
                        LOG.error("Writing config for pipeline {} to {} for further manual investigation.", appId, errorFile.getAbsolutePath());
                        try (OutputStreamWriter outputStreamWriter = new OutputStreamWriter(new FileOutputStream(errorFile))) {
                            outputStreamWriter.write(GSON.toJson(appRequest));
                        } catch (IOException e1) {
                            LOG.error("Error writing config out for manual investigation.", e1);
                        }
                    }
                    return false;
                }
            }
        };
    }
    return upgrader.upgrade(appDetail.getArtifact(), appDetail.getConfiguration(), action);
}
Also used : ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) Upgrader(io.cdap.cdap.etl.tool.config.Upgrader) FileOutputStream(java.io.FileOutputStream) OutputStreamWriter(java.io.OutputStreamWriter) IOException(java.io.IOException) File(java.io.File) IOException(java.io.IOException) BufferedWriter(java.io.BufferedWriter)

Example 13 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by caskdata.

the class SupportBundlePipelineInfoTaskTest method testSupportBundlePipelineInfo.

// Contains two sub-task supportBundleRuntimeInfo and supportBundlePipelineRunLog
// So we will test all three files together
@Test
public void testSupportBundlePipelineInfo() throws Exception {
    String runId = generateWorkflowLog();
    SupportBundleConfiguration supportBundleConfiguration = new SupportBundleConfiguration(namespaceId.getNamespace(), application, runId, programType, workflowName, 1);
    String uuid = UUID.randomUUID().toString();
    File tempFolder = new File(configuration.get(Constants.SupportBundle.LOCAL_DATA_DIR));
    File uuidFile = new File(tempFolder, uuid);
    SupportBundleStatus supportBundleStatus = SupportBundleStatus.builder().setBundleId(uuid).setStartTimestamp(System.currentTimeMillis()).setParameters(supportBundleConfiguration).setStatus(CollectionState.IN_PROGRESS).build();
    SupportBundleJob supportBundleJob = new SupportBundleJob(supportBundleTaskFactorySet, executorService, configuration, supportBundleStatus);
    SupportBundlePipelineInfoTask supportBundlePipelineInfoTask = new SupportBundlePipelineInfoTask(uuid, Collections.singletonList(namespaceId), application, null, uuidFile, remoteApplicationDetailFetcher, remoteProgramRunRecordsFetcher, remoteLogsFetcher, programType, workflowName, remoteMetricsSystemClient, supportBundleJob, 1, remoteProgramRunRecordFetcher);
    supportBundlePipelineInfoTask.collect();
    Set<SupportBundleTaskStatus> supportBundleTaskStatusList = supportBundleStatus.getTasks();
    for (SupportBundleTaskStatus supportBundleTaskStatus : supportBundleTaskStatusList) {
        if (!supportBundleTaskStatus.getName().endsWith("SupportBundleSystemLogTask") && !supportBundleTaskStatus.getName().endsWith("SupportBundlePipelineInfoTask")) {
            Assert.assertEquals(CollectionState.FINISHED, supportBundleTaskStatus.getStatus());
        }
    }
    File pipelineFolder = new File(uuidFile, AppWithWorkflow.NAME);
    File[] pipelineFiles = pipelineFolder.listFiles((dir, name) -> !name.startsWith(".") && !dir.isHidden() && dir.isDirectory());
    Assert.assertEquals(3, pipelineFiles.length);
    File pipelineInfoFile = new File(pipelineFolder, AppWithWorkflow.NAME + ".json");
    try (Reader reader = Files.newBufferedReader(pipelineInfoFile.toPath(), StandardCharsets.UTF_8)) {
        ApplicationDetail pipelineInfo = GSON.fromJson(reader, ApplicationDetail.class);
        Assert.assertEquals(AppWithWorkflow.NAME, pipelineInfo.getName());
        Assert.assertEquals("-SNAPSHOT", pipelineInfo.getAppVersion());
        Assert.assertEquals("Sample application", pipelineInfo.getDescription());
    } catch (Exception e) {
        LOG.error("Can not read pipelineInfo file ", e);
        Assert.fail();
    }
    File runInfoFile = new File(pipelineFolder, runId + ".json");
    try (Reader reader = Files.newBufferedReader(runInfoFile.toPath(), StandardCharsets.UTF_8)) {
        JsonObject runInfo = GSON.fromJson(reader, JsonObject.class);
        Assert.assertEquals("COMPLETED", runInfo.get("status").getAsString());
    } catch (Exception e) {
        LOG.error("Can not read status file ", e);
        Assert.fail();
    }
    File runLogFile = new File(pipelineFolder, runId + SupportBundleFileNames.LOG_SUFFIX_NAME);
    Assert.assertTrue(runLogFile.exists());
}
Also used : SupportBundleTaskStatus(io.cdap.cdap.support.status.SupportBundleTaskStatus) Reader(java.io.Reader) JsonObject(com.google.gson.JsonObject) SupportBundleConfiguration(io.cdap.cdap.support.status.SupportBundleConfiguration) ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) SupportBundlePipelineInfoTask(io.cdap.cdap.support.task.SupportBundlePipelineInfoTask) SupportBundleStatus(io.cdap.cdap.support.status.SupportBundleStatus) SupportBundleJob(io.cdap.cdap.support.job.SupportBundleJob) File(java.io.File) Test(org.junit.Test)

Example 14 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class ApplicationDetailFetcherTest method testGetApplication.

@Test
public void testGetApplication() throws Exception {
    ApplicationDetailFetcher fetcher = getApplicationDetailFetcher(fetcherType);
    String namespace = TEST_NAMESPACE1;
    String appName = AllProgramsApp.NAME;
    // Deploy the application
    deploy(AllProgramsApp.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, namespace);
    // Get and validate the application
    ApplicationId appId = new ApplicationId(namespace, appName);
    ApplicationDetail appDetail = fetcher.get(appId);
    assertAllProgramAppDetail(appDetail);
    // Delete the application
    Assert.assertEquals(200, doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, namespace)).getResponseCode());
}
Also used : ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 15 with ApplicationDetail

use of io.cdap.cdap.proto.ApplicationDetail in project cdap by cdapio.

the class ApplicationLifecycleServiceTest method testCreateAppDetailsArchive.

@Test
public void testCreateAppDetailsArchive() throws Exception {
    createNamespace("ns1");
    createNamespace("ns2");
    createNamespace("ns3");
    deploy(AllProgramsApp.class, HttpResponseStatus.OK.code(), Constants.Gateway.API_VERSION_3_TOKEN, "ns1");
    deploy(AllProgramsApp.class, HttpResponseStatus.OK.code(), Constants.Gateway.API_VERSION_3_TOKEN, "ns2");
    deploy(AllProgramsApp.class, HttpResponseStatus.OK.code(), Constants.Gateway.API_VERSION_3_TOKEN, "ns3");
    File archiveFile = tmpFolder.newFile();
    try (ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(archiveFile))) {
        applicationLifecycleService.createAppDetailsArchive(zipOut);
    }
    // Validate the ZIP file content
    File dir = tmpFolder.newFolder();
    BundleJarUtil.unJar(archiveFile, dir);
    ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
    for (String ns : Arrays.asList("ns1", "ns2", "ns3")) {
        File nsDir = new File(dir, ns);
        Assert.assertTrue(nsDir.isDirectory());
        ApplicationDetail appDetail = GSON.fromJson(Files.toString(new File(nsDir, appSpec.getName() + ".json"), StandardCharsets.UTF_8), ApplicationDetail.class);
        Assert.assertEquals(appSpec.getName(), appDetail.getName());
        // Check if all the programs are there
        int programCount = Arrays.stream(io.cdap.cdap.api.app.ProgramType.values()).map(appSpec::getProgramsByType).mapToInt(Set::size).reduce(0, Integer::sum);
        Assert.assertEquals(programCount, appDetail.getPrograms().size());
        for (ProgramRecord record : appDetail.getPrograms()) {
            Assert.assertTrue(appSpec.getProgramsByType(record.getType().getApiProgramType()).contains(record.getName()));
        }
    }
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ApplicationDetail(io.cdap.cdap.proto.ApplicationDetail) ProgramRecord(io.cdap.cdap.proto.ProgramRecord) ZipOutputStream(java.util.zip.ZipOutputStream) FileOutputStream(java.io.FileOutputStream) AllProgramsApp(io.cdap.cdap.AllProgramsApp) File(java.io.File) Test(org.junit.Test)

Aggregations

ApplicationDetail (io.cdap.cdap.proto.ApplicationDetail)52 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)30 Test (org.junit.Test)30 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)26 File (java.io.File)20 ArrayList (java.util.ArrayList)20 ArtifactSummary (io.cdap.cdap.api.artifact.ArtifactSummary)16 HttpResponse (io.cdap.common.http.HttpResponse)16 HashSet (java.util.HashSet)16 Gson (com.google.gson.Gson)14 Constants (io.cdap.cdap.common.conf.Constants)14 ProgramId (io.cdap.cdap.proto.id.ProgramId)14 HashMap (java.util.HashMap)14 List (java.util.List)14 Map (java.util.Map)14 Assert (org.junit.Assert)14 ImmutableMap (com.google.common.collect.ImmutableMap)12 ImmutableSet (com.google.common.collect.ImmutableSet)12 RunRecord (io.cdap.cdap.proto.RunRecord)12 AppRequest (io.cdap.cdap.proto.artifact.AppRequest)12