Examples with Id co.cask.cdap.proto.Id used on opensource projects

Example 6 with Id

use of co.cask.cdap.proto.Id in project cdap by caskdata.

the class ApplicationLifecycleService method updateApp.

/**
   * Update an existing application. An application's configuration and artifact version can be updated.
   *
   * @param appId the id of the application to update
   * @param appRequest the request to update the application, including new config and artifact
   * @param programTerminator a program terminator that will stop programs that are removed when updating an app.
   *                          For example, if an update removes a flow, the terminator defines how to stop that flow.
   * @return information about the deployed application
   * @throws ApplicationNotFoundException if the specified application does not exist
   * @throws ArtifactNotFoundException if the requested artifact does not exist
   * @throws InvalidArtifactException if the specified artifact is invalid. For example, if the artifact name changed,
   *                                  if the version is an invalid version, or the artifact contains no app classes
   * @throws Exception if there was an exception during the deployment pipeline. This exception will often wrap
   *                   the actual exception
   */
public ApplicationWithPrograms updateApp(ApplicationId appId, AppRequest appRequest, ProgramTerminator programTerminator) throws Exception {
    // check that app exists
    ApplicationSpecification currentSpec = store.getApplication(appId);
    if (currentSpec == null) {
        throw new ApplicationNotFoundException(appId);
    }
    // App exists. Check if the current user has admin privileges on it before updating. The user's write privileges on
    // the namespace will get enforced in the deployApp method.
    authorizationEnforcer.enforce(appId, authenticationContext.getPrincipal(), Action.ADMIN);
    ArtifactId currentArtifact = currentSpec.getArtifactId();
    // if no artifact is given, use the current one.
    ArtifactId newArtifactId = currentArtifact;
    // otherwise, check requested artifact is valid and use it
    ArtifactSummary requestedArtifact = appRequest.getArtifact();
    if (requestedArtifact != null) {
        // cannot change artifact name, only artifact version.
        if (!currentArtifact.getName().equals(requestedArtifact.getName())) {
            throw new InvalidArtifactException(String.format(" Only artifact version updates are allowed. Cannot change from artifact '%s' to '%s'.", currentArtifact.getName(), requestedArtifact.getName()));
        }
        if (!currentArtifact.getScope().equals(requestedArtifact.getScope())) {
            throw new InvalidArtifactException("Only artifact version updates are allowed. " + "Cannot change from a non-system artifact to a system artifact or vice versa.");
        }
        // check requested artifact version is valid
        ArtifactVersion requestedVersion = new ArtifactVersion(requestedArtifact.getVersion());
        if (requestedVersion.getVersion() == null) {
            throw new InvalidArtifactException(String.format("Requested artifact version '%s' is invalid", requestedArtifact.getVersion()));
        }
        newArtifactId = new ArtifactId(currentArtifact.getName(), requestedVersion, currentArtifact.getScope());
    }
    // ownerAdmin.getImpersonationPrincipal will give the owner which will be impersonated for the application
    // irrespective of the version
    SecurityUtil.verifyOwnerPrincipal(appId, appRequest.getOwnerPrincipal(), ownerAdmin);
    Object requestedConfigObj = appRequest.getConfig();
    // if config is null, use the previous config. Shouldn't use a static GSON since the request Config object can
    // be a user class, otherwise there will be ClassLoader leakage.
    String requestedConfigStr = requestedConfigObj == null ? currentSpec.getConfiguration() : new Gson().toJson(requestedConfigObj);
    Id.Artifact artifactId = Artifacts.toArtifactId(appId.getParent(), newArtifactId).toId();
    return deployApp(appId.getParent(), appId.getApplication(), null, artifactId, requestedConfigStr, programTerminator, ownerAdmin.getOwner(appId), appRequest.canUpdateSchedules());
}

Example 7 with Id

use of co.cask.cdap.proto.Id in project cdap by caskdata.

the class ApplicationLifecycleService method deleteApp.

// deletes without performs checks that no programs are running
/**
   * Delete the specified application without performing checks that its programs are stopped.
   *
   * @param appId the id of the application to delete
   * @param spec the spec of the application to delete
   * @throws Exception
   */
private void deleteApp(final ApplicationId appId, ApplicationSpecification spec) throws Exception {
    Id.Application idApplication = appId.toId();
    // enforce ADMIN privileges on the app
    authorizationEnforcer.enforce(appId, authenticationContext.getPrincipal(), Action.ADMIN);
    // first remove all privileges on the app
    revokePrivileges(appId, spec);
    //Delete the schedules
    scheduler.deleteSchedules(appId);
    deleteMetrics(appId);
    //Delete all preferences of the application and of all its programs
    deletePreferences(appId);
    // TODO: This should be unified with the DeletedProgramHandlerStage
    for (final FlowSpecification flowSpecification : spec.getFlows().values()) {
        Id.Program flowProgramId = Id.Program.from(idApplication, ProgramType.FLOW, flowSpecification.getName());
        // Collects stream name to all group ids consuming that stream
        final Multimap<String, Long> streamGroups = HashMultimap.create();
        for (FlowletConnection connection : flowSpecification.getConnections()) {
            if (connection.getSourceType() == FlowletConnection.Type.STREAM) {
                long groupId = FlowUtils.generateConsumerGroupId(flowProgramId, connection.getTargetName());
                streamGroups.put(connection.getSourceName(), groupId);
            }
        }
        // Remove all process states and group states for each stream
        final String namespace = String.format("%s.%s", flowProgramId.getApplicationId(), flowProgramId.getId());
        impersonator.doAs(appId, new Callable<Void>() {

            // TODO: (CDAP-7326) since one worker or flow can only be ran by a single instance of APP, (also a single
            // version), should delete flow for each version
            @Override
            public Void call() throws Exception {
                for (Map.Entry<String, Collection<Long>> entry : streamGroups.asMap().entrySet()) {
                    streamConsumerFactory.dropAll(appId.getParent().stream(entry.getKey()), namespace, entry.getValue());
                }
                queueAdmin.dropAllForFlow(appId.flow(flowSpecification.getName()));
                return null;
            }
        });
    }
    ApplicationSpecification appSpec = store.getApplication(appId);
    deleteAppMetadata(appId, appSpec);
    deleteRouteConfig(appId, appSpec);
    store.deleteWorkflowStats(appId);
    store.removeApplication(appId);
    try {
        // delete the owner as it has already been determined that this is the only version of the app
        ownerAdmin.delete(appId);
    } catch (Exception e) {
        LOG.warn("Failed to delete app owner principal for application {} if one existed while deleting the " + "application.", appId);
    }
    try {
        usageRegistry.unregister(appId);
    } catch (Exception e) {
        LOG.warn("Failed to unregister usage of app: {}", appId, e);
    }
}

Example 8 with Id

use of co.cask.cdap.proto.Id in project cdap by caskdata.

the class ProgramLifecycleHttpHandlerTest method testFlows.

@Test
public void testFlows() throws Exception {
    // deploy WordCountApp in namespace1 and verify
    HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    // check initial flowlet instances
    int initial = getFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME);
    Assert.assertEquals(1, initial);
    // request two more instances
    Assert.assertEquals(200, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 3));
    // verify
    int after = getFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME);
    Assert.assertEquals(3, after);
    // invalid namespace
    Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE2, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 3));
    // invalid app
    Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE1, APP_WITH_SERVICES_APP_ID, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 3));
    // invalid flow
    Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, "random", WORDCOUNT_FLOWLET_NAME, 3));
    // invalid flowlet
    Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, "random", 3));
    // test live info
    // send invalid program type to live info
    response = sendLiveInfoRequest(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, "random", WORDCOUNT_FLOW_NAME);
    Assert.assertEquals(405, response.getStatusLine().getStatusCode());
    // test valid live info
    JsonObject liveInfo = getLiveInfo(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(), WORDCOUNT_FLOW_NAME);
    Assert.assertEquals(WORDCOUNT_APP_NAME, liveInfo.get("app").getAsString());
    Assert.assertEquals(ProgramType.FLOW.getPrettyName(), liveInfo.get("type").getAsString());
    Assert.assertEquals(WORDCOUNT_FLOW_NAME, liveInfo.get("id").getAsString());
    // start flow
    Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME);
    startProgram(wordcountFlow1);
    liveInfo = getLiveInfo(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(), WORDCOUNT_FLOW_NAME);
    Assert.assertEquals(WORDCOUNT_APP_NAME, liveInfo.get("app").getAsString());
    Assert.assertEquals(ProgramType.FLOW.getPrettyName(), liveInfo.get("type").getAsString());
    Assert.assertEquals(WORDCOUNT_FLOW_NAME, liveInfo.get("id").getAsString());
    Assert.assertEquals("in-memory", liveInfo.get("runtime").getAsString());
    // should not delete queues while running
    Assert.assertEquals(403, deleteQueues(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME));
    Assert.assertEquals(403, deleteQueues(TEST_NAMESPACE1));
    // stop
    stopProgram(wordcountFlow1);
    // delete queues
    Assert.assertEquals(200, deleteQueues(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME));
}

Example 9 with Id

use of co.cask.cdap.proto.Id in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testWorkflowSchedules.

@Ignore
@Test
public void testWorkflowSchedules() throws Exception {
    // Steps for the test:
    // 1. Deploy the app
    // 2. Verify the schedules
    // 3. Verify the history after waiting a while
    // 4. Suspend the schedule
    // 5. Verify there are no runs after the suspend by looking at the history
    // 6. Resume the schedule
    // 7. Verify there are runs after the resume by looking at the history
    String appName = AppWithSchedule.NAME;
    String workflowName = AppWithSchedule.WORKFLOW_NAME;
    String sampleSchedule = AppWithSchedule.SCHEDULE;
    // deploy app with schedule in namespace 2
    HttpResponse response = deploy(AppWithSchedule.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    Id.Program programId = Id.Program.from(TEST_NAMESPACE2, appName, ProgramType.WORKFLOW, workflowName);
    Map<String, String> runtimeArguments = ImmutableMap.of("someKey", "someWorkflowValue", "workflowKey", "workflowValue");
    setAndTestRuntimeArgs(programId, runtimeArguments);
    // get schedules
    List<ScheduleDetail> schedules = getSchedules(TEST_NAMESPACE2, appName, workflowName);
    Assert.assertEquals(1, schedules.size());
    String scheduleName = schedules.get(0).getName();
    Assert.assertFalse(scheduleName.isEmpty());
    // get schedules in backward-compatible ScheduleSpecification form
    List<ScheduleSpecification> specs = getScheduleSpecs(TEST_NAMESPACE2, appName, workflowName);
    Assert.assertEquals(1, specs.size());
    String specName = specs.get(0).getSchedule().getName();
    Assert.assertEquals(scheduleName, specName);
    // TODO [CDAP-2327] Sagar Investigate why following check fails sometimes. Mostly test case issue.
    // List<ScheduledRuntime> previousRuntimes = getScheduledRunTime(programId, scheduleName, "previousruntime");
    // Assert.assertTrue(previousRuntimes.size() == 0);
    long current = System.currentTimeMillis();
    // Resume the schedule
    Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, sampleSchedule));
    // Check schedule status
    assertSchedule(programId, scheduleName, true, 30, TimeUnit.SECONDS);
    List<ScheduledRuntime> runtimes = getScheduledRunTime(programId, true);
    String id = runtimes.get(0).getId();
    Assert.assertTrue(String.format("Expected schedule id '%s' to contain schedule name '%s'", id, scheduleName), id.contains(scheduleName));
    Long nextRunTime = runtimes.get(0).getTime();
    Assert.assertTrue(String.format("Expected nextRuntime '%s' to be greater than current runtime '%s'", nextRunTime, current), nextRunTime > current);
    // Verify that at least one program is completed
    verifyProgramRuns(programId, "completed");
    // Suspend the schedule
    Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName));
    // check paused state
    assertSchedule(programId, scheduleName, false, 30, TimeUnit.SECONDS);
    // check that there were at least 1 previous runs
    List<ScheduledRuntime> previousRuntimes = getScheduledRunTime(programId, false);
    int numRuns = previousRuntimes.size();
    Assert.assertTrue(String.format("After sleeping for two seconds, the schedule should have at least triggered " + "once, but found %s previous runs", numRuns), numRuns >= 1);
    // Verify no program running
    verifyNoRunWithStatus(programId, "running");
    // get number of completed runs after schedule is suspended
    int workflowRuns = getProgramRuns(programId, "completed").size();
    // verify that resuming the suspended schedule again has expected behavior (spawns new runs)
    Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, scheduleName));
    //check scheduled state
    assertSchedule(programId, scheduleName, true, 30, TimeUnit.SECONDS);
    // Verify that the program ran after the schedule was resumed
    verifyProgramRuns(programId, "completed", workflowRuns);
    // Suspend the schedule
    Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName));
    //check paused state
    assertSchedule(programId, scheduleName, false, 30, TimeUnit.SECONDS);
    //Check status of a non existing schedule
    try {
        assertSchedule(programId, "invalid", true, 2, TimeUnit.SECONDS);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    //Schedule operations using invalid namespace
    try {
        assertSchedule(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.WORKFLOW, workflowName), scheduleName, true, 2, TimeUnit.SECONDS);
        Assert.fail();
    } catch (Exception e) {
    // expected
    }
    Assert.assertEquals(404, suspendSchedule(TEST_NAMESPACE1, appName, scheduleName));
    Assert.assertEquals(404, resumeSchedule(TEST_NAMESPACE1, appName, scheduleName));
    verifyNoRunWithStatus(programId, "running");
    deleteApp(Id.Application.from(TEST_NAMESPACE2, AppWithSchedule.class.getSimpleName()), 200);
}

Example 10 with Id

use of co.cask.cdap.proto.Id in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testWorkflowTokenPut.

@Test
public void testWorkflowTokenPut() throws Exception {
    Assert.assertEquals(200, deploy(WorkflowTokenTestPutApp.class).getStatusLine().getStatusCode());
    Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, WorkflowTokenTestPutApp.NAME);
    Id.Workflow workflowId = Id.Workflow.from(appId, WorkflowTokenTestPutApp.WorkflowTokenTestPut.NAME);
    Id.Program mapReduceId = Id.Program.from(appId, ProgramType.MAPREDUCE, WorkflowTokenTestPutApp.RecordCounter.NAME);
    Id.Program sparkId = Id.Program.from(appId, ProgramType.SPARK, WorkflowTokenTestPutApp.SparkTestApp.NAME);
    // Start program with inputPath and outputPath arguments.
    // This should succeed. The programs inside the workflow will attempt to write to the workflow token
    // from the Mapper's and Reducer's methods as well as from a Spark closure, and they will throw an exception
    // if that succeeds.
    // The MapReduce's initialize will record the workflow run id in the token, and the destroy as well
    // as the mapper and the reducer will validate that they have the same workflow run id.
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    startProgram(workflowId, ImmutableMap.of("inputPath", createInputForRecordVerification("sixthInput"), "outputPath", outputPath));
    waitState(workflowId, ProgramStatus.RUNNING.name());
    waitState(workflowId, ProgramStatus.STOPPED.name());
    // validate the completed workflow run and validate that it is the same as recorded in the token
    verifyProgramRuns(workflowId, "completed");
    List<RunRecord> runs = getProgramRuns(workflowId, "completed");
    Assert.assertEquals(1, runs.size());
    String wfRunId = runs.get(0).getPid();
    WorkflowTokenDetail tokenDetail = getWorkflowToken(workflowId, wfRunId, null, null);
    List<WorkflowTokenDetail.NodeValueDetail> details = tokenDetail.getTokenData().get("wf.runid");
    Assert.assertEquals(1, details.size());
    Assert.assertEquals(wfRunId, details.get(0).getValue());
    // validate that none of the mapper, reducer or spark closure were able to write to the token
    for (String key : new String[] { "mapper.initialize.key", "map.key", "reducer.initialize.key", "reduce.key", "some.key" }) {
        Assert.assertFalse(tokenDetail.getTokenData().containsKey(key));
    }
    List<RunRecord> sparkProgramRuns = getProgramRuns(sparkId, ProgramRunStatus.COMPLETED.name());
    Assert.assertEquals(1, sparkProgramRuns.size());
}