Search in sources :

Example 21 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class LineageAdminTest method addWorkflowRuns.

/**
 * Adds runs which have workflows associated with them
 *
 * @param store store instance
 * @param workflowName name of the workflow
 * @param workflowRunId run ID associated with all program runs
 * @param runs list of runs to be added
 */
private void addWorkflowRuns(Store store, String workflowName, String workflowRunId, ProgramRunId... runs) {
    Map<String, String> workflowIDMap = new HashMap<>();
    Map<String, String> emptyMap = ImmutableMap.of();
    workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NAME, workflowName);
    workflowIDMap.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    workflowIDMap.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    for (ProgramRunId run : runs) {
        workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NODE_ID, run.getProgram());
        ArtifactId artifactId = run.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId();
        store.setProvisioning(run, emptyMap, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId), artifactId);
        store.setProvisioned(run, 0, AppFabricTestHelper.createSourceId(++sourceId));
        store.setStart(run, null, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId));
        store.setRunning(run, RunIds.getTime(run.getRun(), TimeUnit.SECONDS) + 1, null, AppFabricTestHelper.createSourceId(++sourceId));
    }
}
Also used : ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) HashMap(java.util.HashMap) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId)

Example 22 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ProfileServiceTest method testProfileDeletion.

@Test
public void testProfileDeletion() throws Exception {
    ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile");
    ProfileId myProfile2 = NamespaceId.DEFAULT.profile("MyProfile2");
    Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), ProfileStatus.DISABLED, Profile.NATIVE.getProvisioner());
    profileService.saveProfile(myProfile, profile1);
    // add profile2 and disable it, profile2 can get deleted at any time
    profileService.saveProfile(myProfile2, profile2);
    profileService.disableProfile(myProfile2);
    // Should not be able to delete because the profile is by default enabled
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    // add assignment and disable it, deletion should also fail
    profileService.addProfileAssignment(myProfile, NamespaceId.DEFAULT);
    profileService.disableProfile(myProfile);
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    profileService.removeProfileAssignment(myProfile, NamespaceId.DEFAULT);
    // add an active record to DefaultStore, deletion should still fail
    Store store = getDefaultStore();
    ProgramId programId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram");
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    RunId runId = RunIds.generate(System.currentTimeMillis());
    ProgramRunId programRunId = programId.run(runId.getId());
    Map<String, String> systemArgs = Collections.singletonMap(SystemArguments.PROFILE_NAME, myProfile.getScopedName());
    int sourceId = 0;
    store.setProvisioning(programRunId, Collections.emptyMap(), systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId);
    store.setProvisioned(programRunId, 0, AppFabricTestHelper.createSourceId(++sourceId));
    store.setStart(programRunId, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId));
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    // set the run to stopped then deletion should work
    store.setStop(programRunId, System.currentTimeMillis() + 1000, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    // now profile deletion should succeed
    profileService.deleteProfile(myProfile);
    Assert.assertEquals(Collections.singletonList(profile2), profileService.getProfiles(NamespaceId.DEFAULT, false));
    profileService.saveProfile(myProfile, profile1);
    profileService.disableProfile(myProfile);
    profileService.deleteAllProfiles(NamespaceId.DEFAULT);
    Assert.assertEquals(Collections.emptyList(), profileService.getProfiles(NamespaceId.DEFAULT, false));
}
Also used : ProfileId(io.cdap.cdap.proto.id.ProfileId) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) MetricStore(io.cdap.cdap.api.metrics.MetricStore) Store(io.cdap.cdap.app.store.Store) DefaultStore(io.cdap.cdap.internal.app.store.DefaultStore) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ProgramId(io.cdap.cdap.proto.id.ProgramId) RunId(org.apache.twill.api.RunId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ProfileConflictException(io.cdap.cdap.common.ProfileConflictException) Profile(io.cdap.cdap.proto.profile.Profile) Test(org.junit.Test)

Example 23 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class MessageUtil method constructAndGetProgramRunInfo.

/**
 * Based on the {@link Message} and its ProgramStatus,
 * construct by setting the fields of {@link ProgramRunInfo} and return that.
 * @param message TMS message
 * @param notification
 * @return {@link ProgramRunInfo}
 */
public static ProgramRunInfo constructAndGetProgramRunInfo(Message message, Notification notification) throws IOException {
    ProgramRunInfo programRunInfo = GSON.fromJson(notification.getProperties().get(Constants.Notification.PROGRAM_RUN_ID), ProgramRunInfo.class);
    programRunInfo.setMessageId(message.getId());
    String programStatus = notification.getProperties().get(Constants.Notification.PROGRAM_STATUS);
    programRunInfo.setStatus(programStatus);
    switch(programStatus) {
        case Constants.Notification.Status.STARTING:
            long startTime = getTime(programRunInfo.getRun(), TimeUnit.MILLISECONDS);
            programRunInfo.setTime(startTime);
            ProgramDescriptor programDescriptor = GSON.fromJson(notification.getProperties().get(Constants.Notification.PROGRAM_DESCRIPTOR), ProgramDescriptor.class);
            ArtifactId artifactId = programDescriptor.getArtifactId();
            Map<String, String> systemArguments = GSON.fromJson(notification.getProperties().get(Constants.Notification.SYSTEM_OVERRIDES), MAP_TYPE);
            Map<String, String> userArguments = GSON.fromJson(notification.getProperties().get(Constants.Notification.USER_OVERRIDES), MAP_TYPE);
            String principal = systemArguments.get(Constants.Notification.PRINCIPAL);
            if (principal != null) {
                principal = new KerberosName(principal).getShortName();
            }
            ProgramStartInfo programStartInfo = new ProgramStartInfo(userArguments, artifactId, principal, systemArguments);
            programRunInfo.setStartInfo(programStartInfo);
            break;
        case Constants.Notification.Status.RUNNING:
            programRunInfo.setTime(Long.parseLong(notification.getProperties().get(Constants.Notification.LOGICAL_START_TIME)));
            break;
        case Constants.Notification.Status.KILLED:
        case Constants.Notification.Status.COMPLETED:
        case Constants.Notification.Status.FAILED:
            programRunInfo.setTime(Long.parseLong(notification.getProperties().get(Constants.Notification.END_TIME)));
            break;
        case Constants.Notification.Status.SUSPENDED:
            programRunInfo.setTime(Long.parseLong(notification.getProperties().get(Constants.Notification.SUSPEND_TIME)));
            break;
        case Constants.Notification.Status.RESUMING:
            programRunInfo.setTime(Long.parseLong(notification.getProperties().get(Constants.Notification.RESUME_TIME)));
            break;
    }
    return programRunInfo;
}
Also used : ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) KerberosName(org.apache.hadoop.security.authentication.util.KerberosName)

Example 24 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ApplicationLifecycleService method upgradeApplication.

/**
 * Upgrades an existing application by upgrading application artifact versions and plugin artifact versions.
 *
 * @param appId the id of the application to upgrade.
 * @param allowedArtifactScopes artifact scopes allowed while looking for latest artifacts for upgrade.
 * @param allowSnapshot whether to consider snapshot version of artifacts or not for upgrade.
 * @throws IllegalStateException if something unexpected happened during upgrade.
 * @throws IOException if there was an IO error during initializing application class from artifact.
 * @throws JsonIOException if there was an error in serializing or deserializing app config.
 * @throws UnsupportedOperationException if application does not support upgrade operation.
 * @throws InvalidArtifactException if candidate application artifact is invalid for upgrade purpose.
 * @throws NotFoundException if any object related to upgrade is not found like application/artifact.
 * @throws Exception if there was an exception during the upgrade of application. This exception will often wrap
 *                   the actual exception
 */
public void upgradeApplication(ApplicationId appId, Set<ArtifactScope> allowedArtifactScopes, boolean allowSnapshot) throws Exception {
    // Check if the current user has admin privileges on it before updating.
    accessEnforcer.enforce(appId, authenticationContext.getPrincipal(), StandardPermission.UPDATE);
    // check that app exists
    ApplicationSpecification currentSpec = store.getApplication(appId);
    if (currentSpec == null) {
        LOG.info("Application {} not found for upgrade.", appId);
        throw new NotFoundException(appId);
    }
    ArtifactId currentArtifact = currentSpec.getArtifactId();
    ArtifactSummary candidateArtifact = getLatestAppArtifactForUpgrade(appId, currentArtifact, allowedArtifactScopes, allowSnapshot);
    ArtifactVersion candidateArtifactVersion = new ArtifactVersion(candidateArtifact.getVersion());
    // Current artifact should not have higher version than candidate artifact.
    if (currentArtifact.getVersion().compareTo(candidateArtifactVersion) > 0) {
        String error = String.format("The current artifact has a version higher %s than any existing artifact.", currentArtifact.getVersion());
        throw new InvalidArtifactException(error);
    }
    ArtifactId newArtifactId = new ArtifactId(candidateArtifact.getName(), candidateArtifactVersion, candidateArtifact.getScope());
    Id.Artifact newArtifact = Id.Artifact.fromEntityId(Artifacts.toProtoArtifactId(appId.getParent(), newArtifactId));
    ArtifactDetail newArtifactDetail = artifactRepository.getArtifact(newArtifact);
    updateApplicationInternal(appId, currentSpec.getConfiguration(), programId -> {
    }, newArtifactDetail, Collections.singletonList(ApplicationConfigUpdateAction.UPGRADE_ARTIFACT), allowedArtifactScopes, allowSnapshot, ownerAdmin.getOwner(appId), false);
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ArtifactSummary(io.cdap.cdap.api.artifact.ArtifactSummary) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) ApplicationNotFoundException(io.cdap.cdap.common.ApplicationNotFoundException) NotFoundException(io.cdap.cdap.common.NotFoundException) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) Id(io.cdap.cdap.common.id.Id) InstanceId(io.cdap.cdap.proto.id.InstanceId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) EntityId(io.cdap.cdap.proto.id.EntityId) ProgramId(io.cdap.cdap.proto.id.ProgramId) KerberosPrincipalId(io.cdap.cdap.proto.id.KerberosPrincipalId) InvalidArtifactException(io.cdap.cdap.common.InvalidArtifactException) ArtifactDetail(io.cdap.cdap.internal.app.runtime.artifact.ArtifactDetail)

Example 25 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class CoreSchedulerServiceTest method testProgramEvents.

@Test
@Category(XSlowTests.class)
public void testProgramEvents() throws Exception {
    // Deploy the app
    deploy(AppWithMultipleSchedules.class, 200);
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    TopicId programEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(cConf, messagingService);
    // These notifications should not trigger the program
    ProgramRunId anotherWorkflowRun = ANOTHER_WORKFLOW.run(RunIds.generate());
    ArtifactId artifactId = ANOTHER_WORKFLOW.getNamespaceId().artifact("test", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification(AppWithMultipleSchedules.NAME, ApplicationId.DEFAULT_VERSION, ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(anotherWorkflowRun.getParent(), appSpec);
    BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()));
    ProgramOptions programOptions = new SimpleProgramOptions(anotherWorkflowRun.getParent(), systemArgs, new BasicArguments(), false);
    programStateWriter.start(anotherWorkflowRun, programOptions, null, programDescriptor);
    programStateWriter.running(anotherWorkflowRun, null);
    long lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    programStateWriter.error(anotherWorkflowRun, null);
    waitUntilProcessed(programEventTopic, lastProcessed);
    ProgramRunId someWorkflowRun = SOME_WORKFLOW.run(RunIds.generate());
    programDescriptor = new ProgramDescriptor(someWorkflowRun.getParent(), appSpec);
    programStateWriter.start(someWorkflowRun, new SimpleProgramOptions(someWorkflowRun.getParent(), systemArgs, new BasicArguments()), null, programDescriptor);
    programStateWriter.running(someWorkflowRun, null);
    lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    programStateWriter.killed(someWorkflowRun);
    waitUntilProcessed(programEventTopic, lastProcessed);
    Assert.assertEquals(0, getRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.ALL));
    // Enable the schedule
    scheduler.enableSchedule(APP_MULT_ID.schedule(AppWithMultipleSchedules.WORKFLOW_COMPLETED_SCHEDULE));
    // Start a program with user arguments
    startProgram(ANOTHER_WORKFLOW, ImmutableMap.of(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_KEY, AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE), 200);
    // Wait for a completed run record
    waitForCompleteRuns(1, TRIGGERED_WORKFLOW);
    assertProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED, 1);
    RunRecord run = getProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED).get(0);
    Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = getWorkflowToken(TRIGGERED_WORKFLOW, run.getPid(), null, null).getTokenData();
    // There should be 2 entries in tokenData
    Assert.assertEquals(2, tokenData.size());
    // The value of TRIGGERED_RUNTIME_ARG_KEY should be ANOTHER_RUNTIME_ARG_VALUE from the triggering workflow
    Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_RUNTIME_ARG_KEY).get(0).getValue());
    // The value of TRIGGERED_TOKEN_KEY should be ANOTHER_TOKEN_VALUE from the triggering workflow
    Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_TOKEN_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_TOKEN_KEY).get(0).getValue());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) DefaultApplicationSpecification(io.cdap.cdap.internal.app.DefaultApplicationSpecification) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) RunRecord(io.cdap.cdap.proto.RunRecord) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) TopicId(io.cdap.cdap.proto.id.TopicId) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) DefaultApplicationSpecification(io.cdap.cdap.internal.app.DefaultApplicationSpecification) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) WorkflowTokenDetail(io.cdap.cdap.proto.WorkflowTokenDetail) Category(org.junit.experimental.categories.Category) Test(org.junit.Test)

Aggregations

ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)75 Test (org.junit.Test)43 ProgramId (io.cdap.cdap.proto.id.ProgramId)32 ArtifactVersion (io.cdap.cdap.api.artifact.ArtifactVersion)25 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)25 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)19 RunId (org.apache.twill.api.RunId)18 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)14 File (java.io.File)14 HashMap (java.util.HashMap)14 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)13 Id (io.cdap.cdap.common.id.Id)10 HttpResponse (io.cdap.common.http.HttpResponse)9 Map (java.util.Map)9 Gson (com.google.gson.Gson)8 Location (org.apache.twill.filesystem.Location)8 ApplicationClass (io.cdap.cdap.api.artifact.ApplicationClass)7 PluginClass (io.cdap.cdap.api.plugin.PluginClass)7 AppDeploymentInfo (io.cdap.cdap.internal.app.deploy.pipeline.AppDeploymentInfo)7 DefaultApplicationSpecification (io.cdap.cdap.internal.app.DefaultApplicationSpecification)6