Examples with ProgramRunCluster co.cask.cdap.proto.ProgramRunCluster used on opensource projects

Example 1 with ProgramRunCluster

use of co.cask.cdap.proto.ProgramRunCluster in project cdap by caskdata.

the class MockLogReader method generateLogs.

/**
 * This method is used to generate the logs for program which are used for testing.
 * Single call to this method would add {@link #MAX} number of events.
 * First 20 events are generated without {@link ApplicationLoggingContext#TAG_RUN_ID} tag.
 * For next 40 events, alternate event is tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
 * Last 20 events are not tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
 * All events are alternately marked as {@link Level#ERROR} and {@link Level#WARN}.
 * All events are alternately tagged with "plugin", "program" and "system" as value of MDC property ".origin"
 * All events are alternately tagged with "lifecycle" as value of MDC property "MDC:eventType
 */
private void generateLogs(LoggingContext loggingContext, ProgramId programId, ProgramRunStatus runStatus) throws InterruptedException {
    // All possible values of " MDC property ".origin
    String[] origins = { "plugin", "program", "system" };
    String entityId = LoggingContextHelper.getEntityId(loggingContext).getValue();
    StackTraceElement stackTraceElementNative = new StackTraceElement("co.cask.Test", "testMethod", null, -2);
    RunId runId = null;
    Long stopTs = null;
    for (int i = 0; i < MAX; ++i) {
        // Setup run id for event with ids >= 20
        if (i == 20) {
            runId = RunIds.generate(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        } else if (i == 60 && runStatus != ProgramRunStatus.RUNNING && runStatus != ProgramRunStatus.SUSPENDED) {
            // Record stop time for run for 60th event, but still continue to record run in the other logging events.
            stopTs = getMockTimeSecs(i);
        }
        LoggingEvent event = new LoggingEvent("co.cask.Test", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), i % 2 == 0 ? Level.ERROR : Level.WARN, entityId + "<img>-" + i, null, null);
        event.setTimeStamp(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        // Add runid to logging context
        Map<String, String> tagMap = Maps.newHashMap(Maps.transformValues(loggingContext.getSystemTagsMap(), TAG_TO_STRING_FUNCTION));
        if (runId != null && stopTs == null && i % 2 == 0) {
            tagMap.put(ApplicationLoggingContext.TAG_RUN_ID, runId.getId());
        }
        // Determine the value of ".origin" property by (i % 3)
        tagMap.put(".origin", origins[i % 3]);
        if (i % 2 == 0) {
            tagMap.put("MDC:eventType", "lifecycle");
        }
        if (i == 30) {
            event.setCallerData(new StackTraceElement[] { stackTraceElementNative });
        }
        event.setMDCPropertyMap(tagMap);
        logEvents.add(new LogEvent(event, new LogOffset(i, i)));
    }
    long startTs = RunIds.getTime(runId, TimeUnit.SECONDS);
    if (programId != null) {
        // noinspection ConstantConditions
        runRecordMap.put(programId, RunRecord.builder().setRunId(runId.getId()).setStartTime(startTs).setRunTime(startTs + 1).setStopTime(stopTs).setStatus(runStatus).setCluster(new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONED, null, null)).build());
        setStartAndRunning(programId.run(runId.getId()), startTs);
        if (stopTs != null) {
            store.setStop(programId.run(runId.getId()), stopTs, runStatus, AppFabricTestHelper.createSourceId(++sourceId));
        }
    }
}

Example 2 with ProgramRunCluster

use of co.cask.cdap.proto.ProgramRunCluster in project cdap by caskdata.

the class DefaultStoreTest method testLogProgramRunHistory.

@Test
public void testLogProgramRunHistory() throws Exception {
    Map<String, String> noRuntimeArgsProps = ImmutableMap.of("runtimeArgs", GSON.toJson(ImmutableMap.<String, String>of()));
    // record finished flow
    ProgramId programId = new ProgramId("account1", "application1", ProgramType.FLOW, "flow1");
    long now = System.currentTimeMillis();
    long startTimeSecs = TimeUnit.MILLISECONDS.toSeconds(now);
    // Time after program has been marked started to be marked running
    long startTimeDelaySecs = 1;
    RunId run1 = RunIds.generate(now - 20000);
    setStartAndRunning(programId.run(run1.getId()), runIdToSecs(run1));
    store.setStop(programId.run(run1.getId()), startTimeSecs - 10, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    // record another finished flow
    RunId run2 = RunIds.generate(now - 10000);
    setStartAndRunning(programId.run(run2.getId()), runIdToSecs(run2));
    store.setStop(programId.run(run2.getId()), startTimeSecs - 5, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    // record a suspended flow
    RunId run21 = RunIds.generate(now - 7500);
    setStartAndRunning(programId.run(run21.getId()), runIdToSecs(run21));
    store.setSuspend(programId.run(run21.getId()), AppFabricTestHelper.createSourceId(++sourceId));
    // record not finished flow
    RunId run3 = RunIds.generate(now);
    setStartAndRunning(programId.run(run3.getId()), runIdToSecs(run3));
    // For a RunRecordMeta that has not yet been completed, getStopTs should return null
    RunRecordMeta runRecord = store.getRun(programId.run(run3.getId()));
    Assert.assertNotNull(runRecord);
    Assert.assertNull(runRecord.getStopTs());
    // record run of different program
    ProgramId programId2 = new ProgramId("account1", "application1", ProgramType.FLOW, "flow2");
    RunId run4 = RunIds.generate(now - 5000);
    setStartAndRunning(programId2.run(run4.getId()), runIdToSecs(run4));
    store.setStop(programId2.run(run4.getId()), startTimeSecs - 4, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    // record for different account
    setStartAndRunning(new ProgramId("account2", "application1", ProgramType.FLOW, "flow1").run(run3.getId()), runIdToSecs(run3));
    // we should probably be better with "get" method in DefaultStore interface to do that, but we don't have one
    Map<ProgramRunId, RunRecordMeta> successHistorymap = store.getRuns(programId, ProgramRunStatus.COMPLETED, 0, Long.MAX_VALUE, Integer.MAX_VALUE);
    Map<ProgramRunId, RunRecordMeta> failureHistorymap = store.getRuns(programId, ProgramRunStatus.FAILED, startTimeSecs - 20, startTimeSecs - 10, Integer.MAX_VALUE);
    Assert.assertEquals(failureHistorymap, store.getRuns(programId, ProgramRunStatus.FAILED, 0, Long.MAX_VALUE, Integer.MAX_VALUE));
    Map<ProgramRunId, RunRecordMeta> suspendedHistorymap = store.getRuns(programId, ProgramRunStatus.SUSPENDED, startTimeSecs - 20, startTimeSecs, Integer.MAX_VALUE);
    // only finished + succeeded runs should be returned
    Assert.assertEquals(1, successHistorymap.size());
    // only finished + failed runs should be returned
    Assert.assertEquals(1, failureHistorymap.size());
    // only suspended runs should be returned
    Assert.assertEquals(1, suspendedHistorymap.size());
    // records should be sorted by start time latest to earliest
    RunRecordMeta run = successHistorymap.values().iterator().next();
    Assert.assertEquals(startTimeSecs - 10, run.getStartTs());
    Assert.assertEquals(Long.valueOf(startTimeSecs - 5), run.getStopTs());
    Assert.assertEquals(ProgramController.State.COMPLETED.getRunStatus(), run.getStatus());
    run = failureHistorymap.values().iterator().next();
    Assert.assertEquals(startTimeSecs - 20, run.getStartTs());
    Assert.assertEquals(Long.valueOf(startTimeSecs - 10), run.getStopTs());
    Assert.assertEquals(ProgramController.State.ERROR.getRunStatus(), run.getStatus());
    run = suspendedHistorymap.values().iterator().next();
    Assert.assertEquals(run21.getId(), run.getPid());
    Assert.assertEquals(ProgramController.State.SUSPENDED.getRunStatus(), run.getStatus());
    // Assert all history
    Map<ProgramRunId, RunRecordMeta> allHistorymap = store.getRuns(programId, ProgramRunStatus.ALL, startTimeSecs - 20, startTimeSecs + 1, Integer.MAX_VALUE);
    Assert.assertEquals(allHistorymap.toString(), 4, allHistorymap.size());
    // Assert running programs
    Map<ProgramRunId, RunRecordMeta> runningHistorymap = store.getRuns(programId, ProgramRunStatus.RUNNING, startTimeSecs, startTimeSecs + 1, 100);
    Assert.assertEquals(1, runningHistorymap.size());
    Assert.assertEquals(runningHistorymap, store.getRuns(programId, ProgramRunStatus.RUNNING, 0, Long.MAX_VALUE, 100));
    // Get a run record for running program
    RunRecordMeta expectedRunning = runningHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedRunning);
    RunRecordMeta actualRunning = store.getRun(programId.run(expectedRunning.getPid()));
    Assert.assertEquals(expectedRunning, actualRunning);
    // Get a run record for completed run
    RunRecordMeta expectedCompleted = successHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedCompleted);
    RunRecordMeta actualCompleted = store.getRun(programId.run(expectedCompleted.getPid()));
    Assert.assertEquals(expectedCompleted, actualCompleted);
    // Get a run record for suspended run
    RunRecordMeta expectedSuspended = suspendedHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedSuspended);
    RunRecordMeta actualSuspended = store.getRun(programId.run(expectedSuspended.getPid()));
    Assert.assertEquals(expectedSuspended, actualSuspended);
    // Backwards compatibility test with random UUIDs
    // record finished flow
    RunId run5 = RunIds.fromString(UUID.randomUUID().toString());
    setStartAndRunning(programId.run(run5.getId()), startTimeSecs - 8);
    store.setStop(programId.run(run5.getId()), startTimeSecs - 4, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    ProgramRunCluster emptyCluster = new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONED, null, 0);
    RunRecordMeta expectedRecord5 = RunRecordMeta.builder().setProgramRunId(programId.run(run5)).setStartTime(startTimeSecs - 8).setRunTime(startTimeSecs - 8 + startTimeDelaySecs).setStopTime(startTimeSecs - 4).setStatus(ProgramRunStatus.COMPLETED).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
    // record not finished flow
    RunId run6 = RunIds.fromString(UUID.randomUUID().toString());
    setStartAndRunning(programId.run(run6.getId()), startTimeSecs - 2);
    RunRecordMeta expectedRecord6 = RunRecordMeta.builder().setProgramRunId(programId.run(run6)).setStartTime(startTimeSecs - 2).setRunTime(startTimeSecs - 2 + startTimeDelaySecs).setStatus(ProgramRunStatus.RUNNING).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
    // Get run record for run5
    RunRecordMeta actualRecord5 = store.getRun(programId.run(run5.getId()));
    Assert.assertEquals(expectedRecord5, actualRecord5);
    // Get run record for run6
    RunRecordMeta actualRecord6 = store.getRun(programId.run(run6.getId()));
    Assert.assertEquals(expectedRecord6, actualRecord6);
    // Record flow that starts but encounters error before it runs
    RunId run7 = RunIds.fromString(UUID.randomUUID().toString());
    Map<String, String> emptyArgs = ImmutableMap.of();
    setStart(programId.run(run7.getId()), startTimeSecs, emptyArgs, emptyArgs);
    store.setStop(programId.run(run7.getId()), startTimeSecs + 1, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    RunRecordMeta expectedRunRecord7 = RunRecordMeta.builder().setProgramRunId(programId.run(run7)).setStartTime(startTimeSecs).setStopTime(startTimeSecs + 1).setStatus(ProgramRunStatus.FAILED).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
    RunRecordMeta actualRecord7 = store.getRun(programId.run(run7.getId()));
    Assert.assertEquals(expectedRunRecord7, actualRecord7);
    // Record flow that starts and suspends before it runs
    RunId run8 = RunIds.fromString(UUID.randomUUID().toString());
    setStart(programId.run(run8.getId()), startTimeSecs, emptyArgs, emptyArgs);
    store.setSuspend(programId.run(run8.getId()), AppFabricTestHelper.createSourceId(++sourceId));
    RunRecordMeta expectedRunRecord8 = RunRecordMeta.builder().setProgramRunId(programId.run(run8)).setStartTime(startTimeSecs).setStatus(ProgramRunStatus.SUSPENDED).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
    RunRecordMeta actualRecord8 = store.getRun(programId.run(run8.getId()));
    Assert.assertEquals(expectedRunRecord8, actualRecord8);
    // Record flow that is killed while suspended
    RunId run9 = RunIds.fromString(UUID.randomUUID().toString());
    setStartAndRunning(programId.run(run9.getId()), startTimeSecs);
    store.setSuspend(programId.run(run9.getId()), AppFabricTestHelper.createSourceId(++sourceId));
    store.setStop(programId.run(run9.getId()), startTimeSecs + 5, ProgramRunStatus.KILLED, AppFabricTestHelper.createSourceId(++sourceId));
    RunRecordMeta expectedRunRecord9 = RunRecordMeta.builder().setProgramRunId(programId.run(run9)).setStartTime(startTimeSecs).setRunTime(startTimeSecs + 1).setStopTime(startTimeSecs + 5).setStatus(ProgramRunStatus.KILLED).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
    RunRecordMeta actualRecord9 = store.getRun(programId.run(run9.getId()));
    Assert.assertEquals(expectedRunRecord9, actualRecord9);
    // Non-existent run record should give null
    Assert.assertNull(store.getRun(programId.run(UUID.randomUUID().toString())));
    // Searching for history in wrong time range should give us no results
    Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.COMPLETED, startTimeSecs - 5000, startTimeSecs - 2000, Integer.MAX_VALUE).isEmpty());
    Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.ALL, startTimeSecs - 5000, startTimeSecs - 2000, Integer.MAX_VALUE).isEmpty());
}

Example 3 with ProgramRunCluster

use of co.cask.cdap.proto.ProgramRunCluster in project cdap by caskdata.

the class AppMetadataStore method recordProgramProvisioned.

/**
 * Record that the program run has completed provisioning compute resources for the run. If the current status has
 * a higher source id, this call will be ignored.
 *
 * @param programRunId program run
 * @param numNodes number of cluster nodes provisioned
 * @param sourceId unique id representing the source of program run status, such as the message id of the program
 *                 run status notification in TMS. The source id must increase as the recording time of the program
 *                 run status increases, so that the attempt to persist program run status older than the existing
 *                 program run status will be ignored
 * @return {@link ProgramRunClusterStatus#PROVISIONED} if it is successfully persisted, {@code null} otherwise.
 */
@Nullable
public ProgramRunClusterStatus recordProgramProvisioned(ProgramRunId programRunId, int numNodes, byte[] sourceId) {
    MDSKey key = getProgramKeyBuilder(TYPE_RUN_RECORD_STARTING, programRunId).build();
    RunRecordMeta existing = getRun(programRunId);
    if (existing == null) {
        LOG.warn("Ignoring unexpected request to transition program run {} from non-existent state to cluster state {}.", programRunId, ProgramRunClusterStatus.PROVISIONED);
        return null;
    } else if (!isValid(existing, sourceId, "provisioned")) {
        return null;
    }
    ProgramRunClusterStatus clusterState = existing.getCluster().getStatus();
    if (clusterState != ProgramRunClusterStatus.PROVISIONING) {
        LOG.warn("Ignoring unexpected request to transition program run {} from cluster state {} to cluster state {}.", programRunId, existing.getCluster().getStatus(), ProgramRunClusterStatus.PROVISIONED);
        return null;
    }
    ProgramRunCluster cluster = new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONED, null, numNodes);
    RunRecordMeta meta = RunRecordMeta.builder(existing).setStatus(ProgramRunStatus.STARTING).setCluster(cluster).setSourceId(sourceId).build();
    write(key, meta);
    return ProgramRunClusterStatus.PROVISIONED;
}

Example 4 with ProgramRunCluster

use of co.cask.cdap.proto.ProgramRunCluster in project cdap by caskdata.

the class AppMetadataStore method recordProgramDeprovisioning.

/**
 * Record that the program run has started de-provisioning compute resources for the run. If the current status has
 * a higher source id, this call will be ignored.
 *
 * @param programRunId program run
 * @param sourceId unique id representing the source of program run status, such as the message id of the program
 *                 run status notification in TMS. The source id must increase as the recording time of the program
 *                 run status increases, so that the attempt to persist program run status older than the existing
 *                 program run status will be ignored
 * @return {@link ProgramRunClusterStatus#DEPROVISIONING} if it is successfully persisted, {@code null} otherwise.
 */
@Nullable
public ProgramRunClusterStatus recordProgramDeprovisioning(ProgramRunId programRunId, byte[] sourceId) {
    MDSKey.Builder key = getProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programRunId.getParent());
    RunRecordMeta existing = getRun(programRunId);
    if (existing == null) {
        LOG.debug("Ignoring unexpected transition of program run {} to cluster state {} with no existing run record.", programRunId, ProgramRunClusterStatus.DEPROVISIONING);
        return null;
    } else if (!isValid(existing, sourceId, "deprovisioning")) {
        return null;
    }
    ProgramRunClusterStatus clusterStatus = existing.getCluster().getStatus();
    if (clusterStatus != ProgramRunClusterStatus.PROVISIONED) {
        LOG.warn("Ignoring unexpected request to transition program run {} from cluster state {} to cluster state {}.", programRunId, clusterStatus, ProgramRunClusterStatus.DEPROVISIONING);
        return null;
    }
    if (!existing.getStatus().isEndState()) {
        LOG.warn("Ignoring unexpected request to transition program run {} from program state {} to cluster state {}.", programRunId, existing.getStatus(), ProgramRunClusterStatus.DEPROVISIONING);
        return null;
    }
    key.add(getInvertedTsKeyPart(existing.getStartTs())).add(programRunId.getRun()).build();
    ProgramRunCluster cluster = new ProgramRunCluster(ProgramRunClusterStatus.DEPROVISIONING, null, existing.getCluster().getNumNodes());
    RunRecordMeta meta = RunRecordMeta.builder(existing).setCluster(cluster).setSourceId(sourceId).build();
    write(key.build(), meta);
    return ProgramRunClusterStatus.DEPROVISIONING;
}

Example 5 with ProgramRunCluster

use of co.cask.cdap.proto.ProgramRunCluster in project cdap by caskdata.

the class AppMetadataStore method recordProgramDeprovisioned.

/**
 * Record that the program run has deprovisioned compute resources for the run. If the current status has
 * a higher source id, this call will be ignored.
 *
 * @param programRunId program run
 * @param sourceId unique id representing the source of program run status, such as the message id of the program
 *                 run status notification in TMS. The source id must increase as the recording time of the program
 *                 run status increases, so that the attempt to persist program run status older than the existing
 *                 program run status will be ignored
 * @return {@link ProgramRunClusterStatus#DEPROVISIONED} if it is successfully persisted, {@code null} otherwise.
 */
@Nullable
public ProgramRunClusterStatus recordProgramDeprovisioned(ProgramRunId programRunId, byte[] sourceId) {
    MDSKey.Builder key = getProgramKeyBuilder(TYPE_RUN_RECORD_COMPLETED, programRunId.getParent());
    RunRecordMeta existing = getRun(programRunId);
    if (existing == null) {
        LOG.debug("Ignoring unexpected transition of program run {} to cluster state {} with no existing run record.", programRunId, ProgramRunClusterStatus.DEPROVISIONED);
        return null;
    } else if (!isValid(existing, sourceId, "deprovisioned")) {
        return null;
    }
    ProgramRunClusterStatus clusterStatus = existing.getCluster().getStatus();
    if (clusterStatus != ProgramRunClusterStatus.DEPROVISIONING) {
        LOG.warn("Ignoring unexpected request to transition program run {} from cluster state {} to cluster state {}.", programRunId, clusterStatus, ProgramRunClusterStatus.DEPROVISIONED);
        return null;
    }
    if (!existing.getStatus().isEndState()) {
        LOG.warn("Ignoring unexpected request to transition program run {} from program state {} to cluster state {}.", programRunId, existing.getStatus(), ProgramRunClusterStatus.DEPROVISIONED);
        return null;
    }
    key.add(getInvertedTsKeyPart(existing.getStartTs())).add(programRunId.getRun()).build();
    ProgramRunCluster cluster = new ProgramRunCluster(ProgramRunClusterStatus.DEPROVISIONED, null, existing.getCluster().getNumNodes());
    RunRecordMeta meta = RunRecordMeta.builder(existing).setCluster(cluster).setSourceId(sourceId).build();
    write(key.build(), meta);
    return ProgramRunClusterStatus.DEPROVISIONED;
}