Examples with ProgramRunStatus io.cdap.cdap.proto.ProgramRunStatus used on opensource projects

Example 21 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class RuntimeProgramStatusSubscriberService method processNotification.

/**
 * Processes a given {@link Notification} from TMS and updates the {@link AppMetadataStore}.
 *
 * @param sourceId the message id in TMS
 * @param notification the {@link Notification} to process
 * @param store the {@link AppMetadataStore} to write to
 * @throws IOException if failed to write to the store
 */
private void processNotification(byte[] sourceId, Notification notification, AppMetadataStore store) throws IOException {
    Map<String, String> properties = notification.getProperties();
    ProgramRunId programRunId;
    ProgramRunStatus programRunStatus;
    try {
        programRunId = GSON.fromJson(properties.get(ProgramOptionConstants.PROGRAM_RUN_ID), ProgramRunId.class);
        if (programRunId == null) {
            throw new IllegalArgumentException("Missing program run id from notification");
        }
        programRunStatus = ProgramRunStatus.valueOf(properties.get(ProgramOptionConstants.PROGRAM_STATUS));
    } catch (Exception e) {
        // This shouldn't happen. If it does, we can only log and ignore the event.
        LOG.warn("Ignore notification due to unable to get program run id and program run status from notification {}", notification, e);
        return;
    }
    // Runtime server only needs the programRunId and status. We use the AppMetadataStore to record to help
    // handling state transition correctly, but we omit certain fields when calling those record* methods since
    // they are not used by the runtime server.
    LOG.debug("Received program {} of status {} {}", programRunId, programRunStatus, Bytes.toString(sourceId));
    switch(programRunStatus) {
        case STARTING:
            {
                ProgramOptions programOptions = ProgramOptions.fromNotification(notification, GSON);
                // Strip off user args and trim down system args as runtime only needs the run status for validation purpose.
                // User and system args could be large and store them in local store can lead to unnecessary storage
                // and processing overhead.
                store.recordProgramProvisioning(programRunId, Collections.emptyMap(), RuntimeMonitors.trimSystemArgs(programOptions.getArguments().asMap()), sourceId, null);
                store.recordProgramProvisioned(programRunId, 0, sourceId);
                store.recordProgramStart(programRunId, null, programOptions.getArguments().asMap(), sourceId);
                break;
            }
        case RUNNING:
            store.recordProgramRunning(programRunId, Optional.ofNullable(properties.get(ProgramOptionConstants.LOGICAL_START_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()), null, sourceId);
            break;
        case SUSPENDED:
            store.recordProgramSuspend(programRunId, sourceId, Optional.ofNullable(properties.get(ProgramOptionConstants.SUSPEND_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()));
            break;
        case STOPPING:
            store.recordProgramStopping(programRunId, sourceId, Optional.ofNullable(properties.get(ProgramOptionConstants.STOPPING_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()), Optional.ofNullable(properties.get(ProgramOptionConstants.TERMINATE_TIME)).map(Long::parseLong).orElse(Long.MAX_VALUE));
            break;
        case RESUMING:
            store.recordProgramResumed(programRunId, sourceId, Optional.ofNullable(properties.get(ProgramOptionConstants.RESUME_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()));
            break;
        case COMPLETED:
        case FAILED:
        case KILLED:
            store.recordProgramStop(programRunId, Optional.ofNullable(properties.get(ProgramOptionConstants.END_TIME)).map(Long::parseLong).orElse(System.currentTimeMillis()), programRunStatus, null, sourceId);
            // We don't need to retain records for terminated programs, hence just delete it
            store.deleteRunIfTerminated(programRunId, sourceId);
            break;
        case REJECTED:
            {
                ProgramDescriptor programDescriptor = GSON.fromJson(properties.get(ProgramOptionConstants.PROGRAM_DESCRIPTOR), ProgramDescriptor.class);
                // Strip off user args and trim down system args as runtime only needs the run status for validation purpose.
                // User and system args could be large and store them in local store can lead to unnecessary storage
                // and processing overhead.
                store.recordProgramRejected(programRunId, Collections.emptyMap(), Collections.emptyMap(), sourceId, programDescriptor.getArtifactId().toApiArtifactId());
                // We don't need to retain records for terminated programs, hence just delete it
                store.deleteRunIfTerminated(programRunId, sourceId);
                break;
            }
    }
}

Example 22 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class AppMetadataStoreTest method testGetActiveRuns.

@Test
public void testGetActiveRuns() throws Exception {
    // write a run record for each state for two programs in two apps in two namespaces
    String app1 = "app1";
    String app2 = "app2";
    String program1 = "prog1";
    String program2 = "prog2";
    Collection<NamespaceId> namespaces = Arrays.asList(new NamespaceId("ns1"), new NamespaceId("ns2"));
    Collection<ApplicationId> apps = namespaces.stream().flatMap(ns -> Stream.of(ns.app(app1), ns.app(app2))).collect(Collectors.toList());
    Collection<ProgramId> programs = apps.stream().flatMap(app -> Stream.of(app.mr(program1), app.mr(program2))).collect(Collectors.toList());
    for (ProgramId programId : programs) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            // one run in pending state
            ProgramRunId runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            // one run in starting state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramStart(runId, UUID.randomUUID().toString(), Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            // one run in running state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            String twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            // one in suspended state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramSuspend(runId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), System.currentTimeMillis());
            // one run in stopping state
            runId = programId.run(RunIds.generate());
            store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
            store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            twillRunId = UUID.randomUUID().toString();
            store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramRunning(runId, System.currentTimeMillis(), twillRunId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            store.recordProgramStopping(runId, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), System.currentTimeMillis(), System.currentTimeMillis() + 1000);
            // one run in each stopped state
            for (ProgramRunStatus runStatus : ProgramRunStatus.values()) {
                if (!runStatus.isEndState()) {
                    continue;
                }
                runId = programId.run(RunIds.generate());
                store.recordProgramProvisioning(runId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()), ARTIFACT_ID);
                store.recordProgramProvisioned(runId, 3, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
                twillRunId = UUID.randomUUID().toString();
                store.recordProgramStart(runId, twillRunId, Collections.emptyMap(), AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
                store.recordProgramStop(runId, System.currentTimeMillis(), runStatus, null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            }
        });
    }
    Set<ProgramRunStatus> activeStates = new HashSet<>();
    activeStates.add(ProgramRunStatus.PENDING);
    activeStates.add(ProgramRunStatus.STARTING);
    activeStates.add(ProgramRunStatus.RUNNING);
    activeStates.add(ProgramRunStatus.SUSPENDED);
    activeStates.add(ProgramRunStatus.STOPPING);
    // test the instance level method and namespace level method
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore store = AppMetadataStore.create(context);
        Map<ProgramId, Set<ProgramRunStatus>> allExpected = new HashMap<>();
        Map<ProgramId, Set<ProgramRunStatus>> allActual = new HashMap<>();
        // check active runs per namespace
        for (NamespaceId namespace : namespaces) {
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(namespace);
            // we expect 4 runs per program, with 4 programs in each namespace
            Map<ProgramId, Set<ProgramRunStatus>> expected = new HashMap<>();
            expected.put(namespace.app(app1).mr(program1), activeStates);
            expected.put(namespace.app(app1).mr(program2), activeStates);
            expected.put(namespace.app(app2).mr(program1), activeStates);
            expected.put(namespace.app(app2).mr(program2), activeStates);
            Map<ProgramId, Set<ProgramRunStatus>> actual = new HashMap<>();
            actual.put(namespace.app(app1).mr(program1), new HashSet<>());
            actual.put(namespace.app(app1).mr(program2), new HashSet<>());
            actual.put(namespace.app(app2).mr(program1), new HashSet<>());
            actual.put(namespace.app(app2).mr(program2), new HashSet<>());
            allActual.putAll(actual);
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                ProgramId programId = activeRun.getKey().getParent();
                Assert.assertTrue("Unexpected program returned: " + programId, actual.containsKey(activeRun.getKey().getParent()));
                actual.get(programId).add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(expected, actual);
            allExpected.putAll(expected);
        }
        // test the instance level method
        for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : store.getActiveRuns(x -> true).entrySet()) {
            ProgramId programId = activeRun.getKey().getParent();
            Assert.assertTrue("Unexpected program returned: " + programId, allActual.containsKey(activeRun.getKey().getParent()));
            allActual.get(programId).add(activeRun.getValue().getStatus());
        }
        Assert.assertEquals(allExpected, allActual);
        // test the count-all method
        Assert.assertEquals(store.getActiveRuns(x -> true).size(), store.countActiveRuns(null));
        Assert.assertEquals(store.getActiveRuns(x -> true).size(), store.countActiveRuns(100));
        Assert.assertEquals(2, store.countActiveRuns(2));
    });
    // check active runs per app
    for (ApplicationId app : apps) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(app);
            // we expect 3 runs per program, with 2 programs in each app
            Map<ProgramId, Set<ProgramRunStatus>> expected = new HashMap<>();
            expected.put(app.mr(program1), activeStates);
            expected.put(app.mr(program2), activeStates);
            Map<ProgramId, Set<ProgramRunStatus>> actual = new HashMap<>();
            actual.put(app.mr(program1), new HashSet<>());
            actual.put(app.mr(program2), new HashSet<>());
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                ProgramId programId = activeRun.getKey().getParent();
                Assert.assertTrue("Unexpected program returned: " + programId, actual.containsKey(activeRun.getKey().getParent()));
                actual.get(programId).add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(expected, actual);
        });
    }
    // check active runs per program
    for (ProgramId program : programs) {
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore store = AppMetadataStore.create(context);
            Map<ProgramRunId, RunRecordDetail> activeRuns = store.getActiveRuns(program);
            Set<ProgramRunStatus> actual = new HashSet<>();
            for (Map.Entry<ProgramRunId, RunRecordDetail> activeRun : activeRuns.entrySet()) {
                Assert.assertEquals(program, activeRun.getKey().getParent());
                actual.add(activeRun.getValue().getStatus());
            }
            Assert.assertEquals(activeStates, actual);
        });
    }
}

Example 23 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class RuntimeClientServiceTest method testBasicRelay.

@Test
public void testBasicRelay() throws Exception {
    // Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS.
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Write a non-terminal state to test basic relaying
            programStateWriter.running(PROGRAM_RUN_ID, null);
        } else {
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), entry.getKey(), entry.getKey());
        }
    }
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.RUNNING), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
    // Writes a program terminate message to unblock stopping of the client service
    programStateWriter.completed(PROGRAM_RUN_ID);
}

Example 24 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class DirectRuntimeRequestValidatorTest method testFetcher.

@Test
public void testFetcher() throws BadRequestException {
    ArtifactId artifactId = new ArtifactId("test", new ArtifactVersion("1.0"), ArtifactScope.USER);
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").spark("spark").run(RunIds.generate());
    ProgramRunStatus programRunStatus = ProgramRunStatus.RUNNING;
    RunRecordDetail runRecord = RunRecordDetail.builder().setProgramRunId(programRunId).setStartTime(System.currentTimeMillis()).setArtifactId(artifactId).setStatus(programRunStatus).setSystemArgs(ImmutableMap.of(SystemArguments.PROFILE_NAME, "default", SystemArguments.PROFILE_PROVISIONER, "native")).setProfileId(NamespaceId.DEFAULT.profile("native")).setSourceId(new byte[MessageId.RAW_ID_SIZE]).build();
    MockProgramRunRecordFetcher runRecordFetcher = new MockProgramRunRecordFetcher().setRunRecord(runRecord);
    RuntimeRequestValidator validator = new DirectRuntimeRequestValidator(cConf, txRunner, runRecordFetcher, accessEnforcer, authenticationContext);
    // The first call should be hitting the run record fetching to fetch the run record.
    ProgramRunInfo programRunInfo = validator.getProgramRunStatus(programRunId, new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"));
    Assert.assertEquals(programRunStatus, programRunInfo.getProgramRunStatus());
    // The second call will hit the runtime store, so it shouldn't matter what the run record fetch returns
    runRecordFetcher.setRunRecord(null);
    programRunInfo = validator.getProgramRunStatus(programRunId, new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"));
    Assert.assertEquals(programRunStatus, programRunInfo.getProgramRunStatus());
}

Example 25 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by cdapio.

the class RunRecordCorrectorServiceTest method testFixProgram.

@Test
public void testFixProgram() throws Exception {
    final AtomicInteger sourceId = new AtomicInteger(0);
    // Write 10 services with starting state
    // Write 10 workers with running state
    Map<ProgramRunId, ProgramRunStatus> expectedStates = new HashMap<>();
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    for (int i = 0; i < 10; i++) {
        ProgramRunId serviceId = NamespaceId.DEFAULT.app("test").service("service" + i).run(randomRunId());
        store.setProvisioning(serviceId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(serviceId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(serviceId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(serviceId, ProgramRunStatus.FAILED);
        ProgramRunId workerId = new NamespaceId("ns").app("test").worker("worker" + i).run(randomRunId());
        store.setProvisioning(workerId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(workerId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(workerId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        store.setRunning(workerId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(workerId, ProgramRunStatus.FAILED);
    }
    // Write a service with suspend state
    ProgramRunId flowId = new NamespaceId("ns").app("test").service("flow").run(randomRunId());
    store.setProvisioning(flowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(flowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(flowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(flowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setSuspend(flowId, Bytes.toBytes(sourceId.getAndIncrement()), -1);
    expectedStates.put(flowId, ProgramRunStatus.SUSPENDED);
    // Write two MR in starting state. One with workflow information, one without.
    ProgramRunId mrId = NamespaceId.DEFAULT.app("app").mr("mr").run(randomRunId());
    store.setProvisioning(mrId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(mrId, ProgramRunStatus.FAILED);
    ProgramRunId workflowId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(randomRunId());
    ProgramRunId mrInWorkflowId = workflowId.getParent().getParent().mr("mrInWorkflow").run(randomRunId());
    store.setProvisioning(mrInWorkflowId, Collections.emptyMap(), ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr", SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()), Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrInWorkflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrInWorkflowId, null, ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr"), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.STARTING);
    // Write the workflow in RUNNING state.
    store.setProvisioning(workflowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(workflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(workflowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(workflowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.RUNNING);
    // Use a ProgramRuntimeService that only reports running state based on a set of know ids
    final Map<ProgramId, RunId> runningSet = new HashMap<>();
    ProgramRuntimeService programRuntimeService = new AbstractProgramRuntimeService(cConf, null, null, new NoOpProgramStateWriter(), null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new NotRunningProgramLiveInfo(programId);
        }

        @Override
        public Map<RunId, RuntimeInfo> list(ProgramId program) {
            RunId runId = runningSet.get(program);
            if (runId != null) {
                RuntimeInfo runtimeInfo = new SimpleRuntimeInfo(null, program);
                return Collections.singletonMap(runId, runtimeInfo);
            }
            return Collections.emptyMap();
        }
    };
    // Have both flow and workflow running
    runningSet.put(flowId.getParent(), RunIds.fromString(flowId.getRun()));
    runningSet.put(workflowId.getParent(), RunIds.fromString(workflowId.getRun()));
    ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {

        @Override
        public void error(ProgramRunId programRunId, Throwable failureCause) {
            store.setStop(programRunId, System.currentTimeMillis(), ProgramRunStatus.FAILED, new BasicThrowable(failureCause), Bytes.toBytes(sourceId.getAndIncrement()));
        }
    };
    // Create a run record fixer.
    // Set the start buffer time to -1 so that it fixes right away.
    // Also use a small tx batch size to validate the batching logic.
    RunRecordCorrectorService fixer = new RunRecordCorrectorService(cConf, store, programStateWriter, programRuntimeService, namespaceAdmin, datasetFramework, -1L, 5) {
    };
    fixer.fixRunRecords();
    // Validates all expected states
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
    // Remove the workflow from the running set and mark it as completed
    runningSet.remove(workflowId.getParent());
    store.setStop(workflowId, System.currentTimeMillis(), ProgramRunStatus.COMPLETED, Bytes.toBytes(sourceId.getAndIncrement()));
    fixer.fixRunRecords();
    // Both the workflow and the MR in workflow should be changed to failed state
    expectedStates.put(workflowId, ProgramRunStatus.COMPLETED);
    expectedStates.put(mrInWorkflowId, ProgramRunStatus.FAILED);
    // Validates all expected states again
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
}