Examples with ProgramRunStatus io.cdap.cdap.proto.ProgramRunStatus used on opensource projects

Example 6 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class DirectRuntimeRequestValidatorTest method testFetcher.

@Test
public void testFetcher() throws BadRequestException {
    ArtifactId artifactId = new ArtifactId("test", new ArtifactVersion("1.0"), ArtifactScope.USER);
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").spark("spark").run(RunIds.generate());
    ProgramRunStatus programRunStatus = ProgramRunStatus.RUNNING;
    RunRecordDetail runRecord = RunRecordDetail.builder().setProgramRunId(programRunId).setStartTime(System.currentTimeMillis()).setArtifactId(artifactId).setStatus(programRunStatus).setSystemArgs(ImmutableMap.of(SystemArguments.PROFILE_NAME, "default", SystemArguments.PROFILE_PROVISIONER, "native")).setProfileId(NamespaceId.DEFAULT.profile("native")).setSourceId(new byte[MessageId.RAW_ID_SIZE]).build();
    MockProgramRunRecordFetcher runRecordFetcher = new MockProgramRunRecordFetcher().setRunRecord(runRecord);
    RuntimeRequestValidator validator = new DirectRuntimeRequestValidator(cConf, txRunner, runRecordFetcher, accessEnforcer, authenticationContext);
    // The first call should be hitting the run record fetching to fetch the run record.
    ProgramRunInfo programRunInfo = validator.getProgramRunStatus(programRunId, new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"));
    Assert.assertEquals(programRunStatus, programRunInfo.getProgramRunStatus());
    // The second call will hit the runtime store, so it shouldn't matter what the run record fetch returns
    runRecordFetcher.setRunRecord(null);
    programRunInfo = validator.getProgramRunStatus(programRunId, new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"));
    Assert.assertEquals(programRunStatus, programRunInfo.getProgramRunStatus());
}

Example 7 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class RuntimeClientServiceTest method testBasicRelay.

@Test
public void testBasicRelay() throws Exception {
    // Send some messages to multiple topics in the client side TMS, they should get replicated to the server side TMS.
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Write a non-terminal state to test basic relaying
            programStateWriter.running(PROGRAM_RUN_ID, null);
        } else {
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), entry.getKey(), entry.getKey());
        }
    }
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.RUNNING), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
    // Writes a program terminate message to unblock stopping of the client service
    programStateWriter.completed(PROGRAM_RUN_ID);
}

Example 8 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class RuntimeClientServiceTest method testProgramTerminate.

/**
 * Test for {@link RuntimeClientService} that will terminate itself when seeing program completed message.
 */
@Test
public void testProgramTerminate() throws Exception {
    MessagingContext messagingContext = new MultiThreadMessagingContext(clientMessagingService);
    MessagePublisher messagePublisher = messagingContext.getDirectMessagePublisher();
    ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(clientCConf, clientMessagingService);
    // Send a terminate program state first, wait for the service sees the state change,
    // then publish messages to other topics.
    programStateWriter.completed(PROGRAM_RUN_ID);
    Tasks.waitFor(true, () -> runtimeClientService.getProgramFinishTime() >= 0, 2, TimeUnit.SECONDS);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        // the RuntimeClientService will decode it to watch for program termination
        if (!entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            List<String> payloads = Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey());
            messagePublisher.publish(NamespaceId.SYSTEM.getNamespace(), entry.getValue(), StandardCharsets.UTF_8, payloads.iterator());
        }
    }
    // The client service should get stopped by itself.
    Tasks.waitFor(Service.State.TERMINATED, () -> runtimeClientService.state(), clientCConf.getLong(Constants.RuntimeMonitor.GRACEFUL_SHUTDOWN_MS) + 2000, TimeUnit.MILLISECONDS);
    // All messages should be sent after the runtime client service stopped
    MessagingContext serverMessagingContext = new MultiThreadMessagingContext(messagingService);
    for (Map.Entry<String, String> entry : topicConfigs.entrySet()) {
        if (entry.getKey().equals(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC)) {
            // Extract the program run status from the Notification
            Tasks.waitFor(Collections.singletonList(ProgramRunStatus.COMPLETED), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).map(s -> GSON.fromJson(s, Notification.class)).map(n -> n.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS)).map(ProgramRunStatus::valueOf).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        } else {
            Tasks.waitFor(Arrays.asList(entry.getKey(), entry.getKey(), entry.getKey()), () -> fetchMessages(serverMessagingContext, entry.getValue(), 10, null).stream().map(Message::getPayloadAsString).collect(Collectors.toList()), 5, TimeUnit.SECONDS);
        }
    }
}

Example 9 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class ProgramLifecycleService method getProgramStatus.

/**
 * Returns the program status based on the active run records of a program.
 * A program is RUNNING if there are any RUNNING or SUSPENDED run records.
 * A program is starting if there are any PENDING or STARTING run records and no RUNNING run records.
 * Otherwise, it is STOPPED.
 *
 * @param runRecords run records for the program
 * @return the program status
 */
@VisibleForTesting
static ProgramStatus getProgramStatus(Collection<RunRecordDetail> runRecords) {
    boolean hasStarting = false;
    for (RunRecordDetail runRecord : runRecords) {
        ProgramRunStatus runStatus = runRecord.getStatus();
        if (runStatus == ProgramRunStatus.RUNNING || runStatus == ProgramRunStatus.SUSPENDED) {
            return ProgramStatus.RUNNING;
        }
        hasStarting = hasStarting || runStatus == ProgramRunStatus.STARTING || runStatus == ProgramRunStatus.PENDING;
    }
    return hasStarting ? ProgramStatus.STARTING : ProgramStatus.STOPPED;
}

Example 10 with ProgramRunStatus

use of io.cdap.cdap.proto.ProgramRunStatus in project cdap by caskdata.

the class RunRecordCorrectorServiceTest method testFixProgram.

@Test
public void testFixProgram() throws Exception {
    final AtomicInteger sourceId = new AtomicInteger(0);
    // Write 10 services with starting state
    // Write 10 workers with running state
    Map<ProgramRunId, ProgramRunStatus> expectedStates = new HashMap<>();
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    for (int i = 0; i < 10; i++) {
        ProgramRunId serviceId = NamespaceId.DEFAULT.app("test").service("service" + i).run(randomRunId());
        store.setProvisioning(serviceId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(serviceId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(serviceId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(serviceId, ProgramRunStatus.FAILED);
        ProgramRunId workerId = new NamespaceId("ns").app("test").worker("worker" + i).run(randomRunId());
        store.setProvisioning(workerId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(workerId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(workerId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        store.setRunning(workerId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(workerId, ProgramRunStatus.FAILED);
    }
    // Write a service with suspend state
    ProgramRunId flowId = new NamespaceId("ns").app("test").service("flow").run(randomRunId());
    store.setProvisioning(flowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(flowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(flowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(flowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setSuspend(flowId, Bytes.toBytes(sourceId.getAndIncrement()), -1);
    expectedStates.put(flowId, ProgramRunStatus.SUSPENDED);
    // Write two MR in starting state. One with workflow information, one without.
    ProgramRunId mrId = NamespaceId.DEFAULT.app("app").mr("mr").run(randomRunId());
    store.setProvisioning(mrId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(mrId, ProgramRunStatus.FAILED);
    ProgramRunId workflowId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(randomRunId());
    ProgramRunId mrInWorkflowId = workflowId.getParent().getParent().mr("mrInWorkflow").run(randomRunId());
    store.setProvisioning(mrInWorkflowId, Collections.emptyMap(), ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr", SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()), Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrInWorkflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrInWorkflowId, null, ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr"), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.STARTING);
    // Write the workflow in RUNNING state.
    store.setProvisioning(workflowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(workflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(workflowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(workflowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.RUNNING);
    // Use a ProgramRuntimeService that only reports running state based on a set of know ids
    final Map<ProgramId, RunId> runningSet = new HashMap<>();
    ProgramRuntimeService programRuntimeService = new AbstractProgramRuntimeService(cConf, null, null, new NoOpProgramStateWriter(), null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new NotRunningProgramLiveInfo(programId);
        }

        @Override
        public Map<RunId, RuntimeInfo> list(ProgramId program) {
            RunId runId = runningSet.get(program);
            if (runId != null) {
                RuntimeInfo runtimeInfo = new SimpleRuntimeInfo(null, program);
                return Collections.singletonMap(runId, runtimeInfo);
            }
            return Collections.emptyMap();
        }
    };
    // Have both flow and workflow running
    runningSet.put(flowId.getParent(), RunIds.fromString(flowId.getRun()));
    runningSet.put(workflowId.getParent(), RunIds.fromString(workflowId.getRun()));
    ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {

        @Override
        public void error(ProgramRunId programRunId, Throwable failureCause) {
            store.setStop(programRunId, System.currentTimeMillis(), ProgramRunStatus.FAILED, new BasicThrowable(failureCause), Bytes.toBytes(sourceId.getAndIncrement()));
        }
    };
    // Create a run record fixer.
    // Set the start buffer time to -1 so that it fixes right away.
    // Also use a small tx batch size to validate the batching logic.
    RunRecordCorrectorService fixer = new RunRecordCorrectorService(cConf, store, programStateWriter, programRuntimeService, namespaceAdmin, datasetFramework, -1L, 5) {
    };
    fixer.fixRunRecords();
    // Validates all expected states
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
    // Remove the workflow from the running set and mark it as completed
    runningSet.remove(workflowId.getParent());
    store.setStop(workflowId, System.currentTimeMillis(), ProgramRunStatus.COMPLETED, Bytes.toBytes(sourceId.getAndIncrement()));
    fixer.fixRunRecords();
    // Both the workflow and the MR in workflow should be changed to failed state
    expectedStates.put(workflowId, ProgramRunStatus.COMPLETED);
    expectedStates.put(mrInWorkflowId, ProgramRunStatus.FAILED);
    // Validates all expected states again
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
}