Examples with ArtifactId io.cdap.cdap.api.artifact.ArtifactId used on opensource projects

Example 26 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ProgramLifecycleServiceTest method testProgramStatusFromSingleRun.

@Test
public void testProgramStatusFromSingleRun() {
    RunRecordDetail record = RunRecordDetail.builder().setProgramRunId(NamespaceId.DEFAULT.app("app").mr("mr").run(RunIds.generate())).setStartTime(System.currentTimeMillis()).setArtifactId(new ArtifactId("r", new ArtifactVersion("1.0"), ArtifactScope.USER)).setStatus(ProgramRunStatus.PENDING).setSourceId(new byte[] { 0 }).build();
    // pending or starting -> starting
    ProgramStatus status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record));
    Assert.assertEquals(ProgramStatus.STARTING, status);
    record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.STARTING).build();
    status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record));
    Assert.assertEquals(ProgramStatus.STARTING, status);
    // running, suspended, resuming -> running
    record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.RUNNING).build();
    status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record));
    Assert.assertEquals(ProgramStatus.RUNNING, status);
    record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.SUSPENDED).build();
    status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record));
    Assert.assertEquals(ProgramStatus.RUNNING, status);
    // failed, killed, completed -> stopped
    record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.FAILED).build();
    status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record));
    Assert.assertEquals(ProgramStatus.STOPPED, status);
    record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.KILLED).build();
    status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record));
    Assert.assertEquals(ProgramStatus.STOPPED, status);
    record = RunRecordDetail.builder(record).setStatus(ProgramRunStatus.COMPLETED).build();
    status = ProgramLifecycleService.getProgramStatus(Collections.singleton(record));
    Assert.assertEquals(ProgramStatus.STOPPED, status);
}

Example 27 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ProgramLifecycleServiceTest method testProgramStatusFromMultipleRuns.

@Test
public void testProgramStatusFromMultipleRuns() {
    ProgramId programId = NamespaceId.DEFAULT.app("app").mr("mr");
    RunRecordDetail pending = RunRecordDetail.builder().setProgramRunId(programId.run(RunIds.generate())).setStartTime(System.currentTimeMillis()).setArtifactId(new ArtifactId("r", new ArtifactVersion("1.0"), ArtifactScope.USER)).setStatus(ProgramRunStatus.PENDING).setSourceId(new byte[] { 0 }).build();
    RunRecordDetail starting = RunRecordDetail.builder(pending).setProgramRunId(programId.run(RunIds.generate())).setStatus(ProgramRunStatus.STARTING).build();
    RunRecordDetail running = RunRecordDetail.builder(pending).setProgramRunId(programId.run(RunIds.generate())).setStatus(ProgramRunStatus.RUNNING).build();
    RunRecordDetail killed = RunRecordDetail.builder(pending).setProgramRunId(programId.run(RunIds.generate())).setStatus(ProgramRunStatus.KILLED).build();
    RunRecordDetail failed = RunRecordDetail.builder(pending).setProgramRunId(programId.run(RunIds.generate())).setStatus(ProgramRunStatus.FAILED).build();
    RunRecordDetail completed = RunRecordDetail.builder(pending).setProgramRunId(programId.run(RunIds.generate())).setStatus(ProgramRunStatus.COMPLETED).build();
    // running takes precedence over others
    ProgramStatus status = ProgramLifecycleService.getProgramStatus(Arrays.asList(pending, starting, running, killed, failed, completed));
    Assert.assertEquals(ProgramStatus.RUNNING, status);
    // starting takes precedence over stopped
    status = ProgramLifecycleService.getProgramStatus(Arrays.asList(pending, killed, failed, completed));
    Assert.assertEquals(ProgramStatus.STARTING, status);
    status = ProgramLifecycleService.getProgramStatus(Arrays.asList(starting, killed, failed, completed));
    Assert.assertEquals(ProgramStatus.STARTING, status);
    // end states are stopped
    status = ProgramLifecycleService.getProgramStatus(Arrays.asList(killed, failed, completed));
    Assert.assertEquals(ProgramStatus.STOPPED, status);
}

Example 28 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ProgramNotificationSubscriberServiceTest method testHeartBeatStoreForProgramStatusMessages.

@Test
public void testHeartBeatStoreForProgramStatusMessages() throws Exception {
    ProgramId programId = NamespaceId.DEFAULT.app("someapp", "1.0-SNAPSHOT").program(ProgramType.SERVICE, "s");
    Map<String, String> systemArguments = new HashMap<>();
    systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
    ProgramRunId runId = programId.run(RunIds.generate());
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId, ProgramRunStatus.STARTING);
    long startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    long suspendTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.suspend(runId);
    });
    // perform scan on heart beat store - ensure latest message notification is suspended
    checkProgramStatus(artifactId, runId, ProgramRunStatus.SUSPENDED);
    heartbeatDatasetStatusCheck(suspendTime, ProgramRunStatus.SUSPENDED);
    long resumeTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.resume(runId);
    });
    // app metadata records as RUNNING
    checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
    // heart beat messages wont have been sent due to high interval. resuming program will be recorded as running
    // in run record by app meta
    heartbeatDatasetStatusCheck(resumeTime, ProgramRunStatus.RUNNING);
    // killed status check after error
    long stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.error(runId, new Throwable("Testing"));
    });
    checkProgramStatus(artifactId, runId, ProgramRunStatus.FAILED);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.FAILED);
    ProgramRunId runId2 = programId.run(RunIds.generate());
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId2, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.STARTING);
    startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId2, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    // completed status check
    stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.completed(runId2);
    });
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.COMPLETED);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.COMPLETED);
    ProgramRunId runId3 = programId.run(RunIds.generate());
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId3, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.STARTING);
    startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId3, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    // completed status check
    stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.stop(runId3, 10);
    });
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.STOPPING);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.STOPPING);
}

Example 29 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class RunRecordCorrectorServiceTest method testFixProgram.

@Test
public void testFixProgram() throws Exception {
    final AtomicInteger sourceId = new AtomicInteger(0);
    // Write 10 services with starting state
    // Write 10 workers with running state
    Map<ProgramRunId, ProgramRunStatus> expectedStates = new HashMap<>();
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    for (int i = 0; i < 10; i++) {
        ProgramRunId serviceId = NamespaceId.DEFAULT.app("test").service("service" + i).run(randomRunId());
        store.setProvisioning(serviceId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(serviceId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(serviceId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(serviceId, ProgramRunStatus.FAILED);
        ProgramRunId workerId = new NamespaceId("ns").app("test").worker("worker" + i).run(randomRunId());
        store.setProvisioning(workerId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
        store.setProvisioned(workerId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
        store.setStart(workerId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
        store.setRunning(workerId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
        expectedStates.put(workerId, ProgramRunStatus.FAILED);
    }
    // Write a service with suspend state
    ProgramRunId flowId = new NamespaceId("ns").app("test").service("flow").run(randomRunId());
    store.setProvisioning(flowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(flowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(flowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(flowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setSuspend(flowId, Bytes.toBytes(sourceId.getAndIncrement()), -1);
    expectedStates.put(flowId, ProgramRunStatus.SUSPENDED);
    // Write two MR in starting state. One with workflow information, one without.
    ProgramRunId mrId = NamespaceId.DEFAULT.app("app").mr("mr").run(randomRunId());
    store.setProvisioning(mrId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(mrId, ProgramRunStatus.FAILED);
    ProgramRunId workflowId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(randomRunId());
    ProgramRunId mrInWorkflowId = workflowId.getParent().getParent().mr("mrInWorkflow").run(randomRunId());
    store.setProvisioning(mrInWorkflowId, Collections.emptyMap(), ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr", SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()), Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(mrInWorkflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(mrInWorkflowId, null, ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr"), Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.STARTING);
    // Write the workflow in RUNNING state.
    store.setProvisioning(workflowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(workflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(workflowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(workflowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    expectedStates.put(workflowId, ProgramRunStatus.RUNNING);
    // Use a ProgramRuntimeService that only reports running state based on a set of know ids
    final Map<ProgramId, RunId> runningSet = new HashMap<>();
    ProgramRuntimeService programRuntimeService = new AbstractProgramRuntimeService(cConf, null, null, new NoOpProgramStateWriter(), null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new NotRunningProgramLiveInfo(programId);
        }

        @Override
        public Map<RunId, RuntimeInfo> list(ProgramId program) {
            RunId runId = runningSet.get(program);
            if (runId != null) {
                RuntimeInfo runtimeInfo = new SimpleRuntimeInfo(null, program);
                return Collections.singletonMap(runId, runtimeInfo);
            }
            return Collections.emptyMap();
        }
    };
    // Have both flow and workflow running
    runningSet.put(flowId.getParent(), RunIds.fromString(flowId.getRun()));
    runningSet.put(workflowId.getParent(), RunIds.fromString(workflowId.getRun()));
    ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {

        @Override
        public void error(ProgramRunId programRunId, Throwable failureCause) {
            store.setStop(programRunId, System.currentTimeMillis(), ProgramRunStatus.FAILED, new BasicThrowable(failureCause), Bytes.toBytes(sourceId.getAndIncrement()));
        }
    };
    // Create a run record fixer.
    // Set the start buffer time to -1 so that it fixes right away.
    // Also use a small tx batch size to validate the batching logic.
    RunRecordCorrectorService fixer = new RunRecordCorrectorService(cConf, store, programStateWriter, programRuntimeService, namespaceAdmin, datasetFramework, -1L, 5) {
    };
    fixer.fixRunRecords();
    // Validates all expected states
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
    // Remove the workflow from the running set and mark it as completed
    runningSet.remove(workflowId.getParent());
    store.setStop(workflowId, System.currentTimeMillis(), ProgramRunStatus.COMPLETED, Bytes.toBytes(sourceId.getAndIncrement()));
    fixer.fixRunRecords();
    // Both the workflow and the MR in workflow should be changed to failed state
    expectedStates.put(workflowId, ProgramRunStatus.COMPLETED);
    expectedStates.put(mrInWorkflowId, ProgramRunStatus.FAILED);
    // Validates all expected states again
    for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
        validateExpectedState(entry.getKey(), entry.getValue());
    }
}

Example 30 with ArtifactId

use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class RunRecordCorrectorServiceTest method testFixUnderlyingProgramInWorkflow.

@Test
public void testFixUnderlyingProgramInWorkflow() throws Exception {
    AtomicInteger sourceId = new AtomicInteger(0);
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    // set up a workflow
    Map<String, String> wfSystemArg = ImmutableMap.of(ProgramOptionConstants.CLUSTER_MODE, ClusterMode.ISOLATED.name(), SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    ProgramRunId wfId = NamespaceId.DEFAULT.app("test").workflow("testWF").run(randomRunId());
    store.setProvisioning(wfId, Collections.emptyMap(), wfSystemArg, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(wfId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(wfId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(wfId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    // set up a spark program inside workflow, and in ISOLATED mode
    Map<String, String> spSystemArgs = ImmutableMap.of(ProgramOptionConstants.CLUSTER_MODE, ClusterMode.ISOLATED.name(), SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName(), ProgramOptionConstants.WORKFLOW_NAME, wfId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, wfId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "spark");
    ProgramRunId spId = NamespaceId.DEFAULT.app("test").spark("phase-1").run(randomRunId());
    store.setProvisioning(spId, Collections.emptyMap(), spSystemArgs, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
    store.setProvisioned(spId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
    store.setStart(spId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
    store.setRunning(spId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
    // mark the workflow as finished
    store.setStop(wfId, System.currentTimeMillis(), ProgramRunStatus.COMPLETED, Bytes.toBytes(sourceId.getAndIncrement()));
    ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {

        @Override
        public void error(ProgramRunId programRunId, Throwable failureCause) {
            store.setStop(programRunId, System.currentTimeMillis(), ProgramRunStatus.FAILED, new BasicThrowable(failureCause), Bytes.toBytes(sourceId.getAndIncrement()));
        }
    };
    ProgramRuntimeService noOpRuntimeSerivce = new AbstractProgramRuntimeService(cConf, null, null, new NoOpProgramStateWriter(), null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new NotRunningProgramLiveInfo(programId);
        }

        @Override
        public Map<RunId, RuntimeInfo> list(ProgramId program) {
            return Collections.emptyMap();
        }
    };
    // Create a run record fixer.
    // Set the start buffer time to -1 so that it fixes right away.
    RunRecordCorrectorService fixer = new RunRecordCorrectorService(cConf, store, programStateWriter, noOpRuntimeSerivce, namespaceAdmin, datasetFramework, -1L, 5) {
    };
    fixer.fixRunRecords();
    // check the record is fixed for spark program
    Assert.assertEquals(ProgramRunStatus.FAILED, store.getRun(spId).getStatus());
}