Examples with ProgramRunId co.cask.cdap.proto.id.ProgramRunId used on opensource projects

Example 21 with ProgramRunId

use of co.cask.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class RemoteLineageWriterTest method testSimpleCase.

@Test
public void testSimpleCase() {
    long now = System.currentTimeMillis();
    ApplicationId appId = NamespaceId.DEFAULT.app("test_app");
    ProgramId flowId = appId.flow("test_flow");
    ProgramRunId runId = flowId.run(RunIds.generate(now).getId());
    RunId twillRunId = RunIds.fromString(runId.getRun());
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("test_dataset");
    StreamId streamId = NamespaceId.DEFAULT.stream("test_stream");
    Set<Relation> expectedRelations = new HashSet<>();
    // test null serialization
    remoteLineageWriter.addAccess(runId, datasetId, AccessType.READ, null);
    expectedRelations.add(new Relation(datasetId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(ImmutableSet.of(flowId, datasetId), lineageStore.getEntitiesForRun(runId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.READ);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.WRITE);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.WRITE, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
}

Example 22 with ProgramRunId

use of co.cask.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class DefaultStoreTest method testWorkflowNodeState.

@Test
public void testWorkflowNodeState() throws Exception {
    String namespaceName = "namespace1";
    String appName = "app1";
    String workflowName = "workflow1";
    String mapReduceName = "mapReduce1";
    String sparkName = "spark1";
    ApplicationId appId = Ids.namespace(namespaceName).app(appName);
    ProgramId mapReduceProgram = appId.mr(mapReduceName);
    ProgramId sparkProgram = appId.spark(sparkName);
    long currentTime = System.currentTimeMillis();
    String workflowRunId = RunIds.generate(currentTime).getId();
    ProgramRunId workflowRun = appId.workflow(workflowName).run(workflowRunId);
    // start Workflow
    store.setStart(workflowRun.getParent(), workflowRun.getRun(), currentTime);
    // start MapReduce as a part of Workflow
    Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapReduceName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId mapReduceRunId = RunIds.generate(currentTime + 10);
    store.setStart(mapReduceProgram, mapReduceRunId.getId(), currentTime + 10, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the MapReduce program
    store.setStop(mapReduceProgram, mapReduceRunId.getId(), currentTime + 50, ProgramRunStatus.COMPLETED);
    // start Spark program as a part of Workflow
    systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId sparkRunId = RunIds.generate(currentTime + 60);
    store.setStart(sparkProgram, sparkRunId.getId(), currentTime + 60, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the Spark program with failure
    NullPointerException npe = new NullPointerException("dataset not found");
    IllegalArgumentException iae = new IllegalArgumentException("illegal argument", npe);
    store.setStop(sparkProgram, sparkRunId.getId(), currentTime + 100, ProgramRunStatus.FAILED, new BasicThrowable(iae));
    // stop Workflow
    store.setStop(workflowRun.getParent(), workflowRun.getRun(), currentTime + 110, ProgramRunStatus.FAILED);
    List<WorkflowNodeStateDetail> nodeStateDetails = store.getWorkflowNodeStates(workflowRun);
    Map<String, WorkflowNodeStateDetail> workflowNodeStates = new HashMap<>();
    for (WorkflowNodeStateDetail nodeStateDetail : nodeStateDetails) {
        workflowNodeStates.put(nodeStateDetail.getNodeId(), nodeStateDetail);
    }
    Assert.assertEquals(2, workflowNodeStates.size());
    WorkflowNodeStateDetail nodeStateDetail = workflowNodeStates.get(mapReduceName);
    Assert.assertEquals(mapReduceName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.COMPLETED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(mapReduceRunId.getId(), nodeStateDetail.getRunId());
    Assert.assertNull(nodeStateDetail.getFailureCause());
    nodeStateDetail = workflowNodeStates.get(sparkName);
    Assert.assertEquals(sparkName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.FAILED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(sparkRunId.getId(), nodeStateDetail.getRunId());
    BasicThrowable failureCause = nodeStateDetail.getFailureCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("illegal argument".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.IllegalArgumentException".equals(failureCause.getClassName()));
    failureCause = failureCause.getCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("dataset not found".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.NullPointerException".equals(failureCause.getClassName()));
    Assert.assertNull(failureCause.getCause());
}

Example 23 with ProgramRunId

use of co.cask.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class RemoteRuntimeStoreTest method testWorkflowMethods.

@Test
public void testWorkflowMethods() {
    ProgramId workflowId = new ProgramId(Id.Namespace.DEFAULT.getId(), "test_app", ProgramType.WORKFLOW, "test_workflow");
    long stopTime = System.currentTimeMillis() / 1000;
    long startTime = stopTime - 20;
    String pid = RunIds.generate(startTime * 1000).getId();
    String twillRunId = "twill_run_id";
    Map<String, String> runtimeArgs = ImmutableMap.of();
    Map<String, String> properties = ImmutableMap.of("runtimeArgs", GSON.toJson(runtimeArgs));
    Map<String, String> systemArgs = ImmutableMap.of();
    RunRecordMeta initialRunRecord = new RunRecordMeta(pid, startTime, null, ProgramRunStatus.RUNNING, properties, systemArgs, twillRunId);
    runtimeStore.setStart(workflowId, pid, startTime, twillRunId, runtimeArgs, systemArgs);
    Assert.assertEquals(initialRunRecord, store.getRun(workflowId, pid));
    ProgramId mapreduceId = new ProgramId(workflowId.getNamespace(), workflowId.getApplication(), ProgramType.MAPREDUCE, "test_mr");
    String mapreducePid = RunIds.generate(startTime * 1000).getId();
    // these system properties just have to be set on the system arguments of the program, in order for it to be
    // understood as a program in a workflow node
    Map<String, String> mrSystemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, "test_node_id", ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, pid);
    runtimeStore.setStart(mapreduceId, mapreducePid, startTime, twillRunId, runtimeArgs, mrSystemArgs);
    BasicThrowable failureCause = new BasicThrowable(new IllegalArgumentException("failure", new RuntimeException("oops")));
    runtimeStore.setStop(mapreduceId, mapreducePid, stopTime, ProgramRunStatus.FAILED, failureCause);
    runtimeStore.setStop(workflowId, pid, stopTime, ProgramRunStatus.FAILED);
    RunRecordMeta completedWorkflowRecord = store.getRun(workflowId, pid);
    // we're not comparing properties, since runtime (such as starting/stopping inner programs) modifies it
    Assert.assertEquals(pid, completedWorkflowRecord.getPid());
    Assert.assertEquals(initialRunRecord.getStartTs(), completedWorkflowRecord.getStartTs());
    Assert.assertEquals((Long) stopTime, completedWorkflowRecord.getStopTs());
    Assert.assertEquals(ProgramRunStatus.FAILED, completedWorkflowRecord.getStatus());
    Assert.assertEquals(twillRunId, completedWorkflowRecord.getTwillRunId());
    Assert.assertEquals(systemArgs, completedWorkflowRecord.getSystemArgs());
    // test that the BasicThrowable was serialized properly by RemoteRuntimeStore
    ProgramRunId workflowRunId = workflowId.run(pid);
    List<WorkflowNodeStateDetail> workflowNodeStates = store.getWorkflowNodeStates(workflowRunId);
    Assert.assertEquals(1, workflowNodeStates.size());
    WorkflowNodeStateDetail workflowNodeStateDetail = workflowNodeStates.get(0);
    Assert.assertEquals("test_node_id", workflowNodeStateDetail.getNodeId());
    Assert.assertEquals(mapreducePid, workflowNodeStateDetail.getRunId());
    Assert.assertEquals(NodeStatus.FAILED, workflowNodeStateDetail.getNodeStatus());
    Assert.assertEquals(failureCause, workflowNodeStateDetail.getFailureCause());
}

Example 24 with ProgramRunId

use of co.cask.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class DefaultStoreTest method testLogProgramRunHistory.

@Test
public void testLogProgramRunHistory() throws Exception {
    Map<String, String> noRuntimeArgsProps = ImmutableMap.of("runtimeArgs", GSON.toJson(ImmutableMap.<String, String>of()));
    // record finished flow
    ProgramId programId = new ProgramId("account1", "application1", ProgramType.FLOW, "flow1");
    long now = System.currentTimeMillis();
    long nowSecs = TimeUnit.MILLISECONDS.toSeconds(now);
    RunId run1 = RunIds.generate(now - 20000);
    store.setStart(programId, run1.getId(), runIdToSecs(run1));
    store.setStop(programId, run1.getId(), nowSecs - 10, ProgramController.State.ERROR.getRunStatus());
    // record another finished flow
    RunId run2 = RunIds.generate(now - 10000);
    store.setStart(programId, run2.getId(), runIdToSecs(run2));
    store.setStop(programId, run2.getId(), nowSecs - 5, ProgramController.State.COMPLETED.getRunStatus());
    // record a suspended flow
    RunId run21 = RunIds.generate(now - 7500);
    store.setStart(programId, run21.getId(), runIdToSecs(run21));
    store.setSuspend(programId, run21.getId());
    // record not finished flow
    RunId run3 = RunIds.generate(now);
    store.setStart(programId, run3.getId(), runIdToSecs(run3));
    // For a RunRecordMeta that has not yet been completed, getStopTs should return null
    RunRecordMeta runRecord = store.getRun(programId, run3.getId());
    Assert.assertNotNull(runRecord);
    Assert.assertNull(runRecord.getStopTs());
    // record run of different program
    ProgramId programId2 = new ProgramId("account1", "application1", ProgramType.FLOW, "flow2");
    RunId run4 = RunIds.generate(now - 5000);
    store.setStart(programId2, run4.getId(), runIdToSecs(run4));
    store.setStop(programId2, run4.getId(), nowSecs - 4, ProgramController.State.COMPLETED.getRunStatus());
    // record for different account
    store.setStart(new ProgramId("account2", "application1", ProgramType.FLOW, "flow1"), run3.getId(), RunIds.getTime(run3, TimeUnit.MILLISECONDS));
    // we should probably be better with "get" method in DefaultStore interface to do that, but we don't have one
    Map<ProgramRunId, RunRecordMeta> successHistorymap = store.getRuns(programId, ProgramRunStatus.COMPLETED, 0, Long.MAX_VALUE, Integer.MAX_VALUE);
    Map<ProgramRunId, RunRecordMeta> failureHistorymap = store.getRuns(programId, ProgramRunStatus.FAILED, nowSecs - 20, nowSecs - 10, Integer.MAX_VALUE);
    Assert.assertEquals(failureHistorymap, store.getRuns(programId, ProgramRunStatus.FAILED, 0, Long.MAX_VALUE, Integer.MAX_VALUE));
    Map<ProgramRunId, RunRecordMeta> suspendedHistorymap = store.getRuns(programId, ProgramRunStatus.SUSPENDED, nowSecs - 20, nowSecs, Integer.MAX_VALUE);
    // only finished + succeeded runs should be returned
    Assert.assertEquals(1, successHistorymap.size());
    // only finished + failed runs should be returned
    Assert.assertEquals(1, failureHistorymap.size());
    // only suspended runs should be returned
    Assert.assertEquals(1, suspendedHistorymap.size());
    // records should be sorted by start time latest to earliest
    RunRecordMeta run = successHistorymap.values().iterator().next();
    Assert.assertEquals(nowSecs - 10, run.getStartTs());
    Assert.assertEquals(Long.valueOf(nowSecs - 5), run.getStopTs());
    Assert.assertEquals(ProgramController.State.COMPLETED.getRunStatus(), run.getStatus());
    run = failureHistorymap.values().iterator().next();
    Assert.assertEquals(nowSecs - 20, run.getStartTs());
    Assert.assertEquals(Long.valueOf(nowSecs - 10), run.getStopTs());
    Assert.assertEquals(ProgramController.State.ERROR.getRunStatus(), run.getStatus());
    run = suspendedHistorymap.values().iterator().next();
    Assert.assertEquals(run21.getId(), run.getPid());
    Assert.assertEquals(ProgramController.State.SUSPENDED.getRunStatus(), run.getStatus());
    // Assert all history
    Map<ProgramRunId, RunRecordMeta> allHistorymap = store.getRuns(programId, ProgramRunStatus.ALL, nowSecs - 20, nowSecs + 1, Integer.MAX_VALUE);
    Assert.assertEquals(allHistorymap.toString(), 4, allHistorymap.size());
    // Assert running programs
    Map<ProgramRunId, RunRecordMeta> runningHistorymap = store.getRuns(programId, ProgramRunStatus.RUNNING, nowSecs, nowSecs + 1, 100);
    Assert.assertEquals(1, runningHistorymap.size());
    Assert.assertEquals(runningHistorymap, store.getRuns(programId, ProgramRunStatus.RUNNING, 0, Long.MAX_VALUE, 100));
    // Get a run record for running program
    RunRecordMeta expectedRunning = runningHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedRunning);
    RunRecordMeta actualRunning = store.getRun(programId, expectedRunning.getPid());
    Assert.assertEquals(expectedRunning, actualRunning);
    // Get a run record for completed run
    RunRecordMeta expectedCompleted = successHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedCompleted);
    RunRecordMeta actualCompleted = store.getRun(programId, expectedCompleted.getPid());
    Assert.assertEquals(expectedCompleted, actualCompleted);
    // Get a run record for suspended run
    RunRecordMeta expectedSuspended = suspendedHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedSuspended);
    RunRecordMeta actualSuspended = store.getRun(programId, expectedSuspended.getPid());
    Assert.assertEquals(expectedSuspended, actualSuspended);
    // Backwards compatibility test with random UUIDs
    // record finished flow
    RunId run5 = RunIds.fromString(UUID.randomUUID().toString());
    store.setStart(programId, run5.getId(), nowSecs - 8);
    store.setStop(programId, run5.getId(), nowSecs - 4, ProgramController.State.COMPLETED.getRunStatus());
    // record not finished flow
    RunId run6 = RunIds.fromString(UUID.randomUUID().toString());
    store.setStart(programId, run6.getId(), nowSecs - 2);
    // Get run record for run5
    RunRecordMeta expectedRecord5 = new RunRecordMeta(run5.getId(), nowSecs - 8, nowSecs - 4, ProgramRunStatus.COMPLETED, noRuntimeArgsProps, null, null);
    RunRecordMeta actualRecord5 = store.getRun(programId, run5.getId());
    Assert.assertEquals(expectedRecord5, actualRecord5);
    // Get run record for run6
    RunRecordMeta expectedRecord6 = new RunRecordMeta(run6.getId(), nowSecs - 2, null, ProgramRunStatus.RUNNING, noRuntimeArgsProps, null, null);
    RunRecordMeta actualRecord6 = store.getRun(programId, run6.getId());
    Assert.assertEquals(expectedRecord6, actualRecord6);
    // Non-existent run record should give null
    Assert.assertNull(store.getRun(programId, UUID.randomUUID().toString()));
    // Searching for history in wrong time range should give us no results
    Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.COMPLETED, nowSecs - 5000, nowSecs - 2000, Integer.MAX_VALUE).isEmpty());
    Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.ALL, nowSecs - 5000, nowSecs - 2000, Integer.MAX_VALUE).isEmpty());
}

Example 25 with ProgramRunId

use of co.cask.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testAuditPublish.

@Test
public void testAuditPublish() throws Exception {
    // Clear all audit messages
    inMemoryAuditPublisher.popMessages();
    List<AuditMessage> expectedMessages = new ArrayList<>();
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE2, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    // Update instance
    framework.updateInstance(MY_TABLE, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.UPDATE, AuditPayload.EMPTY_PAYLOAD));
    // Access instance
    ProgramRunId runId = new ProgramId("ns", "app", ProgramType.FLOW, "flow").run(RunIds.generate().getId());
    LineageWriterDatasetFramework lineageFramework = new LineageWriterDatasetFramework(framework, new NoOpLineageWriter(), new NoOpUsageRegistry(), new AuthenticationTestContext(), new NoOpAuthorizer());
    lineageFramework.setContext(new TestProgramContext(runId));
    lineageFramework.setAuditPublisher(inMemoryAuditPublisher);
    lineageFramework.getDataset(MY_TABLE, ImmutableMap.<String, String>of(), getClass().getClassLoader());
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.ACCESS, new AccessPayload(AccessType.UNKNOWN, runId)));
    // Truncate instance
    framework.truncateInstance(MY_TABLE);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.TRUNCATE, AuditPayload.EMPTY_PAYLOAD));
    // Delete instance
    framework.deleteInstance(MY_TABLE);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    // Delete all instances in a namespace
    framework.deleteAllInstances(MY_TABLE2.getParent());
    expectedMessages.add(new AuditMessage(0, MY_TABLE2, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    Assert.assertEquals(expectedMessages, inMemoryAuditPublisher.popMessages());
    // cleanup
    framework.deleteModule(IN_MEMORY);
}