Examples with RunId org.apache.twill.api.RunId used on opensource projects

Example 16 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class RemoteLineageWriterTest method testSimpleCase.

@Test
public void testSimpleCase() {
    long now = System.currentTimeMillis();
    ApplicationId appId = NamespaceId.DEFAULT.app("test_app");
    ProgramId flowId = appId.flow("test_flow");
    ProgramRunId runId = flowId.run(RunIds.generate(now).getId());
    RunId twillRunId = RunIds.fromString(runId.getRun());
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("test_dataset");
    StreamId streamId = NamespaceId.DEFAULT.stream("test_stream");
    Set<Relation> expectedRelations = new HashSet<>();
    // test null serialization
    remoteLineageWriter.addAccess(runId, datasetId, AccessType.READ, null);
    expectedRelations.add(new Relation(datasetId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(ImmutableSet.of(flowId, datasetId), lineageStore.getEntitiesForRun(runId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.READ);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.WRITE);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.WRITE, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
}

Example 17 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class DefaultStoreTest method testRunningInRangeMulti.

@SuppressWarnings("PointlessArithmeticExpression")
@Test
public void testRunningInRangeMulti() throws Exception {
    // Add some run records
    TreeSet<Long> allPrograms = new TreeSet<>();
    TreeSet<Long> stoppedPrograms = new TreeSet<>();
    TreeSet<Long> suspendedPrograms = new TreeSet<>();
    TreeSet<Long> runningPrograms = new TreeSet<>();
    for (int i = 0; i < 99; ++i) {
        Id.Application application = Id.Application.from("default", "app" + i);
        Id.Program program = Id.Program.from(application, ProgramType.values()[i % ProgramType.values().length], "program" + i);
        long startTime = (i + 1) * 10000;
        RunId runId = RunIds.generate(startTime);
        allPrograms.add(startTime);
        Id.Run run = new Id.Run(program, runId.getId());
        writeStartRecord(run);
        // For every 3rd program starting from 0th, write stop record
        if ((i % 3) == 0) {
            writeStopRecord(run, startTime + 10);
            stoppedPrograms.add(startTime);
        }
        // For every 3rd program starting from 1st, write suspended record
        if ((i % 3) == 1) {
            writeSuspendedRecord(run);
            suspendedPrograms.add(startTime);
        }
        // The rest are running programs
        if ((i % 3) == 2) {
            runningPrograms.add(startTime);
        }
    }
    // In all below assertions, TreeSet and metadataStore both have start time inclusive and end time exclusive.
    // querying full range should give all programs
    Assert.assertEquals(allPrograms, runIdsToTime(store.getRunningInRange(0, Long.MAX_VALUE)));
    Assert.assertEquals(allPrograms, runIdsToTime(store.getRunningInRange(1 * 10, 100 * 10)));
    // querying a range before start time of first program should give empty results
    Assert.assertEquals(ImmutableSet.of(), runIdsToTime(store.getRunningInRange(1, 1 * 10)));
    // querying a range after the stop time of the last program should return only running and suspended programs
    Assert.assertEquals(ImmutableSortedSet.copyOf(Iterables.concat(suspendedPrograms, runningPrograms)), runIdsToTime(store.getRunningInRange(100 * 10, Long.MAX_VALUE)));
    // querying a range completely within the start time of the first program and stop time of the last program
    // should give all running and suspended programs started after given start time,
    // and all stopped programs between given start and stop time.
    Assert.assertEquals(ImmutableSortedSet.copyOf(Iterables.concat(stoppedPrograms.subSet(30 * 10000L, 60 * 10000L), suspendedPrograms.subSet(1 * 10000L, 60 * 10000L), runningPrograms.subSet(1 * 10000L, 60 * 10000L))), runIdsToTime(store.getRunningInRange(30 * 10, 60 * 10)));
    // querying a range after start time of first program to after the stop time of the last program
    // should give all running and suspended programs after start time of first program
    // and all stopped programs after the given start time
    Assert.assertEquals(ImmutableSortedSet.copyOf(Iterables.concat(stoppedPrograms.subSet(30 * 10000L, 150 * 10000L), suspendedPrograms.subSet(1 * 10000L, 150 * 10000L), runningPrograms.subSet(1 * 10000L, 150 * 10000L))), runIdsToTime(store.getRunningInRange(30 * 10, 150 * 10)));
    // querying a range before start time of first program to before the stop time of the last program
    // should give all running, suspended, and stopped programs from the first program to the given stop time
    Assert.assertEquals(ImmutableSortedSet.copyOf(Iterables.concat(stoppedPrograms.subSet(1000L, 45 * 10000L), suspendedPrograms.subSet(1000L, 45 * 10000L), runningPrograms.subSet(1000L, 45 * 10000L))), runIdsToTime(store.getRunningInRange(1, 45 * 10)));
}

Example 18 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class DefaultStoreTest method testWorkflowNodeState.

@Test
public void testWorkflowNodeState() throws Exception {
    String namespaceName = "namespace1";
    String appName = "app1";
    String workflowName = "workflow1";
    String mapReduceName = "mapReduce1";
    String sparkName = "spark1";
    ApplicationId appId = Ids.namespace(namespaceName).app(appName);
    ProgramId mapReduceProgram = appId.mr(mapReduceName);
    ProgramId sparkProgram = appId.spark(sparkName);
    long currentTime = System.currentTimeMillis();
    String workflowRunId = RunIds.generate(currentTime).getId();
    ProgramRunId workflowRun = appId.workflow(workflowName).run(workflowRunId);
    // start Workflow
    store.setStart(workflowRun.getParent(), workflowRun.getRun(), currentTime);
    // start MapReduce as a part of Workflow
    Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapReduceName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId mapReduceRunId = RunIds.generate(currentTime + 10);
    store.setStart(mapReduceProgram, mapReduceRunId.getId(), currentTime + 10, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the MapReduce program
    store.setStop(mapReduceProgram, mapReduceRunId.getId(), currentTime + 50, ProgramRunStatus.COMPLETED);
    // start Spark program as a part of Workflow
    systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId sparkRunId = RunIds.generate(currentTime + 60);
    store.setStart(sparkProgram, sparkRunId.getId(), currentTime + 60, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the Spark program with failure
    NullPointerException npe = new NullPointerException("dataset not found");
    IllegalArgumentException iae = new IllegalArgumentException("illegal argument", npe);
    store.setStop(sparkProgram, sparkRunId.getId(), currentTime + 100, ProgramRunStatus.FAILED, new BasicThrowable(iae));
    // stop Workflow
    store.setStop(workflowRun.getParent(), workflowRun.getRun(), currentTime + 110, ProgramRunStatus.FAILED);
    List<WorkflowNodeStateDetail> nodeStateDetails = store.getWorkflowNodeStates(workflowRun);
    Map<String, WorkflowNodeStateDetail> workflowNodeStates = new HashMap<>();
    for (WorkflowNodeStateDetail nodeStateDetail : nodeStateDetails) {
        workflowNodeStates.put(nodeStateDetail.getNodeId(), nodeStateDetail);
    }
    Assert.assertEquals(2, workflowNodeStates.size());
    WorkflowNodeStateDetail nodeStateDetail = workflowNodeStates.get(mapReduceName);
    Assert.assertEquals(mapReduceName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.COMPLETED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(mapReduceRunId.getId(), nodeStateDetail.getRunId());
    Assert.assertNull(nodeStateDetail.getFailureCause());
    nodeStateDetail = workflowNodeStates.get(sparkName);
    Assert.assertEquals(sparkName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.FAILED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(sparkRunId.getId(), nodeStateDetail.getRunId());
    BasicThrowable failureCause = nodeStateDetail.getFailureCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("illegal argument".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.IllegalArgumentException".equals(failureCause.getClassName()));
    failureCause = failureCause.getCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("dataset not found".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.NullPointerException".equals(failureCause.getClassName()));
    Assert.assertNull(failureCause.getCause());
}

Example 19 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class DefaultStoreTest method testLogProgramRunHistory.

@Test
public void testLogProgramRunHistory() throws Exception {
    Map<String, String> noRuntimeArgsProps = ImmutableMap.of("runtimeArgs", GSON.toJson(ImmutableMap.<String, String>of()));
    // record finished flow
    ProgramId programId = new ProgramId("account1", "application1", ProgramType.FLOW, "flow1");
    long now = System.currentTimeMillis();
    long nowSecs = TimeUnit.MILLISECONDS.toSeconds(now);
    RunId run1 = RunIds.generate(now - 20000);
    store.setStart(programId, run1.getId(), runIdToSecs(run1));
    store.setStop(programId, run1.getId(), nowSecs - 10, ProgramController.State.ERROR.getRunStatus());
    // record another finished flow
    RunId run2 = RunIds.generate(now - 10000);
    store.setStart(programId, run2.getId(), runIdToSecs(run2));
    store.setStop(programId, run2.getId(), nowSecs - 5, ProgramController.State.COMPLETED.getRunStatus());
    // record a suspended flow
    RunId run21 = RunIds.generate(now - 7500);
    store.setStart(programId, run21.getId(), runIdToSecs(run21));
    store.setSuspend(programId, run21.getId());
    // record not finished flow
    RunId run3 = RunIds.generate(now);
    store.setStart(programId, run3.getId(), runIdToSecs(run3));
    // For a RunRecordMeta that has not yet been completed, getStopTs should return null
    RunRecordMeta runRecord = store.getRun(programId, run3.getId());
    Assert.assertNotNull(runRecord);
    Assert.assertNull(runRecord.getStopTs());
    // record run of different program
    ProgramId programId2 = new ProgramId("account1", "application1", ProgramType.FLOW, "flow2");
    RunId run4 = RunIds.generate(now - 5000);
    store.setStart(programId2, run4.getId(), runIdToSecs(run4));
    store.setStop(programId2, run4.getId(), nowSecs - 4, ProgramController.State.COMPLETED.getRunStatus());
    // record for different account
    store.setStart(new ProgramId("account2", "application1", ProgramType.FLOW, "flow1"), run3.getId(), RunIds.getTime(run3, TimeUnit.MILLISECONDS));
    // we should probably be better with "get" method in DefaultStore interface to do that, but we don't have one
    Map<ProgramRunId, RunRecordMeta> successHistorymap = store.getRuns(programId, ProgramRunStatus.COMPLETED, 0, Long.MAX_VALUE, Integer.MAX_VALUE);
    Map<ProgramRunId, RunRecordMeta> failureHistorymap = store.getRuns(programId, ProgramRunStatus.FAILED, nowSecs - 20, nowSecs - 10, Integer.MAX_VALUE);
    Assert.assertEquals(failureHistorymap, store.getRuns(programId, ProgramRunStatus.FAILED, 0, Long.MAX_VALUE, Integer.MAX_VALUE));
    Map<ProgramRunId, RunRecordMeta> suspendedHistorymap = store.getRuns(programId, ProgramRunStatus.SUSPENDED, nowSecs - 20, nowSecs, Integer.MAX_VALUE);
    // only finished + succeeded runs should be returned
    Assert.assertEquals(1, successHistorymap.size());
    // only finished + failed runs should be returned
    Assert.assertEquals(1, failureHistorymap.size());
    // only suspended runs should be returned
    Assert.assertEquals(1, suspendedHistorymap.size());
    // records should be sorted by start time latest to earliest
    RunRecordMeta run = successHistorymap.values().iterator().next();
    Assert.assertEquals(nowSecs - 10, run.getStartTs());
    Assert.assertEquals(Long.valueOf(nowSecs - 5), run.getStopTs());
    Assert.assertEquals(ProgramController.State.COMPLETED.getRunStatus(), run.getStatus());
    run = failureHistorymap.values().iterator().next();
    Assert.assertEquals(nowSecs - 20, run.getStartTs());
    Assert.assertEquals(Long.valueOf(nowSecs - 10), run.getStopTs());
    Assert.assertEquals(ProgramController.State.ERROR.getRunStatus(), run.getStatus());
    run = suspendedHistorymap.values().iterator().next();
    Assert.assertEquals(run21.getId(), run.getPid());
    Assert.assertEquals(ProgramController.State.SUSPENDED.getRunStatus(), run.getStatus());
    // Assert all history
    Map<ProgramRunId, RunRecordMeta> allHistorymap = store.getRuns(programId, ProgramRunStatus.ALL, nowSecs - 20, nowSecs + 1, Integer.MAX_VALUE);
    Assert.assertEquals(allHistorymap.toString(), 4, allHistorymap.size());
    // Assert running programs
    Map<ProgramRunId, RunRecordMeta> runningHistorymap = store.getRuns(programId, ProgramRunStatus.RUNNING, nowSecs, nowSecs + 1, 100);
    Assert.assertEquals(1, runningHistorymap.size());
    Assert.assertEquals(runningHistorymap, store.getRuns(programId, ProgramRunStatus.RUNNING, 0, Long.MAX_VALUE, 100));
    // Get a run record for running program
    RunRecordMeta expectedRunning = runningHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedRunning);
    RunRecordMeta actualRunning = store.getRun(programId, expectedRunning.getPid());
    Assert.assertEquals(expectedRunning, actualRunning);
    // Get a run record for completed run
    RunRecordMeta expectedCompleted = successHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedCompleted);
    RunRecordMeta actualCompleted = store.getRun(programId, expectedCompleted.getPid());
    Assert.assertEquals(expectedCompleted, actualCompleted);
    // Get a run record for suspended run
    RunRecordMeta expectedSuspended = suspendedHistorymap.values().iterator().next();
    Assert.assertNotNull(expectedSuspended);
    RunRecordMeta actualSuspended = store.getRun(programId, expectedSuspended.getPid());
    Assert.assertEquals(expectedSuspended, actualSuspended);
    // Backwards compatibility test with random UUIDs
    // record finished flow
    RunId run5 = RunIds.fromString(UUID.randomUUID().toString());
    store.setStart(programId, run5.getId(), nowSecs - 8);
    store.setStop(programId, run5.getId(), nowSecs - 4, ProgramController.State.COMPLETED.getRunStatus());
    // record not finished flow
    RunId run6 = RunIds.fromString(UUID.randomUUID().toString());
    store.setStart(programId, run6.getId(), nowSecs - 2);
    // Get run record for run5
    RunRecordMeta expectedRecord5 = new RunRecordMeta(run5.getId(), nowSecs - 8, nowSecs - 4, ProgramRunStatus.COMPLETED, noRuntimeArgsProps, null, null);
    RunRecordMeta actualRecord5 = store.getRun(programId, run5.getId());
    Assert.assertEquals(expectedRecord5, actualRecord5);
    // Get run record for run6
    RunRecordMeta expectedRecord6 = new RunRecordMeta(run6.getId(), nowSecs - 2, null, ProgramRunStatus.RUNNING, noRuntimeArgsProps, null, null);
    RunRecordMeta actualRecord6 = store.getRun(programId, run6.getId());
    Assert.assertEquals(expectedRecord6, actualRecord6);
    // Non-existent run record should give null
    Assert.assertNull(store.getRun(programId, UUID.randomUUID().toString()));
    // Searching for history in wrong time range should give us no results
    Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.COMPLETED, nowSecs - 5000, nowSecs - 2000, Integer.MAX_VALUE).isEmpty());
    Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.ALL, nowSecs - 5000, nowSecs - 2000, Integer.MAX_VALUE).isEmpty());
}

Example 20 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class DefaultStoreTest method testDeleteSuspendedWorkflow.

@Test
public void testDeleteSuspendedWorkflow() {
    NamespaceId namespaceId = new NamespaceId("namespace1");
    // Test delete application
    ApplicationId appId1 = namespaceId.app("app1");
    ProgramId programId1 = appId1.workflow("pgm1");
    RunId run1 = RunIds.generate();
    store.setStart(programId1, run1.getId(), runIdToSecs(run1));
    store.setSuspend(programId1, run1.getId());
    store.removeApplication(appId1);
    Assert.assertTrue(store.getRuns(programId1, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty());
    // Test delete namespace
    ProgramId programId2 = namespaceId.app("app2").workflow("pgm2");
    RunId run2 = RunIds.generate();
    store.setStart(programId2, run2.getId(), runIdToSecs(run2));
    store.setSuspend(programId2, run2.getId());
    store.removeAll(namespaceId);
    nsStore.delete(namespaceId);
    Assert.assertTrue(store.getRuns(programId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty());
}