Examples with ProgramId co.cask.cdap.proto.id.ProgramId used on opensource projects

Example 66 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class DefaultStoreTest method writeStartRecord.

private void writeStartRecord(Id.Run run) {
    ProgramId programId = run.getProgram().toEntityId();
    store.setStart(programId, run.getId(), RunIds.getTime(RunIds.fromString(run.getId()), TimeUnit.SECONDS));
    Assert.assertNotNull(store.getRun(programId, run.getId()));
}

Example 67 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class DefaultStoreTest method testServiceInstances.

@Test
public void testServiceInstances() throws Exception {
    ApplicationSpecification appSpec = Specifications.from(new AppWithServices());
    ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
    store.addApplication(appId, appSpec);
    // Test setting of service instances
    ProgramId programId = appId.program(ProgramType.SERVICE, "NoOpService");
    int count = store.getServiceInstances(programId);
    Assert.assertEquals(1, count);
    store.setServiceInstances(programId, 10);
    count = store.getServiceInstances(programId);
    Assert.assertEquals(10, count);
    ApplicationSpecification newSpec = store.getApplication(appId);
    Assert.assertNotNull(newSpec);
    Map<String, ServiceSpecification> services = newSpec.getServices();
    Assert.assertEquals(1, services.size());
    ServiceSpecification serviceSpec = services.get("NoOpService");
    Assert.assertEquals(10, serviceSpec.getInstances());
}

Example 68 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class ProgramLifecycleHttpHandlerTest method testBatchStatus.

@Test
public void testBatchStatus() throws Exception {
    final String statusUrl1 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    final String statusUrl2 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    // invalid json must return 400
    Assert.assertEquals(400, doPost(statusUrl1, "").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl2, "").getStatusLine().getStatusCode());
    // empty array is valid args
    Assert.assertEquals(200, doPost(statusUrl1, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode());
    Assert.assertEquals(200, doPost(statusUrl2, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode());
    // deploy an app in namespace1
    deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    // deploy another app in namespace2
    deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    // data requires appId, programId, and programType. Test missing fields/invalid programType
    Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow'}]").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programId':'WordCountFlow'}]").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl1, "[{'programType':'Flow', 'programId':'WordCountFlow'}, {'appId':" + "'AppWithServices', 'programType': 'service', 'programId': 'NoOpService'}]").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow' " + "'programId':'WordCountFlow'}]").getStatusLine().getStatusCode());
    // Test missing app, programType, etc
    List<JsonObject> returnedBody = readResponse(doPost(statusUrl1, "[{'appId':'NotExist', 'programType':'Flow', " + "'programId':'WordCountFlow'}]"), LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace1", "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString());
    returnedBody = readResponse(doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'flow', 'programId':'NotExist'}," + "{'appId':'WordCountApp', 'programType':'flow', 'programId':'WordCountFlow'}]"), LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(new NotFoundException(new ProgramId("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString());
    Assert.assertEquals(new NotFoundException(new ProgramId("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString());
    // The programType should be consistent. Second object should have proper status
    Assert.assertEquals("Flow", returnedBody.get(1).get("programType").getAsString());
    Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString());
    // test valid cases for namespace1
    HttpResponse response = doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': " + "'WordFrequencyService'}]");
    verifyInitialBatchStatusOutput(response);
    // test valid cases for namespace2
    response = doPost(statusUrl2, "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]");
    verifyInitialBatchStatusOutput(response);
    // start the flow
    Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME);
    Id.Program service2 = Id.Program.from(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE, APP_WITH_SERVICES_SERVICE_NAME);
    startProgram(wordcountFlow1);
    // test status API after starting the flow
    response = doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]");
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString());
    Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString());
    // start the service
    startProgram(service2);
    // test status API after starting the service
    response = doPost(statusUrl2, "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]");
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString());
    // stop the flow
    stopProgram(wordcountFlow1);
    waitState(wordcountFlow1, STOPPED);
    // stop the service
    stopProgram(service2);
    waitState(service2, STOPPED);
    // try posting a status request with namespace2 for apps in namespace1
    response = doPost(statusUrl2, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': 'WordFrequencyService'}," + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]");
    returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(0).get("error").getAsString());
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(1).get("error").getAsString());
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(2).get("error").getAsString());
}

Example 69 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class RemoteLineageWriterTest method testSimpleCase.

@Test
public void testSimpleCase() {
    long now = System.currentTimeMillis();
    ApplicationId appId = NamespaceId.DEFAULT.app("test_app");
    ProgramId flowId = appId.flow("test_flow");
    ProgramRunId runId = flowId.run(RunIds.generate(now).getId());
    RunId twillRunId = RunIds.fromString(runId.getRun());
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("test_dataset");
    StreamId streamId = NamespaceId.DEFAULT.stream("test_stream");
    Set<Relation> expectedRelations = new HashSet<>();
    // test null serialization
    remoteLineageWriter.addAccess(runId, datasetId, AccessType.READ, null);
    expectedRelations.add(new Relation(datasetId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(ImmutableSet.of(flowId, datasetId), lineageStore.getEntitiesForRun(runId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.READ);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.WRITE);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.WRITE, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
}

Example 70 with ProgramId

use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.

the class DefaultStoreTest method testWorkflowNodeState.

@Test
public void testWorkflowNodeState() throws Exception {
    String namespaceName = "namespace1";
    String appName = "app1";
    String workflowName = "workflow1";
    String mapReduceName = "mapReduce1";
    String sparkName = "spark1";
    ApplicationId appId = Ids.namespace(namespaceName).app(appName);
    ProgramId mapReduceProgram = appId.mr(mapReduceName);
    ProgramId sparkProgram = appId.spark(sparkName);
    long currentTime = System.currentTimeMillis();
    String workflowRunId = RunIds.generate(currentTime).getId();
    ProgramRunId workflowRun = appId.workflow(workflowName).run(workflowRunId);
    // start Workflow
    store.setStart(workflowRun.getParent(), workflowRun.getRun(), currentTime);
    // start MapReduce as a part of Workflow
    Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapReduceName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId mapReduceRunId = RunIds.generate(currentTime + 10);
    store.setStart(mapReduceProgram, mapReduceRunId.getId(), currentTime + 10, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the MapReduce program
    store.setStop(mapReduceProgram, mapReduceRunId.getId(), currentTime + 50, ProgramRunStatus.COMPLETED);
    // start Spark program as a part of Workflow
    systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId sparkRunId = RunIds.generate(currentTime + 60);
    store.setStart(sparkProgram, sparkRunId.getId(), currentTime + 60, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the Spark program with failure
    NullPointerException npe = new NullPointerException("dataset not found");
    IllegalArgumentException iae = new IllegalArgumentException("illegal argument", npe);
    store.setStop(sparkProgram, sparkRunId.getId(), currentTime + 100, ProgramRunStatus.FAILED, new BasicThrowable(iae));
    // stop Workflow
    store.setStop(workflowRun.getParent(), workflowRun.getRun(), currentTime + 110, ProgramRunStatus.FAILED);
    List<WorkflowNodeStateDetail> nodeStateDetails = store.getWorkflowNodeStates(workflowRun);
    Map<String, WorkflowNodeStateDetail> workflowNodeStates = new HashMap<>();
    for (WorkflowNodeStateDetail nodeStateDetail : nodeStateDetails) {
        workflowNodeStates.put(nodeStateDetail.getNodeId(), nodeStateDetail);
    }
    Assert.assertEquals(2, workflowNodeStates.size());
    WorkflowNodeStateDetail nodeStateDetail = workflowNodeStates.get(mapReduceName);
    Assert.assertEquals(mapReduceName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.COMPLETED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(mapReduceRunId.getId(), nodeStateDetail.getRunId());
    Assert.assertNull(nodeStateDetail.getFailureCause());
    nodeStateDetail = workflowNodeStates.get(sparkName);
    Assert.assertEquals(sparkName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.FAILED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(sparkRunId.getId(), nodeStateDetail.getRunId());
    BasicThrowable failureCause = nodeStateDetail.getFailureCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("illegal argument".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.IllegalArgumentException".equals(failureCause.getClassName()));
    failureCause = failureCause.getCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("dataset not found".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.NullPointerException".equals(failureCause.getClassName()));
    Assert.assertNull(failureCause.getCause());
}