Search in sources :

Example 51 with ApplicationId

use of co.cask.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testServiceInstances.

@Test
public void testServiceInstances() throws Exception {
    ApplicationSpecification appSpec = Specifications.from(new AppWithServices());
    ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
    store.addApplication(appId, appSpec);
    // Test setting of service instances
    ProgramId programId = appId.program(ProgramType.SERVICE, "NoOpService");
    int count = store.getServiceInstances(programId);
    Assert.assertEquals(1, count);
    store.setServiceInstances(programId, 10);
    count = store.getServiceInstances(programId);
    Assert.assertEquals(10, count);
    ApplicationSpecification newSpec = store.getApplication(appId);
    Assert.assertNotNull(newSpec);
    Map<String, ServiceSpecification> services = newSpec.getServices();
    Assert.assertEquals(1, services.size());
    ServiceSpecification serviceSpec = services.get("NoOpService");
    Assert.assertEquals(10, serviceSpec.getInstances());
}
Also used : ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) ServiceSpecification(co.cask.cdap.api.service.ServiceSpecification) AppWithServices(co.cask.cdap.AppWithServices) ApplicationId(co.cask.cdap.proto.id.ApplicationId) ProgramId(co.cask.cdap.proto.id.ProgramId) Test(org.junit.Test)

Example 52 with ApplicationId

use of co.cask.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class ProgramLifecycleHttpHandlerTest method testBatchStatus.

@Test
public void testBatchStatus() throws Exception {
    final String statusUrl1 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    final String statusUrl2 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    // invalid json must return 400
    Assert.assertEquals(400, doPost(statusUrl1, "").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl2, "").getStatusLine().getStatusCode());
    // empty array is valid args
    Assert.assertEquals(200, doPost(statusUrl1, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode());
    Assert.assertEquals(200, doPost(statusUrl2, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode());
    // deploy an app in namespace1
    deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
    // deploy another app in namespace2
    deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    // data requires appId, programId, and programType. Test missing fields/invalid programType
    Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow'}]").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programId':'WordCountFlow'}]").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl1, "[{'programType':'Flow', 'programId':'WordCountFlow'}, {'appId':" + "'AppWithServices', 'programType': 'service', 'programId': 'NoOpService'}]").getStatusLine().getStatusCode());
    Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow' " + "'programId':'WordCountFlow'}]").getStatusLine().getStatusCode());
    // Test missing app, programType, etc
    List<JsonObject> returnedBody = readResponse(doPost(statusUrl1, "[{'appId':'NotExist', 'programType':'Flow', " + "'programId':'WordCountFlow'}]"), LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace1", "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString());
    returnedBody = readResponse(doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'flow', 'programId':'NotExist'}," + "{'appId':'WordCountApp', 'programType':'flow', 'programId':'WordCountFlow'}]"), LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(new NotFoundException(new ProgramId("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString());
    Assert.assertEquals(new NotFoundException(new ProgramId("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString());
    // The programType should be consistent. Second object should have proper status
    Assert.assertEquals("Flow", returnedBody.get(1).get("programType").getAsString());
    Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString());
    // test valid cases for namespace1
    HttpResponse response = doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': " + "'WordFrequencyService'}]");
    verifyInitialBatchStatusOutput(response);
    // test valid cases for namespace2
    response = doPost(statusUrl2, "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]");
    verifyInitialBatchStatusOutput(response);
    // start the flow
    Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME);
    Id.Program service2 = Id.Program.from(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE, APP_WITH_SERVICES_SERVICE_NAME);
    startProgram(wordcountFlow1);
    // test status API after starting the flow
    response = doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]");
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString());
    Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString());
    // start the service
    startProgram(service2);
    // test status API after starting the service
    response = doPost(statusUrl2, "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]");
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString());
    // stop the flow
    stopProgram(wordcountFlow1);
    waitState(wordcountFlow1, STOPPED);
    // stop the service
    stopProgram(service2);
    waitState(service2, STOPPED);
    // try posting a status request with namespace2 for apps in namespace1
    response = doPost(statusUrl2, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': 'WordFrequencyService'}," + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]");
    returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(0).get("error").getAsString());
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(1).get("error").getAsString());
    Assert.assertEquals(new NotFoundException(new ApplicationId("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(2).get("error").getAsString());
}
Also used : JsonObject(com.google.gson.JsonObject) NotFoundException(co.cask.cdap.common.NotFoundException) HttpResponse(org.apache.http.HttpResponse) Id(co.cask.cdap.proto.Id) ProgramId(co.cask.cdap.proto.id.ProgramId) NamespaceId(co.cask.cdap.proto.id.NamespaceId) StreamId(co.cask.cdap.proto.id.StreamId) ApplicationId(co.cask.cdap.proto.id.ApplicationId) ApplicationId(co.cask.cdap.proto.id.ApplicationId) ProgramId(co.cask.cdap.proto.id.ProgramId) Test(org.junit.Test)

Example 53 with ApplicationId

use of co.cask.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class RemoteLineageWriterTest method testSimpleCase.

@Test
public void testSimpleCase() {
    long now = System.currentTimeMillis();
    ApplicationId appId = NamespaceId.DEFAULT.app("test_app");
    ProgramId flowId = appId.flow("test_flow");
    ProgramRunId runId = flowId.run(RunIds.generate(now).getId());
    RunId twillRunId = RunIds.fromString(runId.getRun());
    DatasetId datasetId = NamespaceId.DEFAULT.dataset("test_dataset");
    StreamId streamId = NamespaceId.DEFAULT.stream("test_stream");
    Set<Relation> expectedRelations = new HashSet<>();
    // test null serialization
    remoteLineageWriter.addAccess(runId, datasetId, AccessType.READ, null);
    expectedRelations.add(new Relation(datasetId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(ImmutableSet.of(flowId, datasetId), lineageStore.getEntitiesForRun(runId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.READ);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.READ, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
    remoteLineageWriter.addAccess(runId, streamId, AccessType.WRITE);
    expectedRelations.add(new Relation(streamId, flowId, AccessType.WRITE, twillRunId));
    Assert.assertEquals(expectedRelations, lineageStore.getRelations(flowId, now, now + 1, Predicates.<Relation>alwaysTrue()));
}
Also used : StreamId(co.cask.cdap.proto.id.StreamId) Relation(co.cask.cdap.data2.metadata.lineage.Relation) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) ApplicationId(co.cask.cdap.proto.id.ApplicationId) ProgramId(co.cask.cdap.proto.id.ProgramId) RunId(org.apache.twill.api.RunId) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) DatasetId(co.cask.cdap.proto.id.DatasetId) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 54 with ApplicationId

use of co.cask.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testWorkflowNodeState.

@Test
public void testWorkflowNodeState() throws Exception {
    String namespaceName = "namespace1";
    String appName = "app1";
    String workflowName = "workflow1";
    String mapReduceName = "mapReduce1";
    String sparkName = "spark1";
    ApplicationId appId = Ids.namespace(namespaceName).app(appName);
    ProgramId mapReduceProgram = appId.mr(mapReduceName);
    ProgramId sparkProgram = appId.spark(sparkName);
    long currentTime = System.currentTimeMillis();
    String workflowRunId = RunIds.generate(currentTime).getId();
    ProgramRunId workflowRun = appId.workflow(workflowName).run(workflowRunId);
    // start Workflow
    store.setStart(workflowRun.getParent(), workflowRun.getRun(), currentTime);
    // start MapReduce as a part of Workflow
    Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapReduceName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId mapReduceRunId = RunIds.generate(currentTime + 10);
    store.setStart(mapReduceProgram, mapReduceRunId.getId(), currentTime + 10, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the MapReduce program
    store.setStop(mapReduceProgram, mapReduceRunId.getId(), currentTime + 50, ProgramRunStatus.COMPLETED);
    // start Spark program as a part of Workflow
    systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId sparkRunId = RunIds.generate(currentTime + 60);
    store.setStart(sparkProgram, sparkRunId.getId(), currentTime + 60, null, ImmutableMap.<String, String>of(), systemArgs);
    // stop the Spark program with failure
    NullPointerException npe = new NullPointerException("dataset not found");
    IllegalArgumentException iae = new IllegalArgumentException("illegal argument", npe);
    store.setStop(sparkProgram, sparkRunId.getId(), currentTime + 100, ProgramRunStatus.FAILED, new BasicThrowable(iae));
    // stop Workflow
    store.setStop(workflowRun.getParent(), workflowRun.getRun(), currentTime + 110, ProgramRunStatus.FAILED);
    List<WorkflowNodeStateDetail> nodeStateDetails = store.getWorkflowNodeStates(workflowRun);
    Map<String, WorkflowNodeStateDetail> workflowNodeStates = new HashMap<>();
    for (WorkflowNodeStateDetail nodeStateDetail : nodeStateDetails) {
        workflowNodeStates.put(nodeStateDetail.getNodeId(), nodeStateDetail);
    }
    Assert.assertEquals(2, workflowNodeStates.size());
    WorkflowNodeStateDetail nodeStateDetail = workflowNodeStates.get(mapReduceName);
    Assert.assertEquals(mapReduceName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.COMPLETED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(mapReduceRunId.getId(), nodeStateDetail.getRunId());
    Assert.assertNull(nodeStateDetail.getFailureCause());
    nodeStateDetail = workflowNodeStates.get(sparkName);
    Assert.assertEquals(sparkName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.FAILED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(sparkRunId.getId(), nodeStateDetail.getRunId());
    BasicThrowable failureCause = nodeStateDetail.getFailureCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("illegal argument".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.IllegalArgumentException".equals(failureCause.getClassName()));
    failureCause = failureCause.getCause();
    Assert.assertNotNull(failureCause);
    Assert.assertTrue("dataset not found".equals(failureCause.getMessage()));
    Assert.assertTrue("java.lang.NullPointerException".equals(failureCause.getClassName()));
    Assert.assertNull(failureCause.getCause());
}
Also used : HashMap(java.util.HashMap) ProgramId(co.cask.cdap.proto.id.ProgramId) WorkflowNodeStateDetail(co.cask.cdap.proto.WorkflowNodeStateDetail) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) ApplicationId(co.cask.cdap.proto.id.ApplicationId) RunId(org.apache.twill.api.RunId) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) BasicThrowable(co.cask.cdap.proto.BasicThrowable) Test(org.junit.Test)

Example 55 with ApplicationId

use of co.cask.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class ApplicationVerificationTest method testGoodApplication.

/**
   * Good test
   */
@Test
public void testGoodApplication() throws Exception {
    ApplicationSpecification appSpec = Specifications.from(new WebCrawlApp());
    ApplicationSpecificationAdapter adapter = ApplicationSpecificationAdapter.create(new ReflectionSchemaGenerator());
    ApplicationSpecification newSpec = adapter.fromJson(adapter.toJson(appSpec));
    ApplicationVerification app = new ApplicationVerification();
    VerifyResult result = app.verify(new ApplicationId("test", newSpec.getName()), newSpec);
    Assert.assertTrue(result.getMessage(), result.getStatus() == VerifyResult.Status.SUCCESS);
}
Also used : ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) WebCrawlApp(co.cask.cdap.WebCrawlApp) ApplicationSpecificationAdapter(co.cask.cdap.internal.app.ApplicationSpecificationAdapter) ReflectionSchemaGenerator(co.cask.cdap.internal.io.ReflectionSchemaGenerator) VerifyResult(co.cask.cdap.app.verification.VerifyResult) ApplicationId(co.cask.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Aggregations

ApplicationId (co.cask.cdap.proto.id.ApplicationId)234 Test (org.junit.Test)123 ProgramId (co.cask.cdap.proto.id.ProgramId)73 AppRequest (co.cask.cdap.proto.artifact.AppRequest)64 ApplicationManager (co.cask.cdap.test.ApplicationManager)49 ApplicationSpecification (co.cask.cdap.api.app.ApplicationSpecification)45 ETLStage (co.cask.cdap.etl.proto.v2.ETLStage)44 NamespaceId (co.cask.cdap.proto.id.NamespaceId)43 Table (co.cask.cdap.api.dataset.table.Table)37 StructuredRecord (co.cask.cdap.api.data.format.StructuredRecord)36 Schema (co.cask.cdap.api.data.schema.Schema)34 ETLBatchConfig (co.cask.cdap.etl.proto.v2.ETLBatchConfig)32 WorkflowManager (co.cask.cdap.test.WorkflowManager)31 Path (javax.ws.rs.Path)31 ArtifactSummary (co.cask.cdap.api.artifact.ArtifactSummary)29 StreamId (co.cask.cdap.proto.id.StreamId)28 KeyValueTable (co.cask.cdap.api.dataset.lib.KeyValueTable)26 ArrayList (java.util.ArrayList)25 HashSet (java.util.HashSet)24 NotFoundException (co.cask.cdap.common.NotFoundException)23