Search in sources :

Example 11 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testProgramRunCount.

@Test
public void testProgramRunCount() {
    ApplicationSpecification spec = Specifications.from(new AllProgramsApp());
    ApplicationId appId = NamespaceId.DEFAULT.app(spec.getName());
    ArtifactId testArtifact = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ProgramId workflowId = appId.workflow(AllProgramsApp.NoOpWorkflow.NAME);
    ProgramId serviceId = appId.service(AllProgramsApp.NoOpService.NAME);
    ProgramId nonExistingAppProgramId = NamespaceId.DEFAULT.app("nonExisting").workflow("test");
    ProgramId nonExistingProgramId = appId.workflow("nonExisting");
    // add the application
    store.addApplication(appId, spec);
    // add some run records to workflow and service
    for (int i = 0; i < 5; i++) {
        setStart(workflowId.run(RunIds.generate()), Collections.emptyMap(), Collections.emptyMap(), testArtifact);
        setStart(serviceId.run(RunIds.generate()), Collections.emptyMap(), Collections.emptyMap(), testArtifact);
    }
    List<RunCountResult> result = store.getProgramRunCounts(ImmutableList.of(workflowId, serviceId, nonExistingAppProgramId, nonExistingProgramId));
    // compare the result
    Assert.assertEquals(4, result.size());
    for (RunCountResult runCountResult : result) {
        ProgramId programId = runCountResult.getProgramId();
        Long count = runCountResult.getCount();
        if (programId.equals(nonExistingAppProgramId) || programId.equals(nonExistingProgramId)) {
            Assert.assertNull(count);
            Assert.assertTrue(runCountResult.getException() instanceof NotFoundException);
        } else {
            Assert.assertNotNull(count);
            Assert.assertEquals(5L, count.longValue());
        }
    }
    // remove the app should remove all run count
    store.removeApplication(appId);
    for (RunCountResult runCountResult : store.getProgramRunCounts(ImmutableList.of(workflowId, serviceId))) {
        Assert.assertNull(runCountResult.getCount());
        Assert.assertTrue(runCountResult.getException() instanceof NotFoundException);
    }
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) RunCountResult(io.cdap.cdap.proto.RunCountResult) ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) NotFoundException(io.cdap.cdap.common.NotFoundException) AllProgramsApp(io.cdap.cdap.AllProgramsApp) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ProgramId(io.cdap.cdap.proto.id.ProgramId) Test(org.junit.Test)

Example 12 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testCheckDeletedWorkflow.

@Test
public void testCheckDeletedWorkflow() {
    // Deploy program with all types of programs.
    ApplicationSpecification spec = Specifications.from(new AllProgramsApp());
    ApplicationId appId = NamespaceId.DEFAULT.app(spec.getName());
    store.addApplication(appId, spec);
    Set<String> specsToBeDeleted = Sets.newHashSet();
    specsToBeDeleted.addAll(spec.getWorkflows().keySet());
    Assert.assertEquals(1, specsToBeDeleted.size());
    // Get the spec for app that contains only flow and mapreduce - removing workflows.
    spec = Specifications.from(new DefaultStoreTestApp());
    // Get the deleted program specs by sending a spec with same name as AllProgramsApp but with no programs
    List<ProgramSpecification> deletedSpecs = store.getDeletedProgramSpecifications(appId, spec);
    Assert.assertEquals(2, deletedSpecs.size());
    for (ProgramSpecification specification : deletedSpecs) {
        // Remove the spec that is verified, to check the count later.
        specsToBeDeleted.remove(specification.getName());
    }
    // 2 specs should have been deleted and 0 should be remaining.
    Assert.assertEquals(0, specsToBeDeleted.size());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) DefaultStoreTestApp(io.cdap.cdap.DefaultStoreTestApp) ProgramSpecification(io.cdap.cdap.api.ProgramSpecification) AllProgramsApp(io.cdap.cdap.AllProgramsApp) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 13 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testWorkflowNodeState.

@Test
public void testWorkflowNodeState() {
    String namespaceName = "namespace1";
    String appName = "app1";
    String workflowName = "workflow1";
    String mapReduceName = "mapReduce1";
    String sparkName = "spark1";
    ApplicationId appId = Ids.namespace(namespaceName).app(appName);
    ProgramId mapReduceProgram = appId.mr(mapReduceName);
    ProgramId sparkProgram = appId.spark(sparkName);
    long currentTime = System.currentTimeMillis();
    String workflowRunId = RunIds.generate(currentTime).getId();
    ProgramRunId workflowRun = appId.workflow(workflowName).run(workflowRunId);
    ArtifactId artifactId = appId.getParent().artifact("testArtifact", "1.0").toApiArtifactId();
    // start Workflow
    setStartAndRunning(workflowRun, artifactId);
    // start MapReduce as a part of Workflow
    Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapReduceName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId mapReduceRunId = RunIds.generate(currentTime + 10);
    setStartAndRunning(mapReduceProgram.run(mapReduceRunId.getId()), ImmutableMap.of(), systemArgs, artifactId);
    // stop the MapReduce program
    store.setStop(mapReduceProgram.run(mapReduceRunId.getId()), currentTime + 50, ProgramRunStatus.COMPLETED, AppFabricTestHelper.createSourceId(++sourceId));
    // start Spark program as a part of Workflow
    systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
    RunId sparkRunId = RunIds.generate(currentTime + 60);
    setStartAndRunning(sparkProgram.run(sparkRunId.getId()), ImmutableMap.of(), systemArgs, artifactId);
    // stop the Spark program with failure
    NullPointerException npe = new NullPointerException("dataset not found");
    IllegalArgumentException iae = new IllegalArgumentException("illegal argument", npe);
    store.setStop(sparkProgram.run(sparkRunId.getId()), currentTime + 100, ProgramRunStatus.FAILED, new BasicThrowable(iae), AppFabricTestHelper.createSourceId(++sourceId));
    // stop Workflow
    store.setStop(workflowRun, currentTime + 110, ProgramRunStatus.FAILED, AppFabricTestHelper.createSourceId(++sourceId));
    List<WorkflowNodeStateDetail> nodeStateDetails = store.getWorkflowNodeStates(workflowRun);
    Map<String, WorkflowNodeStateDetail> workflowNodeStates = new HashMap<>();
    for (WorkflowNodeStateDetail nodeStateDetail : nodeStateDetails) {
        workflowNodeStates.put(nodeStateDetail.getNodeId(), nodeStateDetail);
    }
    Assert.assertEquals(2, workflowNodeStates.size());
    WorkflowNodeStateDetail nodeStateDetail = workflowNodeStates.get(mapReduceName);
    Assert.assertEquals(mapReduceName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.COMPLETED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(mapReduceRunId.getId(), nodeStateDetail.getRunId());
    Assert.assertNull(nodeStateDetail.getFailureCause());
    nodeStateDetail = workflowNodeStates.get(sparkName);
    Assert.assertEquals(sparkName, nodeStateDetail.getNodeId());
    Assert.assertEquals(NodeStatus.FAILED, nodeStateDetail.getNodeStatus());
    Assert.assertEquals(sparkRunId.getId(), nodeStateDetail.getRunId());
    BasicThrowable failureCause = nodeStateDetail.getFailureCause();
    Assert.assertNotNull(failureCause);
    Assert.assertEquals("illegal argument", failureCause.getMessage());
    Assert.assertEquals(IllegalArgumentException.class.getName(), failureCause.getClassName());
    failureCause = failureCause.getCause();
    Assert.assertNotNull(failureCause);
    Assert.assertEquals("dataset not found", failureCause.getMessage());
    Assert.assertEquals(NullPointerException.class.getName(), failureCause.getClassName());
    Assert.assertNull(failureCause.getCause());
}
Also used : ArtifactId(io.cdap.cdap.api.artifact.ArtifactId) HashMap(java.util.HashMap) ProgramId(io.cdap.cdap.proto.id.ProgramId) WorkflowNodeStateDetail(io.cdap.cdap.proto.WorkflowNodeStateDetail) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) RunId(org.apache.twill.api.RunId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) BasicThrowable(io.cdap.cdap.proto.BasicThrowable) Test(org.junit.Test)

Example 14 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testScanApplicationsWithNamespace.

public void testScanApplicationsWithNamespace(Store store) {
    ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
    int count = 100;
    for (int i = 0; i < count / 2; i++) {
        String appName = "test" + (2 * i);
        store.addApplication(new ApplicationId(NamespaceId.DEFAULT.getNamespace(), appName), appSpec);
        appName = "test" + (2 * i + 1);
        store.addApplication(new ApplicationId(NamespaceId.CDAP.getNamespace(), appName), appSpec);
    }
    List<ApplicationId> apps = new ArrayList<ApplicationId>();
    ScanApplicationsRequest request = ScanApplicationsRequest.builder().setNamespaceId(NamespaceId.CDAP).build();
    store.scanApplications(request, 20, (appId, spec) -> {
        apps.add(appId);
    });
    Assert.assertEquals(count / 2, apps.size());
    // Reverse
    List<ApplicationId> reverseApps = new ArrayList<>();
    request = ScanApplicationsRequest.builder().setNamespaceId(NamespaceId.CDAP).setSortOrder(SortOrder.DESC).build();
    store.scanApplications(request, 20, (appId, spec) -> reverseApps.add(appId));
    Assert.assertEquals(Lists.reverse(apps), reverseApps);
    // Second page
    int firstPageSize = 10;
    List<ApplicationId> restartApps = new ArrayList<>();
    request = ScanApplicationsRequest.builder().setNamespaceId(NamespaceId.CDAP).setScanFrom(apps.get(firstPageSize - 1)).build();
    store.scanApplications(request, 20, (appId, spec) -> restartApps.add(appId));
    Assert.assertEquals(apps.subList(firstPageSize, apps.size()), restartApps);
}
Also used : ScanApplicationsRequest(io.cdap.cdap.app.store.ScanApplicationsRequest) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ArrayList(java.util.ArrayList) AllProgramsApp(io.cdap.cdap.AllProgramsApp) ApplicationId(io.cdap.cdap.proto.id.ApplicationId)

Example 15 with ApplicationId

use of io.cdap.cdap.proto.id.ApplicationId in project cdap by caskdata.

the class DefaultStoreTest method testServiceDeletion.

@Test
public void testServiceDeletion() {
    // Store the application specification
    AbstractApplication app = new AppWithServices();
    ApplicationSpecification appSpec = Specifications.from(app);
    ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
    store.addApplication(appId, appSpec);
    AbstractApplication newApp = new AppWithNoServices();
    // get the delete program specs after deploying AppWithNoServices
    List<ProgramSpecification> programSpecs = store.getDeletedProgramSpecifications(appId, Specifications.from(newApp));
    // verify the result.
    Assert.assertEquals(1, programSpecs.size());
    Assert.assertEquals("NoOpService", programSpecs.get(0).getName());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ProgramSpecification(io.cdap.cdap.api.ProgramSpecification) AppWithServices(io.cdap.cdap.AppWithServices) AbstractApplication(io.cdap.cdap.api.app.AbstractApplication) AppWithNoServices(io.cdap.cdap.AppWithNoServices) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Aggregations

ApplicationId (io.cdap.cdap.proto.id.ApplicationId)789 Test (org.junit.Test)410 AppRequest (io.cdap.cdap.proto.artifact.AppRequest)279 ProgramId (io.cdap.cdap.proto.id.ProgramId)263 ApplicationManager (io.cdap.cdap.test.ApplicationManager)225 ETLStage (io.cdap.cdap.etl.proto.v2.ETLStage)223 ETLBatchConfig (io.cdap.cdap.etl.proto.v2.ETLBatchConfig)196 StructuredRecord (io.cdap.cdap.api.data.format.StructuredRecord)180 Table (io.cdap.cdap.api.dataset.table.Table)178 WorkflowManager (io.cdap.cdap.test.WorkflowManager)169 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)154 Schema (io.cdap.cdap.api.data.schema.Schema)147 ArrayList (java.util.ArrayList)129 HashSet (java.util.HashSet)126 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)124 HashMap (java.util.HashMap)109 KeyValueTable (io.cdap.cdap.api.dataset.lib.KeyValueTable)107 ArtifactSummary (io.cdap.cdap.api.artifact.ArtifactSummary)88 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)88 Path (javax.ws.rs.Path)75