Examples with RunRecord co.cask.cdap.proto.RunRecord used on opensource projects

Example 16 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class TestFrameworkTestRun method testDeployWorkflowApp.

@Category(XSlowTests.class)
@Test
public void testDeployWorkflowApp() throws Exception {
    ApplicationManager applicationManager = deployApplication(testSpace, AppWithSchedule.class);
    final WorkflowManager wfmanager = applicationManager.getWorkflowManager("SampleWorkflow");
    List<ScheduleDetail> schedules = wfmanager.getProgramSchedules();
    Assert.assertEquals(1, schedules.size());
    String scheduleName = schedules.get(0).getName();
    Assert.assertNotNull(scheduleName);
    Assert.assertFalse(scheduleName.isEmpty());
    final int initialRuns = wfmanager.getHistory().size();
    LOG.info("initialRuns = {}", initialRuns);
    wfmanager.getSchedule(scheduleName).resume();
    String status = wfmanager.getSchedule(scheduleName).status(200);
    Assert.assertEquals("SCHEDULED", status);
    // Make sure something ran before suspending
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return wfmanager.getHistory().size() > initialRuns;
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    wfmanager.getSchedule(scheduleName).suspend();
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SUSPENDED);
    // Sleep for three seconds to make sure scheduled workflows are pending to run
    TimeUnit.SECONDS.sleep(3);
    // All runs should be completed
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            for (RunRecord record : wfmanager.getHistory()) {
                if (record.getStatus() != ProgramRunStatus.COMPLETED) {
                    return false;
                }
            }
            return true;
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    List<RunRecord> history = wfmanager.getHistory();
    int workflowRuns = history.size();
    LOG.info("workflowRuns = {}", workflowRuns);
    Assert.assertTrue(workflowRuns > 0);
    //Sleep for some time and verify there are no more scheduled jobs after the suspend.
    TimeUnit.SECONDS.sleep(5);
    final int workflowRunsAfterSuspend = wfmanager.getHistory().size();
    Assert.assertEquals(workflowRuns, workflowRunsAfterSuspend);
    wfmanager.getSchedule(scheduleName).resume();
    //Check that after resume it goes to "SCHEDULED" state
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SCHEDULED);
    // Make sure new runs happens after resume
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return wfmanager.getHistory().size() > workflowRunsAfterSuspend;
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Check scheduled state
    Assert.assertEquals("SCHEDULED", wfmanager.getSchedule(scheduleName).status(200));
    // Check status of non-existent schedule
    Assert.assertEquals("NOT_FOUND", wfmanager.getSchedule("doesnt exist").status(404));
    // Suspend the schedule
    wfmanager.getSchedule(scheduleName).suspend();
    // Check that after suspend it goes to "SUSPENDED" state
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SUSPENDED);
    // Test workflow token while suspended
    String pid = history.get(0).getPid();
    WorkflowTokenDetail workflowToken = wfmanager.getToken(pid, WorkflowToken.Scope.SYSTEM, null);
    Assert.assertEquals(0, workflowToken.getTokenData().size());
    workflowToken = wfmanager.getToken(pid, null, null);
    Assert.assertEquals(2, workflowToken.getTokenData().size());
    // Wait until workflow finishes execution
    waitForWorkflowStatus(wfmanager, ProgramRunStatus.COMPLETED);
    // Verify workflow token after workflow completion
    WorkflowTokenNodeDetail workflowTokenAtNode = wfmanager.getTokenAtNode(pid, AppWithSchedule.DummyAction.class.getSimpleName(), WorkflowToken.Scope.USER, "finished");
    Assert.assertEquals(true, Boolean.parseBoolean(workflowTokenAtNode.getTokenDataAtNode().get("finished")));
    workflowToken = wfmanager.getToken(pid, null, null);
    Assert.assertEquals(false, Boolean.parseBoolean(workflowToken.getTokenData().get("running").get(0).getValue()));
}

Example 17 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class TestFrameworkTestRun method testAppWithPlugin.

@Test
public void testAppWithPlugin() throws Exception {
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("app-with-plugin", "1.0.0-SNAPSHOT");
    addAppArtifact(artifactId, AppWithPlugin.class);
    ArtifactId pluginArtifactId = NamespaceId.DEFAULT.artifact("test-plugin", "1.0.0-SNAPSHOT");
    addPluginArtifact(pluginArtifactId, artifactId, ToStringPlugin.class);
    ApplicationId appId = NamespaceId.DEFAULT.app("AppWithPlugin");
    AppRequest createRequest = new AppRequest(new ArtifactSummary(artifactId.getArtifact(), artifactId.getVersion()));
    ApplicationManager appManager = deployApplication(appId, createRequest);
    final WorkerManager workerManager = appManager.getWorkerManager(AppWithPlugin.WORKER);
    workerManager.start();
    workerManager.waitForStatus(false, 5, 1);
    Tasks.waitFor(false, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return workerManager.getHistory(ProgramRunStatus.COMPLETED).isEmpty();
        }
    }, 5, TimeUnit.SECONDS, 10, TimeUnit.MILLISECONDS);
    final ServiceManager serviceManager = appManager.getServiceManager(AppWithPlugin.SERVICE);
    serviceManager.start();
    serviceManager.waitForStatus(true, 1, 10);
    URL serviceURL = serviceManager.getServiceURL(5, TimeUnit.SECONDS);
    callServiceGet(serviceURL, "dummy");
    serviceManager.stop();
    serviceManager.waitForStatus(false, 1, 10);
    Tasks.waitFor(false, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return serviceManager.getHistory(ProgramRunStatus.KILLED).isEmpty();
        }
    }, 5, TimeUnit.SECONDS, 10, TimeUnit.MILLISECONDS);
    WorkflowManager workflowManager = appManager.getWorkflowManager(AppWithPlugin.WORKFLOW);
    workflowManager.start();
    workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    List<RunRecord> runRecords = workflowManager.getHistory();
    Assert.assertNotEquals(ProgramRunStatus.FAILED, runRecords.get(0).getStatus());
    DataSetManager<KeyValueTable> workflowTableManager = getDataset(AppWithPlugin.WORKFLOW_TABLE);
    String value = Bytes.toString(workflowTableManager.get().read("val"));
    Assert.assertEquals(AppWithPlugin.TEST, value);
    Map<String, String> workflowTags = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, NamespaceId.DEFAULT.getNamespace(), Constants.Metrics.Tag.APP, "AppWithPlugin", Constants.Metrics.Tag.WORKFLOW, AppWithPlugin.WORKFLOW, Constants.Metrics.Tag.RUN_ID, runRecords.get(0).getPid());
    getMetricsManager().waitForTotalMetricCount(workflowTags, String.format("user.destroy.%s", AppWithPlugin.WORKFLOW), 1, 60, TimeUnit.SECONDS);
    // Testing Spark Plugins. First send some data to stream for the Spark program to process
    StreamManager streamManager = getStreamManager(AppWithPlugin.SPARK_STREAM);
    for (int i = 0; i < 5; i++) {
        streamManager.send("Message " + i);
    }
    SparkManager sparkManager = appManager.getSparkManager(AppWithPlugin.SPARK).start();
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 2, TimeUnit.MINUTES);
    // Verify the Spark result.
    DataSetManager<Table> dataSetManager = getDataset(AppWithPlugin.SPARK_TABLE);
    Table table = dataSetManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        for (int i = 0; i < 5; i++) {
            Row row = scanner.next();
            Assert.assertNotNull(row);
            String expected = "Message " + i + " " + AppWithPlugin.TEST;
            Assert.assertEquals(expected, Bytes.toString(row.getRow()));
            Assert.assertEquals(expected, Bytes.toString(row.get(expected)));
        }
        // There shouldn't be any more rows in the table.
        Assert.assertNull(scanner.next());
    }
}

Example 18 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class TestFrameworkTestRun method testServiceManager.

@Test
public void testServiceManager() throws Exception {
    ApplicationManager applicationManager = deployApplication(FilterAppWithNewFlowAPI.class);
    final ServiceManager countService = applicationManager.getServiceManager("CountService");
    countService.setInstances(2);
    Assert.assertEquals(0, countService.getProvisionedInstances());
    Assert.assertEquals(2, countService.getRequestedInstances());
    Assert.assertFalse(countService.isRunning());
    List<RunRecord> history = countService.getHistory();
    Assert.assertEquals(0, history.size());
    countService.start();
    Assert.assertTrue(countService.isRunning());
    Assert.assertEquals(2, countService.getProvisionedInstances());
    // requesting with ProgramRunStatus.KILLED returns empty list
    history = countService.getHistory(ProgramRunStatus.KILLED);
    Assert.assertEquals(0, history.size());
    // requesting with either RUNNING or ALL will return one record
    Tasks.waitFor(1, new Callable<Integer>() {

        @Override
        public Integer call() throws Exception {
            return countService.getHistory(ProgramRunStatus.RUNNING).size();
        }
    }, 5, TimeUnit.SECONDS);
    history = countService.getHistory(ProgramRunStatus.RUNNING);
    Assert.assertEquals(ProgramRunStatus.RUNNING, history.get(0).getStatus());
    history = countService.getHistory(ProgramRunStatus.ALL);
    Assert.assertEquals(1, history.size());
    Assert.assertEquals(ProgramRunStatus.RUNNING, history.get(0).getStatus());
}

Example 19 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class AuthorizationTest method testScheduleAuth.

@Test
public void testScheduleAuth() throws Exception {
    createAuthNamespace();
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, AppWithSchedule.class);
    ProgramId workflowID = new ProgramId(AUTH_NAMESPACE.getNamespace(), AppWithSchedule.class.getSimpleName(), ProgramType.WORKFLOW, AppWithSchedule.SampleWorkflow.class.getSimpleName());
    final WorkflowManager workflowManager = appManager.getWorkflowManager(AppWithSchedule.SampleWorkflow.class.getSimpleName());
    ScheduleManager scheduleManager = workflowManager.getSchedule(AppWithSchedule.SCHEDULE_NAME);
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    // try to resume schedule as BOB. It should fail since BOB does not have privileges on the programs
    try {
        scheduleManager.resume();
        Assert.fail("Resuming schedule should have failed since BOB does not have EXECUTE on the program");
    } catch (Exception e) {
        Assert.assertTrue(e.getCause() instanceof UnauthorizedException);
    }
    // bob should also not be able see the status of the schedule
    try {
        scheduleManager.status(HttpURLConnection.HTTP_FORBIDDEN);
        Assert.fail("Getting schedule status should have failed since BOB does not have READ on the program");
    } catch (Exception e) {
        Assert.assertTrue(e.getCause() instanceof UnauthorizedException);
    }
    // switch to Alice
    SecurityRequestContext.setUserId(ALICE.getName());
    // give BOB READ permission in the workflow
    grantAndAssertSuccess(workflowID, BOB, EnumSet.of(Action.READ));
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    // try to resume schedule as BOB. It should fail since BOB has READ and not EXECUTE on the workflow
    try {
        scheduleManager.resume();
        Assert.fail("Resuming schedule should have failed since BOB does not have EXECUTE on the program");
    } catch (Exception e) {
        Assert.assertTrue(e.getCause() instanceof UnauthorizedException);
    }
    // but BOB should be able to get schedule status now
    Assert.assertEquals(ProgramScheduleStatus.SUSPENDED.name(), scheduleManager.status(HttpURLConnection.HTTP_OK));
    // switch to Alice
    SecurityRequestContext.setUserId(ALICE.getName());
    // give BOB EXECUTE permission in the workflow
    grantAndAssertSuccess(workflowID, BOB, EnumSet.of(Action.EXECUTE));
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    // try to resume the schedule. This should pass and workflow should run
    scheduleManager.resume();
    Assert.assertEquals(ProgramScheduleStatus.SCHEDULED.name(), scheduleManager.status(HttpURLConnection.HTTP_OK));
    // wait for workflow to start
    workflowManager.waitForStatus(true);
    // suspend the schedule so that it does not start running again
    scheduleManager.suspend();
    // wait for scheduled runs of workflow to run to end
    workflowManager.waitForStatus(false, 2, 3);
    // since the schedule in AppWithSchedule is to  run every second its possible that it will trigger more than one
    // run before the schedule was suspended so check for greater than 0 rather than equal to 1
    Assert.assertTrue(0 < workflowManager.getHistory().size());
    // assert that all run completed
    for (RunRecord runRecord : workflowManager.getHistory()) {
        Assert.assertEquals(ProgramRunStatus.COMPLETED, runRecord.getStatus());
    }
    // switch to Alice
    SecurityRequestContext.setUserId(ALICE.getName());
}

Example 20 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class SparkMetricsIntegrationTestRun method testSparkMetrics.

@Test
public void testSparkMetrics() throws Exception {
    ApplicationManager applicationManager = deployApplication(TestSparkMetricsIntegrationApp.class);
    SparkManager sparkManager = applicationManager.getSparkManager(TestSparkMetricsIntegrationApp.APP_SPARK_NAME).start();
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 120, TimeUnit.SECONDS);
    List<RunRecord> history = sparkManager.getHistory(ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, history.size());
    // Wait for the metrics to get updated
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return getSparkMetric(TestSparkMetricsIntegrationApp.APP_NAME, TestSparkMetricsIntegrationApp.APP_SPARK_NAME, "system.driver.BlockManager.memory.remainingMem_MB") > 0;
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    Tasks.waitFor(2L, new Callable<Long>() {

        @Override
        public Long call() throws Exception {
            return getSparkMetric(TestSparkMetricsIntegrationApp.APP_NAME, TestSparkMetricsIntegrationApp.APP_SPARK_NAME, "user.more.than.30");
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}