Examples with PercentileInformation co.cask.cdap.proto.PercentileInformation used on opensource projects

Example 1 with PercentileInformation

use of co.cask.cdap.proto.PercentileInformation in project cdap by caskdata.

the class WorkflowDataset method getPercentiles.

private List<PercentileInformation> getPercentiles(List<WorkflowRunRecord> workflowRunRecords, List<Double> percentiles) {
    int runs = workflowRunRecords.size();
    List<PercentileInformation> percentileInformationList = new ArrayList<>();
    for (double i : percentiles) {
        List<String> percentileRun = new ArrayList<>();
        int percentileStart = (int) ((i * runs) / 100);
        for (int j = percentileStart; j < runs; j++) {
            percentileRun.add(workflowRunRecords.get(j).getWorkflowRunId());
        }
        percentileInformationList.add(new PercentileInformation(i, workflowRunRecords.get(percentileStart).getTimeTaken(), percentileRun));
    }
    return percentileInformationList;
}

Example 2 with PercentileInformation

use of co.cask.cdap.proto.PercentileInformation in project cdap by caskdata.

the class WorkflowStatsSLAHttpHandlerTest method testStatistics.

@Test
public void testStatistics() throws Exception {
    deploy(WorkflowApp.class);
    String workflowName = "FunWorkflow";
    String mapreduceName = "ClassicWordCount";
    String sparkName = "SparkWorkflowTest";
    ProgramId workflowProgram = WORKFLOW_APP.workflow(workflowName);
    ProgramId mapreduceProgram = WORKFLOW_APP.mr(mapreduceName);
    ProgramId sparkProgram = WORKFLOW_APP.spark(sparkName);
    long startTime = System.currentTimeMillis();
    long currentTimeMillis = startTime;
    String outlierRunId = null;
    for (int i = 0; i < 10; i++) {
        // workflow runs every 5 minutes
        currentTimeMillis = startTime + (i * TimeUnit.MINUTES.toMillis(5));
        RunId workflowRunId = RunIds.generate(currentTimeMillis);
        store.setStart(workflowProgram, workflowRunId.getId(), RunIds.getTime(workflowRunId, TimeUnit.SECONDS));
        // MR job starts 2 seconds after workflow started
        RunId mapreduceRunid = RunIds.generate(currentTimeMillis + TimeUnit.SECONDS.toMillis(2));
        Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapreduceName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getId());
        store.setStart(mapreduceProgram, mapreduceRunid.getId(), RunIds.getTime(mapreduceRunid, TimeUnit.SECONDS), null, ImmutableMap.<String, String>of(), systemArgs);
        store.setStop(mapreduceProgram, mapreduceRunid.getId(), // map-reduce job ran for 17 seconds
        TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 19, ProgramRunStatus.COMPLETED);
        // This makes sure that not all runs have Spark programs in them
        if (i < 5) {
            // spark starts 20 seconds after workflow starts
            RunId sparkRunid = RunIds.generate(currentTimeMillis + TimeUnit.SECONDS.toMillis(20));
            systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkProgram.getProgram(), ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getId());
            store.setStart(sparkProgram, sparkRunid.getId(), RunIds.getTime(sparkRunid, TimeUnit.SECONDS), null, ImmutableMap.<String, String>of(), systemArgs);
            // spark job runs for 38 seconds
            long stopTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 58;
            if (i == 4) {
                // spark job ran for 100 seconds. 62 seconds greater than avg.
                stopTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 120;
            }
            store.setStop(sparkProgram, sparkRunid.getId(), stopTime, ProgramRunStatus.COMPLETED);
        }
        // workflow ran for 1 minute
        long workflowStopTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 60;
        if (i == 4) {
            // spark job ran longer for this run
            workflowStopTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 122;
            outlierRunId = workflowRunId.getId();
        }
        store.setStop(workflowProgram, workflowRunId.getId(), workflowStopTime, ProgramRunStatus.COMPLETED);
    }
    String request = String.format("%s/namespaces/%s/apps/%s/workflows/%s/statistics?start=%s&end=%s" + "&percentile=%s", Constants.Gateway.API_VERSION_3, Id.Namespace.DEFAULT.getId(), WorkflowApp.class.getSimpleName(), workflowProgram.getProgram(), TimeUnit.MILLISECONDS.toSeconds(startTime), TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + TimeUnit.MINUTES.toSeconds(2), "99");
    HttpResponse response = doGet(request);
    WorkflowStatistics workflowStatistics = readResponse(response, new TypeToken<WorkflowStatistics>() {
    }.getType());
    PercentileInformation percentileInformation = workflowStatistics.getPercentileInformationList().get(0);
    Assert.assertEquals(1, percentileInformation.getRunIdsOverPercentile().size());
    Assert.assertEquals(outlierRunId, percentileInformation.getRunIdsOverPercentile().get(0));
    Assert.assertEquals("5", workflowStatistics.getNodes().get(sparkName).get("runs"));
    request = String.format("%s/namespaces/%s/apps/%s/workflows/%s/statistics?start=%s&end=%s" + "&percentile=%s&percentile=%s", Constants.Gateway.API_VERSION_3, Id.Namespace.DEFAULT.getId(), WorkflowApp.class.getSimpleName(), workflowProgram.getProgram(), "now", "0", "90", "95");
    response = doGet(request);
    Assert.assertEquals(HttpResponseStatus.BAD_REQUEST.getCode(), response.getStatusLine().getStatusCode());
    request = String.format("%s/namespaces/%s/apps/%s/workflows/%s/statistics?start=%s&end=%s" + "&percentile=%s&percentile=%s", Constants.Gateway.API_VERSION_3, Id.Namespace.DEFAULT.getId(), WorkflowApp.class.getSimpleName(), workflowProgram.getProgram(), "now", "0", "90.0", "950");
    response = doGet(request);
    Assert.assertEquals(HttpResponseStatus.BAD_REQUEST.getCode(), response.getStatusLine().getStatusCode());
    Id.Application appId = new Id.Application(Id.Namespace.DEFAULT, WorkflowApp.class.getSimpleName());
    deleteApp(appId, HttpResponseStatus.OK.getCode());
    request = String.format("%s/namespaces/%s/apps/%s/workflows/%s/statistics?start=%s&end=%s" + "&percentile=%s", Constants.Gateway.API_VERSION_3, Id.Namespace.DEFAULT.getId(), WorkflowApp.class.getSimpleName(), workflowProgram, 0, System.currentTimeMillis(), "99");
    response = doGet(request);
    Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatusLine().getStatusCode());
    Assert.assertTrue(readResponse(response).startsWith("There are no statistics associated with this workflow : "));
}