Examples with Id io.cdap.cdap.common.id.Id used on opensource projects

Example 6 with Id

use of io.cdap.cdap.common.id.Id in project cdap by caskdata.

the class AppFabricTestBase method getScheduledRunTimes.

/**
 * Returns a list of {@link BatchProgramSchedule}.
 *
 * @param namespace the namespace to query in
 * @param programIds list of programs to query for scheuled run time
 * @param next if true, fetch the list of future run times. If false, fetch the list of past run times.
 * @return a list of {@link BatchProgramSchedule}
 */
protected List<BatchProgramSchedule> getScheduledRunTimes(String namespace, Collection<? extends ProgramId> programIds, boolean next) throws Exception {
    Assert.assertTrue(programIds.stream().map(ProgramId::getNamespace).allMatch(namespace::equals));
    String url = String.format("%sruntime", next ? "next" : "previous");
    String versionedUrl = getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, namespace);
    List<BatchProgram> batchPrograms = programIds.stream().map(id -> new BatchProgram(id.getApplication(), id.getType(), id.getProgram())).collect(Collectors.toList());
    HttpResponse response = doPost(versionedUrl, GSON.toJson(batchPrograms));
    assertResponseCode(200, response);
    return readResponse(response, new TypeToken<List<BatchProgramSchedule>>() {
    }.getType());
}

Example 7 with Id

use of io.cdap.cdap.common.id.Id in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method getStatuses.

/**
 * Returns the status for all programs that are passed into the data. The data is an array of JSON objects
 * where each object must contain the following three elements: appId, programType, and programId
 * (flow name, service name, etc.).
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "MapReduce2"}]
 * </code></pre>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as 2 fields, "status" which maps to the status of the program and "statusCode" which maps to the
 * status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, App2 does not exist), then all JsonObjects for which the
 * parameters have a valid status will have the status field but all JsonObjects for which the parameters do not have
 * a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no App2 in the data above, then the response would be 200 OK with following possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1", "statusCode": 200, "status": "RUNNING"},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2", "statusCode": 200, "status": "STOPPED"}]
 * </code></pre>
 */
@POST
@Path("/status")
@AuditPolicy(AuditDetail.REQUEST_BODY)
public void getStatuses(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> batchPrograms = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    List<ProgramId> programs = batchPrograms.stream().map(p -> new ProgramId(namespaceId, p.getAppId(), p.getProgramType(), p.getProgramId())).collect(Collectors.toList());
    Map<ProgramId, ProgramStatus> statuses = lifecycleService.getProgramStatuses(programs);
    List<BatchProgramStatus> result = new ArrayList<>(programs.size());
    for (BatchProgram program : batchPrograms) {
        ProgramId programId = new ProgramId(namespaceId, program.getAppId(), program.getProgramType(), program.getProgramId());
        ProgramStatus status = statuses.get(programId);
        if (status == null) {
            result.add(new BatchProgramStatus(program, HttpResponseStatus.NOT_FOUND.code(), new NotFoundException(programId).getMessage(), null));
        } else {
            result.add(new BatchProgramStatus(program, HttpResponseStatus.OK.code(), null, status.name()));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result));
}

Example 8 with Id

use of io.cdap.cdap.common.id.Id in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method getLatestRuns.

/**
 * Returns the latest runs for all program runnables that are passed into the data. The data is an array of
 * Json objects where each object must contain the following three elements: appId, programType, and programId.
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 *  {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow"}]
 * </code></pre>
 * </p><p>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as 2 fields, "runs" which is a list of the latest run records and "statusCode" which maps to the
 * status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, workflow in app1 does not exist),
 * then all JsonObjects for which the parameters have a valid status will have the count field but all JsonObjects
 * for which the parameters do not have a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no workflow in App1 in the data above, then the response would be 200 OK with following
 * possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1",
 * "statusCode": 200, "runs": [...]},
 * {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow", "statusCode": 404,
 * "error": "Program 'testWorkflow' is not found"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow", "runnableId": "Flowlet1",
 *  "statusCode": 200, "runs": [...]}]
 * </code></pre>
 */
@POST
@Path("/runs")
public void getLatestRuns(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    List<ProgramId> programIds = programs.stream().map(batchProgram -> new ProgramId(namespaceId, batchProgram.getAppId(), batchProgram.getProgramType(), batchProgram.getProgramId())).collect(Collectors.toList());
    List<BatchProgramHistory> response = new ArrayList<>(programs.size());
    List<ProgramHistory> result = lifecycleService.getRunRecords(programIds, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 1);
    for (ProgramHistory programHistory : result) {
        ProgramId programId = programHistory.getProgramId();
        Exception exception = programHistory.getException();
        BatchProgram batchProgram = new BatchProgram(programId.getApplication(), programId.getType(), programId.getProgram());
        if (exception == null) {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.OK.code(), null, programHistory.getRuns()));
        } else if (exception instanceof NotFoundException) {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.NOT_FOUND.code(), exception.getMessage(), Collections.emptyList()));
        } else if (exception instanceof UnauthorizedException) {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.FORBIDDEN.code(), exception.getMessage(), Collections.emptyList()));
        } else {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), exception.getMessage(), Collections.emptyList()));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(response));
}

Example 9 with Id

use of io.cdap.cdap.common.id.Id in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method getRunCounts.

/**
 * Returns the run counts for all program runnables that are passed into the data. The data is an array of
 * Json objects where each object must contain the following three elements: appId, programType, and programId.
 * The max number of programs in the request is 100.
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 *  {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow"}]
 * </code></pre>
 * </p><p>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as 2 fields, "runCount" which maps to the count of the program and "statusCode" which maps to the
 * status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, workflow in app1 does not exist),
 * then all JsonObjects for which the parameters have a valid status will have the count field but all JsonObjects
 * for which the parameters do not have a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no workflow in App1 in the data above, then the response would be 200 OK with following
 * possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1",
 * "statusCode": 200, "runCount": 20},
 * {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow", "statusCode": 404,
 * "error": "Program 'testWorkflow' is not found"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow",
 *  "statusCode": 200, "runCount": 300}]
 * </code></pre>
 */
@POST
@Path("/runcount")
public void getRunCounts(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    if (programs.size() > 100) {
        throw new BadRequestException(String.format("%d programs found in the request, the maximum number " + "supported is 100", programs.size()));
    }
    List<ProgramId> programIds = programs.stream().map(batchProgram -> new ProgramId(namespaceId, batchProgram.getAppId(), batchProgram.getProgramType(), batchProgram.getProgramId())).collect(Collectors.toList());
    List<BatchProgramCount> counts = new ArrayList<>(programs.size());
    for (RunCountResult runCountResult : lifecycleService.getProgramRunCounts(programIds)) {
        ProgramId programId = runCountResult.getProgramId();
        Exception exception = runCountResult.getException();
        if (exception == null) {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.OK.code(), null, runCountResult.getCount()));
        } else if (exception instanceof NotFoundException) {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.NOT_FOUND.code(), exception.getMessage(), null));
        } else if (exception instanceof UnauthorizedException) {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.FORBIDDEN.code(), exception.getMessage(), null));
        } else {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), exception.getMessage(), null));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(counts));
}

Example 10 with Id

use of io.cdap.cdap.common.id.Id in project cdap by cdapio.

the class WorkflowHttpHandlerTest method testWorkflowTokenPut.

@Test
public void testWorkflowTokenPut() throws Exception {
    deploy(WorkflowTokenTestPutApp.class, 200);
    Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, WorkflowTokenTestPutApp.NAME);
    Id.Workflow workflowId = Id.Workflow.from(appId, WorkflowTokenTestPutApp.WorkflowTokenTestPut.NAME);
    Id.Program sparkId = Id.Program.from(appId, ProgramType.SPARK, WorkflowTokenTestPutApp.SparkTestApp.NAME);
    // Start program with inputPath and outputPath arguments.
    // This should succeed. The programs inside the workflow will attempt to write to the workflow token
    // from the Mapper's and Reducer's methods as well as from a Spark closure, and they will throw an exception
    // if that succeeds.
    // The MapReduce's initialize will record the workflow run id in the token, and the destroy as well
    // as the mapper and the reducer will validate that they have the same workflow run id.
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    startProgram(workflowId, ImmutableMap.of("inputPath", createInputForRecordVerification("sixthInput"), "outputPath", outputPath));
    waitState(workflowId, ProgramStatus.RUNNING.name());
    waitState(workflowId, ProgramStatus.STOPPED.name());
    // validate the completed workflow run and validate that it is the same as recorded in the token
    verifyProgramRuns(workflowId, ProgramRunStatus.COMPLETED);
    List<RunRecord> runs = getProgramRuns(workflowId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, runs.size());
    String wfRunId = runs.get(0).getPid();
    WorkflowTokenDetail tokenDetail = getWorkflowToken(workflowId, wfRunId, null, null);
    List<WorkflowTokenDetail.NodeValueDetail> details = tokenDetail.getTokenData().get("wf.runid");
    Assert.assertEquals(1, details.size());
    Assert.assertEquals(wfRunId, details.get(0).getValue());
    // validate that none of the mapper, reducer or spark closure were able to write to the token
    for (String key : new String[] { "mapper.initialize.key", "map.key", "reducer.initialize.key", "reduce.key", "some.key" }) {
        Assert.assertFalse(tokenDetail.getTokenData().containsKey(key));
    }
    List<RunRecord> sparkProgramRuns = getProgramRuns(sparkId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, sparkProgramRuns.size());
}