Examples with BatchProgram io.cdap.cdap.proto.BatchProgram used on opensource projects

Example 6 with BatchProgram

use of io.cdap.cdap.proto.BatchProgram in project cdap by cdapio.

the class ProgramClientTestRun method testBatchProgramCalls.

@Test
public void testBatchProgramCalls() throws Exception {
    final NamespaceId namespace = NamespaceId.DEFAULT;
    final ApplicationId appId = namespace.app(FakeApp.NAME);
    BatchProgram pingService = new BatchProgram(FakeApp.NAME, ProgramType.SERVICE, PingService.NAME);
    BatchProgram writerService = new BatchProgram(FakeApp.NAME, ProgramType.SERVICE, DatasetWriterService.NAME);
    BatchProgram missing = new BatchProgram(FakeApp.NAME, ProgramType.SERVICE, "not" + PingService.NAME);
    appClient.deploy(namespace, createAppJarFile(FakeApp.class));
    try {
        // make a batch call to start multiple programs, one of which does not exist
        List<BatchProgramStart> programStarts = ImmutableList.of(new BatchProgramStart(pingService), new BatchProgramStart(writerService), new BatchProgramStart(missing));
        List<BatchProgramResult> results = programClient.start(namespace, programStarts);
        // check that we got a 200 for programs that exist, and a 404 for the one that doesn't
        for (BatchProgramResult result : results) {
            if (missing.getProgramId().equals(result.getProgramId())) {
                Assert.assertEquals(404, result.getStatusCode());
            } else {
                Assert.assertEquals(200, result.getStatusCode());
            }
        }
        // wait for all programs to be in RUNNING status
        assertProgramRuns(programClient, namespace.app(pingService.getAppId()).service(pingService.getProgramId()), ProgramRunStatus.RUNNING, 1, 10);
        assertProgramRuns(programClient, namespace.app(writerService.getAppId()).service(writerService.getProgramId()), ProgramRunStatus.RUNNING, 1, 10);
        // make a batch call for status of programs, one of which does not exist
        List<BatchProgram> programs = ImmutableList.of(pingService, writerService, missing);
        List<BatchProgramStatus> statusList = programClient.getStatus(namespace, programs);
        // check status is running for programs that exist, and that we get a 404 for the one that doesn't
        for (BatchProgramStatus status : statusList) {
            if (missing.getProgramId().equals(status.getProgramId())) {
                Assert.assertEquals(404, status.getStatusCode());
            } else {
                Assert.assertEquals(200, status.getStatusCode());
                Assert.assertEquals("RUNNING", status.getStatus());
            }
        }
        // make a batch call to stop programs, one of which does not exist
        results = programClient.stop(namespace, programs);
        // check that we got a 200 for programs that exist, and a 404 for the one that doesn't
        for (BatchProgramResult result : results) {
            if (missing.getProgramId().equals(result.getProgramId())) {
                Assert.assertEquals(404, result.getStatusCode());
            } else {
                Assert.assertEquals(200, result.getStatusCode());
            }
        }
        // check programs are in stopped state
        final List<BatchProgram> stoppedPrograms = ImmutableList.of(pingService, writerService);
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                List<BatchProgramStatus> statusList = programClient.getStatus(namespace, stoppedPrograms);
                for (BatchProgramStatus status : statusList) {
                    if (status.getStatusCode() != 200) {
                        return false;
                    }
                    if (status.getStatus().equals("RUNNING")) {
                        return false;
                    }
                }
                return true;
            }
        }, 10, TimeUnit.SECONDS);
    } finally {
        try {
            appClient.delete(appId);
        } catch (Exception e) {
            LOG.error("Error deleting app {} during test cleanup.", appId, e);
        }
    }
}

Example 7 with BatchProgram

use of io.cdap.cdap.proto.BatchProgram in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method getStatuses.

/**
 * Returns the status for all programs that are passed into the data. The data is an array of JSON objects
 * where each object must contain the following three elements: appId, programType, and programId
 * (flow name, service name, etc.).
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "MapReduce2"}]
 * </code></pre>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as 2 fields, "status" which maps to the status of the program and "statusCode" which maps to the
 * status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, App2 does not exist), then all JsonObjects for which the
 * parameters have a valid status will have the status field but all JsonObjects for which the parameters do not have
 * a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no App2 in the data above, then the response would be 200 OK with following possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1", "statusCode": 200, "status": "RUNNING"},
 * {"appId": "App1", "programType": "Mapreduce", "programId": "Mapreduce2", "statusCode": 200, "status": "STOPPED"}]
 * </code></pre>
 */
@POST
@Path("/status")
@AuditPolicy(AuditDetail.REQUEST_BODY)
public void getStatuses(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> batchPrograms = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    List<ProgramId> programs = batchPrograms.stream().map(p -> new ProgramId(namespaceId, p.getAppId(), p.getProgramType(), p.getProgramId())).collect(Collectors.toList());
    Map<ProgramId, ProgramStatus> statuses = lifecycleService.getProgramStatuses(programs);
    List<BatchProgramStatus> result = new ArrayList<>(programs.size());
    for (BatchProgram program : batchPrograms) {
        ProgramId programId = new ProgramId(namespaceId, program.getAppId(), program.getProgramType(), program.getProgramId());
        ProgramStatus status = statuses.get(programId);
        if (status == null) {
            result.add(new BatchProgramStatus(program, HttpResponseStatus.NOT_FOUND.code(), new NotFoundException(programId).getMessage(), null));
        } else {
            result.add(new BatchProgramStatus(program, HttpResponseStatus.OK.code(), null, status.name()));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(result));
}

Example 8 with BatchProgram

use of io.cdap.cdap.proto.BatchProgram in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method getLatestRuns.

/**
 * Returns the latest runs for all program runnables that are passed into the data. The data is an array of
 * Json objects where each object must contain the following three elements: appId, programType, and programId.
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 *  {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow"}]
 * </code></pre>
 * </p><p>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as 2 fields, "runs" which is a list of the latest run records and "statusCode" which maps to the
 * status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, workflow in app1 does not exist),
 * then all JsonObjects for which the parameters have a valid status will have the count field but all JsonObjects
 * for which the parameters do not have a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no workflow in App1 in the data above, then the response would be 200 OK with following
 * possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1",
 * "statusCode": 200, "runs": [...]},
 * {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow", "statusCode": 404,
 * "error": "Program 'testWorkflow' is not found"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow", "runnableId": "Flowlet1",
 *  "statusCode": 200, "runs": [...]}]
 * </code></pre>
 */
@POST
@Path("/runs")
public void getLatestRuns(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    List<ProgramId> programIds = programs.stream().map(batchProgram -> new ProgramId(namespaceId, batchProgram.getAppId(), batchProgram.getProgramType(), batchProgram.getProgramId())).collect(Collectors.toList());
    List<BatchProgramHistory> response = new ArrayList<>(programs.size());
    List<ProgramHistory> result = lifecycleService.getRunRecords(programIds, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 1);
    for (ProgramHistory programHistory : result) {
        ProgramId programId = programHistory.getProgramId();
        Exception exception = programHistory.getException();
        BatchProgram batchProgram = new BatchProgram(programId.getApplication(), programId.getType(), programId.getProgram());
        if (exception == null) {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.OK.code(), null, programHistory.getRuns()));
        } else if (exception instanceof NotFoundException) {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.NOT_FOUND.code(), exception.getMessage(), Collections.emptyList()));
        } else if (exception instanceof UnauthorizedException) {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.FORBIDDEN.code(), exception.getMessage(), Collections.emptyList()));
        } else {
            response.add(new BatchProgramHistory(batchProgram, HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), exception.getMessage(), Collections.emptyList()));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(response));
}

Example 9 with BatchProgram

use of io.cdap.cdap.proto.BatchProgram in project cdap by caskdata.

the class ProgramLifecycleHttpHandler method getRunCounts.

/**
 * Returns the run counts for all program runnables that are passed into the data. The data is an array of
 * Json objects where each object must contain the following three elements: appId, programType, and programId.
 * The max number of programs in the request is 100.
 * <p>
 * Example input:
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
 *  {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow"}]
 * </code></pre>
 * </p><p>
 * </p><p>
 * The response will be an array of JsonObjects each of which will contain the three input parameters
 * as well as 2 fields, "runCount" which maps to the count of the program and "statusCode" which maps to the
 * status code for the data in that JsonObjects.
 * </p><p>
 * If an error occurs in the input (for the example above, workflow in app1 does not exist),
 * then all JsonObjects for which the parameters have a valid status will have the count field but all JsonObjects
 * for which the parameters do not have a valid status will have an error message and statusCode.
 * </p><p>
 * For example, if there is no workflow in App1 in the data above, then the response would be 200 OK with following
 * possible data:
 * </p>
 * <pre><code>
 * [{"appId": "App1", "programType": "Service", "programId": "Service1",
 * "statusCode": 200, "runCount": 20},
 * {"appId": "App1", "programType": "Workflow", "programId": "testWorkflow", "statusCode": 404,
 * "error": "Program 'testWorkflow' is not found"},
 *  {"appId": "App2", "programType": "Workflow", "programId": "DataPipelineWorkflow",
 *  "statusCode": 200, "runCount": 300}]
 * </code></pre>
 */
@POST
@Path("/runcount")
public void getRunCounts(FullHttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId) throws Exception {
    List<BatchProgram> programs = validateAndGetBatchInput(request, BATCH_PROGRAMS_TYPE);
    if (programs.size() > 100) {
        throw new BadRequestException(String.format("%d programs found in the request, the maximum number " + "supported is 100", programs.size()));
    }
    List<ProgramId> programIds = programs.stream().map(batchProgram -> new ProgramId(namespaceId, batchProgram.getAppId(), batchProgram.getProgramType(), batchProgram.getProgramId())).collect(Collectors.toList());
    List<BatchProgramCount> counts = new ArrayList<>(programs.size());
    for (RunCountResult runCountResult : lifecycleService.getProgramRunCounts(programIds)) {
        ProgramId programId = runCountResult.getProgramId();
        Exception exception = runCountResult.getException();
        if (exception == null) {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.OK.code(), null, runCountResult.getCount()));
        } else if (exception instanceof NotFoundException) {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.NOT_FOUND.code(), exception.getMessage(), null));
        } else if (exception instanceof UnauthorizedException) {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.FORBIDDEN.code(), exception.getMessage(), null));
        } else {
            counts.add(new BatchProgramCount(programId, HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), exception.getMessage(), null));
        }
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(counts));
}

Example 10 with BatchProgram

use of io.cdap.cdap.proto.BatchProgram in project cdap by caskdata.

the class ProgramClientTestRun method testBatchProgramCalls.

@Test
public void testBatchProgramCalls() throws Exception {
    final NamespaceId namespace = NamespaceId.DEFAULT;
    final ApplicationId appId = namespace.app(FakeApp.NAME);
    BatchProgram pingService = new BatchProgram(FakeApp.NAME, ProgramType.SERVICE, PingService.NAME);
    BatchProgram writerService = new BatchProgram(FakeApp.NAME, ProgramType.SERVICE, DatasetWriterService.NAME);
    BatchProgram missing = new BatchProgram(FakeApp.NAME, ProgramType.SERVICE, "not" + PingService.NAME);
    appClient.deploy(namespace, createAppJarFile(FakeApp.class));
    try {
        // make a batch call to start multiple programs, one of which does not exist
        List<BatchProgramStart> programStarts = ImmutableList.of(new BatchProgramStart(pingService), new BatchProgramStart(writerService), new BatchProgramStart(missing));
        List<BatchProgramResult> results = programClient.start(namespace, programStarts);
        // check that we got a 200 for programs that exist, and a 404 for the one that doesn't
        for (BatchProgramResult result : results) {
            if (missing.getProgramId().equals(result.getProgramId())) {
                Assert.assertEquals(404, result.getStatusCode());
            } else {
                Assert.assertEquals(200, result.getStatusCode());
            }
        }
        // wait for all programs to be in RUNNING status
        assertProgramRuns(programClient, namespace.app(pingService.getAppId()).service(pingService.getProgramId()), ProgramRunStatus.RUNNING, 1, 10);
        assertProgramRuns(programClient, namespace.app(writerService.getAppId()).service(writerService.getProgramId()), ProgramRunStatus.RUNNING, 1, 10);
        // make a batch call for status of programs, one of which does not exist
        List<BatchProgram> programs = ImmutableList.of(pingService, writerService, missing);
        List<BatchProgramStatus> statusList = programClient.getStatus(namespace, programs);
        // check status is running for programs that exist, and that we get a 404 for the one that doesn't
        for (BatchProgramStatus status : statusList) {
            if (missing.getProgramId().equals(status.getProgramId())) {
                Assert.assertEquals(404, status.getStatusCode());
            } else {
                Assert.assertEquals(200, status.getStatusCode());
                Assert.assertEquals("RUNNING", status.getStatus());
            }
        }
        // make a batch call to stop programs, one of which does not exist
        results = programClient.stop(namespace, programs);
        // check that we got a 200 for programs that exist, and a 404 for the one that doesn't
        for (BatchProgramResult result : results) {
            if (missing.getProgramId().equals(result.getProgramId())) {
                Assert.assertEquals(404, result.getStatusCode());
            } else {
                Assert.assertEquals(200, result.getStatusCode());
            }
        }
        // check programs are in stopped state
        final List<BatchProgram> stoppedPrograms = ImmutableList.of(pingService, writerService);
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                List<BatchProgramStatus> statusList = programClient.getStatus(namespace, stoppedPrograms);
                for (BatchProgramStatus status : statusList) {
                    if (status.getStatusCode() != 200) {
                        return false;
                    }
                    if (status.getStatus().equals("RUNNING")) {
                        return false;
                    }
                }
                return true;
            }
        }, 10, TimeUnit.SECONDS);
    } finally {
        try {
            appClient.delete(appId);
        } catch (Exception e) {
            LOG.error("Error deleting app {} during test cleanup.", appId, e);
        }
    }
}