Examples with TransformJob com.thinkbiganalytics.spark.metadata.TransformJob used on opensource projects

Example 1 with TransformJob

use of com.thinkbiganalytics.spark.metadata.TransformJob in project kylo by Teradata.

the class TransformService method submitTransformJob.

/**
 * Submits the specified task to be executed and returns the result.
 */
@Nonnull
private TransformResponse submitTransformJob(@Nonnull final Supplier<TransformResult> task, @Nullable final FieldPolicy[] policies) throws ScriptException {
    log.entry(task, policies);
    // Prepare script
    Supplier<TransformResult> result = task;
    if (policies != null && policies.length > 0 && validator != null) {
        result = Suppliers.compose(new ValidationStage(policies, validator), result);
    }
    if (profiler != null) {
        result = Suppliers.compose(new ProfileStage(profiler), result);
    }
    // Execute script
    final String table = newTableName();
    final TransformJob job = new TransformJob(table, Suppliers.compose(new ResponseStage(table), result), engine.getSparkContext());
    tracker.submitJob(job);
    // Build response
    TransformResponse response;
    try {
        response = job.get(500, TimeUnit.MILLISECONDS);
        tracker.removeJob(table);
    } catch (final ExecutionException cause) {
        throw log.throwing(new ScriptException(cause));
    } catch (final InterruptedException | TimeoutException e) {
        log.trace("Timeout waiting for script result", e);
        response = new TransformResponse();
        response.setProgress(0.0);
        response.setStatus(TransformResponse.Status.PENDING);
        response.setTable(table);
    }
    return log.exit(response);
}

Example 2 with TransformJob

use of com.thinkbiganalytics.spark.metadata.TransformJob in project kylo by Teradata.

the class TransformService method submitTransformJob.

/**
 * Submits the specified task to be executed and returns the result.
 */
@Nonnull
private TransformResponse submitTransformJob(final Supplier<TransformResult> result, @Nonnull final PageSpec pageSpec) throws ScriptException {
    // Execute script
    final String table = newTableName();
    final TransformJob job = new TransformJob(table, Suppliers.compose(new ResponseStage(table, converterService, pageSpec), result), engine.getSparkContext());
    tracker.submitJob(job);
    // Build response
    TransformResponse response;
    try {
        response = job.get(500, TimeUnit.MILLISECONDS);
        tracker.removeJob(table);
    } catch (final ExecutionException cause) {
        throw log.throwing(new ScriptException(cause));
    } catch (final InterruptedException | TimeoutException e) {
        log.trace("Timeout waiting for script result", e);
        response = new TransformResponse();
        response.setProgress(0.0);
        response.setStatus(TransformResponse.Status.PENDING);
        response.setTable(table);
    }
    return log.exit(response);
}

Example 3 with TransformJob

use of com.thinkbiganalytics.spark.metadata.TransformJob in project kylo by Teradata.

the class AbstractTransformController method getTableTransformResponse.

public TransformResponse getTableTransformResponse(final String id) throws ExecutionException, InterruptedException {
    TransformJob job = transformService.getTransformJob(id);
    if (job.isDone()) {
        return job.get();
    } else {
        TransformResponse response = new TransformResponse();
        response.setProgress(job.progress());
        response.setStatus(TransformResponse.Status.PENDING);
        response.setTable(job.getGroupId());
        return response;
    }
}

Example 4 with TransformJob

use of com.thinkbiganalytics.spark.metadata.TransformJob in project kylo by Teradata.

the class AbstractTransformController method getTable.

/**
 * Requests the status of a transformation.
 *
 * @param id the destination table name
 * @return the transformation status
 */
@GET
@Path("{table}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Fetches the status of a transformation.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the status of the transformation.", response = TransformResponse.class), @ApiResponse(code = 404, message = "The transformation does not exist.", response = TransformResponse.class), @ApiResponse(code = 500, message = "There was a problem accessing the data.", response = TransformResponse.class) })
@Nonnull
public Response getTable(@Nonnull @PathParam("table") final String id) {
    try {
        TransformJob job = transformService.getTransformJob(id);
        if (job.isDone()) {
            return Response.ok(job.get()).build();
        } else {
            TransformResponse response = new TransformResponse();
            response.setProgress(job.progress());
            response.setStatus(TransformResponse.Status.PENDING);
            response.setTable(job.getGroupId());
            return Response.ok(response).build();
        }
    } catch (final IllegalArgumentException e) {
        return error(Response.Status.NOT_FOUND, "getTable.notFound");
    } catch (final Exception e) {
        return error(Response.Status.INTERNAL_SERVER_ERROR, e);
    }
}

Example 5 with TransformJob

use of com.thinkbiganalytics.spark.metadata.TransformJob in project kylo by Teradata.

the class AbstractTransformControllerTest method getTable.

/**
 * Verify requesting a transformation status.
 */
@Test
public void getTable() throws Exception {
    // Mock transform objects
    TransformJob pendingJob = Mockito.mock(TransformJob.class);
    Mockito.when(pendingJob.getGroupId()).thenReturn("PendingJob");
    Mockito.when(pendingJob.progress()).thenReturn(0.5);
    TransformJob successJob = Mockito.mock(TransformJob.class);
    TransformResponse successResponse = new TransformResponse();
    Mockito.when(successJob.get()).thenReturn(successResponse);
    Mockito.when(successJob.isDone()).thenReturn(true);
    TransformService transformService = Mockito.mock(TransformService.class);
    Mockito.when(transformService.getTransformJob("PendingJob")).thenReturn(pendingJob);
    Mockito.when(transformService.getTransformJob("SuccessJob")).thenReturn(successJob);
    // Test with pending job
    AbstractTransformController controller = new AbstractTransformController() {
    };
    controller.transformService = transformService;
    Response response = controller.getTable("PendingJob");
    Assert.assertEquals(Response.Status.OK, response.getStatusInfo());
    TransformResponse transformResponse = (TransformResponse) response.getEntity();
    Assert.assertEquals(0.5, transformResponse.getProgress(), 0.001);
    Assert.assertEquals(TransformResponse.Status.PENDING, transformResponse.getStatus());
    Assert.assertEquals("PendingJob", transformResponse.getTable());
    // Test with success job
    response = controller.getTable("SuccessJob");
    Assert.assertEquals(successResponse, response.getEntity());
    Assert.assertEquals(Response.Status.OK, response.getStatusInfo());
}