Examples with ProfileStage com.thinkbiganalytics.spark.metadata.ProfileStage used on opensource projects

Example 1 with ProfileStage

use of com.thinkbiganalytics.spark.metadata.ProfileStage in project kylo by Teradata.

the class TransformService method submitTransformJob.

/**
 * Submits the specified task to be executed and returns the result.
 */
@Nonnull
private TransformResponse submitTransformJob(@Nonnull final Supplier<TransformResult> task, @Nullable final FieldPolicy[] policies) throws ScriptException {
    log.entry(task, policies);
    // Prepare script
    Supplier<TransformResult> result = task;
    if (policies != null && policies.length > 0 && validator != null) {
        result = Suppliers.compose(new ValidationStage(policies, validator), result);
    }
    if (profiler != null) {
        result = Suppliers.compose(new ProfileStage(profiler), result);
    }
    // Execute script
    final String table = newTableName();
    final TransformJob job = new TransformJob(table, Suppliers.compose(new ResponseStage(table), result), engine.getSparkContext());
    tracker.submitJob(job);
    // Build response
    TransformResponse response;
    try {
        response = job.get(500, TimeUnit.MILLISECONDS);
        tracker.removeJob(table);
    } catch (final ExecutionException cause) {
        throw log.throwing(new ScriptException(cause));
    } catch (final InterruptedException | TimeoutException e) {
        log.trace("Timeout waiting for script result", e);
        response = new TransformResponse();
        response.setProgress(0.0);
        response.setStatus(TransformResponse.Status.PENDING);
        response.setTable(table);
    }
    return log.exit(response);
}