Examples with SQLEngineException io.cdap.cdap.etl.api.engine.sql.SQLEngineException used on opensource projects

Example 1 with SQLEngineException

use of io.cdap.cdap.etl.api.engine.sql.SQLEngineException in project cdap by caskdata.

the class BatchSQLEngineAdapter method pullInternal.

/**
 * Pull implementation. This method has blocking calls and should be executed in a separate thread.
 *
 * @param dataset the dataset to pull.
 * @return {@link JavaRDD} representing the records contained in this dataset.
 * @throws SQLEngineException if the pull process fails.
 */
@SuppressWarnings("unchecked,raw")
private <T> JavaRDD<T> pullInternal(SQLDataset dataset) throws SQLEngineException {
    // Create pull operation for this dataset and wait until completion
    SQLPullRequest pullRequest = new SQLPullRequest(dataset);
    // If so, we will process this request using a producer.
    for (PullCapability capability : sqlEngine.getPullCapabilities()) {
        SQLDatasetProducer producer = sqlEngine.getProducer(pullRequest, capability);
        // If a producer is able to produce records for this pull request, extract the RDD from this request.
        if (producer != null) {
            RecordCollection recordCollection = producer.produce(dataset);
            // If the collection that got generarted is not an instance of a SparkRecordCollection, skip.
            if (recordCollection instanceof SparkRecordCollection) {
                Schema schema = dataset.getSchema();
                return (JavaRDD<T>) ((SparkRecordCollection) recordCollection).getDataFrame().javaRDD().map(r -> DataFrames.fromRow((Row) r, schema));
            }
        }
    }
    // If no capabilities could be used to produce records, proceed using the Pull Provider.
    SQLPullDataset<StructuredRecord, ?, ?> sqlPullDataset = sqlEngine.getPullProvider(pullRequest);
    // Run operation to read from the InputFormatProvider supplied by this operation.
    ClassLoader classLoader = Objects.firstNonNull(Thread.currentThread().getContextClassLoader(), getClass().getClassLoader());
    JavaPairRDD pairRDD = RDDUtils.readUsingInputFormat(jsc, sqlPullDataset, classLoader, Object.class, Object.class);
    return pairRDD.flatMap(new TransformFromPairFunction(sqlPullDataset.fromKeyValue()));
}

Example 2 with SQLEngineException

use of io.cdap.cdap.etl.api.engine.sql.SQLEngineException in project cdap by caskdata.

the class BatchSQLEngineAdapter method waitForJobAndThrowException.

/**
 * Method used to handle execution exceptions from a job.
 * <p>
 * Any error during execution will be wrapped in a {@link SQLEngineException} if needed.
 *
 * @param job the job to wait for.
 * @throws SQLEngineException if the internal task threw an exception.
 */
public void waitForJobAndThrowException(SQLEngineJob<?> job) throws SQLEngineException {
    SQLEngineException ex = null;
    try {
        job.waitFor();
    } catch (CompletionException ce) {
        LOG.error("SQL Engine Task completed exceptionally");
        try {
            throw ce.getCause();
        } catch (SQLEngineException see) {
            // If the source of this was an SQL exception, just rethrow.
            LOG.error("SQL Engine Task failed with exception.", see);
            ex = see;
        } catch (Throwable t) {
            // Wrap any other exception in SQL exception
            LOG.error("SQL Engine Task failed with exception.", t);
            ex = new SQLEngineException(t);
        }
    } catch (CancellationException ce) {
        LOG.error("SQL Engine Task was cancelled", ce);
        ex = new SQLEngineException(ce);
    } catch (Throwable t) {
        LOG.error("SQL Engine Task failed with unexpected throwable", t);
        ex = new SQLEngineException(t);
    }
    // Throw SQL Exception if needed.
    if (ex != null) {
        throw ex;
    }
}

Example 3 with SQLEngineException

use of io.cdap.cdap.etl.api.engine.sql.SQLEngineException in project cdap by caskdata.

the class SQLEngineCollection method tryStoreDirect.

@Override
public boolean tryStoreDirect(StageSpec stageSpec) {
    String stageName = stageSpec.getName();
    // Check if this stage should be excluded from executing in the SQL engine
    if (adapter.getExcludedStageNames().contains(stageName)) {
        return false;
    }
    // Get SQLEngineOutput instance for this stage
    SQLEngineOutput sqlEngineOutput = sinkFactory.getSQLEngineOutput(stageName);
    if (sqlEngineOutput != null) {
        // Exceptions are handled and logged so standard sink flow takes over in case of failure.
        try {
            SQLEngineJob<Boolean> writeJob = adapter.write(datasetName, sqlEngineOutput);
            adapter.waitForJobAndThrowException(writeJob);
            return writeJob.waitFor();
        } catch (SQLEngineException e) {
            LOG.warn(DIRECT_WRITE_ERROR, stageName, e);
        }
    }
    return false;
}