Examples with SaveResponse com.thinkbiganalytics.spark.rest.model.SaveResponse used on opensource projects

Example 6 with SaveResponse

use of com.thinkbiganalytics.spark.rest.model.SaveResponse in project kylo by Teradata.

the class TransformService method saveSql.

/**
 * Executes and saves a Spark SQL request.
 */
@Nonnull
public SaveResponse saveSql(@Nonnull final String id, @Nonnull final SaveRequest save) {
    log.entry(id, save);
    // Create task
    final Supplier<SaveResult> task;
    final TransformRequest transform = getTransformRequest(id);
    final JdbcDatasource transformDatasource = (transform.getDatasources() != null && transform.getDatasources().size() == 1 && transform.getDatasources().get(0) instanceof JdbcDatasource) ? (JdbcDatasource) transform.getDatasources().get(0) : null;
    if (transformDatasource != null && save.getJdbc() != null && Objects.equal(transformDatasource.getId(), save.getJdbc().getId())) {
        Preconditions.checkArgument(save.getTableName() != null, "Missing target table name.");
        task = new SaveSqlStage(save.getTableName(), transform.getScript(), save.getJdbc());
    } else {
        task = createSaveTask(save, createSqlTask(transform));
    }
    // Submit job
    final SaveResponse response = submitSaveJob(task);
    return log.exit(response);
}

Example 7 with SaveResponse

use of com.thinkbiganalytics.spark.rest.model.SaveResponse in project kylo by Teradata.

the class TransformService method saveShell.

/**
 * Executes and saves a Spark shell request.
 */
@Nonnull
public SaveResponse saveShell(@Nonnull final String id, @Nonnull final SaveRequest save) throws ScriptException {
    log.entry(id, save);
    final DataSet dataSet = createShellTask(getTransformRequest(id));
    final SaveResponse response = submitSaveJob(createSaveTask(save, new ShellTransformStage(dataSet)));
    return log.exit(response);
}

Example 8 with SaveResponse

use of com.thinkbiganalytics.spark.rest.model.SaveResponse in project kylo by Teradata.

the class SparkShellIT method testScriptSave.

/**
 * Verify saving the result of a script.
 */
@Test
public void testScriptSave() {
    final TransformResponse transform = executeScript(SCRIPT, true);
    final String tableName = newTableName();
    final SaveResponse save = saveScript(transform.getTable(), "orc", null, tableName);
    Assert.assertEquals(SaveResponse.Status.SUCCESS, save.getStatus());
    try {
        final List<HashMap<String, String>> table = getHiveQuery("SELECT * FROM " + tableName);
        Assert.assertEquals(ImmutableList.of(ImmutableMap.of("id", 1, "value", "a"), ImmutableMap.of("id", 2, "value", "b"), ImmutableMap.of("id", 3, "value", "c")), table);
    } finally {
        ssh("hive -e \"drop table " + tableName + "\"");
    }
}

Example 9 with SaveResponse

use of com.thinkbiganalytics.spark.rest.model.SaveResponse in project kylo by Teradata.

the class SparkShellIT method save.

/**
 * Saves the result of a script or query.
 *
 * @param id          the transform or query identifier
 * @param requestPath the URL path to initiate the save
 * @param resultPath  the URL path to get the result
 * @param request     the save request
 * @return the save response
 */
private SaveResponse save(@Nonnull final String id, @Nonnull final String requestPath, @Nonnull final String resultPath, @Nonnull final SaveRequest request) {
    // Execute request
    Response response = given(SparkShellProxyController.BASE).body(request).when().post(requestPath, id);
    SaveResponse save = null;
    Stopwatch timeout = Stopwatch.createStarted();
    do {
        if (save != null) {
            response = given(SparkShellProxyController.BASE).get(resultPath, id, save.getId());
        }
        save = response.as(SaveResponse.class);
        if (save.getStatus() == SaveResponse.Status.PENDING && TIMEOUT - timeout.elapsed(TimeUnit.SECONDS) > 1) {
            waitFor(1, TimeUnit.SECONDS, "for save to finish");
        }
    } while (save.getStatus() == SaveResponse.Status.PENDING && timeout.elapsed(TimeUnit.SECONDS) < TIMEOUT);
    // Verify response
    response.then().statusCode(HTTP_OK);
    return response.as(SaveResponse.class);
}