Examples with DataFrameWriter org.apache.spark.sql.DataFrameWriter used on opensource projects

Example 1 with DataFrameWriter

use of org.apache.spark.sql.DataFrameWriter in project kylo by Teradata.

the class SaveDataSetStage method apply.

@Nonnull
@Override
public SaveResult apply(@Nullable final TransformResult transform) {
    Preconditions.checkNotNull(transform);
    // Configure writer
    final DataFrameWriter writer = getDataSet(transform).write();
    if (request.getFormat() != null) {
        writer.format(request.getFormat());
    }
    if (request.getMode() != null) {
        writer.mode(request.getMode());
    }
    if (request.getOptions() != null) {
        writer.options(request.getOptions());
    }
    // Save transformation
    final SaveResult result = new SaveResult();
    if (request.getJdbc() != null) {
        final Properties properties = new Properties();
        properties.setProperty("driver", request.getJdbc().getDatabaseDriverClassName());
        properties.setProperty("user", request.getJdbc().getDatabaseUser());
        properties.setProperty("password", request.getJdbc().getPassword());
        writer.jdbc(request.getJdbc().getDatabaseConnectionUrl(), request.getTableName(), properties);
    } else if (request.getTableName() != null) {
        writer.saveAsTable(request.getTableName());
    } else {
        final String hadoopTmpDir = fs.getConf().get("hadoop.tmp.dir", "/tmp");
        final Path absolutePath = new Path(hadoopTmpDir, UUID.randomUUID().toString());
        final Path qualifiedPath = fs.makeQualified(absolutePath);
        result.setPath(qualifiedPath);
        writer.save(qualifiedPath.toString());
    }
    return result;
}