Examples with SimpleRecordFormatAvroIO org.talend.components.simplefileio.runtime.SimpleRecordFormatAvroIO used on opensource projects

Example 1 with SimpleRecordFormatAvroIO

use of org.talend.components.simplefileio.runtime.SimpleRecordFormatAvroIO in project components by Talend.

the class S3InputRuntime method expand.

@Override
public PCollection<IndexedRecord> expand(PBegin in) {
    // The UGI does not control security for S3.
    UgiDoAs doAs = UgiDoAs.ofNone();
    String path = S3Connection.getUriPath(properties.getDatasetProperties());
    // overwrite is ignored for reads.
    boolean overwrite = false;
    int limit = properties.limit.getValue();
    // mergeOutput is ignored for reads.
    boolean mergeOutput = false;
    SimpleRecordFormatBase rf = null;
    switch(properties.getDatasetProperties().format.getValue()) {
        case AVRO:
            rf = new SimpleRecordFormatAvroIO(doAs, path, overwrite, limit, mergeOutput);
            break;
        case CSV:
            rf = new SimpleRecordFormatCsvIO(doAs, path, overwrite, limit, properties.getDatasetProperties().getRecordDelimiter(), properties.getDatasetProperties().getFieldDelimiter(), mergeOutput);
            break;
        case PARQUET:
            rf = new SimpleRecordFormatParquetIO(doAs, path, overwrite, limit, mergeOutput);
            break;
    }
    if (rf == null) {
        throw new RuntimeException("To be implemented: " + properties.getDatasetProperties().format.getValue());
    }
    S3Connection.setS3Configuration(rf.getExtraHadoopConfiguration(), properties.getDatasetProperties());
    return rf.read(in);
}

Example 2 with SimpleRecordFormatAvroIO

use of org.talend.components.simplefileio.runtime.SimpleRecordFormatAvroIO in project components by Talend.

the class S3OutputRuntime method expand.

@Override
public PDone expand(PCollection<IndexedRecord> in) {
    // The UGI does not control security for S3.
    UgiDoAs doAs = UgiDoAs.ofNone();
    String path = S3Connection.getUriPath(properties.getDatasetProperties());
    boolean overwrite = properties.overwrite.getValue();
    // limit is ignored for sinks
    int limit = -1;
    boolean mergeOutput = properties.mergeOutput.getValue();
    SimpleRecordFormatBase rf = null;
    switch(properties.getDatasetProperties().format.getValue()) {
        case AVRO:
            rf = new SimpleRecordFormatAvroIO(doAs, path, overwrite, limit, mergeOutput);
            break;
        case CSV:
            rf = new SimpleRecordFormatCsvIO(doAs, path, overwrite, limit, properties.getDatasetProperties().getRecordDelimiter(), properties.getDatasetProperties().getFieldDelimiter(), mergeOutput);
            break;
        case PARQUET:
            rf = new SimpleRecordFormatParquetIO(doAs, path, overwrite, limit, mergeOutput);
            break;
    }
    if (rf == null) {
        throw new RuntimeException("To be implemented: " + properties.getDatasetProperties().format.getValue());
    }
    S3Connection.setS3Configuration(rf.getExtraHadoopConfiguration(), properties.getDatasetProperties());
    return rf.write(in);
}