Examples with ExtraHadoopConfiguration org.talend.components.simplefileio.runtime.ExtraHadoopConfiguration used on opensource projects

Example 1 with ExtraHadoopConfiguration

use of org.talend.components.simplefileio.runtime.ExtraHadoopConfiguration in project components by Talend.

the class S3Connection method createFileSystem.

public static S3AFileSystem createFileSystem(S3DatasetProperties properties) throws IOException {
    Configuration config = new Configuration(true);
    ExtraHadoopConfiguration extraConfig = new ExtraHadoopConfiguration();
    S3Connection.setS3Configuration(extraConfig, properties);
    extraConfig.addTo(config);
    try {
        return (S3AFileSystem) FileSystem.get(new URI(Constants.FS_S3A + "://" + properties.bucket.getValue()), config);
    } catch (URISyntaxException e) {
        // The URI is constant, so this exception should never occur.
        throw new RuntimeException(e);
    }
}

Example 2 with ExtraHadoopConfiguration

use of org.talend.components.simplefileio.runtime.ExtraHadoopConfiguration in project components by Talend.

the class S3OutputRuntime method runAtDriver.

/**
 * If overwriting a file, causes any any files at the existing location to be deleted.
 *
 * This action occurs before running the job.
 */
@Override
public void runAtDriver(RuntimeContainer container) {
    if (properties.overwrite.getValue()) {
        try {
            Path p = new Path(S3Connection.getUriPath(properties.getDatasetProperties()));
            // Add the AWS configuration to the Hadoop filesystem.
            ExtraHadoopConfiguration awsConf = new ExtraHadoopConfiguration();
            S3Connection.setS3Configuration(awsConf, properties.getDatasetProperties());
            Configuration hadoopConf = new Configuration();
            awsConf.addTo(hadoopConf);
            FileSystem fs = p.getFileSystem(hadoopConf);
            if (fs.exists(p)) {
                boolean deleted = fs.delete(p, true);
                if (!deleted)
                    throw SimpleFileIOErrorCode.createOutputNotAuthorized(null, null, p.toString());
            }
        } catch (RuntimeException e) {
            throw e;
        } catch (Exception e) {
            throw TalendRuntimeException.createUnexpectedException(e);
        }
    }
}