Examples with FileInputStream java.io.FileInputStream used on opensource projects

Example 91 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class AliyunOSSFileSystemStore method multipartUploadObject.

/**
   * Upload a file as an OSS object, using multipart upload.
   *
   * @param key object key.
   * @param file local file to upload.
   * @throws IOException if failed to upload object.
   */
public void multipartUploadObject(String key, File file) throws IOException {
    File object = file.getAbsoluteFile();
    long dataLen = object.length();
    long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
    int partNum = (int) (dataLen / realPartSize);
    if (dataLen % realPartSize != 0) {
        partNum += 1;
    }
    InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, key);
    ObjectMetadata meta = new ObjectMetadata();
    if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
        meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
    }
    initiateMultipartUploadRequest.setObjectMetadata(meta);
    InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
    List<PartETag> partETags = new ArrayList<PartETag>();
    String uploadId = initiateMultipartUploadResult.getUploadId();
    try {
        for (int i = 0; i < partNum; i++) {
            // TODO: Optimize this, avoid opening the object multiple times
            FileInputStream fis = new FileInputStream(object);
            try {
                long skipBytes = realPartSize * i;
                AliyunOSSUtils.skipFully(fis, skipBytes);
                long size = (realPartSize < dataLen - skipBytes) ? realPartSize : dataLen - skipBytes;
                UploadPartRequest uploadPartRequest = new UploadPartRequest();
                uploadPartRequest.setBucketName(bucketName);
                uploadPartRequest.setKey(key);
                uploadPartRequest.setUploadId(uploadId);
                uploadPartRequest.setInputStream(fis);
                uploadPartRequest.setPartSize(size);
                uploadPartRequest.setPartNumber(i + 1);
                UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
                statistics.incrementWriteOps(1);
                partETags.add(uploadPartResult.getPartETag());
            } finally {
                fis.close();
            }
        }
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
        CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        LOG.debug(completeMultipartUploadResult.getETag());
    } catch (OSSException | ClientException e) {
        AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId);
        ossClient.abortMultipartUpload(abortMultipartUploadRequest);
    }
}

Example 92 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class RegexCopyFilter method initialize.

/**
   * Loads a list of filter patterns for use in shouldCopy.
   */
@Override
public void initialize() {
    BufferedReader reader = null;
    try {
        InputStream is = new FileInputStream(filtersFile);
        reader = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
        String line;
        while ((line = reader.readLine()) != null) {
            Pattern pattern = Pattern.compile(line);
            filters.add(pattern);
        }
    } catch (FileNotFoundException notFound) {
        LOG.error("Can't find filters file " + filtersFile);
    } catch (IOException cantRead) {
        LOG.error("An error occurred while attempting to read from " + filtersFile);
    } finally {
        IOUtils.cleanup(LOG, reader);
    }
}

Example 93 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class RumenToSLSConverter method generateSLSLoadFile.

private static void generateSLSLoadFile(String inputFile, String outputFile) throws IOException {
    try (Reader input = new InputStreamReader(new FileInputStream(inputFile), "UTF-8")) {
        try (Writer output = new OutputStreamWriter(new FileOutputStream(outputFile), "UTF-8")) {
            ObjectMapper mapper = new ObjectMapper();
            ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
            Iterator<Map> i = mapper.readValues(new JsonFactory().createParser(input), Map.class);
            while (i.hasNext()) {
                Map m = i.next();
                output.write(writer.writeValueAsString(createSLSJob(m)) + EOL);
            }
        }
    }
}

Example 94 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class TestFileOutputCommitter method slurp.

public static String slurp(File f) throws IOException {
    int len = (int) f.length();
    byte[] buf = new byte[len];
    FileInputStream in = new FileInputStream(f);
    String contents = null;
    try {
        in.read(buf, 0, len);
        contents = new String(buf, "UTF-8");
    } finally {
        in.close();
    }
    return contents;
}

Example 95 with FileInputStream

use of java.io.FileInputStream in project hadoop by apache.

the class TestLineRecordReader method readRecordsDirectly.

// Gather the records by just splitting on new lines
public String[] readRecordsDirectly(URL testFileUrl, boolean bzip) throws IOException {
    int MAX_DATA_SIZE = 1024 * 1024;
    byte[] data = new byte[MAX_DATA_SIZE];
    FileInputStream fis = new FileInputStream(testFileUrl.getFile());
    int count;
    if (bzip) {
        BZip2CompressorInputStream bzIn = new BZip2CompressorInputStream(fis);
        count = bzIn.read(data);
        bzIn.close();
    } else {
        count = fis.read(data);
    }
    fis.close();
    assertTrue("Test file data too big for buffer", count < data.length);
    return new String(data, 0, count, "UTF-8").split("\n");
}