Examples with BufferedOutputStream java.io.BufferedOutputStream used on opensource projects

Example 56 with BufferedOutputStream

use of java.io.BufferedOutputStream in project Gaffer by gchq.

the class IngestUtils method createSplitsFile.

/**
     * Get the existing splits from a table in Accumulo and write a splits file.
     * The number of splits is returned.
     *
     * @param conn       - An existing connection to an Accumulo instance
     * @param table      - The table name
     * @param fs         - The FileSystem in which to create the splits file
     * @param splitsFile - A Path for the output splits file
     * @param maxSplits  - The maximum number of splits
     * @return The number of splits in the table
     * @throws IOException for any IO issues reading from the file system. Other accumulo exceptions are caught and wrapped in an IOException.
     */
public static int createSplitsFile(final Connector conn, final String table, final FileSystem fs, final Path splitsFile, final int maxSplits) throws IOException {
    LOGGER.info("Creating splits file in location {} from table {} with maximum splits {}", splitsFile, table, maxSplits);
    // Get the splits from the table
    Collection<Text> splits;
    try {
        splits = conn.tableOperations().listSplits(table, maxSplits);
    } catch (TableNotFoundException | AccumuloSecurityException | AccumuloException e) {
        throw new IOException(e.getMessage(), e);
    }
    // This should have returned at most maxSplits splits, but this is not implemented properly in MockInstance.
    if (splits.size() > maxSplits) {
        if (conn instanceof MockConnector) {
            LOGGER.info("Manually reducing the number of splits to {} due to MockInstance not implementing" + " listSplits(table, maxSplits) properly", maxSplits);
        } else {
            LOGGER.info("Manually reducing the number of splits to {} (number of splits was {})", maxSplits, splits.size());
        }
        final Collection<Text> filteredSplits = new TreeSet<>();
        final int outputEveryNth = splits.size() / maxSplits;
        LOGGER.info("Outputting every {}-th split from {} total", outputEveryNth, splits.size());
        int i = 0;
        for (final Text text : splits) {
            if (i % outputEveryNth == 0) {
                filteredSplits.add(text);
            }
            i++;
            if (filteredSplits.size() >= maxSplits) {
                break;
            }
        }
        splits = filteredSplits;
    }
    LOGGER.info("Found {} splits from table {}", splits.size(), table);
    try (final PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(splitsFile, true)), false, CommonConstants.UTF_8)) {
        // Write the splits to file
        if (splits.isEmpty()) {
            out.close();
            return 0;
        }
        for (final Text split : splits) {
            out.println(new String(Base64.encodeBase64(split.getBytes()), CommonConstants.UTF_8));
        }
    }
    return splits.size();
}

Example 57 with BufferedOutputStream

use of java.io.BufferedOutputStream in project Gaffer by gchq.

the class SampleDataAndCreateSplitsFileTool method run.

@Override
public int run(final String[] strings) throws OperationException {
    try {
        LOGGER.info("Creating job using SampleDataForSplitPointsJobFactory");
        job = new SampleDataForSplitPointsJobFactory().createJob(operation, store);
    } catch (final IOException e) {
        LOGGER.error("Failed to create Hadoop job: {}", e.getMessage());
        throw new OperationException("Failed to create the Hadoop job: " + e.getMessage(), e);
    }
    try {
        LOGGER.info("Running SampleDataForSplitPoints job (job name is {})", job.getJobName());
        job.waitForCompletion(true);
    } catch (final IOException | InterruptedException | ClassNotFoundException e) {
        LOGGER.error("Exception running job: {}", e.getMessage());
        throw new OperationException("Error while waiting for job to complete: " + e.getMessage(), e);
    }
    try {
        if (!job.isSuccessful()) {
            LOGGER.error("Job was not successful (job name is {})", job.getJobName());
            throw new OperationException("Error running job");
        }
    } catch (final IOException e) {
        LOGGER.error("Exception running job: {}", e.getMessage());
        throw new OperationException("Error running job" + e.getMessage(), e);
    }
    // Find the number of records output
    // NB In the following line use mapred.Task.Counter.REDUCE_OUTPUT_RECORDS rather than
    // mapreduce.TaskCounter.REDUCE_OUTPUT_RECORDS as this is more compatible with earlier
    // versions of Hadoop.
    Counter counter;
    try {
        counter = job.getCounters().findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS);
        LOGGER.info("Number of records output = {}", counter);
    } catch (final IOException e) {
        LOGGER.error("Failed to get counter org.apache.hadoop.mapred.Task.Counter.REDUCE_OUTPUT_RECORDS from job: {}", e.getMessage());
        throw new OperationException("Failed to get counter: " + Task.Counter.REDUCE_OUTPUT_RECORDS, e);
    }
    int numberTabletServers;
    try {
        numberTabletServers = store.getConnection().instanceOperations().getTabletServers().size();
        LOGGER.info("Number of tablet servers is {}", numberTabletServers);
    } catch (final StoreException e) {
        LOGGER.error("Exception thrown getting number of tablet servers: {}", e.getMessage());
        throw new OperationException(e.getMessage(), e);
    }
    long outputEveryNthRecord = counter.getValue() / (numberTabletServers - 1);
    final Path resultsFile = new Path(operation.getOutputPath(), "part-r-00000");
    LOGGER.info("Will output every {}-th record from {}", outputEveryNthRecord, resultsFile);
    // Read through resulting file, pick out the split points and write to file.
    final Configuration conf = getConf();
    final FileSystem fs;
    try {
        fs = FileSystem.get(conf);
    } catch (final IOException e) {
        LOGGER.error("Exception getting filesystem: {}", e.getMessage());
        throw new OperationException("Failed to get filesystem from configuration: " + e.getMessage(), e);
    }
    LOGGER.info("Writing splits to {}", operation.getResultingSplitsFilePath());
    final Key key = new Key();
    final Value value = new Value();
    long count = 0;
    int numberSplitPointsOutput = 0;
    try (final SequenceFile.Reader reader = new SequenceFile.Reader(fs, resultsFile, conf);
        final PrintStream splitsWriter = new PrintStream(new BufferedOutputStream(fs.create(new Path(operation.getResultingSplitsFilePath()), true)), false, CommonConstants.UTF_8)) {
        while (reader.next(key, value) && numberSplitPointsOutput < numberTabletServers - 1) {
            count++;
            if (count % outputEveryNthRecord == 0) {
                LOGGER.debug("Outputting split point number {} ({})", numberSplitPointsOutput, Base64.encodeBase64(key.getRow().getBytes()));
                numberSplitPointsOutput++;
                splitsWriter.println(new String(Base64.encodeBase64(key.getRow().getBytes()), CommonConstants.UTF_8));
            }
        }
        LOGGER.info("Total number of records read was {}", count);
    } catch (final IOException e) {
        LOGGER.error("Exception reading results file and outputting split points: {}", e.getMessage());
        throw new OperationException(e.getMessage(), e);
    }
    try {
        fs.delete(resultsFile, true);
        LOGGER.info("Deleted the results file {}", resultsFile);
    } catch (final IOException e) {
        LOGGER.error("Failed to delete the results file {}", resultsFile);
        throw new OperationException("Failed to delete the results file: " + e.getMessage(), e);
    }
    return SUCCESS_RESPONSE;
}

Example 58 with BufferedOutputStream

use of java.io.BufferedOutputStream in project fresco by facebook.

the class LocalFileFetchProducerTest method setUp.

@Before
public void setUp() throws Exception {
    MockitoAnnotations.initMocks(this);
    mExecutor = new TestExecutorService(new FakeClock());
    mLocalFileFetchProducer = new LocalFileFetchProducer(mExecutor, mPooledByteBufferFactory);
    mFile = new File(RuntimeEnvironment.application.getExternalFilesDir(null), TEST_FILENAME);
    BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(mFile));
    bos.write(new byte[INPUT_STREAM_LENGTH], 0, INPUT_STREAM_LENGTH);
    bos.close();
    mProducerContext = new SettableProducerContext(mImageRequest, mRequestId, mProducerListener, mock(Object.class), ImageRequest.RequestLevel.FULL_FETCH, false, true, Priority.MEDIUM);
    when(mImageRequest.getSourceFile()).thenReturn(mFile);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            mCapturedEncodedImage = EncodedImage.cloneOrNull((EncodedImage) invocation.getArguments()[0]);
            return null;
        }
    }).when(mConsumer).onNewResult(notNull(EncodedImage.class), anyBoolean());
}

Example 59 with BufferedOutputStream

use of java.io.BufferedOutputStream in project bazel by bazelbuild.

the class JavacTurbine method emitClassJar.

/** Write the class output from a successful compilation to the output jar. */
private static void emitClassJar(Path outputJar, ImmutableMap<String, OutputFileObject> files) throws IOException {
    try (OutputStream fos = Files.newOutputStream(outputJar);
        ZipOutputStream zipOut = new ZipOutputStream(new BufferedOutputStream(fos, ZIPFILE_BUFFER_SIZE))) {
        for (Map.Entry<String, OutputFileObject> entry : files.entrySet()) {
            if (entry.getValue().location != StandardLocation.CLASS_OUTPUT) {
                continue;
            }
            String name = entry.getKey();
            byte[] bytes = entry.getValue().asBytes();
            if (bytes == null) {
                continue;
            }
            if (name.endsWith(".class")) {
                bytes = processBytecode(bytes);
            }
            ZipUtil.storeEntry(name, bytes, zipOut);
        }
    }
}

Example 60 with BufferedOutputStream

use of java.io.BufferedOutputStream in project coursera-android by aporter.

the class ExternalFileWriteReadActivity method copyImageToMemory.

private void copyImageToMemory(File outFile) {
    try {
        BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(outFile));
        BufferedInputStream is = new BufferedInputStream(getResources().openRawResource(R.raw.painter));
        copy(is, os);
    } catch (FileNotFoundException e) {
        Log.e(TAG, "FileNotFoundException");
    }
}