Examples with AsyncWriterManager org.apache.gobblin.writer.AsyncWriterManager used on opensource projects

Example 1 with AsyncWriterManager

use of org.apache.gobblin.writer.AsyncWriterManager in project incubator-gobblin by apache.

the class CouchbaseWriterTest method writeRecordsWithAsyncWriter.

/**
 * Uses the {@link AsyncWriterManager} to write records through a couchbase writer
 * It keeps a copy of the key, value combinations written and checks after all the writes have gone through.
 * @param recordIterator
 * @throws IOException
 */
private void writeRecordsWithAsyncWriter(Iterator<AbstractDocument> recordIterator) throws IOException {
    boolean verbose = false;
    Properties props = new Properties();
    props.setProperty(CouchbaseWriterConfigurationKeys.BUCKET, "default");
    Config config = ConfigFactory.parseProperties(props);
    CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, config);
    try {
        AsyncWriterManager asyncWriterManager = AsyncWriterManager.builder().asyncDataWriter(writer).maxOutstandingWrites(100000).retriesEnabled(true).numRetries(5).build();
        if (verbose) {
            // Create a reporter for metrics. This reporter will write metrics to STDOUT.
            OutputStreamReporter.Factory.newBuilder().build(new Properties());
            // Start all metric reporters.
            RootMetricContext.get().startReporting();
        }
        Verifier verifier = new Verifier();
        while (recordIterator.hasNext()) {
            AbstractDocument doc = recordIterator.next();
            verifier.onWrite(doc);
            asyncWriterManager.write(doc);
        }
        asyncWriterManager.commit();
        verifier.verify(writer.getBucket());
    } finally {
        writer.close();
    }
}