Examples with AbstractDocument com.couchbase.client.java.document.AbstractDocument used on opensource projects

Example 1 with AbstractDocument

use of com.couchbase.client.java.document.AbstractDocument in project incubator-gobblin by apache.

the class CouchbaseWriterTest method writeRecordsWithAsyncWriter.

/**
 * Uses the {@link AsyncWriterManager} to write records through a couchbase writer
 * It keeps a copy of the key, value combinations written and checks after all the writes have gone through.
 * @param recordIterator
 * @throws IOException
 */
private void writeRecordsWithAsyncWriter(Iterator<AbstractDocument> recordIterator) throws IOException {
    boolean verbose = false;
    Properties props = new Properties();
    props.setProperty(CouchbaseWriterConfigurationKeys.BUCKET, "default");
    Config config = ConfigFactory.parseProperties(props);
    CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, config);
    try {
        AsyncWriterManager asyncWriterManager = AsyncWriterManager.builder().asyncDataWriter(writer).maxOutstandingWrites(100000).retriesEnabled(true).numRetries(5).build();
        if (verbose) {
            // Create a reporter for metrics. This reporter will write metrics to STDOUT.
            OutputStreamReporter.Factory.newBuilder().build(new Properties());
            // Start all metric reporters.
            RootMetricContext.get().startReporting();
        }
        Verifier verifier = new Verifier();
        while (recordIterator.hasNext()) {
            AbstractDocument doc = recordIterator.next();
            verifier.onWrite(doc);
            asyncWriterManager.write(doc);
        }
        asyncWriterManager.commit();
        verifier.verify(writer.getBucket());
    } finally {
        writer.close();
    }
}

Example 2 with AbstractDocument

use of com.couchbase.client.java.document.AbstractDocument in project incubator-gobblin by apache.

the class CouchbaseWriterTest method writeRecords.

private List<Pair<AbstractDocument, Future>> writeRecords(Iterator<AbstractDocument> recordIterator, CouchbaseWriter writer, int outstandingRequests, long kvTimeout, TimeUnit kvTimeoutUnit) throws DataConversionException, UnsupportedEncodingException {
    final BlockingQueue<Pair<AbstractDocument, Future>> outstandingCallQueue = new LinkedBlockingDeque<>(outstandingRequests);
    final List<Pair<AbstractDocument, Future>> failedFutures = new ArrayList<>(outstandingRequests);
    int index = 0;
    long runTime = 0;
    final AtomicInteger callbackSuccesses = new AtomicInteger(0);
    final AtomicInteger callbackFailures = new AtomicInteger(0);
    final ConcurrentLinkedDeque<Throwable> callbackExceptions = new ConcurrentLinkedDeque<>();
    Verifier verifier = new Verifier();
    while (recordIterator.hasNext()) {
        AbstractDocument doc = recordIterator.next();
        index++;
        verifier.onWrite(doc);
        final long startTime = System.nanoTime();
        Future callFuture = writer.write(doc, new WriteCallback<TupleDocument>() {

            @Override
            public void onSuccess(WriteResponse<TupleDocument> writeResponse) {
                callbackSuccesses.incrementAndGet();
            }

            @Override
            public void onFailure(Throwable throwable) {
                callbackFailures.incrementAndGet();
                callbackExceptions.add(throwable);
            }
        });
        drainQueue(outstandingCallQueue, 1, kvTimeout, kvTimeoutUnit, failedFutures);
        outstandingCallQueue.add(new Pair<>(doc, callFuture));
        runTime += System.nanoTime() - startTime;
    }
    int failedWrites = 0;
    long responseStartTime = System.nanoTime();
    drainQueue(outstandingCallQueue, outstandingRequests, kvTimeout, kvTimeoutUnit, failedFutures);
    runTime += System.nanoTime() - responseStartTime;
    for (Throwable failure : callbackExceptions) {
        System.out.println(failure.getClass() + " : " + failure.getMessage());
    }
    failedWrites += failedFutures.size();
    System.out.println("Total time to send " + index + " records = " + runTime / 1000000.0 + "ms, " + "Failed writes = " + failedWrites + " Callback Successes = " + callbackSuccesses.get() + "Callback Failures = " + callbackFailures.get());
    verifier.verify(writer.getBucket());
    return failedFutures;
}