Examples with TupleDocument org.apache.gobblin.couchbase.common.TupleDocument used on opensource projects

Example 1 with TupleDocument

use of org.apache.gobblin.couchbase.common.TupleDocument in project incubator-gobblin by apache.

the class AvroToCouchbaseTupleConverterTest method testBasicConvert.

@Test
public void testBasicConvert() throws Exception {
    Schema dataRecordSchema = SchemaBuilder.record("Data").fields().name("data").type().bytesType().noDefault().name("flags").type().intType().noDefault().endRecord();
    Schema schema = SchemaBuilder.record("TestRecord").fields().name("key").type().stringType().noDefault().name("data").type(dataRecordSchema).noDefault().endRecord();
    GenericData.Record testRecord = new GenericData.Record(schema);
    String testContent = "hello world";
    GenericData.Record dataRecord = new GenericData.Record(dataRecordSchema);
    dataRecord.put("data", ByteBuffer.wrap(testContent.getBytes(Charset.forName("UTF-8"))));
    dataRecord.put("flags", 0);
    testRecord.put("key", "hello");
    testRecord.put("data", dataRecord);
    Converter<Schema, String, GenericRecord, TupleDocument> recordConverter = new AvroToCouchbaseTupleConverter();
    TupleDocument returnDoc = recordConverter.convertRecord("", testRecord, null).iterator().next();
    byte[] returnedBytes = new byte[returnDoc.content().value1().readableBytes()];
    returnDoc.content().value1().readBytes(returnedBytes);
    Assert.assertEquals(returnedBytes, testContent.getBytes(Charset.forName("UTF-8")));
    int returnedFlags = returnDoc.content().value2();
    Assert.assertEquals(returnedFlags, 0);
}

Example 2 with TupleDocument

use of org.apache.gobblin.couchbase.common.TupleDocument in project incubator-gobblin by apache.

the class CouchbaseWriterTest method testTupleDocumentWrite.

/**
 * Test that a single tuple document can be written successfully.
 * @throws IOException
 * @throws DataConversionException
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Test
public void testTupleDocumentWrite() throws IOException, DataConversionException, ExecutionException, InterruptedException {
    Properties props = new Properties();
    props.setProperty(CouchbaseWriterConfigurationKeys.BUCKET, "default");
    Config config = ConfigFactory.parseProperties(props);
    CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, config);
    try {
        Schema dataRecordSchema = SchemaBuilder.record("Data").fields().name("data").type().bytesType().noDefault().name("flags").type().intType().noDefault().endRecord();
        Schema schema = SchemaBuilder.record("TestRecord").fields().name("key").type().stringType().noDefault().name("data").type(dataRecordSchema).noDefault().endRecord();
        GenericData.Record testRecord = new GenericData.Record(schema);
        String testContent = "hello world";
        GenericData.Record dataRecord = new GenericData.Record(dataRecordSchema);
        dataRecord.put("data", ByteBuffer.wrap(testContent.getBytes(Charset.forName("UTF-8"))));
        dataRecord.put("flags", 0);
        testRecord.put("key", "hello");
        testRecord.put("data", dataRecord);
        Converter<Schema, String, GenericRecord, TupleDocument> recordConverter = new AvroToCouchbaseTupleConverter();
        TupleDocument doc = recordConverter.convertRecord("", testRecord, null).iterator().next();
        writer.write(doc, null).get();
        TupleDocument returnDoc = writer.getBucket().get("hello", TupleDocument.class);
        byte[] returnedBytes = new byte[returnDoc.content().value1().readableBytes()];
        returnDoc.content().value1().readBytes(returnedBytes);
        Assert.assertEquals(returnedBytes, testContent.getBytes(Charset.forName("UTF-8")));
        int returnedFlags = returnDoc.content().value2();
        Assert.assertEquals(returnedFlags, 0);
    } finally {
        writer.close();
    }
}

Example 3 with TupleDocument

use of org.apache.gobblin.couchbase.common.TupleDocument in project incubator-gobblin by apache.

the class CouchbaseWriter method write.

@Override
public Future<WriteResponse> write(final D record, final WriteCallback callback) {
    assertRecordWritable(record);
    if (record instanceof TupleDocument) {
        ((TupleDocument) record).content().value1().retain();
    }
    Observable<D> observable = _bucket.async().upsert(record);
    if (callback == null) {
        return new WriteResponseFuture<>(observable.timeout(_operationTimeout, _operationTimeunit).toBlocking().toFuture(), _defaultWriteResponseMapper);
    } else {
        final AtomicBoolean callbackFired = new AtomicBoolean(false);
        final BlockingQueue<Pair<WriteResponse, Throwable>> writeResponseQueue = new ArrayBlockingQueue<>(1);
        final Future<WriteResponse> writeResponseFuture = new Future<WriteResponse>() {

            @Override
            public boolean cancel(boolean mayInterruptIfRunning) {
                return false;
            }

            @Override
            public boolean isCancelled() {
                return false;
            }

            @Override
            public boolean isDone() {
                return callbackFired.get();
            }

            @Override
            public WriteResponse get() throws InterruptedException, ExecutionException {
                Pair<WriteResponse, Throwable> writeResponseThrowablePair = writeResponseQueue.take();
                return getWriteResponseorThrow(writeResponseThrowablePair);
            }

            @Override
            public WriteResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
                Pair<WriteResponse, Throwable> writeResponseThrowablePair = writeResponseQueue.poll(timeout, unit);
                if (writeResponseThrowablePair == null) {
                    throw new TimeoutException("Timeout exceeded while waiting for future to be done");
                } else {
                    return getWriteResponseorThrow(writeResponseThrowablePair);
                }
            }
        };
        observable.timeout(_operationTimeout, _operationTimeunit).subscribe(new Subscriber<D>() {

            @Override
            public void onCompleted() {
            }

            @Override
            public void onError(Throwable e) {
                callbackFired.set(true);
                writeResponseQueue.add(new Pair<WriteResponse, Throwable>(null, e));
                callback.onFailure(e);
            }

            @Override
            public void onNext(D doc) {
                try {
                    callbackFired.set(true);
                    WriteResponse writeResponse = new GenericWriteResponse<D>(doc);
                    writeResponseQueue.add(new Pair<WriteResponse, Throwable>(writeResponse, null));
                    callback.onSuccess(writeResponse);
                } finally {
                    if (doc instanceof TupleDocument) {
                        ((TupleDocument) doc).content().value1().release();
                    }
                }
            }
        });
        return writeResponseFuture;
    }
}

Example 4 with TupleDocument

use of org.apache.gobblin.couchbase.common.TupleDocument in project incubator-gobblin by apache.

the class CouchbaseWriterTest method writeRecords.

private List<Pair<AbstractDocument, Future>> writeRecords(Iterator<AbstractDocument> recordIterator, CouchbaseWriter writer, int outstandingRequests, long kvTimeout, TimeUnit kvTimeoutUnit) throws DataConversionException, UnsupportedEncodingException {
    final BlockingQueue<Pair<AbstractDocument, Future>> outstandingCallQueue = new LinkedBlockingDeque<>(outstandingRequests);
    final List<Pair<AbstractDocument, Future>> failedFutures = new ArrayList<>(outstandingRequests);
    int index = 0;
    long runTime = 0;
    final AtomicInteger callbackSuccesses = new AtomicInteger(0);
    final AtomicInteger callbackFailures = new AtomicInteger(0);
    final ConcurrentLinkedDeque<Throwable> callbackExceptions = new ConcurrentLinkedDeque<>();
    Verifier verifier = new Verifier();
    while (recordIterator.hasNext()) {
        AbstractDocument doc = recordIterator.next();
        index++;
        verifier.onWrite(doc);
        final long startTime = System.nanoTime();
        Future callFuture = writer.write(doc, new WriteCallback<TupleDocument>() {

            @Override
            public void onSuccess(WriteResponse<TupleDocument> writeResponse) {
                callbackSuccesses.incrementAndGet();
            }

            @Override
            public void onFailure(Throwable throwable) {
                callbackFailures.incrementAndGet();
                callbackExceptions.add(throwable);
            }
        });
        drainQueue(outstandingCallQueue, 1, kvTimeout, kvTimeoutUnit, failedFutures);
        outstandingCallQueue.add(new Pair<>(doc, callFuture));
        runTime += System.nanoTime() - startTime;
    }
    int failedWrites = 0;
    long responseStartTime = System.nanoTime();
    drainQueue(outstandingCallQueue, outstandingRequests, kvTimeout, kvTimeoutUnit, failedFutures);
    runTime += System.nanoTime() - responseStartTime;
    for (Throwable failure : callbackExceptions) {
        System.out.println(failure.getClass() + " : " + failure.getMessage());
    }
    failedWrites += failedFutures.size();
    System.out.println("Total time to send " + index + " records = " + runTime / 1000000.0 + "ms, " + "Failed writes = " + failedWrites + " Callback Successes = " + callbackSuccesses.get() + "Callback Failures = " + callbackFailures.get());
    verifier.verify(writer.getBucket());
    return failedFutures;
}

Example 5 with TupleDocument

use of org.apache.gobblin.couchbase.common.TupleDocument in project incubator-gobblin by apache.

the class AvroToCouchbaseTupleConverter method convertRecord.

@Override
public Iterable<TupleDocument> convertRecord(String outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException {
    String key = inputRecord.get(keyField).toString();
    GenericRecord data = (GenericRecord) inputRecord.get(dataRecordField);
    ByteBuffer dataBytes = (ByteBuffer) data.get(valueField);
    Integer flags = (Integer) data.get(flagsField);
    ByteBuf buffer = Unpooled.copiedBuffer(dataBytes);
    return new SingleRecordIterable<>(new TupleDocument(key, Tuple.create(buffer, flags)));
}