Examples with BufferedRecord org.apache.gobblin.async.BufferedRecord used on opensource projects

Example 1 with BufferedRecord

use of org.apache.gobblin.async.BufferedRecord in project incubator-gobblin by apache.

the class AbstractAsyncDataWriter method write.

/**
 * Asynchronously write the record with a callback
 */
@Override
public final Future<WriteResponse> write(D record, @Nullable WriteCallback callback) {
    FutureWrappedWriteCallback wrappedWriteCallback = new FutureWrappedWriteCallback(callback);
    BufferedRecord<D> bufferedRecord = new BufferedRecord<>(record, wrappedWriteCallback);
    put(bufferedRecord);
    return wrappedWriteCallback;
}

Example 2 with BufferedRecord

use of org.apache.gobblin.async.BufferedRecord in project incubator-gobblin by apache.

the class ApacheHttpRequestBuilderTest method testBuildWriteRequest.

/**
 * Build a {@link HttpUriRequest} from a {@link GenericRecord}
 */
public void testBuildWriteRequest() throws IOException {
    String urlTemplate = "http://www.test.com/a/part1:${part1}/a/part2:${part2}";
    String verb = "post";
    ApacheHttpRequestBuilder builder = spy(new ApacheHttpRequestBuilder(urlTemplate, verb, "application/json"));
    ArgumentCaptor<RequestBuilder> requestBuilderArgument = ArgumentCaptor.forClass(RequestBuilder.class);
    Queue<BufferedRecord<GenericRecord>> queue = HttpTestUtils.createQueue(1, false);
    AsyncRequest<GenericRecord, HttpUriRequest> request = builder.buildRequest(queue);
    verify(builder).build(requestBuilderArgument.capture());
    RequestBuilder expected = RequestBuilder.post();
    expected.setUri("http://www.test.com/a/part1:01/a/part2:02?param1=01");
    String payloadStr = "{\"id\":\"id0\"}";
    expected.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()).setEntity(new StringEntity(payloadStr, ContentType.APPLICATION_JSON));
    // Compare HttpUriRequest
    HttpTestUtils.assertEqual(requestBuilderArgument.getValue(), expected);
    Assert.assertEquals(request.getRecordCount(), 1);
    Assert.assertEquals(queue.size(), 0);
}

Example 3 with BufferedRecord

use of org.apache.gobblin.async.BufferedRecord in project incubator-gobblin by apache.

the class R2RestRequestBuilderTest method testBuildWriteRequest.

/**
 * Build a {@link RestRequest} from a {@link GenericRecord}
 */
public void testBuildWriteRequest() throws URISyntaxException, IOException {
    String urlTemplate = "http://www.test.com/a/part1:${part1}/a/part2:${part2}";
    String verb = "update";
    String protocolVersion = "2.0.0";
    R2RestRequestBuilder builder = spy(new R2RestRequestBuilder(urlTemplate, verb, protocolVersion));
    ArgumentCaptor<RestRequestBuilder> requestBuilderArgument = ArgumentCaptor.forClass(RestRequestBuilder.class);
    Queue<BufferedRecord<GenericRecord>> queue = HttpTestUtils.createQueue(1, false);
    AsyncRequest<GenericRecord, RestRequest> request = builder.buildRequest(queue);
    verify(builder).build(requestBuilderArgument.capture());
    RestRequestBuilder expected = new RestRequestBuilder(new URI("http://www.test.com/a/part1:01/a/part2:02?param1=01"));
    expected.setMethod("PUT");
    expected.setHeader(RestConstants.HEADER_RESTLI_PROTOCOL_VERSION, protocolVersion);
    expected.setHeader(RestConstants.HEADER_RESTLI_REQUEST_METHOD, verb.toLowerCase());
    expected.setHeader(RestConstants.HEADER_CONTENT_TYPE, RestConstants.HEADER_VALUE_APPLICATION_JSON);
    DataMap data = new DataMap();
    data.put("id", "id0");
    expected.setEntity(JACKSON_DATA_CODEC.mapToBytes(data));
    HttpTestUtils.assertEqual(requestBuilderArgument.getValue(), expected);
    Assert.assertEquals(request.getRecordCount(), 1);
    Assert.assertEquals(queue.size(), 0);
}

Example 4 with BufferedRecord

use of org.apache.gobblin.async.BufferedRecord in project incubator-gobblin by apache.

the class HttpTestUtils method createQueue.

public static Queue<BufferedRecord<GenericRecord>> createQueue(int size, boolean isHttpOperation) {
    Queue<BufferedRecord<GenericRecord>> queue = new ArrayDeque<>(size);
    for (int i = 0; i < size; i++) {
        Map<String, String> keys = new HashMap<>();
        keys.put("part1", i + "1");
        keys.put("part2", i + "2");
        Map<String, String> queryParams = new HashMap<>();
        queryParams.put("param1", i + "1");
        GenericRecord record = isHttpOperation ? new HttpOperation() : new MockGenericRecord();
        record.put("keys", keys);
        record.put("queryParams", queryParams);
        record.put("body", "{\"id\":\"id" + i + "\"}");
        BufferedRecord<GenericRecord> item = new BufferedRecord<>(record, null);
        queue.add(item);
    }
    return queue;
}

Example 5 with BufferedRecord

use of org.apache.gobblin.async.BufferedRecord in project incubator-gobblin by apache.

the class HttpJoinConverter method convertRecord.

@Override
public final Iterable<DO> convertRecord(SO outputSchema, DI inputRecord, WorkUnitState workUnitState) throws DataConversionException {
    // Convert DI to HttpOperation
    HttpOperation operation = generateHttpOperation(inputRecord, workUnitState);
    BufferedRecord<GenericRecord> bufferedRecord = new BufferedRecord<>(operation, WriteCallback.EMPTY);
    // Convert HttpOperation to RQ
    Queue<BufferedRecord<GenericRecord>> buffer = new LinkedBlockingDeque<>();
    buffer.add(bufferedRecord);
    AsyncRequest<GenericRecord, RQ> request = this.requestBuilder.buildRequest(buffer);
    RQ rawRequest = request.getRawRequest();
    try {
        RP response = httpClient.sendRequest(rawRequest);
        ResponseStatus status = responseHandler.handleResponse(request, response);
        switch(status.getType()) {
            case OK:
            case CLIENT_ERROR:
                // Convert (DI, RQ, RP etc..) to output DO
                log.debug("{} send with status type {}", rawRequest, status.getType());
                DO output = convertRecordImpl(outputSchema, inputRecord, rawRequest, status);
                return new SingleRecordIterable<>(output);
            case SERVER_ERROR:
                // Server side error. Retry
                throw new DataConversionException(rawRequest + " send failed due to server error");
            default:
                throw new DataConversionException(rawRequest + " Should not reach here");
        }
    } catch (IOException e) {
        throw new DataConversionException(e);
    }
}