Examples with HttpOperation org.apache.gobblin.http.HttpOperation used on opensource projects

Example 1 with HttpOperation

use of org.apache.gobblin.http.HttpOperation in project incubator-gobblin by apache.

the class AvroHttpJoinConverter method generateHttpOperation.

/**
 * Extract user defined keys by looking at "gobblin.converter.http.keys"
 * If keys are defined, extract key-value pair from inputRecord and set it to HttpOperation
 * If keys are not defined, generate HttpOperation by HttpUtils.toHttpOperation
 */
@Override
protected HttpOperation generateHttpOperation(GenericRecord inputRecord, State state) {
    Map<String, String> keyAndValue = new HashMap<>();
    Optional<Iterable<String>> keys = getKeys(state);
    HttpOperation operation;
    if (keys.isPresent()) {
        for (String key : keys.get()) {
            String value = inputRecord.get(key).toString();
            log.debug("Http join converter: key is {}, value is {}", key, value);
            keyAndValue.put(key, value);
        }
        operation = new HttpOperation();
        operation.setKeys(keyAndValue);
    } else {
        operation = HttpUtils.toHttpOperation(inputRecord);
    }
    return operation;
}

Example 2 with HttpOperation

use of org.apache.gobblin.http.HttpOperation in project incubator-gobblin by apache.

the class R2RestRequestBuilder method buildWriteRequest.

/**
 * Build a request from a single record
 */
private R2Request<GenericRecord> buildWriteRequest(BufferedRecord<GenericRecord> record) {
    if (record == null) {
        return null;
    }
    R2Request<GenericRecord> request = new R2Request<>();
    HttpOperation httpOperation = HttpUtils.toHttpOperation(record.getRecord());
    // Set uri
    URI uri = HttpUtils.buildURI(urlTemplate, httpOperation.getKeys(), httpOperation.getQueryParams());
    if (uri == null) {
        return null;
    }
    RestRequestBuilder builder = new RestRequestBuilder(uri).setMethod(method.getHttpMethod().toString());
    // Set headers
    Map<String, String> headers = httpOperation.getHeaders();
    if (headers != null && headers.size() != 0) {
        builder.setHeaders(headers);
    }
    builder.setHeader(RestConstants.HEADER_RESTLI_PROTOCOL_VERSION, protocolVersion);
    builder.setHeader(RestConstants.HEADER_RESTLI_REQUEST_METHOD, method.toString());
    // Add payload
    int bytesWritten = addPayload(builder, httpOperation.getBody());
    if (bytesWritten == -1) {
        throw new RuntimeException("Fail to write payload into request");
    }
    request.markRecord(record, bytesWritten);
    request.setRawRequest(build(builder));
    return request;
}

Example 3 with HttpOperation

use of org.apache.gobblin.http.HttpOperation in project incubator-gobblin by apache.

the class AsyncHttpJoinConverter method convertRecordAsync.

/**
 * Convert an input record to a future object where an output record will be filled in sometime later
 * Sequence:
 *    Convert input (DI) to an http request
 *    Send http request asynchronously, and registers an http callback
 *    Create an {@link CompletableFuture} object. When the callback is invoked, this future object is filled in by an output record which is converted from http response.
 *    Return the future object.
 */
@Override
public final CompletableFuture<DO> convertRecordAsync(SO outputSchema, DI inputRecord, WorkUnitState workUnitState) throws DataConversionException {
    // Convert DI to HttpOperation
    HttpOperation operation = generateHttpOperation(inputRecord, workUnitState);
    BufferedRecord<GenericRecord> bufferedRecord = new BufferedRecord<>(operation, WriteCallback.EMPTY);
    // Convert HttpOperation to RQ
    Queue<BufferedRecord<GenericRecord>> buffer = new LinkedBlockingDeque<>();
    buffer.add(bufferedRecord);
    AsyncRequest<GenericRecord, RQ> request = this.requestBuilder.buildRequest(buffer);
    RQ rawRequest = request.getRawRequest();
    // Execute query and get response
    AsyncHttpJoinConverterContext context = new AsyncHttpJoinConverterContext(this, outputSchema, inputRecord, request);
    try {
        httpClient.sendAsyncRequest(rawRequest, context.getCallback());
    } catch (IOException e) {
        throw new DataConversionException(e);
    }
    return context.future;
}

Example 4 with HttpOperation

use of org.apache.gobblin.http.HttpOperation in project incubator-gobblin by apache.

the class HttpJoinConverter method convertRecord.

@Override
public final Iterable<DO> convertRecord(SO outputSchema, DI inputRecord, WorkUnitState workUnitState) throws DataConversionException {
    // Convert DI to HttpOperation
    HttpOperation operation = generateHttpOperation(inputRecord, workUnitState);
    BufferedRecord<GenericRecord> bufferedRecord = new BufferedRecord<>(operation, WriteCallback.EMPTY);
    // Convert HttpOperation to RQ
    Queue<BufferedRecord<GenericRecord>> buffer = new LinkedBlockingDeque<>();
    buffer.add(bufferedRecord);
    AsyncRequest<GenericRecord, RQ> request = this.requestBuilder.buildRequest(buffer);
    RQ rawRequest = request.getRawRequest();
    try {
        RP response = httpClient.sendRequest(rawRequest);
        ResponseStatus status = responseHandler.handleResponse(request, response);
        switch(status.getType()) {
            case OK:
            case CLIENT_ERROR:
                // Convert (DI, RQ, RP etc..) to output DO
                log.debug("{} send with status type {}", rawRequest, status.getType());
                DO output = convertRecordImpl(outputSchema, inputRecord, rawRequest, status);
                return new SingleRecordIterable<>(output);
            case SERVER_ERROR:
                // Server side error. Retry
                throw new DataConversionException(rawRequest + " send failed due to server error");
            default:
                throw new DataConversionException(rawRequest + " Should not reach here");
        }
    } catch (IOException e) {
        throw new DataConversionException(e);
    }
}

Example 5 with HttpOperation

use of org.apache.gobblin.http.HttpOperation in project incubator-gobblin by apache.

the class HttpTestUtils method createQueue.

public static Queue<BufferedRecord<GenericRecord>> createQueue(int size, boolean isHttpOperation) {
    Queue<BufferedRecord<GenericRecord>> queue = new ArrayDeque<>(size);
    for (int i = 0; i < size; i++) {
        Map<String, String> keys = new HashMap<>();
        keys.put("part1", i + "1");
        keys.put("part2", i + "2");
        Map<String, String> queryParams = new HashMap<>();
        queryParams.put("param1", i + "1");
        GenericRecord record = isHttpOperation ? new HttpOperation() : new MockGenericRecord();
        record.put("keys", keys);
        record.put("queryParams", queryParams);
        record.put("body", "{\"id\":\"id" + i + "\"}");
        BufferedRecord<GenericRecord> item = new BufferedRecord<>(record, null);
        queue.add(item);
    }
    return queue;
}