Examples with RestApiProcessingException org.apache.gobblin.source.extractor.exception.RestApiProcessingException used on opensource projects

Example 1 with RestApiProcessingException

use of org.apache.gobblin.source.extractor.exception.RestApiProcessingException in project incubator-gobblin by apache.

the class RestApiExtractor method extractMetadata.

@Override
public void extractMetadata(String schema, String entity, WorkUnit workUnit) throws SchemaException {
    log.info("Extract Metadata using Rest Api");
    JsonArray columnArray = new JsonArray();
    String inputQuery = workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_QUERY);
    List<String> columnListInQuery = null;
    JsonArray array = null;
    if (!Strings.isNullOrEmpty(inputQuery)) {
        columnListInQuery = Utils.getColumnListFromQuery(inputQuery);
    }
    String excludedColumns = workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_EXCLUDED_COLUMNS);
    List<String> columnListExcluded = ImmutableList.<String>of();
    if (Strings.isNullOrEmpty(inputQuery) && !Strings.isNullOrEmpty(excludedColumns)) {
        Splitter splitter = Splitter.on(",").omitEmptyStrings().trimResults();
        columnListExcluded = splitter.splitToList(excludedColumns.toLowerCase());
    }
    try {
        boolean success = this.connector.connect();
        if (!success) {
            throw new SchemaException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        List<Command> cmds = this.getSchemaMetadata(schema, entity);
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        array = this.getSchema(response);
        for (JsonElement columnElement : array) {
            Schema obj = GSON.fromJson(columnElement, Schema.class);
            String columnName = obj.getColumnName();
            obj.setWaterMark(this.isWatermarkColumn(workUnitState.getProp("extract.delta.fields"), columnName));
            if (this.isWatermarkColumn(workUnitState.getProp("extract.delta.fields"), columnName)) {
                obj.setNullable(false);
            } else if (this.getPrimarykeyIndex(workUnitState.getProp("extract.primary.key.fields"), columnName) == 0) {
                // set all columns as nullable except primary key and watermark columns
                obj.setNullable(true);
            }
            obj.setPrimaryKey(this.getPrimarykeyIndex(workUnitState.getProp("extract.primary.key.fields"), columnName));
            String jsonStr = GSON.toJson(obj);
            JsonObject jsonObject = GSON.fromJson(jsonStr, JsonObject.class).getAsJsonObject();
            // Else, consider only the columns mentioned in the column list
            if (inputQuery == null || columnListInQuery == null || (columnListInQuery.size() == 1 && columnListInQuery.get(0).equals("*")) || (columnListInQuery.size() >= 1 && this.isMetadataColumn(columnName, columnListInQuery))) {
                if (!columnListExcluded.contains(columnName.trim().toLowerCase())) {
                    this.columnList.add(columnName);
                    columnArray.add(jsonObject);
                }
            }
        }
        this.updatedQuery = buildDataQuery(inputQuery, entity);
        log.info("Schema:" + columnArray);
        this.setOutputSchema(columnArray);
    } catch (RuntimeException | RestApiConnectionException | RestApiProcessingException | IOException | SchemaException e) {
        throw new SchemaException("Failed to get schema using rest api; error - " + e.getMessage(), e);
    }
}

Example 2 with RestApiProcessingException

use of org.apache.gobblin.source.extractor.exception.RestApiProcessingException in project incubator-gobblin by apache.

the class RestApiConnector method getResponse.

/**
 * get http response in json format using url
 * @return json string with the response
 */
public CommandOutput<?, ?> getResponse(List<Command> cmds) throws RestApiProcessingException {
    String url = cmds.get(0).getParams().get(0);
    log.info("URL: " + url);
    String jsonStr = null;
    HttpRequestBase httpRequest = new HttpGet(url);
    addHeaders(httpRequest);
    HttpEntity httpEntity = null;
    HttpResponse httpResponse = null;
    try {
        httpResponse = this.httpClient.execute(httpRequest);
        StatusLine status = httpResponse.getStatusLine();
        httpEntity = httpResponse.getEntity();
        if (httpEntity != null) {
            jsonStr = EntityUtils.toString(httpEntity);
        }
        if (status.getStatusCode() >= 400) {
            log.info("Unable to get response using: " + url);
            JsonElement jsonRet = GSON.fromJson(jsonStr, JsonArray.class);
            throw new RestApiProcessingException(getFirstErrorMessage("Failed to retrieve response from", jsonRet));
        }
    } catch (Exception e) {
        throw new RestApiProcessingException("Failed to process rest api request; error - " + e.getMessage(), e);
    } finally {
        try {
            if (httpEntity != null) {
                EntityUtils.consume(httpEntity);
            }
        // httpResponse.close();
        } catch (Exception e) {
            throw new RestApiProcessingException("Failed to consume httpEntity; error - " + e.getMessage(), e);
        }
    }
    CommandOutput<RestApiCommand, String> output = new RestApiCommandOutput();
    output.put((RestApiCommand) cmds.get(0), jsonStr);
    return output;
}