Examples with RestApiConnectionException org.apache.gobblin.source.extractor.exception.RestApiConnectionException used on opensource projects

Example 1 with RestApiConnectionException

use of org.apache.gobblin.source.extractor.exception.RestApiConnectionException in project incubator-gobblin by apache.

the class RestApiConnector method connect.

/**
 * get http connection
 * @return true if the connection is success else false
 */
public boolean connect() throws RestApiConnectionException {
    if (this.autoEstablishAuthToken) {
        if (this.authTokenTimeout <= 0) {
            return false;
        } else if ((System.currentTimeMillis() - this.createdAt) > this.authTokenTimeout) {
            return false;
        }
    }
    HttpEntity httpEntity = null;
    try {
        httpEntity = getAuthentication();
        if (httpEntity != null) {
            JsonElement json = GSON.fromJson(EntityUtils.toString(httpEntity), JsonObject.class);
            if (json == null) {
                log.error("Http entity: " + httpEntity);
                log.error("entity class: " + httpEntity.getClass().getName());
                log.error("entity string size: " + EntityUtils.toString(httpEntity).length());
                log.error("content length: " + httpEntity.getContentLength());
                log.error("content: " + IOUtils.toString(httpEntity.getContent(), Charsets.UTF_8));
                throw new RestApiConnectionException("JSON is NULL ! Failed on authentication with the following HTTP response received:\n" + EntityUtils.toString(httpEntity));
            }
            JsonObject jsonRet = json.getAsJsonObject();
            log.info("jsonRet: " + jsonRet.toString());
            parseAuthenticationResponse(jsonRet);
        }
    } catch (IOException e) {
        throw new RestApiConnectionException("Failed to get rest api connection; error - " + e.getMessage(), e);
    } finally {
        if (httpEntity != null) {
            try {
                EntityUtils.consume(httpEntity);
            } catch (IOException e) {
                throw new RestApiConnectionException("Failed to consume httpEntity; error - " + e.getMessage(), e);
            }
        }
    }
    return true;
}

Example 2 with RestApiConnectionException

use of org.apache.gobblin.source.extractor.exception.RestApiConnectionException in project incubator-gobblin by apache.

the class RestApiExtractor method extractMetadata.

@Override
public void extractMetadata(String schema, String entity, WorkUnit workUnit) throws SchemaException {
    log.info("Extract Metadata using Rest Api");
    JsonArray columnArray = new JsonArray();
    String inputQuery = workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_QUERY);
    List<String> columnListInQuery = null;
    JsonArray array = null;
    if (!Strings.isNullOrEmpty(inputQuery)) {
        columnListInQuery = Utils.getColumnListFromQuery(inputQuery);
    }
    String excludedColumns = workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_EXCLUDED_COLUMNS);
    List<String> columnListExcluded = ImmutableList.<String>of();
    if (Strings.isNullOrEmpty(inputQuery) && !Strings.isNullOrEmpty(excludedColumns)) {
        Splitter splitter = Splitter.on(",").omitEmptyStrings().trimResults();
        columnListExcluded = splitter.splitToList(excludedColumns.toLowerCase());
    }
    try {
        boolean success = this.connector.connect();
        if (!success) {
            throw new SchemaException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        List<Command> cmds = this.getSchemaMetadata(schema, entity);
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        array = this.getSchema(response);
        for (JsonElement columnElement : array) {
            Schema obj = GSON.fromJson(columnElement, Schema.class);
            String columnName = obj.getColumnName();
            obj.setWaterMark(this.isWatermarkColumn(workUnitState.getProp("extract.delta.fields"), columnName));
            if (this.isWatermarkColumn(workUnitState.getProp("extract.delta.fields"), columnName)) {
                obj.setNullable(false);
            } else if (this.getPrimarykeyIndex(workUnitState.getProp("extract.primary.key.fields"), columnName) == 0) {
                // set all columns as nullable except primary key and watermark columns
                obj.setNullable(true);
            }
            obj.setPrimaryKey(this.getPrimarykeyIndex(workUnitState.getProp("extract.primary.key.fields"), columnName));
            String jsonStr = GSON.toJson(obj);
            JsonObject jsonObject = GSON.fromJson(jsonStr, JsonObject.class).getAsJsonObject();
            // Else, consider only the columns mentioned in the column list
            if (inputQuery == null || columnListInQuery == null || (columnListInQuery.size() == 1 && columnListInQuery.get(0).equals("*")) || (columnListInQuery.size() >= 1 && this.isMetadataColumn(columnName, columnListInQuery))) {
                if (!columnListExcluded.contains(columnName.trim().toLowerCase())) {
                    this.columnList.add(columnName);
                    columnArray.add(jsonObject);
                }
            }
        }
        this.updatedQuery = buildDataQuery(inputQuery, entity);
        log.info("Schema:" + columnArray);
        this.setOutputSchema(columnArray);
    } catch (RuntimeException | RestApiConnectionException | RestApiProcessingException | IOException | SchemaException e) {
        throw new SchemaException("Failed to get schema using rest api; error - " + e.getMessage(), e);
    }
}

Example 3 with RestApiConnectionException

use of org.apache.gobblin.source.extractor.exception.RestApiConnectionException in project incubator-gobblin by apache.

the class SalesforceConnector method getAuthentication.

@Override
public HttpEntity getAuthentication() throws RestApiConnectionException {
    log.debug("Authenticating salesforce");
    String clientId = this.state.getProp(ConfigurationKeys.SOURCE_CONN_CLIENT_ID);
    String clientSecret = this.state.getProp(ConfigurationKeys.SOURCE_CONN_CLIENT_SECRET);
    String host = this.state.getProp(ConfigurationKeys.SOURCE_CONN_HOST_NAME);
    List<NameValuePair> formParams = Lists.newArrayList();
    formParams.add(new BasicNameValuePair("client_id", clientId));
    formParams.add(new BasicNameValuePair("client_secret", clientSecret));
    if (refreshToken == null) {
        log.info("Authenticating salesforce with username/password");
        String userName = this.state.getProp(ConfigurationKeys.SOURCE_CONN_USERNAME);
        String password = PasswordManager.getInstance(this.state).readPassword(this.state.getProp(ConfigurationKeys.SOURCE_CONN_PASSWORD));
        String securityToken = this.state.getProp(ConfigurationKeys.SOURCE_CONN_SECURITY_TOKEN);
        formParams.add(new BasicNameValuePair("grant_type", "password"));
        formParams.add(new BasicNameValuePair("username", userName));
        formParams.add(new BasicNameValuePair("password", password + securityToken));
    } else {
        log.info("Authenticating salesforce with refresh_token");
        formParams.add(new BasicNameValuePair("grant_type", "refresh_token"));
        formParams.add(new BasicNameValuePair("refresh_token", refreshToken));
    }
    try {
        HttpPost post = new HttpPost(host + DEFAULT_AUTH_TOKEN_PATH);
        post.setEntity(new UrlEncodedFormEntity(formParams));
        HttpResponse httpResponse = getHttpClient().execute(post);
        HttpEntity httpEntity = httpResponse.getEntity();
        return httpEntity;
    } catch (Exception e) {
        throw new RestApiConnectionException("Failed to authenticate salesforce host:" + host + "; error-" + e.getMessage(), e);
    }
}

Example 4 with RestApiConnectionException

use of org.apache.gobblin.source.extractor.exception.RestApiConnectionException in project incubator-gobblin by apache.

the class SalesforceSource method getHistogram.

/**
 * Generate the histogram
 */
private Histogram getHistogram(String entity, String watermarkColumn, SourceState state, Partition partition) {
    SalesforceConnector connector = new SalesforceConnector(state);
    try {
        if (!connector.connect()) {
            throw new RuntimeException("Failed to connect.");
        }
    } catch (RestApiConnectionException e) {
        throw new RuntimeException("Failed to connect.", e);
    }
    Histogram histogram = getHistogramByDayBucketing(connector, entity, watermarkColumn, partition);
    // exchange the first histogram group key with the global low watermark to ensure that the low watermark is captured
    // in the range of generated partitions
    HistogramGroup firstGroup = histogram.get(0);
    Date lwmDate = Utils.toDate(partition.getLowWatermark(), Partitioner.WATERMARKTIMEFORMAT);
    histogram.getGroups().set(0, new HistogramGroup(Utils.epochToDate(lwmDate.getTime(), SECONDS_FORMAT), firstGroup.getCount()));
    // refine the histogram
    if (state.getPropAsBoolean(ENABLE_DYNAMIC_PROBING)) {
        histogram = getRefinedHistogram(connector, entity, watermarkColumn, state, partition, histogram);
    }
    return histogram;
}