Examples with BatchInfo com.sforce.async.BatchInfo used on opensource projects

Example 6 with BatchInfo

use of com.sforce.async.BatchInfo in project tdi-studio-se by Talend.

the class SalesforceBulkAPI method getBatchLog.

public List<Map<String, String>> getBatchLog(int batchNum) throws AsyncApiException, IOException, ConnectionException {
    // batchInfoList was populated when batches were created and submitted
    List<Map<String, String>> resultInfoList = new ArrayList<Map<String, String>>();
    Map<String, String> resultInfo;
    BatchInfo b = batchInfoList.get(batchNum);
    CSVReader rdr = new CSVReader(connection.getBatchResultStream(job.getId(), b.getId()));
    List<String> resultHeader = rdr.nextRecord();
    int resultCols = resultHeader.size();
    List<String> row;
    while ((row = rdr.nextRecord()) != null) {
        resultInfo = new HashMap<String, String>();
        resultInfo.putAll(getBaseFileRow());
        for (int i = 0; i < resultCols; i++) {
            resultInfo.put(resultHeader.get(i), row.get(i));
        }
        resultInfoList.add(resultInfo);
    // boolean success = Boolean.valueOf(resultInfo.get("Success"));
    // boolean created = Boolean.valueOf(resultInfo.get("Created"));
    // String id = resultInfo.get("Id");
    // String error = resultInfo.get("Error");
    // if (success && created) {
    // System.out.println("Created row with id " + id);
    // } else if (!success) {
    // System.out.println("Failed with error: " + error);
    // }
    }
    return resultInfoList;
}

Example 7 with BatchInfo

use of com.sforce.async.BatchInfo in project tdi-studio-se by Talend.

the class SalesforceBulkAPI method awaitCompletion.

private void awaitCompletion() throws AsyncApiException, ConnectionException {
    long sleepTime = 0L;
    Set<String> incomplete = new HashSet<String>();
    for (BatchInfo bi : batchInfoList) {
        incomplete.add(bi.getId());
    }
    while (!incomplete.isEmpty()) {
        try {
            Thread.sleep(sleepTime);
        } catch (InterruptedException e) {
        }
        // System.out.println("Awaiting results..." + incomplete.size());
        sleepTime = awaitTime;
        BatchInfo[] statusList = connection.getBatchInfoList(job.getId()).getBatchInfo();
        for (BatchInfo b : statusList) {
            if (b.getState() == BatchStateEnum.Completed || b.getState() == BatchStateEnum.Failed) {
                incomplete.remove(b.getId());
            }
        }
    }
}

Example 8 with BatchInfo

use of com.sforce.async.BatchInfo in project tdi-studio-se by Talend.

the class SalesforceBulkAPI method doBulkQuery.

public void doBulkQuery(String moduleName, String queryStatement, int secToWait) throws AsyncApiException, InterruptedException, ConnectionException {
    job = new JobInfo();
    job.setObject(moduleName);
    job.setOperation(OperationEnum.query);
    if (concurrencyMode != null) {
        job.setConcurrencyMode(concurrencyMode);
    }
    job.setContentType(ContentType.CSV);
    job = connection.createJob(job);
    job = connection.getJobStatus(job.getId());
    batchInfoList = new ArrayList<BatchInfo>();
    BatchInfo info = null;
    ByteArrayInputStream bout = new ByteArrayInputStream(queryStatement.getBytes());
    info = connection.createBatchFromStream(job, bout);
    while (true) {
        // default is 30 sec
        Thread.sleep(secToWait * 1000);
        info = connection.getBatchInfo(job.getId(), info.getId());
        if (info.getState() == BatchStateEnum.Completed) {
            QueryResultList list = connection.getQueryResultList(job.getId(), info.getId());
            queryResultIDs = list.getResult();
            break;
        } else if (info.getState() == BatchStateEnum.Failed) {
            throw new RuntimeException("-------------- failed ----------" + info);
        } else {
            System.out.println("-------------- waiting ----------" + info);
        }
    }
    batchInfoList.add(info);
    // For TDI-27909
    closeJob();
}

Example 9 with BatchInfo

use of com.sforce.async.BatchInfo in project incubator-gobblin by apache.

the class SalesforceExtractor method waitForPkBatches.

/**
 * Waits for the PK batches to complete. The wait will stop after all batches are complete or on the first failed batch
 * @param batchInfoList list of batch info
 * @param retryInterval the polling interval
 * @return the last {@link BatchInfo} processed
 * @throws InterruptedException
 * @throws AsyncApiException
 */
private BatchInfo waitForPkBatches(BatchInfoList batchInfoList, int retryInterval) throws InterruptedException, AsyncApiException {
    BatchInfo batchInfo = null;
    BatchInfo[] batchInfos = batchInfoList.getBatchInfo();
    // Wait for all batches other than the first one. The first one is not processed in PK chunking mode
    for (int i = 1; i < batchInfos.length; i++) {
        BatchInfo bi = batchInfos[i];
        // get refreshed job status
        bi = this.bulkConnection.getBatchInfo(this.bulkJob.getId(), bi.getId());
        while ((bi.getState() != BatchStateEnum.Completed) && (bi.getState() != BatchStateEnum.Failed)) {
            Thread.sleep(retryInterval * 1000);
            bi = this.bulkConnection.getBatchInfo(this.bulkJob.getId(), bi.getId());
            log.debug("Bulk Api Batch Info:" + bi);
            log.info("Waiting for bulk resultSetIds");
        }
        batchInfo = bi;
        // exit if there was a failure
        if (batchInfo.getState() == BatchStateEnum.Failed) {
            break;
        }
    }
    return batchInfo;
}

Example 10 with BatchInfo

use of com.sforce.async.BatchInfo in project components by Talend.

the class SalesforceBulkRuntime method retrieveResultsOfQuery.

/**
 * Retrieve resultId(-s) from job batches info.
 * Results will be retrieved only from completed batches.
 *
 * When pk chunking is enabled, we need to go through all batches in the job.
 * More information on Salesforce documentation:
 * https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/
 * asynch_api_code_curl_walkthrough_pk_chunking.htm
 *
 * If some batches were queued or in progress, we must wait till they completed or failed/notprocessed.
 * Quick instructions for primary key chunking flow may be read here:
 * https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/asynch_api_bulk_query_processing.htm
 *
 * @param info - batch info from created job.
 * @throws AsyncApiException
 * @throws ConnectionException
 * @throws InterruptedException
 */
private void retrieveResultsOfQuery(BatchInfo info) throws AsyncApiException, ConnectionException, InterruptedException {
    if (BatchStateEnum.Completed == info.getState()) {
        QueryResultList list = getQueryResultList(job.getId(), info.getId());
        queryResultIDs = new HashSet<String>(Arrays.asList(list.getResult())).iterator();
        this.batchInfoList = Collections.singletonList(info);
        return;
    }
    BatchInfoList batchInfoList = null;
    Set<String> resultSet = new HashSet<>();
    boolean isInProgress = true;
    while (isInProgress) {
        batchInfoList = getBatchInfoList(job.getId());
        isInProgress = isJobBatchesInProgress(batchInfoList, info);
        if (isInProgress) {
            Thread.sleep(chunkSleepTime);
            long processingTime = System.currentTimeMillis() - job.getCreatedDate().getTimeInMillis();
            if (processingTime > MAX_BATCH_EXECUTION_TIME) {
                // Break processing and return processed data if any batch was processed.
                LOGGER.warn(MESSAGES.getMessage("warn.batch.timeout"));
                break;
            }
        }
    }
    for (BatchInfo batch : batchInfoList.getBatchInfo()) {
        if (batch.getId().equals(info.getId())) {
            continue;
        }
        resultSet.addAll(Arrays.asList(getQueryResultList(job.getId(), batch.getId()).getResult()));
    }
    queryResultIDs = resultSet.iterator();
    this.batchInfoList = Arrays.asList(batchInfoList.getBatchInfo());
}