Examples with QueryObjectsResult com.emc.object.s3.bean.QueryObjectsResult used on opensource projects

Example 1 with QueryObjectsResult

use of com.emc.object.s3.bean.QueryObjectsResult in project ecs-dashboard by carone1.

the class QueryObjectsCollection method queryObjects.

public boolean queryObjects() {
    String queryCriteria = collectionConfig.getQueryCriteria();
    // create request
    QueryObjectsRequest queryRequest;
    if (queryCriteria != null && !queryCriteria.isEmpty()) {
        // there is a criteria defined
        queryRequest = createQueryObjectRequest(queryCriteria);
    } else {
        // no criteria we will just collect all possible objects
        queryRequest = createQueryObjectRequest();
    }
    if (queryRequest == null) {
        // will be triggered
        return false;
    }
    Long bucketObjectCount = 0L;
    long startTime = System.currentTimeMillis();
    // Query Objects
    try {
        QueryObjectsResult queryResult = collectionConfig.getS3JerseyClient().queryObjects(queryRequest);
        long stopTime = System.currentTimeMillis();
        Double elapsedTime = Double.valueOf(stopTime - startTime) / 1000;
        if (queryResult != null) {
            Long collected = (long) queryResult.getObjects().size();
            // increase local counter
            bucketObjectCount += collected;
            // Increase central counter
            this.collectionConfig.getObjectCount().getAndAdd(collected);
            logger.info("Took: " + elapsedTime + " seconds to query " + collected + " objects from namespace: " + collectionConfig.getNamespace() + " bucket: " + queryResult.getBucketName());
            if (collectionConfig.getObjectDAO() != null) {
                collectionConfig.getObjectDAO().insert(queryResult, collectionConfig.getNamespace(), queryResult.getBucketName(), collectionConfig.getCollectionTime());
            }
            // extra pages to collect
            while (queryResult.isTruncated()) {
                // Move marker to beginning of next batch
                queryRequest.setMarker(queryResult.getNextMarker());
                startTime = System.currentTimeMillis();
                queryResult = this.collectionConfig.getS3JerseyClient().queryObjects(queryRequest);
                stopTime = System.currentTimeMillis();
                elapsedTime = Double.valueOf(stopTime - startTime) / 1000;
                collected = (long) queryResult.getObjects().size();
                // increase local counter
                bucketObjectCount += collected;
                // Increase central counter
                collectionConfig.getObjectCount().getAndAdd(collected);
                logger.info("Took: " + elapsedTime + " seconds to query " + collected + " objects from namespace: " + this.collectionConfig.getNamespace() + " bucket: " + queryResult.getBucketName());
                if (this.collectionConfig.getObjectDAO() != null) {
                    this.collectionConfig.getObjectDAO().insert(queryResult, collectionConfig.getNamespace(), queryResult.getBucketName(), collectionConfig.getCollectionTime());
                }
            }
        }
    } catch (Exception ex) {
        // known issue ECs returns this error when a bucket has MD keys but has not objects
        if (ex.getMessage().contains("Invalid search index value format or operator used")) {
            // just silently let this go. This error will eventually be fixed by ECS
            logger.error("Error: Namespace: " + collectionConfig.getNamespace() + " bucket: " + objectBucket.getName() + " Query string: `" + queryRequest.getQuery() + "`" + ex.getMessage());
        } else if (ex.getMessage().contains("We encountered an internal error. Please try again")) {
            // Here we could try again
            logger.error("Error: Namespace: " + collectionConfig.getNamespace() + " bucket: " + objectBucket.getName() + " Query string: `" + queryRequest.getQuery() + "`" + ex.getMessage());
        }
    }
    if (bucketObjectCount.compareTo(0L) == 0) {
        // will be triggered
        return false;
    }
    return true;
}

Example 2 with QueryObjectsResult

use of com.emc.object.s3.bean.QueryObjectsResult in project ecs-dashboard by carone1.

the class ElasticS3ObjectDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(QueryObjectsResult queryObjectsResult, String namespace, String bucketName, Date collectionTime) {
    if (queryObjectsResult == null || queryObjectsResult.getObjects() == null || queryObjectsResult.getObjects().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object buckets info
    for (QueryObject queryObject : queryObjectsResult.getObjects()) {
        XContentBuilder s3ObjectBuilder = toJsonFormat(queryObject, namespace, bucketName, collectionTime);
        IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectIndexDayName).setType(S3_OBJECT_INDEX_TYPE).setSource(s3ObjectBuilder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failure(s) occured while items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    }
}