Examples with QueryObjectsRequest com.emc.object.s3.request.QueryObjectsRequest used on opensource projects

Example 1 with QueryObjectsRequest

use of com.emc.object.s3.request.QueryObjectsRequest in project ecs-dashboard by carone1.

the class QueryObjectsCollection method queryObjects.

public boolean queryObjects() {
    String queryCriteria = collectionConfig.getQueryCriteria();
    // create request
    QueryObjectsRequest queryRequest;
    if (queryCriteria != null && !queryCriteria.isEmpty()) {
        // there is a criteria defined
        queryRequest = createQueryObjectRequest(queryCriteria);
    } else {
        // no criteria we will just collect all possible objects
        queryRequest = createQueryObjectRequest();
    }
    if (queryRequest == null) {
        // will be triggered
        return false;
    }
    Long bucketObjectCount = 0L;
    long startTime = System.currentTimeMillis();
    // Query Objects
    try {
        QueryObjectsResult queryResult = collectionConfig.getS3JerseyClient().queryObjects(queryRequest);
        long stopTime = System.currentTimeMillis();
        Double elapsedTime = Double.valueOf(stopTime - startTime) / 1000;
        if (queryResult != null) {
            Long collected = (long) queryResult.getObjects().size();
            // increase local counter
            bucketObjectCount += collected;
            // Increase central counter
            this.collectionConfig.getObjectCount().getAndAdd(collected);
            logger.info("Took: " + elapsedTime + " seconds to query " + collected + " objects from namespace: " + collectionConfig.getNamespace() + " bucket: " + queryResult.getBucketName());
            if (collectionConfig.getObjectDAO() != null) {
                collectionConfig.getObjectDAO().insert(queryResult, collectionConfig.getNamespace(), queryResult.getBucketName(), collectionConfig.getCollectionTime());
            }
            // extra pages to collect
            while (queryResult.isTruncated()) {
                // Move marker to beginning of next batch
                queryRequest.setMarker(queryResult.getNextMarker());
                startTime = System.currentTimeMillis();
                queryResult = this.collectionConfig.getS3JerseyClient().queryObjects(queryRequest);
                stopTime = System.currentTimeMillis();
                elapsedTime = Double.valueOf(stopTime - startTime) / 1000;
                collected = (long) queryResult.getObjects().size();
                // increase local counter
                bucketObjectCount += collected;
                // Increase central counter
                collectionConfig.getObjectCount().getAndAdd(collected);
                logger.info("Took: " + elapsedTime + " seconds to query " + collected + " objects from namespace: " + this.collectionConfig.getNamespace() + " bucket: " + queryResult.getBucketName());
                if (this.collectionConfig.getObjectDAO() != null) {
                    this.collectionConfig.getObjectDAO().insert(queryResult, collectionConfig.getNamespace(), queryResult.getBucketName(), collectionConfig.getCollectionTime());
                }
            }
        }
    } catch (Exception ex) {
        // known issue ECs returns this error when a bucket has MD keys but has not objects
        if (ex.getMessage().contains("Invalid search index value format or operator used")) {
            // just silently let this go. This error will eventually be fixed by ECS
            logger.error("Error: Namespace: " + collectionConfig.getNamespace() + " bucket: " + objectBucket.getName() + " Query string: `" + queryRequest.getQuery() + "`" + ex.getMessage());
        } else if (ex.getMessage().contains("We encountered an internal error. Please try again")) {
            // Here we could try again
            logger.error("Error: Namespace: " + collectionConfig.getNamespace() + " bucket: " + objectBucket.getName() + " Query string: `" + queryRequest.getQuery() + "`" + ex.getMessage());
        }
    }
    if (bucketObjectCount.compareTo(0L) == 0) {
        // will be triggered
        return false;
    }
    return true;
}

Example 2 with QueryObjectsRequest

use of com.emc.object.s3.request.QueryObjectsRequest in project ecs-dashboard by carone1.

the class QueryObjectsCollection method createQueryObjectRequest.

private QueryObjectsRequest createQueryObjectRequest(String queryString) {
    // create request
    QueryObjectsRequest queryRequest = new QueryObjectsRequest(objectBucket.getName());
    List<String> attributeList = new ArrayList<String>();
    List<String> attributeListDetails = new ArrayList<String>();
    StringBuilder queryBufferString = new StringBuilder();
    // add all attribute to attr list
    for (Metadata metadata : objectBucket.getSearchMetadata()) {
        if (metadata.getName() == null || metadata.getName().isEmpty()) {
            continue;
        }
        attributeList.add(metadata.getName());
        String attributeDetails = "Name: " + metadata.getName() + " Type:" + metadata.getDataType().trim().toLowerCase();
        attributeListDetails.add(attributeDetails);
    }
    // one search metadata key is defined
    if (attributeList.isEmpty()) {
        // return null to prevent any querying
        return null;
    } else {
        queryBufferString.append(queryString);
    }
    // append ( at beginning
    queryBufferString.insert(0, "( ");
    // append ) at end
    queryBufferString.append(" )");
    queryRequest.withQuery(queryString.toString());
    queryRequest.withAttributes(attributeList);
    queryRequest.setMaxKeys(maxObjectPerRequest);
    queryRequest.setNamespace(collectionConfig.getNamespace());
    logger.info("QueryObject Collection for Namespace: " + collectionConfig.getNamespace() + " Bucket: " + objectBucket.getName());
    logger.info("MD Keys details: " + attributeListDetails.toString());
    logger.info(" Using query string: `" + queryString.toString() + "`");
    return queryRequest;
}

Example 3 with QueryObjectsRequest

use of com.emc.object.s3.request.QueryObjectsRequest in project ecs-dashboard by carone1.

the class QueryObjectsCollection method createQueryObjectRequest.

private QueryObjectsRequest createQueryObjectRequest() {
    // create request
    QueryObjectsRequest queryRequest = new QueryObjectsRequest(objectBucket.getName());
    List<String> attributeList = new ArrayList<String>();
    StringBuilder queryString = new StringBuilder();
    // add index keys to the search query
    for (Metadata metadata : objectBucket.getSearchMetadata()) {
        if (metadata.getName() == null || metadata.getName().isEmpty()) {
            continue;
        }
        attributeList.add(metadata.getName());
        // Only want to use MD keys (Last Modified Time or Size) which have
        // the better chance of being present on all objects
        // if( LAST_MODIFIED_KEY.equals(metadata.getName())
        // SIZE_KEY.equals(metadata.getName()) ) {
        String dataType = metadata.getDataType().trim().toLowerCase();
        if (dataType.equals("string")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("(" + metadata.getName() + " <= '') or (" + metadata.getName() + " >= '' )");
        } else if (dataType.equals("decimal")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("(" + metadata.getName() + " <= 1.0) or (" + metadata.getName() + " >= 1.0 )");
        } else if (dataType.equals("integer")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("(" + metadata.getName() + " <= 1) or (" + metadata.getName() + " >= 1 )");
        } else if (dataType.equals("datetime")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("( " + metadata.getName() + " > '1970-01-01T00:00:00Z' )");
        } else {
            logger.error("Unhandled data type: " + dataType);
        }
    }
    if (queryString.length() == 0) {
        // return null to trigger list operation instead
        return null;
    }
    // append ( at beginning
    queryString.insert(0, "( ");
    // append ) at end
    queryString.append(" )");
    queryRequest.withQuery(queryString.toString());
    queryRequest.withAttributes(attributeList);
    queryRequest.setMaxKeys(maxObjectPerRequest);
    queryRequest.setNamespace(collectionConfig.getNamespace());
    logger.info("QueryObject Collection for Namespace: " + collectionConfig.getNamespace() + " Bucket: " + objectBucket.getName());
    logger.debug(" Using query string: `" + queryString.toString() + "`");
    return queryRequest;
}