Examples with QueryObject com.emc.object.s3.bean.QueryObject used on opensource projects

Example 1 with QueryObject

use of com.emc.object.s3.bean.QueryObject in project ecs-dashboard by carone1.

the class ElasticS3ObjectDAO method toJsonFormat.

/**
 * Converts Query Object data into JSON
 *
 * @param queryObject - Query Object
 * @param namespace - Namespace
 * @param bucket - Bucket
 * @param collectionTime - Collection Time
 * @param builder - Builder
 * @return XContentBuilder
 */
private static XContentBuilder toJsonFormat(QueryObject queryObject, String namespace, String bucket, Date collectionTime, XContentBuilder builder) {
    try {
        if (builder == null) {
            builder = XContentFactory.jsonBuilder();
        }
        // add known basic fields
        builder = builder.startObject().field(KEY_TAG, queryObject.getObjectName()).field(KEY_TAG + ANALYZED_TAG, queryObject.getObjectName()).field(ETAG_TAG, queryObject.getObjectId()).field(NAMESPACE_TAG, namespace).field(BUCKET_TAG, bucket).field(COLLECTION_TIME, collectionTime);
        // Add custom MS Key values as dynamic fields
        for (QueryMetadata metadata : queryObject.getQueryMds()) {
            for (Entry<String, String> entry : metadata.getMdMap().entrySet()) {
                builder.field(entry.getKey(), entry.getValue());
            }
        }
        builder.endObject();
    } catch (IOException e) {
        throw new RuntimeException(e.getLocalizedMessage());
    }
    return builder;
}

Example 2 with QueryObject

use of com.emc.object.s3.bean.QueryObject in project ecs-dashboard by carone1.

the class ElasticS3ObjectDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(QueryObjectsResult queryObjectsResult, String namespace, String bucketName, Date collectionTime) {
    if (queryObjectsResult == null || queryObjectsResult.getObjects() == null || queryObjectsResult.getObjects().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object buckets info
    for (QueryObject queryObject : queryObjectsResult.getObjects()) {
        XContentBuilder s3ObjectBuilder = toJsonFormat(queryObject, namespace, bucketName, collectionTime);
        IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectIndexDayName).setType(S3_OBJECT_INDEX_TYPE).setSource(s3ObjectBuilder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failure(s) occured while items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    }
}

Example 3 with QueryObject

use of com.emc.object.s3.bean.QueryObject in project ecs-dashboard by carone1.

the class QueryObjectsCollection method createQueryObjectRequest.

private QueryObjectsRequest createQueryObjectRequest(String queryString) {
    // create request
    QueryObjectsRequest queryRequest = new QueryObjectsRequest(objectBucket.getName());
    List<String> attributeList = new ArrayList<String>();
    List<String> attributeListDetails = new ArrayList<String>();
    StringBuilder queryBufferString = new StringBuilder();
    // add all attribute to attr list
    for (Metadata metadata : objectBucket.getSearchMetadata()) {
        if (metadata.getName() == null || metadata.getName().isEmpty()) {
            continue;
        }
        attributeList.add(metadata.getName());
        String attributeDetails = "Name: " + metadata.getName() + " Type:" + metadata.getDataType().trim().toLowerCase();
        attributeListDetails.add(attributeDetails);
    }
    // one search metadata key is defined
    if (attributeList.isEmpty()) {
        // return null to prevent any querying
        return null;
    } else {
        queryBufferString.append(queryString);
    }
    // append ( at beginning
    queryBufferString.insert(0, "( ");
    // append ) at end
    queryBufferString.append(" )");
    queryRequest.withQuery(queryString.toString());
    queryRequest.withAttributes(attributeList);
    queryRequest.setMaxKeys(maxObjectPerRequest);
    queryRequest.setNamespace(collectionConfig.getNamespace());
    logger.info("QueryObject Collection for Namespace: " + collectionConfig.getNamespace() + " Bucket: " + objectBucket.getName());
    logger.info("MD Keys details: " + attributeListDetails.toString());
    logger.info(" Using query string: `" + queryString.toString() + "`");
    return queryRequest;
}

Example 4 with QueryObject

use of com.emc.object.s3.bean.QueryObject in project ecs-dashboard by carone1.

the class QueryObjectsCollection method createQueryObjectRequest.

private QueryObjectsRequest createQueryObjectRequest() {
    // create request
    QueryObjectsRequest queryRequest = new QueryObjectsRequest(objectBucket.getName());
    List<String> attributeList = new ArrayList<String>();
    StringBuilder queryString = new StringBuilder();
    // add index keys to the search query
    for (Metadata metadata : objectBucket.getSearchMetadata()) {
        if (metadata.getName() == null || metadata.getName().isEmpty()) {
            continue;
        }
        attributeList.add(metadata.getName());
        // Only want to use MD keys (Last Modified Time or Size) which have
        // the better chance of being present on all objects
        // if( LAST_MODIFIED_KEY.equals(metadata.getName())
        // SIZE_KEY.equals(metadata.getName()) ) {
        String dataType = metadata.getDataType().trim().toLowerCase();
        if (dataType.equals("string")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("(" + metadata.getName() + " <= '') or (" + metadata.getName() + " >= '' )");
        } else if (dataType.equals("decimal")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("(" + metadata.getName() + " <= 1.0) or (" + metadata.getName() + " >= 1.0 )");
        } else if (dataType.equals("integer")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("(" + metadata.getName() + " <= 1) or (" + metadata.getName() + " >= 1 )");
        } else if (dataType.equals("datetime")) {
            if (queryString.length() > 0) {
                queryString.append(" or ");
            }
            queryString.append("( " + metadata.getName() + " > '1970-01-01T00:00:00Z' )");
        } else {
            logger.error("Unhandled data type: " + dataType);
        }
    }
    if (queryString.length() == 0) {
        // return null to trigger list operation instead
        return null;
    }
    // append ( at beginning
    queryString.insert(0, "( ");
    // append ) at end
    queryString.append(" )");
    queryRequest.withQuery(queryString.toString());
    queryRequest.withAttributes(attributeList);
    queryRequest.setMaxKeys(maxObjectPerRequest);
    queryRequest.setNamespace(collectionConfig.getNamespace());
    logger.info("QueryObject Collection for Namespace: " + collectionConfig.getNamespace() + " Bucket: " + objectBucket.getName());
    logger.debug(" Using query string: `" + queryString.toString() + "`");
    return queryRequest;
}