Examples with ObjectBucket com.emc.ecs.management.entity.ObjectBucket used on opensource projects

Example 1 with ObjectBucket

use of com.emc.ecs.management.entity.ObjectBucket in project ecs-dashboard by carone1.

the class ElasticBillingDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(ObjectBuckets objectBuckets, Date collectionTime) {
    if (objectBuckets == null || objectBuckets.getObjectBucket() == null || objectBuckets.getObjectBucket().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object buckets info
    for (ObjectBucket objectBucket : objectBuckets.getObjectBucket()) {
        XContentBuilder objectBucketBuilder = toJsonFormat(objectBucket, collectionTime);
        IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(objectBucketIndexDayName).setType(OBJECT_BUCKET_INDEX_TYPE).setSource(objectBucketBuilder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in ElasticSearch" + "index: " + objectBucketIndexDayName + " index type: " + OBJECT_BUCKET_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failures occured while items in ElasticSearch " + "index: " + objectBucketIndexDayName + " index type: " + OBJECT_BUCKET_INDEX_TYPE);
    }
}

Example 2 with ObjectBucket

use of com.emc.ecs.management.entity.ObjectBucket in project ecs-dashboard by carone1.

the class BillingBO method collectObjectBucketData.

/**
 * Collects Bucket metadata for all namespace defined on a cluster
 * @param objectBucketMap - Object Bucket Map
 * @param collectionTime - Collection Time
 * @param billDAO - Billing DAO Object
 */
private void collectObjectBucketData(Map<String, Set<ObjectBucket>> objectPerNamespaceMap, Map<NamespaceBucketKey, ObjectBucket> objectBucketMap, Date collectionTime, BillingDAO billDAO) {
    // Start collecting billing data from ECS systems
    List<Namespace> namespaceList = getNamespaces();
    // At this point we should have all the namespace supported by the ECS system
    long objCounter = 0;
    for (Namespace namespace : namespaceList) {
        // ===============================================
        // Initial billing request for current namespace
        // ===============================================
        NamespaceRequest namespaceRequest = new NamespaceRequest();
        namespaceRequest.setName(namespace.getName());
        ObjectBuckets objectBucketsResponse = client.getNamespaceBucketInfo(namespaceRequest);
        if (objectBucketsResponse == null) {
            continue;
        }
        logger.info("Collect Billing Data for namespace: " + namespace.getName());
        objCounter += (objectBucketsResponse.getObjectBucket() != null) ? objectBucketsResponse.getObjectBucket().size() : 0;
        // Push collected info into datastore
        if (billDAO != null) {
            // insert something
            billDAO.insert(objectBucketsResponse, collectionTime);
        }
        // Add to return map per namespace and bucket key
        if (objectBucketsResponse.getObjectBucket() != null && objectBucketMap != null) {
            for (ObjectBucket objectBucket : objectBucketsResponse.getObjectBucket()) {
                NamespaceBucketKey key = new NamespaceBucketKey(namespace.getName(), objectBucket.getName());
                objectBucketMap.put(key, objectBucket);
            }
        }
        // Add to return map per namespace key
        if (objectBucketsResponse.getObjectBucket() != null && objectPerNamespaceMap != null) {
            Set<ObjectBucket> objectSet = objectPerNamespaceMap.get(namespace.getName());
            if (objectSet == null) {
                // there isn;t already a set present for that namespace
                // create one
                objectSet = new HashSet<ObjectBucket>();
                // add reference of set to map
                objectPerNamespaceMap.put(namespace.getName(), objectSet);
            }
            for (ObjectBucket objectBucket : objectBucketsResponse.getObjectBucket()) {
                // add all object to set
                objectSet.add(objectBucket);
            }
        }
        // collect n subsequent pages
        while (namespaceRequest.getNextMarker() != null) {
            objectBucketsResponse = client.getNamespaceBucketInfo(namespaceRequest);
            if (objectBucketsResponse != null) {
                objCounter += (objectBucketsResponse.getObjectBucket() != null) ? objectBucketsResponse.getObjectBucket().size() : 0;
                namespaceRequest.setNextMarker(objectBucketsResponse.getNextMarker());
                // Push collected info into datastore
                if (billDAO != null) {
                    // insert something
                    billDAO.insert(objectBucketsResponse, collectionTime);
                }
                // Add to return map
                if (objectBucketsResponse.getObjectBucket() != null && objectBucketMap != null) {
                    for (ObjectBucket objectBucket : objectBucketsResponse.getObjectBucket()) {
                        NamespaceBucketKey key = new NamespaceBucketKey(namespace.getName(), objectBucket.getName());
                        objectBucketMap.put(key, objectBucket);
                    }
                }
                // Add to return map per namespace key
                if (objectBucketsResponse.getObjectBucket() != null && objectPerNamespaceMap != null) {
                    Set<ObjectBucket> objectSet = objectPerNamespaceMap.get(namespace.getName());
                    if (objectSet == null) {
                        // there isn;t already a set present for that namespace
                        // create one
                        objectSet = new HashSet<ObjectBucket>();
                        // add reference of set to map
                        objectPerNamespaceMap.put(namespace.getName(), objectSet);
                    }
                    for (ObjectBucket objectBucket : objectBucketsResponse.getObjectBucket()) {
                        // add all object to set
                        objectSet.add(objectBucket);
                    }
                }
            } else {
                // stop the loop
                namespaceRequest.setNextMarker(null);
            }
        }
    }
    // peg global counter
    this.objectCount.getAndAdd(objCounter);
}

Example 3 with ObjectBucket

use of com.emc.ecs.management.entity.ObjectBucket in project ecs-dashboard by carone1.

the class BillingBO method collectBillingData.

/**
 * Collects Billing metadata for all namespace defined on a cluster
 * @param collectionTime - Collection Time
 */
public void collectBillingData(Date collectionTime) {
    // Collect the object bucket data first in order to use some of
    // the fields from object bucket
    Map<NamespaceBucketKey, ObjectBucket> objectBuckets = new HashMap<NamespaceBucketKey, ObjectBucket>();
    Map<String, Set<ObjectBucket>> objectBucketsPerNamespace = new HashMap<String, Set<ObjectBucket>>();
    // Collect bucket data and write to datastore
    // secondly returns classified data per namespace
    // and also per namespace+bucketname
    collectObjectBucketData(objectBucketsPerNamespace, objectBuckets, collectionTime, billingDAO);
    // Start collecting billing data from ECS systems
    List<Namespace> namespaceList = getNamespaces();
    // At this point we should have all namespaces in the ECS system
    long objCounter = 0;
    for (Namespace namespace : namespaceList) {
        // ===============================================
        // Initial billing request for current namespace
        // ===============================================
        NamespaceRequest namespaceRequest = new NamespaceRequest();
        namespaceRequest.setName(namespace.getName());
        if (objectBucketsPerNamespace.get(namespace.getName()) != null && !objectBucketsPerNamespace.get(namespace.getName()).isEmpty()) {
            // There are buckets in that namespace
            // indicate to client to include bucket data
            namespaceRequest.setIncludeBuckets(true);
        }
        NamespaceBillingInfo namespaceBillingResponse = client.getNamespaceBillingInfo(namespaceRequest);
        if (namespaceBillingResponse == null) {
            continue;
        }
        // add object bucket attributes
        for (BucketBillingInfo bucketBillingInfo : namespaceBillingResponse.getBucketBillingInfo()) {
            NamespaceBucketKey namespaceBucketKey = new NamespaceBucketKey(namespace.getName(), bucketBillingInfo.getName());
            ObjectBucket objectBucket = objectBuckets.get(namespaceBucketKey);
            if (objectBucket != null) {
                // set api type
                bucketBillingInfo.setApiType(objectBucket.getApiType());
                // set namespace
                bucketBillingInfo.setNamespace(namespace.getName());
            } else {
                // set api type
                bucketBillingInfo.setApiType("unknown");
                // set namespace
                bucketBillingInfo.setNamespace(namespace.getName());
            }
            objCounter++;
        }
        // Push collected info into datastore
        if (this.billingDAO != null) {
            // insert something
            billingDAO.insert(namespaceBillingResponse, collectionTime);
        }
        // collect n subsequent pages
        while (namespaceRequest.getNextMarker() != null) {
            namespaceBillingResponse = client.getNamespaceBillingInfo(namespaceRequest);
            if (namespaceBillingResponse != null) {
                namespaceRequest.setNextMarker(namespaceBillingResponse.getNextMarker());
                // add object bucket attributes
                for (BucketBillingInfo bucketBillingInfo : namespaceBillingResponse.getBucketBillingInfo()) {
                    ObjectBucket objectBucket = objectBuckets.get(bucketBillingInfo.getName());
                    if (objectBucket != null) {
                        // set api type
                        bucketBillingInfo.setApiType(objectBucket.getApiType());
                        // set namespace
                        bucketBillingInfo.setNamespace(namespace.getName());
                    }
                    objCounter++;
                }
                // Push collected info into datastore
                if (this.billingDAO != null) {
                    // insert something
                    billingDAO.insert(namespaceBillingResponse, collectionTime);
                }
            } else {
                namespaceRequest.setNextMarker(null);
            }
        }
    }
    // peg global counter
    this.objectCount.getAndAdd(objCounter);
}

Example 4 with ObjectBucket

use of com.emc.ecs.management.entity.ObjectBucket in project ecs-dashboard by carone1.

the class ObjectBO method collectObjectVersionData.

public void collectObjectVersionData(Date collectionTime) {
    // collect S3 user Id and credentials
    List<ObjectUserDetails> objectUserDetailsList = billingBO.getObjectUserSecretKeys();
    // Collect bucket details
    Map<NamespaceBucketKey, ObjectBucket> objectBucketMap = new HashMap<>();
    billingBO.getObjectBucketData(objectBucketMap);
    Map<String, S3JerseyClient> s3ObjectClientMap = null;
    try {
        // create all required S3 jersey clients for very S3 users
        s3ObjectClientMap = createS3ObjectClients(objectUserDetailsList, this.ecsObjectHosts);
        // collect objects for all users
        for (ObjectUserDetails objectUserDetails : objectUserDetailsList) {
            if (objectUserDetails.getObjectUser().getUserId() == null || objectUserDetails.getSecretKeys().getSecretKey1() == null) {
                // in that case we just skip over that user
                continue;
            }
            String userId = objectUserDetails.getObjectUser().getUserId().toString();
            S3JerseyClient s3JerseyClient = s3ObjectClientMap.get(userId);
            String namespace = objectUserDetails.getObjectUser().getNamespace().toString();
            if (s3JerseyClient != null && namespace != null) {
                ObjectCollectionConfig collectionConfig = new ObjectCollectionConfig(s3JerseyClient, namespace, objectDAO, objectBucketMap, collectionTime, objectCount, threadPoolExecutor, futures, // no criteria required here
                null);
                NamespaceObjectVersionCollection namespaceObjectVersionCollection = new NamespaceObjectVersionCollection(collectionConfig);
                try {
                    // submit namespace collection to thread pool
                    futures.add(threadPoolExecutor.submit(namespaceObjectVersionCollection));
                } catch (RejectedExecutionException e) {
                    // Thread pool didn't accept bucket collection
                    // running in the current thread
                    logger.error("Thread pool didn't accept bucket collection - running in current thread");
                    try {
                        namespaceObjectVersionCollection.call();
                    } catch (Exception e1) {
                        logger.error("Error occured during namespace object version collection operation - message: " + e.getLocalizedMessage());
                    }
                }
            }
        }
    } finally {
        // ensure to clean up S3 jersey clients
        if (s3ObjectClientMap != null) {
            for (S3JerseyClient s3JerseyClient : s3ObjectClientMap.values()) {
                s3JerseyClient.destroy();
            }
        }
    }
}

Example 5 with ObjectBucket

use of com.emc.ecs.management.entity.ObjectBucket in project ecs-dashboard by carone1.

the class ObjectBO method collectObjectData.

public void collectObjectData(Date collectionTime, String queryCriteria) {
    // collect S3 user Id and credentials
    List<ObjectUserDetails> objectUserDetailsList = billingBO.getObjectUserSecretKeys();
    // Collect bucket details
    Map<NamespaceBucketKey, ObjectBucket> objectBucketMap = new HashMap<>();
    billingBO.getObjectBucketData(objectBucketMap);
    Map<String, S3JerseyClient> s3ObjectClientMap = null;
    try {
        // create all required S3 jersey clients for very S3 users
        s3ObjectClientMap = createS3ObjectClients(objectUserDetailsList, this.ecsObjectHosts);
        // collect objects for all users
        for (ObjectUserDetails objectUserDetails : objectUserDetailsList) {
            if (objectUserDetails.getObjectUser().getUserId() == null || objectUserDetails.getSecretKeys().getSecretKey1() == null) {
                // in that case we just skip over that user
                continue;
            }
            String userId = objectUserDetails.getObjectUser().getUserId().toString();
            S3JerseyClient s3JerseyClient = s3ObjectClientMap.get(userId);
            String namespace = objectUserDetails.getObjectUser().getNamespace().toString();
            if (s3JerseyClient != null && namespace != null) {
                ObjectCollectionConfig collectionConfig = new ObjectCollectionConfig(s3JerseyClient, namespace, this.objectDAO, objectBucketMap, collectionTime, objectCount, threadPoolExecutor, futures, queryCriteria);
                NamespaceObjectCollection namespaceObjectCollection = new NamespaceObjectCollection(collectionConfig);
                try {
                    // submit namespace collection to thread pool
                    futures.add(threadPoolExecutor.submit(namespaceObjectCollection));
                } catch (RejectedExecutionException e) {
                    // Thread pool didn't accept bucket collection
                    // running in the current thread
                    logger.error("Thread pool didn't accept bucket collection - running in current thread");
                    try {
                        namespaceObjectCollection.call();
                    } catch (Exception e1) {
                        logger.error("Error occured during namespace object collection operation - message: " + e.getLocalizedMessage());
                    }
                }
            }
        }
    } finally {
        // ensure to clean up S3 jersey clients
        if (s3ObjectClientMap != null) {
            for (S3JerseyClient s3JerseyClient : s3ObjectClientMap.values()) {
                s3JerseyClient.destroy();
            }
        }
    }
}