Examples with Version com.emc.object.s3.bean.Version used on opensource projects

Example 1 with Version

use of com.emc.object.s3.bean.Version in project ecs-dashboard by carone1.

the class ObjectBO method collectObjectVersionData.

public void collectObjectVersionData(Date collectionTime) {
    // collect S3 user Id and credentials
    List<ObjectUserDetails> objectUserDetailsList = billingBO.getObjectUserSecretKeys();
    // Collect bucket details
    Map<NamespaceBucketKey, ObjectBucket> objectBucketMap = new HashMap<>();
    billingBO.getObjectBucketData(objectBucketMap);
    Map<String, S3JerseyClient> s3ObjectClientMap = null;
    try {
        // create all required S3 jersey clients for very S3 users
        s3ObjectClientMap = createS3ObjectClients(objectUserDetailsList, this.ecsObjectHosts);
        // collect objects for all users
        for (ObjectUserDetails objectUserDetails : objectUserDetailsList) {
            if (objectUserDetails.getObjectUser().getUserId() == null || objectUserDetails.getSecretKeys().getSecretKey1() == null) {
                // in that case we just skip over that user
                continue;
            }
            String userId = objectUserDetails.getObjectUser().getUserId().toString();
            S3JerseyClient s3JerseyClient = s3ObjectClientMap.get(userId);
            String namespace = objectUserDetails.getObjectUser().getNamespace().toString();
            if (s3JerseyClient != null && namespace != null) {
                ObjectCollectionConfig collectionConfig = new ObjectCollectionConfig(s3JerseyClient, namespace, objectDAO, objectBucketMap, collectionTime, objectCount, threadPoolExecutor, futures, // no criteria required here
                null);
                NamespaceObjectVersionCollection namespaceObjectVersionCollection = new NamespaceObjectVersionCollection(collectionConfig);
                try {
                    // submit namespace collection to thread pool
                    futures.add(threadPoolExecutor.submit(namespaceObjectVersionCollection));
                } catch (RejectedExecutionException e) {
                    // Thread pool didn't accept bucket collection
                    // running in the current thread
                    logger.error("Thread pool didn't accept bucket collection - running in current thread");
                    try {
                        namespaceObjectVersionCollection.call();
                    } catch (Exception e1) {
                        logger.error("Error occured during namespace object version collection operation - message: " + e.getLocalizedMessage());
                    }
                }
            }
        }
    } finally {
        // ensure to clean up S3 jersey clients
        if (s3ObjectClientMap != null) {
            for (S3JerseyClient s3JerseyClient : s3ObjectClientMap.values()) {
                s3JerseyClient.destroy();
            }
        }
    }
}

Example 2 with Version

use of com.emc.object.s3.bean.Version in project ecs-dashboard by carone1.

the class ElasticS3ObjectDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(ListVersionsResult listVersionsResult, String namespace, String bucketName, Date collectionTime) {
    if (listVersionsResult == null || listVersionsResult.getVersions() == null || listVersionsResult.getVersions().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object version info
    for (AbstractVersion abstractVersion : listVersionsResult.getVersions()) {
        if (abstractVersion instanceof Version) {
            XContentBuilder s3ObjectVersionBuilder = toJsonFormat((Version) abstractVersion, namespace, bucketName, collectionTime);
            IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectVersionIndexDayName).setType(S3_OBJECT_VERSION_INDEX_TYPE).setSource(s3ObjectVersionBuilder);
            requestBuilder.add(request);
        } else if (abstractVersion instanceof DeleteMarker) {
            XContentBuilder s3ObjectVersionBuilder = toJsonFormat((DeleteMarker) abstractVersion, namespace, bucketName, collectionTime);
            IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectVersionIndexDayName).setType(S3_OBJECT_VERSION_INDEX_TYPE).setSource(s3ObjectVersionBuilder);
            requestBuilder.add(request);
        }
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in Elasticsearch " + "index: " + s3ObjectVersionIndexDayName + " index type: " + S3_OBJECT_VERSION_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failure(s) occured while items in Elasticsearch " + "index: " + s3ObjectVersionIndexDayName + " index type: " + S3_OBJECT_VERSION_INDEX_TYPE);
    }
}

Example 3 with Version

use of com.emc.object.s3.bean.Version in project ecs-dashboard by carone1.

the class ListObjectsVersionsCollection method collectObjectVersions.

// =============================
// Public methods
// =============================
public void collectObjectVersions() {
    // Verify if versioning is enabled on bucket
    if (!getBucketVersioningConfiguration().equals(VersioningConfiguration.Status.Enabled)) {
        logger.info("No object version collected from namespace: " + collectionConfig.getNamespace() + " bucket: " + objectBucket.getName() + " because versioning is disabled on bucket");
        return;
    }
    // prepare request object
    ListVersionsRequest listVersionsRequest = new ListVersionsRequest(objectBucket.getName());
    listVersionsRequest.setMaxKeys(maxObjectPerRequest);
    listVersionsRequest.setNamespace(collectionConfig.getNamespace());
    long startTime = System.currentTimeMillis();
    try {
        // collect objects
        ListVersionsResult listVersionsResult = collectionConfig.getS3JerseyClient().listVersions(listVersionsRequest);
        long stopTime = System.currentTimeMillis();
        Double elapsedTime = Double.valueOf(stopTime - startTime) / 1000;
        if (listVersionsResult != null) {
            Long collected = (long) listVersionsResult.getVersions().size();
            this.collectionConfig.getObjectCount().getAndAdd(collected);
            logger.info("Took: " + elapsedTime + " seconds to collect " + collected + " objects from namespace: " + collectionConfig.getNamespace() + " bucket: " + objectBucket.getName());
            // add collected entries into datastore
            if (collectionConfig.getObjectDAO() != null) {
                collectionConfig.getObjectDAO().insert(listVersionsResult, collectionConfig.getNamespace(), objectBucket.getName(), collectionConfig.getCollectionTime());
            }
            // process extra pages of objects
            while (listVersionsResult.isTruncated()) {
                listVersionsRequest.setVersionIdMarker(listVersionsResult.getNextVersionIdMarker());
                startTime = System.currentTimeMillis();
                listVersionsResult = collectionConfig.getS3JerseyClient().listVersions(listVersionsRequest);
                stopTime = System.currentTimeMillis();
                elapsedTime = Double.valueOf(stopTime - startTime) / 1000;
                collected = (long) listVersionsResult.getVersions().size();
                collectionConfig.getObjectCount().getAndAdd(collected);
                logger.info("Took: " + elapsedTime + " seconds to collect " + collected + " objects versions from namespace: " + collectionConfig.getNamespace() + " bucket: " + objectBucket.getName());
                // add collected entries into datastore
                if (collectionConfig.getObjectDAO() != null) {
                    collectionConfig.getObjectDAO().insert(listVersionsResult, collectionConfig.getNamespace(), objectBucket.getName(), collectionConfig.getCollectionTime());
                }
            }
        }
    } catch (Exception ex) {
        logger.error("Error occur while listing object versions from namespace: " + collectionConfig.getNamespace() + " Bucket: " + objectBucket.getName() + " " + ex.getLocalizedMessage());
        throw new RuntimeException(ex.getLocalizedMessage());
    }
}