Examples with TestFailException com.sequenceiq.it.cloudbreak.exception.TestFailException used on opensource projects

Example 56 with TestFailException

use of com.sequenceiq.it.cloudbreak.exception.TestFailException in project cloudbreak by hortonworks.

the class S3ClientActions method getLoggingUrl.

public String getLoggingUrl(String baseLocation, String clusterLogPath) {
    AmazonS3 s3Client = buildS3Client();
    AmazonS3URI amazonS3URI = new AmazonS3URI(getBaseLocationUri());
    String bucketName = amazonS3URI.getBucket();
    String keyPrefix = StringUtils.substringAfterLast(baseLocation, "/");
    Log.log(LOGGER, format(" Amazon S3 URI: %s", amazonS3URI));
    Log.log(LOGGER, format(" Amazon S3 Bucket: %s", bucketName));
    Log.log(LOGGER, format(" Amazon S3 Key Prefix: %s", keyPrefix));
    Log.log(LOGGER, format(" Amazon S3 Cluster Logs: %s", clusterLogPath));
    if (s3Client.doesBucketExistV2(bucketName)) {
        try {
            return String.format("https://s3.console.aws.amazon.com/s3/buckets/%s?region=%s&prefix=%s%s/&showversions=false", bucketName, amazonS3URI.getRegion(), keyPrefix, clusterLogPath);
        } catch (AmazonServiceException e) {
            LOGGER.error("Amazon S3 couldn't process the call. So it has been returned with error!", e);
            throw new TestFailException("Amazon S3 couldn't process the call.", e);
        } catch (SdkClientException e) {
            LOGGER.error("Amazon S3 response could not been parsed, because of error!", e);
            throw new TestFailException("Amazon S3 response could not been parsed", e);
        } finally {
            s3Client.shutdown();
        }
    } else {
        LOGGER.error("Amazon S3 bucket is NOT present with name: {}", bucketName);
        throw new TestFailException("Amazon S3 bucket is NOT present with name: " + bucketName);
    }
}

Example 57 with TestFailException

use of com.sequenceiq.it.cloudbreak.exception.TestFailException in project cloudbreak by hortonworks.

the class S3ClientActions method listBucketSelectedObject.

public void listBucketSelectedObject(String baseLocation, String selectedObject, boolean zeroContent) {
    AmazonS3 s3Client = buildS3Client();
    String bucketName = getBucketName();
    AmazonS3URI amazonS3URI = new AmazonS3URI(getBaseLocationUri());
    List<S3ObjectSummary> filteredObjectSummaries;
    String keyPrefix = StringUtils.substringAfterLast(baseLocation, "/");
    if (StringUtils.isEmpty(selectedObject)) {
        selectedObject = "ranger";
    }
    Log.log(LOGGER, format(" Amazon S3 URI: %s", amazonS3URI));
    Log.log(LOGGER, format(" Amazon S3 Bucket: %s", bucketName));
    Log.log(LOGGER, format(" Amazon S3 Key Prefix: %s", keyPrefix));
    Log.log(LOGGER, format(" Amazon S3 Object: %s", selectedObject));
    if (s3Client.doesBucketExistV2(bucketName)) {
        try {
            ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(keyPrefix);
            ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
            do {
                String finalSelectedObject = selectedObject;
                filteredObjectSummaries = objectListing.getObjectSummaries().stream().filter(objectSummary -> objectSummary.getKey().contains(finalSelectedObject)).collect(Collectors.toList());
                if (objectListing.isTruncated()) {
                    objectListing = s3Client.listNextBatchOfObjects(objectListing);
                }
            } while (filteredObjectSummaries.isEmpty() && objectListing.isTruncated());
            if (filteredObjectSummaries.size() == 0) {
                LOGGER.error("Amazon S3 object: {} has 0 sub-objects!", selectedObject);
                throw new TestFailException(String.format("Amazon S3 object: %s has 0 sub-objects!", selectedObject));
            } else {
                Log.log(LOGGER, format(" Amazon S3 object: %s contains %d sub-objects.", selectedObject, filteredObjectSummaries.size()));
            }
            for (S3ObjectSummary objectSummary : filteredObjectSummaries.stream().limit(10).collect(Collectors.toList())) {
                S3Object object = s3Client.getObject(bucketName, objectSummary.getKey());
                S3ObjectInputStream inputStream = object.getObjectContent();
                if (!zeroContent && object.getObjectMetadata().getContentLength() == 0) {
                    LOGGER.error("Amazon S3 path: {} has 0 bytes of content!", object.getKey());
                    throw new TestFailException(String.format("Amazon S3 path: %s has 0 bytes of content!", object.getKey()));
                }
                try {
                    inputStream.abort();
                    object.close();
                } catch (IOException e) {
                    LOGGER.error("Unable to close Amazon S3 object {}. It has been returned with error!", object.getKey(), e);
                }
            }
        } catch (AmazonServiceException e) {
            LOGGER.error("Amazon S3 couldn't process the call. So it has been returned with error!", e);
            throw new TestFailException("Amazon S3 couldn't process the call.", e);
        } catch (SdkClientException e) {
            LOGGER.error("Amazon S3 response could not been parsed, because of error!", e);
            throw new TestFailException("Amazon S3 response could not been parsed", e);
        } finally {
            s3Client.shutdown();
        }
    } else {
        LOGGER.error("Amazon S3 bucket is NOT present with name: {}", bucketName);
        throw new TestFailException("Amazon S3 bucket is NOT present with name: " + bucketName);
    }
}

Example 58 with TestFailException

use of com.sequenceiq.it.cloudbreak.exception.TestFailException in project cloudbreak by hortonworks.

the class AzureCloudBlobClientActions method listAllFolders.

public void listAllFolders(String baseLocation) {
    String containerName = getContainerName(baseLocation);
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);
    String keyPrefix = StringUtils.substringAfterLast(baseLocation, "/");
    Log.log(LOGGER, format(" Azure Blob Storage URI: %s", cloudBlobContainer.getStorageUri()));
    Log.log(LOGGER, format(" Azure Blob Container: %s", cloudBlobContainer.getName()));
    Log.log(LOGGER, format(" Azure Blob Key Prefix: %s", keyPrefix));
    Log.log(LOGGER, format(" Azure Blob Location: %s", baseLocation));
    try {
        CloudBlobDirectory storageDirectory = cloudBlobContainer.getDirectoryReference(keyPrefix);
        Iterable<ListBlobItem> blobListing = storageDirectory.listBlobs();
        List<ListBlobItem> listBlobItems = StreamSupport.stream(blobListing.spliterator(), false).collect(Collectors.toList());
        Log.log(LOGGER, format(" Azure Blob Directory: %s contains %d sub-objects.", keyPrefix, listBlobItems.size()));
        for (ListBlobItem blob : blobListing) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();
            if (blob instanceof CloudBlob) {
                if (((CloudBlob) blob).exists()) {
                    Log.log(LOGGER, format(" Azure Adls Gen 2 Blob is present with Name: %s and with bytes of content: %d at URI: %s ", ((CloudBlob) blob).getName(), ((CloudBlob) blob).getProperties().getLength(), blobUriPath));
                }
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = storageDirectory.getDirectoryReference(blobName);
                listBlobsInDirectory(blobDirectory);
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw new TestFailException("Azure Adls Gen 2 Blob couldn't process the call.", e);
    }
}

Example 59 with TestFailException

use of com.sequenceiq.it.cloudbreak.exception.TestFailException in project cloudbreak by hortonworks.

the class AzureCloudBlobClientActions method listSelectedDirectory.

public void listSelectedDirectory(String baseLocation, String selectedDirectory, boolean zeroContent) {
    String containerName = getContainerName(baseLocation);
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);
    String keyPrefix = StringUtils.substringAfterLast(baseLocation, "/");
    Log.log(LOGGER, format(" Azure Blob Storage URI: %s", cloudBlobContainer.getStorageUri()));
    Log.log(LOGGER, format(" Azure Blob Container: %s", cloudBlobContainer.getName()));
    Log.log(LOGGER, format(" Azure Blob Key Prefix: %s", keyPrefix));
    Log.log(LOGGER, format(" Azure Blob Directory: %s", selectedDirectory));
    try {
        CloudBlobDirectory selectedStorageDirectory = cloudBlobContainer.getDirectoryReference(keyPrefix);
        CloudBlobDirectory selectedBlobDirectory = selectedStorageDirectory.getDirectoryReference(selectedDirectory);
        Set<String> blobsWithZeroLength = new HashSet<>();
        Iterable<ListBlobItem> blobListing = selectedBlobDirectory.listBlobs();
        List<ListBlobItem> listBlobItems = StreamSupport.stream(blobListing.spliterator(), false).collect(Collectors.toList());
        Log.log(LOGGER, format(" Azure Blob Directory: %s contains %d sub-objects.", selectedDirectory, listBlobItems.size()));
        for (ListBlobItem blob : blobListing) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();
            if (blob instanceof CloudBlob) {
                if (((CloudBlob) blob).exists()) {
                    validateBlobItemLength(blob, zeroContent, blobsWithZeroLength);
                } else {
                    LOGGER.error("Azure Adls Gen 2 Blob is NOT present with Name: {} and with bytes of content: {} at URI: {}", ((CloudBlob) blob).getName(), ((CloudBlob) blob).getProperties().getLength(), blobUriPath);
                    throw new TestFailException(String.format("Azure Adls Gen 2 Blob is NOT present with Name: %s", ((CloudBlob) blob).getName()));
                }
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = selectedBlobDirectory.getDirectoryReference(blobName);
                listBlobsInDirectoryWithValidation(blobDirectory, zeroContent);
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw new TestFailException("Azure Adls Gen 2 Blob couldn't process the call.", e);
    }
}

Example 60 with TestFailException

use of com.sequenceiq.it.cloudbreak.exception.TestFailException in project cloudbreak by hortonworks.

the class AzureCloudBlobClientActions method cleanupContainer.

public void cleanupContainer(String baseLocation) {
    String containerName = getContainerName(baseLocation);
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);
    try {
        Log.log(LOGGER, format(" Removing Azure Adls Gen 2 Blob Container with Name: %s at Base Location: %s", containerName, baseLocation));
        cloudBlobContainer.deleteIfExists();
        Log.log(LOGGER, format(" Azure Adls Gen 2 Blob Container: %s delete has been initiated. ", containerName));
    } catch (StorageException e) {
        if (e.getHttpStatusCode() == 404) {
            LOGGER.error("Azure Adls Gen2 Blob Container does not present with name: {} at Base Location: {}", containerName, baseLocation);
            throw new TestFailException("Azure Adls Gen2 Blob Container does not present with name: " + containerName + " at Base Location: " + baseLocation);
        } else {
            LOGGER.error("Azure Adls Gen2 Blob Container delete cannot be succeed!", e);
            throw new TestFailException("Azure Adls Gen2 Blob Container delete cannot be succeed!", e);
        }
    } finally {
        createCloudBlobContainer(containerName);
    }
}