Examples with AmazonS3Exception com.amazonaws.services.s3.model.AmazonS3Exception used on opensource projects

Example 61 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project stocator by SparkTC.

the class COSAPIClient method getFileStatus.

@Override
public FileStatus getFileStatus(String hostName, Path f, String msg) throws IOException, FileNotFoundException {
    Path path = f;
    boolean originalTempTarget = false;
    if (path.toString().contains(HADOOP_TEMPORARY)) {
        if (!path.toString().contains(HADOOP_PART)) {
            LOG.debug("getFileStatus on temp object {}. Return not found", path.toString());
            throw new FileNotFoundException("Not found " + path.toString());
        }
        path = stocatorPath.modifyPathToFinalDestination(f);
        LOG.debug("getFileStatus on temp object {}. Modify to final name {}", f.toString(), path.toString());
        originalTempTarget = true;
    }
    FileStatus res = null;
    FileStatus cached = memoryCache.getFileStatus(path.toString());
    if (cached != null) {
        return cached;
    }
    LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName);
    /*
     * The requested path is equal to hostName. HostName is equal to
     * hostNameScheme, thus the container. Therefore we have no object to look
     * for and we return the FileStatus as a directory. Containers have to
     * lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.trace("getFileStatus(completed) {}", path);
        res = new FileStatus(0L, true, 1, mBlockSize, 0L, path);
        memoryCache.putFileStatus(path.toString(), res);
        return res;
    }
    String key = pathToKey(path);
    LOG.debug("getFileStatus: on original key {}", key);
    FileStatus fileStatus = null;
    try {
        fileStatus = getFileStatusKeyBased(key, path);
    } catch (AmazonS3Exception e) {
        LOG.warn("file status {} returned {}", key, e.getStatusCode());
        if (e.getStatusCode() != 404) {
            LOG.warn("Throw IOException for {}. Most likely authentication failed", key);
            throw new IOException(e);
        } else if (originalTempTarget && e.getStatusCode() == 404) {
            throw new FileNotFoundException("Not found " + f.toString());
        }
    }
    if (fileStatus != null) {
        LOG.trace("getFileStatus(completed) {}", path);
        memoryCache.putFileStatus(path.toString(), fileStatus);
        return fileStatus;
    }
    // probably not needed this call
    if (!key.endsWith("/")) {
        String newKey = key + "/";
        try {
            LOG.debug("getFileStatus: original key not found. Alternative key {}", newKey);
            fileStatus = getFileStatusKeyBased(newKey, path);
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() != 404) {
                throw new IOException(e);
            }
        }
        if (fileStatus != null) {
            LOG.trace("getFileStatus(completed) {}", path);
            memoryCache.putFileStatus(path.toString(), fileStatus);
            return fileStatus;
        } else {
            // if here: both key and key/ returned not found.
            // trying to see if pseudo directory of the form
            // a/b/key/d/e (a/b/key/ doesn't exists by itself)
            // perform listing on the key
            LOG.debug("getFileStatus: Modifined key {} not found. Trying to list", key);
            key = maybeAddTrailingSlash(key);
            ListObjectsRequest request = new ListObjectsRequest();
            request.setBucketName(mBucket);
            request.setPrefix(key);
            request.withEncodingType("url");
            request.setDelimiter("/");
            request.setMaxKeys(1);
            ObjectListing objects = mClient.listObjects(request);
            if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            } else if (key.isEmpty()) {
                LOG.trace("Found root directory");
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            }
        }
    }
    LOG.debug("Not found {}. Throw FNF exception", path.toString());
    throw new FileNotFoundException("Not found " + path.toString());
}

Example 62 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project stocator by SparkTC.

the class COSUtils method translateException.

/**
 * Translate an exception raised in an operation into an IOException. The
 * specific type of IOException depends on the class of
 * {@link AmazonClientException} passed in, and any status codes included in
 * the operation. That is: HTTP error codes are examined and can be used to
 * build a more specific response.
 *
 * @param operation operation
 * @param path path operated on (may be null)
 * @param exception amazon exception raised
 * @return an IOE which wraps the caught exception
 */
@SuppressWarnings("ThrowableInstanceNeverThrown")
public static IOException translateException(String operation, String path, AmazonClientException exception) {
    String message = String.format("%s%s: %s", operation, path != null ? (" on " + path) : "", exception);
    if (!(exception instanceof AmazonServiceException)) {
        if (containsInterruptedException(exception)) {
            return (IOException) new InterruptedIOException(message).initCause(exception);
        }
        return new COSClientIOException(message, exception);
    } else {
        IOException ioe;
        AmazonServiceException ase = (AmazonServiceException) exception;
        // this exception is non-null if the service exception is an COS one
        AmazonS3Exception s3Exception = ase instanceof AmazonS3Exception ? (AmazonS3Exception) ase : null;
        int status = ase.getStatusCode();
        switch(status) {
            case 301:
                if (s3Exception != null) {
                    if (s3Exception.getAdditionalDetails() != null && s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) {
                        message = String.format("Received permanent redirect response to " + "endpoint %s.  This likely indicates that the COS endpoint " + "configured in %s does not match the region containing " + "the bucket.", s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT_URL);
                    }
                    ioe = new COSIOException(message, s3Exception);
                } else {
                    ioe = new COSServiceIOException(message, ase);
                }
                break;
            // permissions
            case 401:
            case 403:
                ioe = new AccessDeniedException(path, null, message);
                ioe.initCause(ase);
                break;
            // the object isn't there
            case 404:
            case 410:
                ioe = new FileNotFoundException(message);
                ioe.initCause(ase);
                break;
            // a shorter one while it is being read.
            case 416:
                ioe = new EOFException(message);
                break;
            default:
                // no specific exit code. Choose an IOE subclass based on the class
                // of the caught exception
                ioe = s3Exception != null ? new COSIOException(message, s3Exception) : new COSServiceIOException(message, ase);
                break;
        }
        return ioe;
    }
}

Example 63 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project presto by prestodb.

the class MockAmazonS3 method getObjectMetadata.

@Override
public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest) {
    this.getObjectMetadataRequest = getObjectMetadataRequest;
    if (getObjectMetadataHttpCode != HTTP_OK) {
        AmazonS3Exception exception = new AmazonS3Exception("Failing getObjectMetadata call with " + getObjectMetadataHttpCode);
        exception.setStatusCode(getObjectMetadataHttpCode);
        throw exception;
    }
    return null;
}

Example 64 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project presto by prestodb.

the class TestPrestoS3FileSystem method testReadRetryCounters.

@SuppressWarnings({ "ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions" })
@Test
public void testReadRetryCounters() throws Exception {
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        int maxRetries = 2;
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(HTTP_INTERNAL_ERROR);
        Configuration configuration = new Configuration();
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        } catch (Throwable expected) {
            assertInstanceOf(expected, AmazonS3Exception.class);
            assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries);
        }
    }
}

Example 65 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project alluxio by Alluxio.

the class S3AInputStream method openStream.

/**
 * Opens a new stream at mPos if the wrapped stream mIn is null.
 */
private void openStream() throws IOException {
    if (mIn != null) {
        // stream is already open
        return;
    }
    GetObjectRequest getReq = new GetObjectRequest(mBucketName, mKey);
    // If the position is 0, setting range is redundant and causes an error if the file is 0 length
    if (mPos > 0) {
        getReq.setRange(mPos);
    }
    AmazonS3Exception lastException = null;
    String errorMessage = String.format("Failed to open key: %s bucket: %s, left retry:%d", mKey, mBucketName, mRetryPolicy.getAttemptCount());
    while (mRetryPolicy.attempt()) {
        try {
            mIn = getClient().getObject(getReq).getObjectContent();
            return;
        } catch (AmazonS3Exception e) {
            errorMessage = String.format("Failed to open key: %s bucket: %s attempts: %d error: %s", mKey, mBucketName, mRetryPolicy.getAttemptCount(), e.getMessage());
            if (e.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
                throw new IOException(errorMessage, e);
            }
            // Key does not exist
            lastException = e;
        }
    }
    // Failed after retrying key does not exist
    throw new IOException(errorMessage, lastException);
}