Examples with AmazonClientException com.amazonaws.AmazonClientException used on opensource projects

Example 1 with AmazonClientException

use of com.amazonaws.AmazonClientException in project elasticsearch by elastic.

the class AwsEc2ServiceImpl method buildConfiguration.

protected static ClientConfiguration buildConfiguration(Logger logger, Settings settings) {
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    // the response metadata cache is only there for diagnostics purposes,
    // but can force objects from every response to the old generation.
    clientConfiguration.setResponseMetadataCacheSize(0);
    clientConfiguration.setProtocol(CLOUD_EC2.PROTOCOL_SETTING.get(settings));
    if (PROXY_HOST_SETTING.exists(settings) || CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) {
        String proxyHost = CLOUD_EC2.PROXY_HOST_SETTING.get(settings);
        Integer proxyPort = CLOUD_EC2.PROXY_PORT_SETTING.get(settings);
        String proxyUsername = CLOUD_EC2.PROXY_USERNAME_SETTING.get(settings);
        String proxyPassword = CLOUD_EC2.PROXY_PASSWORD_SETTING.get(settings);
        clientConfiguration.withProxyHost(proxyHost).withProxyPort(proxyPort).withProxyUsername(proxyUsername).withProxyPassword(proxyPassword);
    }
    // #155: we might have 3rd party users using older EC2 API version
    String awsSigner = CLOUD_EC2.SIGNER_SETTING.get(settings);
    if (Strings.hasText(awsSigner)) {
        logger.debug("using AWS API signer [{}]", awsSigner);
        AwsSigner.configureSigner(awsSigner, clientConfiguration);
    }
    // Increase the number of retries in case of 5xx API responses
    final Random rand = Randomness.get();
    RetryPolicy retryPolicy = new RetryPolicy(RetryPolicy.RetryCondition.NO_RETRY_CONDITION, new RetryPolicy.BackoffStrategy() {

        @Override
        public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception, int retriesAttempted) {
            // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
            logger.warn("EC2 API request failed, retry again. Reason was:", exception);
            return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
        }
    }, 10, false);
    clientConfiguration.setRetryPolicy(retryPolicy);
    clientConfiguration.setSocketTimeout((int) CLOUD_EC2.READ_TIMEOUT.get(settings).millis());
    return clientConfiguration;
}

Example 2 with AmazonClientException

use of com.amazonaws.AmazonClientException in project elasticsearch by elastic.

the class AwsEc2UnicastHostsProvider method fetchDynamicNodes.

protected List<DiscoveryNode> fetchDynamicNodes() {
    List<DiscoveryNode> discoNodes = new ArrayList<>();
    DescribeInstancesResult descInstances;
    try {
        // Query EC2 API based on AZ, instance state, and tag.
        // NOTE: we don't filter by security group during the describe instances request for two reasons:
        // 1. differences in VPCs require different parameters during query (ID vs Name)
        // 2. We want to use two different strategies: (all security groups vs. any security groups)
        descInstances = SocketAccess.doPrivileged(() -> client.describeInstances(buildDescribeInstancesRequest()));
    } catch (AmazonClientException e) {
        logger.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
        logger.debug("Full exception:", e);
        return discoNodes;
    }
    logger.trace("building dynamic unicast discovery nodes...");
    for (Reservation reservation : descInstances.getReservations()) {
        for (Instance instance : reservation.getInstances()) {
            // lets see if we can filter based on groups
            if (!groups.isEmpty()) {
                List<GroupIdentifier> instanceSecurityGroups = instance.getSecurityGroups();
                ArrayList<String> securityGroupNames = new ArrayList<String>();
                ArrayList<String> securityGroupIds = new ArrayList<String>();
                for (GroupIdentifier sg : instanceSecurityGroups) {
                    securityGroupNames.add(sg.getGroupName());
                    securityGroupIds.add(sg.getGroupId());
                }
                if (bindAnyGroup) {
                    // We check if we can find at least one group name or one group id in groups.
                    if (disjoint(securityGroupNames, groups) && disjoint(securityGroupIds, groups)) {
                        logger.trace("filtering out instance {} based on groups {}, not part of {}", instance.getInstanceId(), instanceSecurityGroups, groups);
                        // continue to the next instance
                        continue;
                    }
                } else {
                    // We need tp match all group names or group ids, otherwise we ignore this instance
                    if (!(securityGroupNames.containsAll(groups) || securityGroupIds.containsAll(groups))) {
                        logger.trace("filtering out instance {} based on groups {}, does not include all of {}", instance.getInstanceId(), instanceSecurityGroups, groups);
                        // continue to the next instance
                        continue;
                    }
                }
            }
            String address = null;
            if (hostType.equals(PRIVATE_DNS)) {
                address = instance.getPrivateDnsName();
            } else if (hostType.equals(PRIVATE_IP)) {
                address = instance.getPrivateIpAddress();
            } else if (hostType.equals(PUBLIC_DNS)) {
                address = instance.getPublicDnsName();
            } else if (hostType.equals(PUBLIC_IP)) {
                address = instance.getPublicIpAddress();
            } else if (hostType.startsWith(TAG_PREFIX)) {
                // Reading the node host from its metadata
                String tagName = hostType.substring(TAG_PREFIX.length());
                logger.debug("reading hostname from [{}] instance tag", tagName);
                List<Tag> tags = instance.getTags();
                for (Tag tag : tags) {
                    if (tag.getKey().equals(tagName)) {
                        address = tag.getValue();
                        logger.debug("using [{}] as the instance address", address);
                    }
                }
            } else {
                throw new IllegalArgumentException(hostType + " is unknown for discovery.ec2.host_type");
            }
            if (address != null) {
                try {
                    // we only limit to 1 port per address, makes no sense to ping 100 ports
                    TransportAddress[] addresses = transportService.addressesFromString(address, 1);
                    for (int i = 0; i < addresses.length; i++) {
                        logger.trace("adding {}, address {}, transport_address {}", instance.getInstanceId(), address, addresses[i]);
                        discoNodes.add(new DiscoveryNode(instance.getInstanceId(), "#cloud-" + instance.getInstanceId() + "-" + i, addresses[i], emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
                    }
                } catch (Exception e) {
                    final String finalAddress = address;
                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to add {}, address {}", instance.getInstanceId(), finalAddress), e);
                }
            } else {
                logger.trace("not adding {}, address is null, host_type {}", instance.getInstanceId(), hostType);
            }
        }
    }
    logger.debug("using dynamic discovery nodes {}", discoNodes);
    return discoNodes;
}

Example 3 with AmazonClientException

use of com.amazonaws.AmazonClientException in project elasticsearch by elastic.

the class DefaultS3OutputStream method uploadMultipart.

private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) throws IOException {
    try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) {
        int retry = 0;
        while (retry <= getNumberOfRetries()) {
            try {
                PartETag partETag = doUploadMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, is, len, lastPart);
                multiparts.add(partETag);
                multipartChunks++;
                return;
            } catch (AmazonClientException e) {
                if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) {
                    is.reset();
                    retry++;
                } else {
                    abortMultipart();
                    throw e;
                }
            }
        }
    }
}

Example 4 with AmazonClientException

use of com.amazonaws.AmazonClientException in project zeppelin by apache.

the class S3NotebookRepo method getNote.

private Note getNote(String key) throws IOException {
    GsonBuilder gsonBuilder = new GsonBuilder();
    gsonBuilder.setPrettyPrinting();
    Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();
    S3Object s3object;
    try {
        s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }
    Note note;
    try (InputStream ins = s3object.getObjectContent()) {
        String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
        note = gson.fromJson(json, Note.class);
    }
    for (Paragraph p : note.getParagraphs()) {
        if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
            p.setStatus(Status.ABORT);
        }
    }
    return note;
}

Example 5 with AmazonClientException

use of com.amazonaws.AmazonClientException in project hadoop by apache.

the class S3AFileSystem method listLocatedStatus.

/**
   * {@inheritDoc}.
   *
   * S3 Optimized directory listing. The initial operation performs the
   * first bulk listing; extra listings will take place
   * when all the current set of results are used up.
   * @param f a path
   * @param filter a path filter
   * @return an iterator that traverses statuses of the files/directories
   *         in the given path
   * @throws FileNotFoundException if {@code path} does not exist
   * @throws IOException if any I/O error occurred
   */
@Override
public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f, final PathFilter filter) throws FileNotFoundException, IOException {
    incrementStatistic(INVOCATION_LIST_LOCATED_STATUS);
    Path path = qualify(f);
    LOG.debug("listLocatedStatus({}, {}", path, filter);
    try {
        // lookup dir triggers existence check
        final FileStatus fileStatus = getFileStatus(path);
        if (fileStatus.isFile()) {
            // simple case: File
            LOG.debug("Path is a file");
            return new Listing.SingleStatusRemoteIterator(filter.accept(path) ? toLocatedFileStatus(fileStatus) : null);
        } else {
            // directory: trigger a lookup
            String key = maybeAddTrailingSlash(pathToKey(path));
            return listing.createLocatedFileStatusIterator(listing.createFileStatusListingIterator(path, createListObjectsRequest(key, "/"), filter, new Listing.AcceptAllButSelfAndS3nDirs(path)));
        }
    } catch (AmazonClientException e) {
        throw translateException("listLocatedStatus", path, e);
    }
}