use of com.amazonaws.services.s3.model.Tag in project crate by crate.
the class AwsEc2SeedHostsProvider method fetchDynamicNodes.
private List<TransportAddress> fetchDynamicNodes() {
final List<TransportAddress> dynamicHosts = new ArrayList<>();
final DescribeInstancesResult descInstances;
try (AmazonEc2Reference clientReference = awsEc2Service.client()) {
// Query EC2 API based on AZ, instance state, and tag.
// NOTE: we don't filter by security group during the describe instances request for two reasons:
// 1. differences in VPCs require different parameters during query (ID vs Name)
// 2. We want to use two different strategies: (all security groups vs. any security groups)
descInstances = clientReference.client().describeInstances(buildDescribeInstancesRequest());
} catch (final AmazonClientException e) {
LOGGER.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
LOGGER.debug("Full exception:", e);
return dynamicHosts;
}
LOGGER.trace("finding seed nodes...");
for (final Reservation reservation : descInstances.getReservations()) {
for (final Instance instance : reservation.getInstances()) {
// lets see if we can filter based on groups
if (!groups.isEmpty()) {
final List<GroupIdentifier> instanceSecurityGroups = instance.getSecurityGroups();
final List<String> securityGroupNames = new ArrayList<>(instanceSecurityGroups.size());
final List<String> securityGroupIds = new ArrayList<>(instanceSecurityGroups.size());
for (final GroupIdentifier sg : instanceSecurityGroups) {
securityGroupNames.add(sg.getGroupName());
securityGroupIds.add(sg.getGroupId());
}
if (bindAnyGroup) {
// We check if we can find at least one group name or one group id in groups.
if (disjoint(securityGroupNames, groups) && disjoint(securityGroupIds, groups)) {
LOGGER.trace("filtering out instance {} based on groups {}, not part of {}", instance.getInstanceId(), instanceSecurityGroups, groups);
// continue to the next instance
continue;
}
} else {
// We need tp match all group names or group ids, otherwise we ignore this instance
if (!(securityGroupNames.containsAll(groups) || securityGroupIds.containsAll(groups))) {
LOGGER.trace("filtering out instance {} based on groups {}, does not include all of {}", instance.getInstanceId(), instanceSecurityGroups, groups);
// continue to the next instance
continue;
}
}
}
String address = null;
if (hostType.equals(PRIVATE_DNS)) {
address = instance.getPrivateDnsName();
} else if (hostType.equals(PRIVATE_IP)) {
address = instance.getPrivateIpAddress();
} else if (hostType.equals(PUBLIC_DNS)) {
address = instance.getPublicDnsName();
} else if (hostType.equals(PUBLIC_IP)) {
address = instance.getPublicIpAddress();
} else if (hostType.startsWith(TAG_PREFIX)) {
// Reading the node host from its metadata
final String tagName = hostType.substring(TAG_PREFIX.length());
LOGGER.debug("reading hostname from [{}] instance tag", tagName);
final List<Tag> tags = instance.getTags();
for (final Tag tag : tags) {
if (tag.getKey().equals(tagName)) {
address = tag.getValue();
LOGGER.debug("using [{}] as the instance address", address);
}
}
} else {
throw new IllegalArgumentException(hostType + " is unknown for discovery.ec2.host_type");
}
if (address != null) {
try {
final TransportAddress[] addresses = transportService.addressesFromString(address);
for (int i = 0; i < addresses.length; i++) {
LOGGER.trace("adding {}, address {}, transport_address {}", instance.getInstanceId(), address, addresses[i]);
dynamicHosts.add(addresses[i]);
}
} catch (final Exception e) {
final String finalAddress = address;
LOGGER.warn((Supplier<?>) () -> new ParameterizedMessage("failed to add {}, address {}", instance.getInstanceId(), finalAddress), e);
}
} else {
LOGGER.trace("not adding {}, address is null, host_type {}", instance.getInstanceId(), hostType);
}
}
}
LOGGER.debug("using dynamic transport addresses {}", dynamicHosts);
return dynamicHosts;
}
use of com.amazonaws.services.s3.model.Tag in project elasticsearch by elastic.
the class AmazonEC2Mock method describeInstances.
@Override
public DescribeInstancesResult describeInstances(DescribeInstancesRequest describeInstancesRequest) throws AmazonServiceException, AmazonClientException {
Collection<Instance> filteredInstances = new ArrayList<>();
logger.debug("--> mocking describeInstances");
for (Instance instance : instances) {
boolean tagFiltered = false;
boolean instanceFound = false;
Map<String, List<String>> expectedTags = new HashMap<>();
Map<String, List<String>> instanceTags = new HashMap<>();
for (Tag tag : instance.getTags()) {
List<String> tags = instanceTags.get(tag.getKey());
if (tags == null) {
tags = new ArrayList<>();
instanceTags.put(tag.getKey(), tags);
}
tags.add(tag.getValue());
}
for (Filter filter : describeInstancesRequest.getFilters()) {
// If we have the same tag name and one of the values, we add the instance
if (filter.getName().startsWith("tag:")) {
tagFiltered = true;
String tagName = filter.getName().substring(4);
// if we have more than one value for the same key, then the key is appended with .x
Pattern p = Pattern.compile("\\.\\d+", Pattern.DOTALL);
Matcher m = p.matcher(tagName);
if (m.find()) {
int i = tagName.lastIndexOf(".");
tagName = tagName.substring(0, i);
}
List<String> tags = expectedTags.get(tagName);
if (tags == null) {
tags = new ArrayList<>();
expectedTags.put(tagName, tags);
}
tags.addAll(filter.getValues());
}
}
if (tagFiltered) {
logger.debug("--> expected tags: [{}]", expectedTags);
logger.debug("--> instance tags: [{}]", instanceTags);
instanceFound = true;
for (Map.Entry<String, List<String>> expectedTagsEntry : expectedTags.entrySet()) {
List<String> instanceTagValues = instanceTags.get(expectedTagsEntry.getKey());
if (instanceTagValues == null) {
instanceFound = false;
break;
}
for (String expectedValue : expectedTagsEntry.getValue()) {
boolean valueFound = false;
for (String instanceTagValue : instanceTagValues) {
if (instanceTagValue.equals(expectedValue)) {
valueFound = true;
}
}
if (valueFound == false) {
instanceFound = false;
}
}
}
}
if (tagFiltered == false || instanceFound) {
logger.debug("--> instance added");
filteredInstances.add(instance);
} else {
logger.debug("--> instance filtered");
}
}
return new DescribeInstancesResult().withReservations(new Reservation().withInstances(filteredInstances));
}
use of com.amazonaws.services.s3.model.Tag in project GNS by MobilityFirst.
the class AWSEC2 method addInstanceTag.
/**
* Adds the key and value as a 'tag' for the instance.
*
* @param ec2
* @param createdInstanceId
* @param key
* @param value
*/
public static void addInstanceTag(AmazonEC2 ec2, String createdInstanceId, String key, String value) {
List<String> resources = new LinkedList<>();
resources.add(createdInstanceId);
List<Tag> tags = new LinkedList<>();
Tag nameTag = new Tag(key, value);
tags.add(nameTag);
CreateTagsRequest ctr = new CreateTagsRequest(resources, tags);
ec2.createTags(ctr);
}
use of com.amazonaws.services.s3.model.Tag in project herd by FINRAOS.
the class S3DaoImplTest method testTagObjectsS3FilesListEmpty.
@Test
public void testTagObjectsS3FilesListEmpty() {
// Create an S3 file transfer request parameters DTO to access S3 objects without specifying S3 files.
S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
s3FileTransferRequestParamsDto.setFiles(new ArrayList<>());
// Create an S3 file transfer request parameters DTO to tag S3 objects.
S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
// Create an S3 object tag.
Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);
// Call the method under test.
s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, tag);
// Verify the external calls.
verifyNoMoreInteractionsHelper();
}
use of com.amazonaws.services.s3.model.Tag in project herd by FINRAOS.
the class S3DaoImplTest method testTagObjects.
@Test
public void testTagObjects() {
// Create an S3 file transfer request parameters DTO to access S3 objects.
S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
s3FileTransferRequestParamsDto.setFiles(Arrays.asList(new File(S3_KEY_PREFIX + "/" + LOCAL_FILE)));
// Create an S3 file transfer request parameters DTO to tag S3 objects.
S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
// Create an S3 object tag.
Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);
// Create a retry policy.
RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);
// Create a get object tagging result.
GetObjectTaggingResult getObjectTaggingResult = new GetObjectTaggingResult(null);
// Create a set object tagging result.
SetObjectTaggingResult setObjectTaggingResult = new SetObjectTaggingResult();
// Mock the external calls.
when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
when(s3Operations.getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(getObjectTaggingResult);
when(s3Operations.setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(setObjectTaggingResult);
// Call the method under test.
s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, tag);
// Verify the external calls.
verify(retryPolicyFactory, times(2)).getRetryPolicy();
verify(s3Operations).getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class));
verify(s3Operations).setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class));
verifyNoMoreInteractionsHelper();
}
Aggregations