Examples with ObjectMetadata com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata used on opensource projects

Example 41 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project jaqy by Teradata.

the class S3Path method getLength.

private void getLength() {
    try {
        ObjectMetadata meta = m_s3.getObjectMetadata(m_bucket, m_file);
        m_length = meta.getContentLength();
    } catch (Exception ex) {
        m_interpreter.getGlobals().log(Level.INFO, ex);
        m_length = Long.MIN_VALUE;
    }
}

Example 42 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project bender by Nextdoor.

the class GeoIpOperationFactoryTest method setup.

@Before
public void setup() {
    /*
     * Patch the handler to use this test's factory which produces a mock client.
     */
    S3MockClientFactory f;
    try {
        f = new S3MockClientFactory(tmpFolder);
    } catch (Exception e) {
        throw new RuntimeException("unable to start s3proxy", e);
    }
    this.clientFactory = f;
    this.client = f.newInstance();
    /*
     * Populate S3
     */
    this.client.createBucket(BUCKET);
    InputStream is = this.getClass().getResourceAsStream("my-ip-data.mmdb");
    this.client.putObject(BUCKET, "my-ip-data.mmdb", is, new ObjectMetadata());
    this.opFactory = new GeoIpOperationFactory();
    this.opFactory.s3Factory = this.clientFactory;
}

Example 43 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project bender by Nextdoor.

the class S3SnsNotifier method main.

public static void main(String[] args) throws ParseException, InterruptedException, IOException {
    formatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC();
    /*
     * Parse cli arguments
     */
    Options options = new Options();
    options.addOption(Option.builder().longOpt("bucket").hasArg().required().desc("Name of S3 bucket to list s3 objects from").build());
    options.addOption(Option.builder().longOpt("key-file").hasArg().required().desc("Local file of S3 keys to process").build());
    options.addOption(Option.builder().longOpt("sns-arn").hasArg().required().desc("SNS arn to publish to").build());
    options.addOption(Option.builder().longOpt("throttle-ms").hasArg().desc("Amount of ms to wait between publishing to SNS").build());
    options.addOption(Option.builder().longOpt("processed-file").hasArg().desc("Local file to use to store procssed S3 object names").build());
    options.addOption(Option.builder().longOpt("skip-processed").hasArg(false).desc("Whether to skip S3 objects that have been processed").build());
    options.addOption(Option.builder().longOpt("dry-run").hasArg(false).desc("If set do not publish to SNS").build());
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = parser.parse(options, args);
    String bucket = cmd.getOptionValue("bucket");
    String keyFile = cmd.getOptionValue("key-file");
    String snsArn = cmd.getOptionValue("sns-arn");
    String processedFile = cmd.getOptionValue("processed-file", null);
    boolean skipProcessed = cmd.hasOption("skip-processed");
    dryRun = cmd.hasOption("dry-run");
    long throttle = Long.parseLong(cmd.getOptionValue("throttle-ms", "-1"));
    if (processedFile != null) {
        File file = new File(processedFile);
        if (!file.exists()) {
            logger.debug("creating local file to store processed s3 object names: " + processedFile);
            file.createNewFile();
        }
    }
    /*
     * Import S3 keys that have been processed
     */
    if (skipProcessed && processedFile != null) {
        try (BufferedReader br = new BufferedReader(new FileReader(processedFile))) {
            String line;
            while ((line = br.readLine()) != null) {
                alreadyPublished.add(line.trim());
            }
        }
    }
    /*
     * Setup writer for file containing processed S3 keys
     */
    FileWriter fw = null;
    BufferedWriter bw = null;
    if (processedFile != null) {
        fw = new FileWriter(processedFile, true);
        bw = new BufferedWriter(fw);
    }
    /*
     * Create clients
     */
    AmazonS3Client s3Client = new AmazonS3Client();
    AmazonSNSClient snsClient = new AmazonSNSClient();
    /*
     * Get S3 object list
     */
    try (BufferedReader br = new BufferedReader(new FileReader(keyFile))) {
        String line;
        while ((line = br.readLine()) != null) {
            String key = line.trim();
            if (alreadyPublished.contains(key)) {
                logger.info("skipping " + key);
            }
            ObjectMetadata om = s3Client.getObjectMetadata(bucket, key);
            S3EventNotification s3Notification = getS3Notification(key, bucket, om.getContentLength());
            String json = s3Notification.toJson();
            /*
         * Publish to SNS
         */
            if (publish(snsArn, json, snsClient, key) && processedFile != null) {
                bw.write(key + "\n");
                bw.flush();
            }
            if (throttle != -1) {
                Thread.sleep(throttle);
            }
        }
    }
    if (processedFile != null) {
        bw.close();
        fw.close();
    }
}

Example 44 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project Synapse-Stack-Builder by Sage-Bionetworks.

the class BuildStackMainTest method before.

@Before
public void before() throws IOException {
    inputProps = TestHelper.createInputProperties("dev");
    InputConfiguration config = TestHelper.createTestConfig("dev");
    defaultProps = TestHelper.createDefaultProperties();
    clientFactory = new MockAmazonClientFactory();
    AmazonS3Client mockS3Client = clientFactory.createS3Client();
    AmazonEC2Client mockEC2Client = clientFactory.createEC2Client();
    AmazonSNSClient mockSNSnsClient = clientFactory.createSNSClient();
    AmazonRDSClient mockRdsClient = clientFactory.createRDSClient();
    // Write the default properties.
    when(mockS3Client.getObject(any(GetObjectRequest.class), any(File.class))).thenAnswer(new Answer<ObjectMetadata>() {

        public ObjectMetadata answer(InvocationOnMock invocation) throws Throwable {
            // Write the property file
            File file = (File) invocation.getArguments()[1];
            FileWriter writer = new FileWriter(file);
            try {
                defaultProps.store(writer, "test generated");
            } finally {
                writer.close();
            }
            return new ObjectMetadata();
        }
    });
    // Return a valid EC2 security group.
    DescribeSecurityGroupsRequest dsgr = new DescribeSecurityGroupsRequest().withGroupNames(config.getElasticSecurityGroupName());
    when(mockEC2Client.describeSecurityGroups(dsgr)).thenReturn(new DescribeSecurityGroupsResult().withSecurityGroups(new SecurityGroup().withGroupName(config.getElasticSecurityGroupName())));
    // Return a valid topic
    String topicArn = "some:arn";
    when(mockSNSnsClient.createTopic(new CreateTopicRequest(config.getRDSAlertTopicName()))).thenReturn(new CreateTopicResult().withTopicArn(topicArn));
    when(mockSNSnsClient.listSubscriptionsByTopic(new ListSubscriptionsByTopicRequest(topicArn))).thenReturn(new ListSubscriptionsByTopicResult().withSubscriptions(new Subscription()));
    // return a valid group
    when(mockRdsClient.describeDBParameterGroups(new DescribeDBParameterGroupsRequest().withDBParameterGroupName(config.getDatabaseParameterGroupName()))).thenReturn(new DescribeDBParameterGroupsResult().withDBParameterGroups(new DBParameterGroup().withDBParameterGroupName(config.getDatabaseParameterGroupName())));
    when(mockRdsClient.describeDBParameters(new DescribeDBParametersRequest().withDBParameterGroupName(config.getDatabaseParameterGroupName()))).thenReturn(new DescribeDBParametersResult().withParameters(new Parameter().withParameterName(Constants.DB_PARAM_KEY_SLOW_QUERY_LOG)).withParameters(new Parameter().withParameterName(Constants.DB_PARAM_KEY_LONG_QUERY_TIME)));
}

Example 45 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project sic by belluccifranco.

the class AmazonServiceImpl method saveFileIntoS3Bucket.

@Override
public String saveFileIntoS3Bucket(String key, InputStream file, String contentType) {
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentType(contentType);
    try {
        this.getAmazonS3Client().putObject(new PutObjectRequest(bucketName, key, file, objectMetadata).withCannedAcl(CannedAccessControlList.PublicRead));
    } catch (SdkClientException ex) {
        LOGGER.error(ex.getMessage());
        throw new ServiceException(ResourceBundle.getBundle("Mensajes").getString("mensaje_acceso_S3_error"), ex);
    }
    return amazonS3url + bucketName + "/" + key;
}