Examples with S3EventNotification com.amazonaws.services.s3.event.S3EventNotification used on opensource projects

Example 6 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project bender by Nextdoor.

the class S3HandlerTest method testSourceRegexFail.

@Test
public void testSourceRegexFail() throws Throwable {
    BaseHandler.CONFIG_FILE = "/com/nextdoor/bender/handler/config_s3_source.json";
    TestContext ctx = new TestContext();
    ctx.setFunctionName("unittest");
    ctx.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test-function:staging");
    BaseHandler<S3EventNotification> handler = (BaseHandler) getHandler();
    handler.init(ctx);
    handler.handler(getTestEvent("foo", false), ctx);
    assertEquals(0, DummyTransportHelper.BufferedTransporter.output.size());
}

Example 7 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project bender by Nextdoor.

the class SNSS3HandlerTest method getTestEvent.

private SNSEvent getTestEvent(String bucket, boolean doPut) throws Exception {
    /*
     * Upload a test resoruce to the mock S3
     */
    if (doPut) {
        String payload = IOUtils.toString(new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
        this.client.putObject(bucket, "basic_input.log", payload);
    }
    /*
     * Create a S3EventNotification event
     */
    S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
    S3BucketEntity bucketEntity = new S3BucketEntity(bucket, null, null);
    S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);
    S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null, "1970-01-01T00:00:00.000Z", null, null, null, entity, null);
    List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
    notifications.add(rec);
    /*
     * Wrap as an SNS Event
     */
    S3EventNotification event = new S3EventNotification(notifications);
    SNSEvent.SNS sns = new SNSEvent.SNS();
    sns.setMessage(event.toJson());
    SNSEvent snsEvent = new SNSEvent();
    ArrayList<SNSRecord> snsRecords = new ArrayList<SNSRecord>(1);
    SNSRecord snsRecord = new SNSRecord();
    snsRecord.setEventSource("aws:sns");
    snsRecord.setEventVersion("1.0");
    snsRecord.setEventSubscriptionArn("arn");
    snsRecord.setSns(sns);
    snsRecords.add(snsRecord);
    snsEvent.setRecords(snsRecords);
    return snsEvent;
}

Example 8 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project bender by Nextdoor.

the class S3SnsNotifier method main.

public static void main(String[] args) throws ParseException, InterruptedException, IOException {
    formatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC();
    /*
     * Parse cli arguments
     */
    Options options = new Options();
    options.addOption(Option.builder().longOpt("bucket").hasArg().required().desc("Name of S3 bucket to list s3 objects from").build());
    options.addOption(Option.builder().longOpt("key-file").hasArg().required().desc("Local file of S3 keys to process").build());
    options.addOption(Option.builder().longOpt("sns-arn").hasArg().required().desc("SNS arn to publish to").build());
    options.addOption(Option.builder().longOpt("throttle-ms").hasArg().desc("Amount of ms to wait between publishing to SNS").build());
    options.addOption(Option.builder().longOpt("processed-file").hasArg().desc("Local file to use to store procssed S3 object names").build());
    options.addOption(Option.builder().longOpt("skip-processed").hasArg(false).desc("Whether to skip S3 objects that have been processed").build());
    options.addOption(Option.builder().longOpt("dry-run").hasArg(false).desc("If set do not publish to SNS").build());
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = parser.parse(options, args);
    String bucket = cmd.getOptionValue("bucket");
    String keyFile = cmd.getOptionValue("key-file");
    String snsArn = cmd.getOptionValue("sns-arn");
    String processedFile = cmd.getOptionValue("processed-file", null);
    boolean skipProcessed = cmd.hasOption("skip-processed");
    dryRun = cmd.hasOption("dry-run");
    long throttle = Long.parseLong(cmd.getOptionValue("throttle-ms", "-1"));
    if (processedFile != null) {
        File file = new File(processedFile);
        if (!file.exists()) {
            logger.debug("creating local file to store processed s3 object names: " + processedFile);
            file.createNewFile();
        }
    }
    /*
     * Import S3 keys that have been processed
     */
    if (skipProcessed && processedFile != null) {
        try (BufferedReader br = new BufferedReader(new FileReader(processedFile))) {
            String line;
            while ((line = br.readLine()) != null) {
                alreadyPublished.add(line.trim());
            }
        }
    }
    /*
     * Setup writer for file containing processed S3 keys
     */
    FileWriter fw = null;
    BufferedWriter bw = null;
    if (processedFile != null) {
        fw = new FileWriter(processedFile, true);
        bw = new BufferedWriter(fw);
    }
    /*
     * Create clients
     */
    AmazonS3Client s3Client = new AmazonS3Client();
    AmazonSNSClient snsClient = new AmazonSNSClient();
    /*
     * Get S3 object list
     */
    try (BufferedReader br = new BufferedReader(new FileReader(keyFile))) {
        String line;
        while ((line = br.readLine()) != null) {
            String key = line.trim();
            if (alreadyPublished.contains(key)) {
                logger.info("skipping " + key);
            }
            ObjectMetadata om = s3Client.getObjectMetadata(bucket, key);
            S3EventNotification s3Notification = getS3Notification(key, bucket, om.getContentLength());
            String json = s3Notification.toJson();
            /*
         * Publish to SNS
         */
            if (publish(snsArn, json, snsClient, key) && processedFile != null) {
                bw.write(key + "\n");
                bw.flush();
            }
            if (throttle != -1) {
                Thread.sleep(throttle);
            }
        }
    }
    if (processedFile != null) {
        bw.close();
        fw.close();
    }
}

Example 9 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project bender by Nextdoor.

the class S3SnsNotifier method getS3Notification.

public static S3EventNotification getS3Notification(String key, String bucket, long size) {
    S3ObjectEntity objEntity = new S3ObjectEntity(key, size, null, null);
    S3BucketEntity bucketEntity = new S3BucketEntity(bucket, null, null);
    S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);
    String timestamp = formatter.print(System.currentTimeMillis());
    S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null, timestamp, null, null, null, entity, null);
    List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(1);
    notifications.add(rec);
    return new S3EventNotification(notifications);
}

Example 10 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project herd by FINRAOS.

the class HerdJmsMessageListenerTest method testS3MessageS3FileNoExists.

@Test
public void testS3MessageS3FileNoExists() throws Exception {
    setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF);
    uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting();
    UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper.createUploadSingleInitiationRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, TARGET_S3_KEY));
    String filePath = resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath();
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, 0L, null, null), null);
    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));
    S3EventNotification s3EventNotification = new S3EventNotification(records);
    setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF);
    setLogLevel(HerdJmsMessageListener.class, LogLevel.OFF);
    // Try to process an S3 JMS message, when source S3 file does not exist.
    herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
}