Examples with S3EventNotificationRecord com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord used on opensource projects

Example 1 with S3EventNotificationRecord

use of com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord in project bender by Nextdoor.

the class S3HandlerTest method getTestEvent.

@Override
public S3EventNotification getTestEvent() throws Exception {
    /*
     * Upload a test resoruce to the mock S3
     */
    String payload = IOUtils.toString(new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
    this.client.putObject(S3_BUCKET, "basic_input.log", payload);
    /*
     * Create a S3EventNotification event
     */
    S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
    S3BucketEntity bucketEntity = new S3BucketEntity(S3_BUCKET, null, null);
    S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);
    S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null, "1970-01-01T00:00:00.000Z", null, null, null, entity, null);
    List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
    notifications.add(rec);
    return new S3EventNotification(notifications);
}

Example 2 with S3EventNotificationRecord

use of com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord in project bender by Nextdoor.

the class SNSS3HandlerTest method getTestEvent.

@Override
public SNSEvent getTestEvent() throws Exception {
    /*
     * Upload a test resoruce to the mock S3
     */
    String payload = IOUtils.toString(new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
    this.client.putObject(S3_BUCKET, "basic_input.log", payload);
    /*
     * Create a S3EventNotification event
     */
    S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
    S3BucketEntity bucketEntity = new S3BucketEntity(S3_BUCKET, null, null);
    S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);
    S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null, "1970-01-01T00:00:00.000Z", null, null, null, entity, null);
    List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
    notifications.add(rec);
    /*
     * Wrap as an SNS Event
     */
    S3EventNotification event = new S3EventNotification(notifications);
    SNSEvent.SNS sns = new SNSEvent.SNS();
    sns.setMessage(event.toJson());
    SNSEvent snsEvent = new SNSEvent();
    ArrayList<SNSRecord> snsRecords = new ArrayList<SNSRecord>(1);
    SNSRecord snsRecord = new SNSRecord();
    snsRecord.setEventSource("aws:sns");
    snsRecord.setEventVersion("1.0");
    snsRecord.setEventSubscriptionArn("arn");
    snsRecord.setSns(sns);
    snsRecords.add(snsRecord);
    snsEvent.setRecords(snsRecords);
    return snsEvent;
}

Example 3 with S3EventNotificationRecord

use of com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord in project bender by Nextdoor.

the class S3EventIterator method updateCursor.

private void updateCursor() {
    if (this.currentIndex == 0 || (this.currentIndex < this.records.size() && !this.lineIterator.hasNext())) {
        /*
       * The previous reader must be closed in order to prevent S3 connection leaking
       */
        closeCurrentReader();
        /*
       * Use the S3 trigger event time for arrival time of records in file. This is less precise but
       * avoids making a call to the S3 api to find file creation time. Note that if the
       * deserializer creates a {@link com.nextdoor.bender.deserializer.DeserializedTimeSeriesEvent}
       * then this arrival time is not used.
       */
        S3EventNotificationRecord event = this.records.get(currentIndex);
        this.arrivalTime = event.getEventTime().toDate().getTime();
        this.currentS3Entity = event.getS3();
        /*
       * The S3 Object key is URL encoded and must be decoded before it can be used by the
       * AmazonS3Client
       */
        String key;
        try {
            key = URLDecoder.decode(this.currentS3Entity.getObject().getKey(), "UTF-8");
        } catch (UnsupportedEncodingException e) {
            throw new RuntimeException(e);
        }
        /*
       * Stream object back from S3 into a reader
       */
        String bucketName = this.currentS3Entity.getBucket().getName();
        logger.debug("opening s3://" + bucketName + "/" + key);
        GetObjectRequest req = new GetObjectRequest(bucketName, key);
        S3Object obj = client.getObject(req);
        logger.trace("s3 get request id: " + client.getCachedResponseMetadata(req).getRequestId() + " host: " + client.getCachedResponseMetadata(req).getHostId() + " cloudfrontid: " + client.getCachedResponseMetadata(req).getCloudFrontId());
        // TODO: support different types of compressions
        if (key.endsWith(".gz")) {
            GZIPInputStream gzip;
            try {
                gzip = new GZIPInputStream(obj.getObjectContent());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            reader = new BufferedReader(new InputStreamReader(gzip));
        } else {
            reader = new BufferedReader(new InputStreamReader(obj.getObjectContent()));
        }
        /*
       * Note the BufferedReader is lazy and so is the iterator. The object is directly streamed
       * from S3, fed into an input stream and consumed line by line by the iterator.
       */
        this.lineIterator = reader.lines().iterator();
        currentIndex++;
    }
}

Example 4 with S3EventNotificationRecord

use of com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord in project bender by Nextdoor.

the class SNSS3Handler method handler.

@Override
public void handler(SNSEvent event, Context context) throws HandlerException {
    if (!initialized) {
        init(context);
        SNSS3HandlerConfig handlerConfig = (SNSS3HandlerConfig) this.config.getHandlerConfig();
        this.logTrigger = handlerConfig.getLogSnsTrigger();
    }
    this.source = this.sources.get(0);
    this.inputFiles = new ArrayList<String>(0);
    if (this.logTrigger) {
        logger.info("trigger: " + gson.toJson(event));
    }
    for (SNSRecord record : event.getRecords()) {
        /*
       * Parse SNS as a S3 notification
       */
        String json = record.getSNS().getMessage();
        S3EventNotification s3Event = S3EventNotification.parseJson(json);
        /*
       * Validate the S3 file matches the regex
       */
        List<S3EventNotificationRecord> toProcess = new ArrayList<S3EventNotificationRecord>(s3Event.getRecords());
        for (S3EventNotificationRecord s3Record : s3Event.getRecords()) {
            String s3Path = String.format("s3://%s/%s", s3Record.getS3().getBucket().getName(), s3Record.getS3().getObject().getKey());
            try {
                this.source = SourceUtils.getSource(s3Path, this.sources);
            } catch (SourceNotFoundException e) {
                logger.warn("skipping processing " + s3Path);
                toProcess.remove(s3Record);
            }
        }
        if (toProcess.size() == 0) {
            logger.warn("Nothing to process");
            return;
        }
        this.inputFiles.addAll(toProcess.stream().map(m -> {
            return m.getS3().getObject().getKey();
        }).collect(Collectors.toList()));
        this.recordIterator = new S3EventIterator(context, toProcess, s3ClientFactory);
        super.process(context);
    }
}

Example 5 with S3EventNotificationRecord

use of com.amazonaws.services.s3.event.S3EventNotification.S3EventNotificationRecord in project bender by Nextdoor.

the class SNSS3HandlerTest method getTestEvent.

private SNSEvent getTestEvent(String bucket, boolean doPut) throws Exception {
    /*
     * Upload a test resoruce to the mock S3
     */
    if (doPut) {
        String payload = IOUtils.toString(new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
        this.client.putObject(bucket, "basic_input.log", payload);
    }
    /*
     * Create a S3EventNotification event
     */
    S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
    S3BucketEntity bucketEntity = new S3BucketEntity(bucket, null, null);
    S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);
    S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null, "1970-01-01T00:00:00.000Z", null, null, null, entity, null);
    List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
    notifications.add(rec);
    /*
     * Wrap as an SNS Event
     */
    S3EventNotification event = new S3EventNotification(notifications);
    SNSEvent.SNS sns = new SNSEvent.SNS();
    sns.setMessage(event.toJson());
    SNSEvent snsEvent = new SNSEvent();
    ArrayList<SNSRecord> snsRecords = new ArrayList<SNSRecord>(1);
    SNSRecord snsRecord = new SNSRecord();
    snsRecord.setEventSource("aws:sns");
    snsRecord.setEventVersion("1.0");
    snsRecord.setEventSubscriptionArn("arn");
    snsRecord.setSns(sns);
    snsRecords.add(snsRecord);
    snsEvent.setRecords(snsRecords);
    return snsEvent;
}