Examples with S3EventNotification com.amazonaws.services.s3.event.S3EventNotification used on opensource projects

Example 1 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project herd by FINRAOS.

the class SampleDataJmsMessageListener method processMessage.

/**
 * Processes a JMS message.
 *
 * @param payload the message payload
 * @param allHeaders the JMS headers
 */
@JmsListener(id = HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, containerFactory = "jmsListenerContainerFactory", destination = HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE)
public void processMessage(String payload, @Headers Map<Object, Object> allHeaders) {
    LOGGER.info("Message received from the JMS queue. jmsQueueName=\"{}\" jmsMessageHeaders=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, allHeaders, payload);
    try {
        // Process messages coming from S3 bucket.
        S3EventNotification s3EventNotification = S3EventNotification.parseJson(payload);
        String objectKey = URLDecoder.decode(s3EventNotification.getRecords().get(0).getS3().getObject().getKey(), CharEncoding.UTF_8);
        long fileSize = s3EventNotification.getRecords().get(0).getS3().getObject().getSizeAsLong();
        // parse the objectKey, it should be in the format of namespace/businessObjectDefinitionName/fileName
        String[] objectKeyArrays = objectKey.split("/");
        Assert.isTrue(objectKeyArrays.length == 3, String.format("S3 notification message %s is not in expected format", objectKey));
        String namespace = objectKeyArrays[0];
        String businessObjectDefinitionName = objectKeyArrays[1];
        String fileName = objectKeyArrays[2];
        String path = namespace + "/" + businessObjectDefinitionName + "/";
        BusinessObjectDefinitionSampleFileUpdateDto businessObjectDefinitionSampleFileUpdateDto = new BusinessObjectDefinitionSampleFileUpdateDto(path, fileName, fileSize);
        String convertedNamespaece = convertS3KeyFormat(namespace);
        String convertedBusinessObjectDefinitionName = convertS3KeyFormat(businessObjectDefinitionName);
        BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(convertedNamespaece, convertedBusinessObjectDefinitionName);
        try {
            businessObjectDefinitionService.updateBusinessObjectDefinitionEntitySampleFile(businessObjectDefinitionKey, businessObjectDefinitionSampleFileUpdateDto);
        } catch (ObjectNotFoundException ex) {
            LOGGER.info("Failed to find the business object definition, next try the original namespace and business oject defination name " + ex);
            // if Business object definition is not found, use the original name space and bdef name
            businessObjectDefinitionKey = new BusinessObjectDefinitionKey(namespace, businessObjectDefinitionName);
            businessObjectDefinitionService.updateBusinessObjectDefinitionEntitySampleFile(businessObjectDefinitionKey, businessObjectDefinitionSampleFileUpdateDto);
        }
    } catch (RuntimeException | IOException e) {
        LOGGER.error("Failed to process message from the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, payload, e);
    }
}

Example 2 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project herd by FINRAOS.

the class SampleDataJmsMessageListenerTest method testS3Message.

@Test
public void testS3Message() throws Exception {
    // Create and persist database entities required for testing.
    businessObjectDefinitionServiceTestHelper.createDatabaseEntitiesForBusinessObjectDefinitionTesting();
    storageDaoTestHelper.createStorageEntity(StorageEntity.SAMPLE_DATA_FILE_STORAGE, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME)));
    // Create a business object definition.
    BusinessObjectDefinitionCreateRequest request = new BusinessObjectDefinitionCreateRequest(NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION, BDEF_DISPLAY_NAME, businessObjectDefinitionServiceTestHelper.getNewAttributes());
    businessObjectDefinitionService.createBusinessObjectDefinition(request);
    // Get the business object definition entity.
    BusinessObjectDefinitionEntity businessObjectDefinitionEntity = businessObjectDefinitionDao.getBusinessObjectDefinitionByKey(new BusinessObjectDefinitionKey(NAMESPACE, BDEF_NAME));
    assertNotNull(businessObjectDefinitionEntity);
    String fileName = "test1.csv";
    String filePath = NAMESPACE + "/" + BDEF_NAME + "/" + fileName;
    long fileSize = 1024L;
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, fileSize, null, null), null);
    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));
    S3EventNotification s3EventNotification = new S3EventNotification(records);
    sampleDataJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
    BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(NAMESPACE, BDEF_NAME);
    BusinessObjectDefinition updatedBusinessObjectDefinition = businessObjectDefinitionService.getBusinessObjectDefinition(businessObjectDefinitionKey, false);
    List<SampleDataFile> sampleDataFiles = Arrays.asList(new SampleDataFile(NAMESPACE + "/" + BDEF_NAME + "/", fileName));
    // Validate the returned object.
    assertEquals(new BusinessObjectDefinition(updatedBusinessObjectDefinition.getId(), NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION, NO_BDEF_SHORT_DESCRIPTION, BDEF_DISPLAY_NAME, businessObjectDefinitionServiceTestHelper.getNewAttributes(), NO_DESCRIPTIVE_BUSINESS_OBJECT_FORMAT, sampleDataFiles, businessObjectDefinitionEntity.getCreatedBy(), businessObjectDefinitionEntity.getUpdatedBy(), HerdDateUtils.getXMLGregorianCalendarValue(businessObjectDefinitionEntity.getUpdatedOn()), NO_BUSINESS_OBJECT_DEFINITION_CHANGE_EVENTS), updatedBusinessObjectDefinition);
}

Example 3 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project bender by Nextdoor.

the class S3HandlerTest method getTestEvent.

@Override
public S3EventNotification getTestEvent() throws Exception {
    /*
     * Upload a test resoruce to the mock S3
     */
    String payload = IOUtils.toString(new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
    this.client.putObject(S3_BUCKET, "basic_input.log", payload);
    /*
     * Create a S3EventNotification event
     */
    S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
    S3BucketEntity bucketEntity = new S3BucketEntity(S3_BUCKET, null, null);
    S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);
    S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null, "1970-01-01T00:00:00.000Z", null, null, null, entity, null);
    List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
    notifications.add(rec);
    return new S3EventNotification(notifications);
}

Example 4 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project bender by Nextdoor.

the class SNSS3HandlerTest method getTestEvent.

@Override
public SNSEvent getTestEvent() throws Exception {
    /*
     * Upload a test resoruce to the mock S3
     */
    String payload = IOUtils.toString(new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
    this.client.putObject(S3_BUCKET, "basic_input.log", payload);
    /*
     * Create a S3EventNotification event
     */
    S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
    S3BucketEntity bucketEntity = new S3BucketEntity(S3_BUCKET, null, null);
    S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);
    S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null, "1970-01-01T00:00:00.000Z", null, null, null, entity, null);
    List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
    notifications.add(rec);
    /*
     * Wrap as an SNS Event
     */
    S3EventNotification event = new S3EventNotification(notifications);
    SNSEvent.SNS sns = new SNSEvent.SNS();
    sns.setMessage(event.toJson());
    SNSEvent snsEvent = new SNSEvent();
    ArrayList<SNSRecord> snsRecords = new ArrayList<SNSRecord>(1);
    SNSRecord snsRecord = new SNSRecord();
    snsRecord.setEventSource("aws:sns");
    snsRecord.setEventVersion("1.0");
    snsRecord.setEventSubscriptionArn("arn");
    snsRecord.setSns(sns);
    snsRecords.add(snsRecord);
    snsEvent.setRecords(snsRecords);
    return snsEvent;
}

Example 5 with S3EventNotification

use of com.amazonaws.services.s3.event.S3EventNotification in project bender by Nextdoor.

the class SNSS3Handler method handler.

@Override
public void handler(SNSEvent event, Context context) throws HandlerException {
    if (!initialized) {
        init(context);
        SNSS3HandlerConfig handlerConfig = (SNSS3HandlerConfig) this.config.getHandlerConfig();
        this.logTrigger = handlerConfig.getLogSnsTrigger();
    }
    this.source = this.sources.get(0);
    this.inputFiles = new ArrayList<String>(0);
    if (this.logTrigger) {
        logger.info("trigger: " + gson.toJson(event));
    }
    for (SNSRecord record : event.getRecords()) {
        /*
       * Parse SNS as a S3 notification
       */
        String json = record.getSNS().getMessage();
        S3EventNotification s3Event = S3EventNotification.parseJson(json);
        /*
       * Validate the S3 file matches the regex
       */
        List<S3EventNotificationRecord> toProcess = new ArrayList<S3EventNotificationRecord>(s3Event.getRecords());
        for (S3EventNotificationRecord s3Record : s3Event.getRecords()) {
            String s3Path = String.format("s3://%s/%s", s3Record.getS3().getBucket().getName(), s3Record.getS3().getObject().getKey());
            try {
                this.source = SourceUtils.getSource(s3Path, this.sources);
            } catch (SourceNotFoundException e) {
                logger.warn("skipping processing " + s3Path);
                toProcess.remove(s3Record);
            }
        }
        if (toProcess.size() == 0) {
            logger.warn("Nothing to process");
            return;
        }
        this.inputFiles.addAll(toProcess.stream().map(m -> {
            return m.getS3().getObject().getKey();
        }).collect(Collectors.toList()));
        this.recordIterator = new S3EventIterator(context, toProcess, s3ClientFactory);
        super.process(context);
    }
}