Examples with MockFlowFile org.apache.nifi.util.MockFlowFile used on opensource projects

Example 46 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class ITPutS3Object method testDynamicProperty.

@Test
public void testDynamicProperty() throws IOException {
    final String DYNAMIC_ATTRIB_KEY = "fs.runTimestamp";
    final String DYNAMIC_ATTRIB_VALUE = "${now():toNumber()}";
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    runner.setProperty(PutS3Object.MULTIPART_PART_SIZE, TEST_PARTSIZE_STRING);
    PropertyDescriptor testAttrib = processor.getSupportedDynamicPropertyDescriptor(DYNAMIC_ATTRIB_KEY);
    runner.setProperty(testAttrib, DYNAMIC_ATTRIB_VALUE);
    final String FILE1_NAME = "file1";
    Map<String, String> attribs = new HashMap<>();
    attribs.put(CoreAttributes.FILENAME.key(), FILE1_NAME);
    runner.enqueue("123".getBytes(), attribs);
    runner.assertValid();
    processor.getPropertyDescriptor(DYNAMIC_ATTRIB_KEY);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS);
    final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    Assert.assertEquals(1, successFiles.size());
    MockFlowFile ff1 = successFiles.get(0);
    Long now = System.currentTimeMillis();
    String millisNow = Long.toString(now);
    String millisOneSecAgo = Long.toString(now - 1000L);
    String usermeta = ff1.getAttribute(PutS3Object.S3_USERMETA_ATTR_KEY);
    String[] usermetaLine0 = usermeta.split(System.lineSeparator())[0].split("=");
    String usermetaKey0 = usermetaLine0[0];
    String usermetaValue0 = usermetaLine0[1];
    Assert.assertEquals(DYNAMIC_ATTRIB_KEY, usermetaKey0);
    Assert.assertTrue(usermetaValue0.compareTo(millisOneSecAgo) >= 0 && usermetaValue0.compareTo(millisNow) <= 0);
}

Example 47 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class TestListS3 method testListIgnoreByMinAge.

@Test
public void testListIgnoreByMinAge() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.MIN_AGE, "30 sec");
    Date lastModifiedNow = new Date();
    Date lastModifiedMinus1Hour = DateUtils.addHours(lastModifiedNow, -1);
    Date lastModifiedMinus3Hour = DateUtils.addHours(lastModifiedNow, -3);
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("minus-3hour");
    objectSummary1.setLastModified(lastModifiedMinus3Hour);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("minus-1hour");
    objectSummary2.setLastModified(lastModifiedMinus1Hour);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("now");
    objectSummary3.setLastModified(lastModifiedNow);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);
    Map<String, String> stateMap = new HashMap<>();
    String previousTimestamp = String.valueOf(lastModifiedMinus3Hour.getTime());
    stateMap.put(ListS3.CURRENT_TIMESTAMP, previousTimestamp);
    stateMap.put(ListS3.CURRENT_KEY_PREFIX + "0", "minus-3hour");
    runner.getStateManager().setState(stateMap, Scope.CLUSTER);
    runner.run();
    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "minus-1hour");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModifiedMinus1Hour.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}

Example 48 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class TestListS3 method testListVersions.

@Test
public void testListVersions() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.USE_VERSIONS, "true");
    Date lastModified = new Date();
    VersionListing versionListing = new VersionListing();
    S3VersionSummary versionSummary1 = new S3VersionSummary();
    versionSummary1.setBucketName("test-bucket");
    versionSummary1.setKey("test-key");
    versionSummary1.setVersionId("1");
    versionSummary1.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary1);
    S3VersionSummary versionSummary2 = new S3VersionSummary();
    versionSummary2.setBucketName("test-bucket");
    versionSummary2.setKey("test-key");
    versionSummary2.setVersionId("2");
    versionSummary2.setLastModified(lastModified);
    versionListing.getVersionSummaries().add(versionSummary2);
    Mockito.when(mockS3Client.listVersions(Mockito.any(ListVersionsRequest.class))).thenReturn(versionListing);
    runner.run();
    ArgumentCaptor<ListVersionsRequest> captureRequest = ArgumentCaptor.forClass(ListVersionsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listVersions(captureRequest.capture());
    ListVersionsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listObjects(Mockito.any(ListObjectsRequest.class));
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 2);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "test-key");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    ff0.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff0.assertAttributeEquals("s3.version", "1");
    MockFlowFile ff1 = flowFiles.get(1);
    ff1.assertAttributeEquals("filename", "test-key");
    ff1.assertAttributeEquals("s3.bucket", "test-bucket");
    ff1.assertAttributeEquals("s3.lastModified", String.valueOf(lastModified.getTime()));
    ff1.assertAttributeEquals("s3.version", "2");
}

Example 49 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class TestPutSNS method testPublish.

@Test
public void testPublish() throws IOException {
    runner.setValidateExpressionUsage(false);
    runner.setProperty(PutSNS.CREDENTIALS_FILE, "src/test/resources/mock-aws-credentials.properties");
    runner.setProperty(PutSNS.ARN, "arn:aws:sns:us-west-2:123456789012:test-topic-1");
    runner.setProperty(PutSNS.SUBJECT, "${eval.subject}");
    assertTrue(runner.setProperty("DynamicProperty", "hello!").isValid());
    final Map<String, String> ffAttributes = new HashMap<>();
    ffAttributes.put("filename", "1.txt");
    ffAttributes.put("eval.subject", "test-subject");
    runner.enqueue("Test Message Content", ffAttributes);
    PublishResult mockPublishResult = new PublishResult();
    Mockito.when(mockSNSClient.publish(Mockito.any(PublishRequest.class))).thenReturn(mockPublishResult);
    runner.run();
    ArgumentCaptor<PublishRequest> captureRequest = ArgumentCaptor.forClass(PublishRequest.class);
    Mockito.verify(mockSNSClient, Mockito.times(1)).publish(captureRequest.capture());
    PublishRequest request = captureRequest.getValue();
    assertEquals("arn:aws:sns:us-west-2:123456789012:test-topic-1", request.getTopicArn());
    assertEquals("Test Message Content", request.getMessage());
    assertEquals("test-subject", request.getSubject());
    assertEquals("hello!", request.getMessageAttributes().get("DynamicProperty").getStringValue());
    runner.assertAllFlowFilesTransferred(PutSNS.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutSNS.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals(CoreAttributes.FILENAME.key(), "1.txt");
}

Example 50 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class ITGetSQS method testSimpleGetUsingCredentialsProviderService.

@Test
public void testSimpleGetUsingCredentialsProviderService() throws Throwable {
    final TestRunner runner = TestRunners.newTestRunner(new GetSQS());
    runner.setProperty(GetSQS.TIMEOUT, "30 secs");
    String queueUrl = "Add queue url here";
    runner.setProperty(GetSQS.QUEUE_URL, queueUrl);
    final AWSCredentialsProviderControllerService serviceImpl = new AWSCredentialsProviderControllerService();
    runner.addControllerService("awsCredentialsProvider", serviceImpl);
    runner.setProperty(serviceImpl, AbstractAWSProcessor.CREDENTIALS_FILE, System.getProperty("user.home") + "/aws-credentials.properties");
    runner.enableControllerService(serviceImpl);
    runner.assertValid(serviceImpl);
    runner.setProperty(GetSQS.AWS_CREDENTIALS_PROVIDER_SERVICE, "awsCredentialsProvider");
    runner.run(1);
    final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetSQS.REL_SUCCESS);
    for (final MockFlowFile mff : flowFiles) {
        System.out.println(mff.getAttributes());
        System.out.println(new String(mff.toByteArray()));
    }
}