Examples with MockFlowFile org.apache.nifi.util.MockFlowFile used on opensource projects

Example 41 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class ITListS3 method testSimpleListWithDelimiter.

@Test
public void testSimpleListWithDelimiter() throws Throwable {
    putTestFile("a", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("b/c", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("d/e", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    final TestRunner runner = TestRunners.newTestRunner(new ListS3());
    runner.setProperty(ListS3.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(ListS3.REGION, REGION);
    runner.setProperty(ListS3.BUCKET, BUCKET_NAME);
    runner.setProperty(ListS3.DELIMITER, "/");
    runner.run();
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "a");
}

Example 42 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class ITListS3 method testSimpleListUsingCredentialsProviderService.

@Test
public void testSimpleListUsingCredentialsProviderService() throws Throwable {
    putTestFile("a", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("b/c", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("d/e", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    final TestRunner runner = TestRunners.newTestRunner(new ListS3());
    final AWSCredentialsProviderControllerService serviceImpl = new AWSCredentialsProviderControllerService();
    runner.addControllerService("awsCredentialsProvider", serviceImpl);
    runner.setProperty(serviceImpl, AbstractAWSProcessor.CREDENTIALS_FILE, System.getProperty("user.home") + "/aws-credentials.properties");
    runner.enableControllerService(serviceImpl);
    runner.assertValid(serviceImpl);
    runner.setProperty(ListS3.AWS_CREDENTIALS_PROVIDER_SERVICE, "awsCredentialsProvider");
    runner.setProperty(ListS3.REGION, REGION);
    runner.setProperty(ListS3.BUCKET, BUCKET_NAME);
    runner.run();
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "a");
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
}

Example 43 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class ITPutS3Object method testContentType.

@Test
public void testContentType() throws IOException {
    PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    runner.setProperty(PutS3Object.CONTENT_TYPE, "text/plain");
    runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    MockFlowFile ff1 = flowFiles.get(0);
    ff1.assertAttributeEquals(PutS3Object.S3_CONTENT_TYPE, "text/plain");
}

Example 44 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class ITPutS3Object method testMultipartSmallerThanMinimum.

@Test
public void testMultipartSmallerThanMinimum() throws IOException {
    final String FILE1_NAME = "file1";
    final byte[] megabyte = new byte[1024 * 1024];
    final Path tempFile = Files.createTempFile("s3mulitpart", "tmp");
    final FileOutputStream tempOut = new FileOutputStream(tempFile.toFile());
    long tempByteCount = 0;
    for (int i = 0; i < 5; i++) {
        tempOut.write(megabyte);
        tempByteCount += megabyte.length;
    }
    tempOut.close();
    System.out.println("file size: " + tempByteCount);
    Assert.assertTrue(tempByteCount < S3_MINIMUM_PART_SIZE);
    Assert.assertTrue(megabyte.length < S3_MINIMUM_PART_SIZE);
    Assert.assertTrue(TEST_PARTSIZE_LONG >= S3_MINIMUM_PART_SIZE && TEST_PARTSIZE_LONG <= S3_MAXIMUM_OBJECT_SIZE);
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    runner.setProperty(PutS3Object.MULTIPART_PART_SIZE, TEST_PARTSIZE_STRING);
    Map<String, String> attributes = new HashMap<>();
    attributes.put(CoreAttributes.FILENAME.key(), FILE1_NAME);
    runner.enqueue(new FileInputStream(tempFile.toFile()), attributes);
    runner.assertValid();
    runner.run();
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS);
    final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    Assert.assertEquals(1, successFiles.size());
    final List<MockFlowFile> failureFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_FAILURE);
    Assert.assertEquals(0, failureFiles.size());
    MockFlowFile ff1 = successFiles.get(0);
    Assert.assertEquals(PutS3Object.S3_API_METHOD_PUTOBJECT, ff1.getAttribute(PutS3Object.S3_API_METHOD_ATTR_KEY));
    Assert.assertEquals(FILE1_NAME, ff1.getAttribute(CoreAttributes.FILENAME.key()));
    Assert.assertEquals(BUCKET_NAME, ff1.getAttribute(PutS3Object.S3_BUCKET_KEY));
    Assert.assertEquals(FILE1_NAME, ff1.getAttribute(PutS3Object.S3_OBJECT_KEY));
    Assert.assertTrue(reS3ETag.matcher(ff1.getAttribute(PutS3Object.S3_ETAG_ATTR_KEY)).matches());
    Assert.assertEquals(tempByteCount, ff1.getSize());
}

Example 45 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class ITPutS3Object method testPutThenFetch.

private void testPutThenFetch(String sseAlgorithm) throws IOException {
    // Put
    TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    if (ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION.equals(sseAlgorithm)) {
        runner.setProperty(PutS3Object.SERVER_SIDE_ENCRYPTION, sseAlgorithm);
    }
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "filename-on-s3.txt");
    runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
    List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    if (ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION.equals(sseAlgorithm)) {
        ffs.get(0).assertAttributeEquals(PutS3Object.S3_SSE_ALGORITHM, ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    } else {
        ffs.get(0).assertAttributeNotExists(PutS3Object.S3_SSE_ALGORITHM);
    }
    // Fetch
    runner = TestRunners.newTestRunner(new FetchS3Object());
    runner.setProperty(FetchS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(FetchS3Object.REGION, REGION);
    runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);
    runner.enqueue(new byte[0], attrs);
    runner.run(1);
    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertContentEquals(getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    if (ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION.equals(sseAlgorithm)) {
        ff.assertAttributeEquals(PutS3Object.S3_SSE_ALGORITHM, ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    } else {
        ff.assertAttributeNotExists(PutS3Object.S3_SSE_ALGORITHM);
    }
}