Examples with TestRunner org.apache.nifi.util.TestRunner used on opensource projects

Example 11 with TestRunner

use of org.apache.nifi.util.TestRunner in project nifi by apache.

the class ITListS3 method testSimpleListWithDelimiter.

@Test
public void testSimpleListWithDelimiter() throws Throwable {
    putTestFile("a", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("b/c", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("d/e", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    final TestRunner runner = TestRunners.newTestRunner(new ListS3());
    runner.setProperty(ListS3.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(ListS3.REGION, REGION);
    runner.setProperty(ListS3.BUCKET, BUCKET_NAME);
    runner.setProperty(ListS3.DELIMITER, "/");
    runner.run();
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "a");
}

Example 12 with TestRunner

use of org.apache.nifi.util.TestRunner in project nifi by apache.

the class ITListS3 method testSimpleListUsingCredentialsProviderService.

@Test
public void testSimpleListUsingCredentialsProviderService() throws Throwable {
    putTestFile("a", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("b/c", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("d/e", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    final TestRunner runner = TestRunners.newTestRunner(new ListS3());
    final AWSCredentialsProviderControllerService serviceImpl = new AWSCredentialsProviderControllerService();
    runner.addControllerService("awsCredentialsProvider", serviceImpl);
    runner.setProperty(serviceImpl, AbstractAWSProcessor.CREDENTIALS_FILE, System.getProperty("user.home") + "/aws-credentials.properties");
    runner.enableControllerService(serviceImpl);
    runner.assertValid(serviceImpl);
    runner.setProperty(ListS3.AWS_CREDENTIALS_PROVIDER_SERVICE, "awsCredentialsProvider");
    runner.setProperty(ListS3.REGION, REGION);
    runner.setProperty(ListS3.BUCKET, BUCKET_NAME);
    runner.run();
    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "a");
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
}

Example 13 with TestRunner

use of org.apache.nifi.util.TestRunner in project nifi by apache.

the class ITPutS3Object method testContentType.

@Test
public void testContentType() throws IOException {
    PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    runner.setProperty(PutS3Object.CONTENT_TYPE, "text/plain");
    runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    MockFlowFile ff1 = flowFiles.get(0);
    ff1.assertAttributeEquals(PutS3Object.S3_CONTENT_TYPE, "text/plain");
}

Example 14 with TestRunner

use of org.apache.nifi.util.TestRunner in project nifi by apache.

the class ITPutS3Object method testSimplePut.

@Test
public void testSimplePut() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());
    for (int i = 0; i < 3; i++) {
        final Map<String, String> attrs = new HashMap<>();
        attrs.put("filename", String.valueOf(i) + ".txt");
        runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
    }
    runner.run(3);
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 3);
}

Example 15 with TestRunner

use of org.apache.nifi.util.TestRunner in project nifi by apache.

the class ITPutS3Object method testS3MultipartAgeoff.

@Ignore
@Test
public void testS3MultipartAgeoff() throws InterruptedException, IOException {
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    final ProcessContext context = runner.getProcessContext();
    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    // set check interval and age off to minimum values
    runner.setProperty(PutS3Object.MULTIPART_S3_AGEOFF_INTERVAL, "1 milli");
    runner.setProperty(PutS3Object.MULTIPART_S3_MAX_AGE, "1 milli");
    // create some dummy uploads
    for (Integer i = 0; i < 3; i++) {
        final InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(BUCKET_NAME, "file" + i.toString() + ".txt");
        try {
            client.initiateMultipartUpload(initiateRequest);
        } catch (AmazonClientException e) {
            Assert.fail("Failed to initiate upload: " + e.getMessage());
        }
    }
    // Age off is time dependent, so test has some timing constraints.  This
    // sleep() delays long enough to satisfy interval and age intervals.
    Thread.sleep(2000L);
    // System millis are used for timing, but it is incremented on each
    // call to circumvent what appears to be caching in the AWS library.
    // The increments are 1000 millis because AWS returns upload
    // initiation times in whole seconds.
    Long now = System.currentTimeMillis();
    MultipartUploadListing uploadList = processor.getS3AgeoffListAndAgeoffLocalState(context, client, now);
    Assert.assertEquals(3, uploadList.getMultipartUploads().size());
    MultipartUpload upload0 = uploadList.getMultipartUploads().get(0);
    processor.abortS3MultipartUpload(client, BUCKET_NAME, upload0);
    uploadList = processor.getS3AgeoffListAndAgeoffLocalState(context, client, now + 1000);
    Assert.assertEquals(2, uploadList.getMultipartUploads().size());
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "test-upload.txt");
    runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);
    runner.run();
    uploadList = processor.getS3AgeoffListAndAgeoffLocalState(context, client, now + 2000);
    Assert.assertEquals(0, uploadList.getMultipartUploads().size());
}