Search in sources :

Example 11 with MockRecordWriter

use of org.apache.nifi.serialization.record.MockRecordWriter in project nifi by apache.

the class TestQueryRecord method testHandlingWithInvalidSchema.

@Test
public void testHandlingWithInvalidSchema() throws InitializationException {
    final MockRecordParser parser = new MockRecordParser();
    parser.addSchemaField("name", RecordFieldType.STRING);
    parser.addSchemaField("favorite_color", RecordFieldType.STRING);
    parser.addSchemaField("address", RecordFieldType.STRING);
    parser.addRecord("Tom", "blue", null);
    parser.addRecord("Jerry", "red", null);
    final MockRecordWriter writer = new MockRecordWriter("\"name\",\"points\"");
    final TestRunner runner = TestRunners.newTestRunner(QueryRecord.class);
    runner.enforceReadStreamsClosed(false);
    runner.addControllerService("parser", parser);
    runner.enableControllerService(parser);
    runner.addControllerService("writer", writer);
    runner.enableControllerService(writer);
    runner.setProperty(QueryRecord.INCLUDE_ZERO_RECORD_FLOWFILES, "false");
    runner.setProperty("rel1", "select * from FLOWFILE where address IS NOT NULL");
    runner.setProperty("rel2", "select name, CAST(favorite_color AS DOUBLE) AS num from FLOWFILE");
    runner.setProperty("rel3", "select * from FLOWFILE where address IS NOT NULL");
    runner.setProperty(QueryRecord.RECORD_READER_FACTORY, "parser");
    runner.setProperty(QueryRecord.RECORD_WRITER_FACTORY, "writer");
    runner.enqueue("");
    runner.run();
    runner.assertAllFlowFilesTransferred(QueryRecord.REL_FAILURE, 1);
}
Also used : TestRunner(org.apache.nifi.util.TestRunner) MockRecordWriter(org.apache.nifi.serialization.record.MockRecordWriter) MockRecordParser(org.apache.nifi.serialization.record.MockRecordParser) Test(org.junit.Test)

Example 12 with MockRecordWriter

use of org.apache.nifi.serialization.record.MockRecordWriter in project nifi by apache.

the class TestSplitRecord method testAllSplitsOneDesintation.

@Test
public void testAllSplitsOneDesintation() throws InitializationException {
    final MockRecordParser readerService = new MockRecordParser();
    final MockRecordWriter writerService = new MockRecordWriter("header", false);
    final TestRunner runner = TestRunners.newTestRunner(SplitRecord.class);
    runner.addControllerService("reader", readerService);
    runner.enableControllerService(readerService);
    runner.addControllerService("writer", writerService);
    runner.enableControllerService(writerService);
    runner.setProperty(SplitRecord.RECORD_READER, "reader");
    runner.setProperty(SplitRecord.RECORD_WRITER, "writer");
    runner.setProperty(SplitRecord.RECORDS_PER_SPLIT, "3");
    readerService.addSchemaField("name", RecordFieldType.STRING);
    readerService.addSchemaField("age", RecordFieldType.INT);
    readerService.addRecord("John Doe", 48);
    readerService.addRecord("Jane Doe", 47);
    readerService.addRecord("Jimmy Doe", 14);
    runner.enqueue("");
    runner.run();
    runner.assertTransferCount(SplitRecord.REL_SPLITS, 1);
    runner.assertTransferCount(SplitRecord.REL_ORIGINAL, 1);
    runner.assertTransferCount(SplitRecord.REL_FAILURE, 0);
    final MockFlowFile out = runner.getFlowFilesForRelationship(SplitRecord.REL_SPLITS).get(0);
    out.assertAttributeEquals("record.count", "3");
    out.assertAttributeEquals("mime.type", "text/plain");
    out.assertContentEquals("header\nJohn Doe,48\nJane Doe,47\nJimmy Doe,14\n");
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) TestRunner(org.apache.nifi.util.TestRunner) MockRecordWriter(org.apache.nifi.serialization.record.MockRecordWriter) MockRecordParser(org.apache.nifi.serialization.record.MockRecordParser) Test(org.junit.Test)

Example 13 with MockRecordWriter

use of org.apache.nifi.serialization.record.MockRecordWriter in project nifi by apache.

the class TestSplitRecord method testMultipleRecordsPerSplit.

@Test
public void testMultipleRecordsPerSplit() throws InitializationException {
    final MockRecordParser readerService = new MockRecordParser();
    final MockRecordWriter writerService = new MockRecordWriter("header", false);
    final TestRunner runner = TestRunners.newTestRunner(SplitRecord.class);
    runner.addControllerService("reader", readerService);
    runner.enableControllerService(readerService);
    runner.addControllerService("writer", writerService);
    runner.enableControllerService(writerService);
    runner.setProperty(SplitRecord.RECORD_READER, "reader");
    runner.setProperty(SplitRecord.RECORD_WRITER, "writer");
    runner.setProperty(SplitRecord.RECORDS_PER_SPLIT, "2");
    readerService.addSchemaField("name", RecordFieldType.STRING);
    readerService.addSchemaField("age", RecordFieldType.INT);
    readerService.addRecord("John Doe", 48);
    readerService.addRecord("Jane Doe", 47);
    readerService.addRecord("Jimmy Doe", 14);
    runner.enqueue("");
    runner.run();
    runner.assertTransferCount(SplitRecord.REL_SPLITS, 2);
    runner.assertTransferCount(SplitRecord.REL_ORIGINAL, 1);
    runner.assertTransferCount(SplitRecord.REL_FAILURE, 0);
    final List<MockFlowFile> out = runner.getFlowFilesForRelationship(SplitRecord.REL_SPLITS);
    assertEquals(1, out.stream().filter(mff -> mff.getAttribute("record.count").equals("1")).count());
    assertTrue(out.stream().allMatch(mff -> mff.getAttribute("mime.type").equals("text/plain")));
    assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJohn Doe,48\nJane Doe,47\n")).count());
    assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJimmy Doe,14\n")).count());
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) MockRecordParser(org.apache.nifi.serialization.record.MockRecordParser) List(java.util.List) TestRunner(org.apache.nifi.util.TestRunner) InitializationException(org.apache.nifi.reporting.InitializationException) MockRecordWriter(org.apache.nifi.serialization.record.MockRecordWriter) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) TestRunners(org.apache.nifi.util.TestRunners) Assert.assertEquals(org.junit.Assert.assertEquals) MockFlowFile(org.apache.nifi.util.MockFlowFile) TestRunner(org.apache.nifi.util.TestRunner) MockRecordWriter(org.apache.nifi.serialization.record.MockRecordWriter) MockRecordParser(org.apache.nifi.serialization.record.MockRecordParser) Test(org.junit.Test)

Example 14 with MockRecordWriter

use of org.apache.nifi.serialization.record.MockRecordWriter in project nifi by apache.

the class TestSplitRecord method testReadFailure.

@Test
public void testReadFailure() throws InitializationException {
    final MockRecordParser readerService = new MockRecordParser(2);
    final MockRecordWriter writerService = new MockRecordWriter("header", false);
    final TestRunner runner = TestRunners.newTestRunner(SplitRecord.class);
    runner.addControllerService("reader", readerService);
    runner.enableControllerService(readerService);
    runner.addControllerService("writer", writerService);
    runner.enableControllerService(writerService);
    runner.setProperty(SplitRecord.RECORD_READER, "reader");
    runner.setProperty(SplitRecord.RECORD_WRITER, "writer");
    runner.setProperty(SplitRecord.RECORDS_PER_SPLIT, "1");
    readerService.addSchemaField("name", RecordFieldType.STRING);
    readerService.addSchemaField("age", RecordFieldType.INT);
    readerService.addRecord("John Doe", 48);
    readerService.addRecord("Jane Doe", 47);
    readerService.addRecord("Jimmy Doe", 14);
    final MockFlowFile original = runner.enqueue("");
    runner.run();
    runner.assertAllFlowFilesTransferred(SplitRecord.REL_FAILURE, 1);
    final MockFlowFile failed = runner.getFlowFilesForRelationship(SplitRecord.REL_FAILURE).get(0);
    assertTrue(original == failed);
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) TestRunner(org.apache.nifi.util.TestRunner) MockRecordWriter(org.apache.nifi.serialization.record.MockRecordWriter) MockRecordParser(org.apache.nifi.serialization.record.MockRecordParser) Test(org.junit.Test)

Example 15 with MockRecordWriter

use of org.apache.nifi.serialization.record.MockRecordWriter in project nifi by apache.

the class TestSplitRecord method testIndividualRecordPerSplit.

@Test
public void testIndividualRecordPerSplit() throws InitializationException {
    final MockRecordParser readerService = new MockRecordParser();
    final MockRecordWriter writerService = new MockRecordWriter("header", false);
    final TestRunner runner = TestRunners.newTestRunner(SplitRecord.class);
    runner.addControllerService("reader", readerService);
    runner.enableControllerService(readerService);
    runner.addControllerService("writer", writerService);
    runner.enableControllerService(writerService);
    runner.setProperty(SplitRecord.RECORD_READER, "reader");
    runner.setProperty(SplitRecord.RECORD_WRITER, "writer");
    runner.setProperty(SplitRecord.RECORDS_PER_SPLIT, "1");
    readerService.addSchemaField("name", RecordFieldType.STRING);
    readerService.addSchemaField("age", RecordFieldType.INT);
    readerService.addRecord("John Doe", 48);
    readerService.addRecord("Jane Doe", 47);
    readerService.addRecord("Jimmy Doe", 14);
    runner.enqueue("");
    runner.run();
    runner.assertTransferCount(SplitRecord.REL_SPLITS, 3);
    runner.assertTransferCount(SplitRecord.REL_ORIGINAL, 1);
    runner.assertTransferCount(SplitRecord.REL_FAILURE, 0);
    final List<MockFlowFile> out = runner.getFlowFilesForRelationship(SplitRecord.REL_SPLITS);
    for (final MockFlowFile mff : out) {
        mff.assertAttributeEquals("record.count", "1");
        mff.assertAttributeEquals("mime.type", "text/plain");
    }
    assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJohn Doe,48\n")).count());
    assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJane Doe,47\n")).count());
    assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJimmy Doe,14\n")).count());
}
Also used : MockFlowFile(org.apache.nifi.util.MockFlowFile) TestRunner(org.apache.nifi.util.TestRunner) MockRecordWriter(org.apache.nifi.serialization.record.MockRecordWriter) MockRecordParser(org.apache.nifi.serialization.record.MockRecordParser) Test(org.junit.Test)

Aggregations

MockRecordWriter (org.apache.nifi.serialization.record.MockRecordWriter)21 MockRecordParser (org.apache.nifi.serialization.record.MockRecordParser)16 Test (org.junit.Test)13 TestRunner (org.apache.nifi.util.TestRunner)12 MockFlowFile (org.apache.nifi.util.MockFlowFile)11 Before (org.junit.Before)7 JsonTreeReader (org.apache.nifi.json.JsonTreeReader)2 RecordReaderFactory (org.apache.nifi.serialization.RecordReaderFactory)2 RecordSetWriterFactory (org.apache.nifi.serialization.RecordSetWriterFactory)2 List (java.util.List)1 MockDruidTranquilityController (org.apache.nifi.controller.druid.MockDruidTranquilityController)1 StandardEvent (org.apache.nifi.processor.util.listen.event.StandardEvent)1 InitializationException (org.apache.nifi.reporting.InitializationException)1 CommaSeparatedRecordReader (org.apache.nifi.serialization.record.CommaSeparatedRecordReader)1 RecordFieldType (org.apache.nifi.serialization.record.RecordFieldType)1 TestRunners (org.apache.nifi.util.TestRunners)1 Assert.assertEquals (org.junit.Assert.assertEquals)1 Assert.assertTrue (org.junit.Assert.assertTrue)1