use of org.apache.nifi.serialization.record.MockRecordParser in project nifi by apache.
the class TestSplitRecord method testMultipleRecordsPerSplit.
@Test
public void testMultipleRecordsPerSplit() throws InitializationException {
final MockRecordParser readerService = new MockRecordParser();
final MockRecordWriter writerService = new MockRecordWriter("header", false);
final TestRunner runner = TestRunners.newTestRunner(SplitRecord.class);
runner.addControllerService("reader", readerService);
runner.enableControllerService(readerService);
runner.addControllerService("writer", writerService);
runner.enableControllerService(writerService);
runner.setProperty(SplitRecord.RECORD_READER, "reader");
runner.setProperty(SplitRecord.RECORD_WRITER, "writer");
runner.setProperty(SplitRecord.RECORDS_PER_SPLIT, "2");
readerService.addSchemaField("name", RecordFieldType.STRING);
readerService.addSchemaField("age", RecordFieldType.INT);
readerService.addRecord("John Doe", 48);
readerService.addRecord("Jane Doe", 47);
readerService.addRecord("Jimmy Doe", 14);
runner.enqueue("");
runner.run();
runner.assertTransferCount(SplitRecord.REL_SPLITS, 2);
runner.assertTransferCount(SplitRecord.REL_ORIGINAL, 1);
runner.assertTransferCount(SplitRecord.REL_FAILURE, 0);
final List<MockFlowFile> out = runner.getFlowFilesForRelationship(SplitRecord.REL_SPLITS);
assertEquals(1, out.stream().filter(mff -> mff.getAttribute("record.count").equals("1")).count());
assertTrue(out.stream().allMatch(mff -> mff.getAttribute("mime.type").equals("text/plain")));
assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJohn Doe,48\nJane Doe,47\n")).count());
assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJimmy Doe,14\n")).count());
}
use of org.apache.nifi.serialization.record.MockRecordParser in project nifi by apache.
the class TestSplitRecord method testReadFailure.
@Test
public void testReadFailure() throws InitializationException {
final MockRecordParser readerService = new MockRecordParser(2);
final MockRecordWriter writerService = new MockRecordWriter("header", false);
final TestRunner runner = TestRunners.newTestRunner(SplitRecord.class);
runner.addControllerService("reader", readerService);
runner.enableControllerService(readerService);
runner.addControllerService("writer", writerService);
runner.enableControllerService(writerService);
runner.setProperty(SplitRecord.RECORD_READER, "reader");
runner.setProperty(SplitRecord.RECORD_WRITER, "writer");
runner.setProperty(SplitRecord.RECORDS_PER_SPLIT, "1");
readerService.addSchemaField("name", RecordFieldType.STRING);
readerService.addSchemaField("age", RecordFieldType.INT);
readerService.addRecord("John Doe", 48);
readerService.addRecord("Jane Doe", 47);
readerService.addRecord("Jimmy Doe", 14);
final MockFlowFile original = runner.enqueue("");
runner.run();
runner.assertAllFlowFilesTransferred(SplitRecord.REL_FAILURE, 1);
final MockFlowFile failed = runner.getFlowFilesForRelationship(SplitRecord.REL_FAILURE).get(0);
assertTrue(original == failed);
}
use of org.apache.nifi.serialization.record.MockRecordParser in project nifi by apache.
the class TestSplitRecord method testIndividualRecordPerSplit.
@Test
public void testIndividualRecordPerSplit() throws InitializationException {
final MockRecordParser readerService = new MockRecordParser();
final MockRecordWriter writerService = new MockRecordWriter("header", false);
final TestRunner runner = TestRunners.newTestRunner(SplitRecord.class);
runner.addControllerService("reader", readerService);
runner.enableControllerService(readerService);
runner.addControllerService("writer", writerService);
runner.enableControllerService(writerService);
runner.setProperty(SplitRecord.RECORD_READER, "reader");
runner.setProperty(SplitRecord.RECORD_WRITER, "writer");
runner.setProperty(SplitRecord.RECORDS_PER_SPLIT, "1");
readerService.addSchemaField("name", RecordFieldType.STRING);
readerService.addSchemaField("age", RecordFieldType.INT);
readerService.addRecord("John Doe", 48);
readerService.addRecord("Jane Doe", 47);
readerService.addRecord("Jimmy Doe", 14);
runner.enqueue("");
runner.run();
runner.assertTransferCount(SplitRecord.REL_SPLITS, 3);
runner.assertTransferCount(SplitRecord.REL_ORIGINAL, 1);
runner.assertTransferCount(SplitRecord.REL_FAILURE, 0);
final List<MockFlowFile> out = runner.getFlowFilesForRelationship(SplitRecord.REL_SPLITS);
for (final MockFlowFile mff : out) {
mff.assertAttributeEquals("record.count", "1");
mff.assertAttributeEquals("mime.type", "text/plain");
}
assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJohn Doe,48\n")).count());
assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJane Doe,47\n")).count());
assertEquals(1, out.stream().filter(mff -> mff.isContentEqual("header\nJimmy Doe,14\n")).count());
}
use of org.apache.nifi.serialization.record.MockRecordParser in project nifi by apache.
the class TestPutHBaseRecord method generateTestData.
private void generateTestData(TestRunner runner) throws IOException {
final MockRecordParser parser = new MockRecordParser();
try {
runner.addControllerService("parser", parser);
} catch (InitializationException e) {
throw new IOException(e);
}
runner.enableControllerService(parser);
runner.setProperty(PutHBaseRecord.RECORD_READER_FACTORY, "parser");
parser.addSchemaField("id", RecordFieldType.INT);
parser.addSchemaField("name", RecordFieldType.STRING);
parser.addSchemaField("code", RecordFieldType.LONG);
for (int x = 0; x < KEYS.size(); x++) {
parser.addRecord(KEYS.get(x), NAMES.get(x), CODES.get(x));
}
}
use of org.apache.nifi.serialization.record.MockRecordParser in project nifi by apache.
the class PutElasticsearchHttpRecordIT method setup.
@Before
public void setup() throws Exception {
recordReader = new MockRecordParser();
recordReader.addSchemaField("id", RecordFieldType.INT);
recordReader.addSchemaField("person", RecordFieldType.RECORD);
runner = TestRunners.newTestRunner(PutElasticsearchHttpRecord.class);
runner.addControllerService("reader", recordReader);
runner.enableControllerService(recordReader);
runner.setProperty(PutElasticsearchHttpRecord.RECORD_READER, "reader");
runner.setProperty(PutElasticsearchHttpRecord.ES_URL, "http://localhost:9200");
runner.setProperty(PutElasticsearchHttpRecord.INDEX, "people_test");
runner.setProperty(PutElasticsearchHttpRecord.TYPE, "person");
runner.setProperty(PutElasticsearchHttpRecord.ID_RECORD_PATH, "/id");
runner.assertValid();
}
Aggregations