Examples with PutFlowFile org.apache.nifi.hbase.put.PutFlowFile used on opensource projects

Example 11 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseCell method testMultipleFlowFilesSameTableSameRow.

@Test
public void testMultipleFlowFilesSameTableSameRow() throws IOException, InitializationException {
    final String tableName = "nifi";
    final String row = "row1";
    final String columnFamily = "family1";
    final String columnQualifier = "qualifier1";
    final PutHBaseCell proc = new PutHBaseCell();
    final TestRunner runner = getTestRunnerWithEL(proc);
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    final String content1 = "some content1";
    final Map<String, String> attributes1 = getAttributeMapWithEL(tableName, row, columnFamily, columnQualifier);
    runner.enqueue(content1.getBytes("UTF-8"), attributes1);
    final String content2 = "some content1";
    runner.enqueue(content2.getBytes("UTF-8"), attributes1);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content1);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(tableName);
    assertEquals(2, puts.size());
    verifyPut(row, columnFamily, columnQualifier, null, content1, puts.get(0));
    verifyPut(row, columnFamily, columnQualifier, null, content2, puts.get(1));
    assertEquals(2, runner.getProvenanceEvents().size());
}

Example 12 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseCell method testSingleFlowFileWithBinaryRowKey.

@Test
public void testSingleFlowFileWithBinaryRowKey() throws IOException, InitializationException {
    final String tableName = "nifi";
    final String row = "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\x00\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00" + "\\x01\\x00\\x00\\x01\\x00\\x00\\x00\\x00\\x01\\x01\\x01\\x00\\x01\\x00\\x01\\x01\\x01\\x00\\x00\\x00" + "\\x00\\x00\\x00\\x01\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x01\\x00\\x01\\x00\\x01\\x00" + "\\x00\\x01\\x01\\x01\\x01\\x00\\x00\\x01\\x01\\x01\\x00\\x01\\x00\\x00";
    final String columnFamily = "family1";
    final String columnQualifier = "qualifier1";
    final TestRunner runner = TestRunners.newTestRunner(PutHBaseCell.class);
    runner.setProperty(PutHBaseCell.TABLE_NAME, tableName);
    runner.setProperty(PutHBaseCell.ROW_ID, row);
    runner.setProperty(PutHBaseCell.ROW_ID_ENCODING_STRATEGY, PutHBaseCell.ROW_ID_ENCODING_BINARY.getValue());
    runner.setProperty(PutHBaseCell.COLUMN_FAMILY, columnFamily);
    runner.setProperty(PutHBaseCell.COLUMN_QUALIFIER, columnQualifier);
    runner.setProperty(PutHBaseCell.BATCH_SIZE, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    final byte[] expectedRowKey = hBaseClient.toBytesBinary(row);
    final String content = "some content";
    runner.enqueue(content.getBytes("UTF-8"));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(tableName);
    assertEquals(1, puts.size());
    verifyPut(expectedRowKey, columnFamily.getBytes(StandardCharsets.UTF_8), columnQualifier.getBytes(StandardCharsets.UTF_8), null, content, puts.get(0));
    assertEquals(1, runner.getProvenanceEvents().size());
}

Example 13 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testSingleJsonDocAndProvidedRowId.

@Test
public void testSingleJsonDocAndProvidedRowId() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes("UTF-8"));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes("value1"));
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    final ProvenanceEventRecord event = events.get(0);
    assertEquals("hbase://" + DEFAULT_TABLE_NAME + "/" + DEFAULT_ROW, event.getTransitUri());
}

Example 14 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testELWithExtractedRowId.

@Test
public void testELWithExtractedRowId() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner("${hbase.table}", "${hbase.colFamily}", "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "${hbase.rowField}");
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("hbase.table", "myTable");
    attributes.put("hbase.colFamily", "myColFamily");
    attributes.put("hbase.rowField", "field1");
    final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes("UTF-8"), attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get("myTable");
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut("value1", "myColFamily", expectedColumns, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://myTable/value1", ProvenanceEventType.SEND);
}

Example 15 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testNullAndArrayElementsWithIgnoreStrategy.

@Test
public void testNullAndArrayElementsWithIgnoreStrategy() throws InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_IGNORE.getValue());
    // should route to success because there is at least one valid field
    final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : \"value2\", \"field3\" : null }";
    runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    // should have skipped field1 and field3
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
}