Examples with PutFlowFile org.apache.nifi.hbase.put.PutFlowFile used on opensource projects

Example 16 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testNullAndArrayElementsWithWarnStrategy.

@Test
public void testNullAndArrayElementsWithWarnStrategy() throws InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_WARN.getValue());
    // should route to success because there is at least one valid field
    final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : \"value2\", \"field3\" : null }";
    runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    // should have skipped field1 and field3
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
}

Example 17 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testSingleJsonDocAndProvidedRowIdwithNonString.

@Test
public void testSingleJsonDocAndProvidedRowIdwithNonString() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    runner.setProperty(PutHBaseJSON.FIELD_ENCODING_STRATEGY, PutHBaseJSON.BYTES_ENCODING_VALUE);
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    final String content = "{ \"field1\" : 1.23456, \"field2\" : 2345235, \"field3\" : false }";
    runner.enqueue(content.getBytes("UTF-8"));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes(1.23456d));
    expectedColumns.put("field2", hBaseClient.toBytes(2345235l));
    expectedColumns.put("field3", hBaseClient.toBytes(false));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    final ProvenanceEventRecord event = events.get(0);
    assertEquals("hbase://" + DEFAULT_TABLE_NAME + "/" + DEFAULT_ROW, event.getTransitUri());
}

Example 18 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testELWithProvidedRowId.

@Test
public void testELWithProvidedRowId() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner("${hbase.table}", "${hbase.colFamily}", "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, "${hbase.rowId}");
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("hbase.table", "myTable");
    attributes.put("hbase.colFamily", "myColFamily");
    attributes.put("hbase.rowId", "myRowId");
    final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes("UTF-8"), attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get("myTable");
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes("value1"));
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut("myRowId", "myColFamily", expectedColumns, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://myTable/myRowId", ProvenanceEventType.SEND);
}

Example 19 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testNestedDocWithTextStrategy.

@Test
public void testNestedDocWithTextStrategy() throws InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_TEXT.getValue());
    // should route to success because there is at least one valid field
    final String content = "{ \"field1\" : { \"child_field1\" : \"child_value1\" }, \"field2\" : \"value2\", \"field3\" : null }";
    runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    // should have skipped field1 and field3
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes("{\"child_field1\":\"child_value1\"}"));
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
}

Example 20 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseRecord method basicPutSetup.

private void basicPutSetup(String encodingStrategy, PutValidator validator, String batchSize, int expectedPuts) throws Exception {
    Assert.assertEquals(1L, 1L);
    TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, batchSize);
    runner.setProperty(PutHBaseRecord.ROW_FIELD_NAME, "id");
    runner.setProperty(PutHBaseRecord.FIELD_ENCODING_STRATEGY, encodingStrategy);
    MockHBaseClientService client = getHBaseClientService(runner);
    generateTestData(runner);
    // This is to coax the processor into reading the data in the reader.l
    runner.enqueue("Test".getBytes("UTF-8"));
    runner.run();
    List<MockFlowFile> results = runner.getFlowFilesForRelationship(PutHBaseRecord.REL_SUCCESS);
    Assert.assertTrue("Wrong count", results.size() == 1);
    Assert.assertEquals("Wrong number of PutFlowFiles ", client.getFlowFilePuts().get("nifi").size(), expectedPuts);
    for (PutFlowFile putFlowFile : client.getFlowFilePuts().get("nifi")) {
        Iterator<PutColumn> columnIterator = putFlowFile.getColumns().iterator();
        PutColumn name = columnIterator.next();
        PutColumn code = columnIterator.next();
        Assert.assertNotNull("Name was null", name);
        Assert.assertNotNull("Code was null", code);
        String nFamName = new String(name.getColumnFamily());
        String cFamName = new String(code.getColumnFamily());
        String nQual = new String(name.getColumnQualifier());
        String cQual = new String(code.getColumnQualifier());
        Assert.assertEquals("Name column family didn't match", nFamName, DEFAULT_COLUMN_FAMILY);
        Assert.assertEquals("Code column family didn't match", cFamName, DEFAULT_COLUMN_FAMILY);
        Assert.assertEquals("Name qualifier didn't match", nQual, "name");
        Assert.assertEquals("Code qualifier didn't match", cQual, "code");
        validator.handle(name, code);
    }
}