Examples with PutFlowFile org.apache.nifi.hbase.put.PutFlowFile used on opensource projects

Example 26 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testSingJsonDocAndExtractedRowId.

@Test
public void testSingJsonDocAndExtractedRowId() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "rowField");
    final String content = "{ \"rowField\" : \"myRowId\", \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    // should be a put with row id of myRowId, and rowField shouldn't end up in the columns
    final Map<String, byte[]> expectedColumns1 = new HashMap<>();
    expectedColumns1.put("field1", hBaseClient.toBytes("value1"));
    expectedColumns1.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut("myRowId", DEFAULT_COLUMN_FAMILY, expectedColumns1, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://" + DEFAULT_TABLE_NAME + "/myRowId", ProvenanceEventType.SEND);
}

Example 27 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testTimestamp.

@Test
public void testTimestamp() throws UnsupportedEncodingException, InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    runner.setProperty(PutHBaseJSON.TIMESTAMP, DEFAULT_TIMESTAMP.toString());
    final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes("UTF-8"));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes("value1"));
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, DEFAULT_TIMESTAMP, expectedColumns, puts);
}

Example 28 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testNullAndArrayElementsWithTextStrategy.

@Test
public void testNullAndArrayElementsWithTextStrategy() throws InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    runner.setProperty(PutHBaseJSON.COMPLEX_FIELD_STRATEGY, PutHBaseJSON.COMPLEX_FIELD_TEXT.getValue());
    // should route to success because there is at least one valid field
    final String content = "{ \"field1\" : [{ \"child_field1\" : \"child_value1\" }], \"field2\" : \"value2\", \"field3\" : null }";
    runner.enqueue(content.getBytes(StandardCharsets.UTF_8));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    // should have skipped field1 and field3
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes("[{\"child_field1\":\"child_value1\"}]"));
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
}

Example 29 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseJSON method testTimestampWithEL.

@Test
public void testTimestampWithEL() throws UnsupportedEncodingException, InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    runner.setProperty(PutHBaseJSON.TIMESTAMP, "${hbase.timestamp}");
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("hbase.timestamp", DEFAULT_TIMESTAMP.toString());
    final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes("UTF-8"), attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes("value1"));
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, DEFAULT_TIMESTAMP, expectedColumns, puts);
}

Example 30 with PutFlowFile

use of org.apache.nifi.hbase.put.PutFlowFile in project nifi by apache.

the class TestPutHBaseRecord method testFailure.

@Test
public void testFailure() throws Exception {
    Assert.assertEquals(1L, 1L);
    TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "2");
    runner.setProperty(PutHBaseRecord.ROW_FIELD_NAME, "id");
    runner.setProperty(PutHBaseRecord.FIELD_ENCODING_STRATEGY, PutHBaseRecord.STRING_ENCODING_VALUE);
    MockHBaseClientService client = getHBaseClientService(runner);
    client.setTestFailure(true);
    client.setFailureThreshold(2);
    generateTestData(runner);
    // This is to coax the processor into reading the data in the reader.
    runner.enqueue("Test".getBytes("UTF-8"));
    runner.run();
    List<MockFlowFile> result = runner.getFlowFilesForRelationship(PutHBaseRecord.REL_FAILURE);
    Assert.assertEquals("Size was wrong", result.size(), 1);
    Assert.assertEquals("Wrong # of PutFlowFiles", client.getFlowFilePuts().get("nifi").size(), 2);
    Assert.assertTrue(runner.getFlowFilesForRelationship(PutHBaseRecord.REL_SUCCESS).size() == 0);
    MockFlowFile mff = result.get(0);
    Assert.assertNotNull("Missing restart index attribute", mff.getAttribute("restart.index"));
    List<PutFlowFile> old = client.getFlowFilePuts().get("nifi");
    client.setTestFailure(false);
    runner.enqueue("test");
    runner.run();
    Assert.assertEquals("Size was wrong", result.size(), 1);
    Assert.assertEquals("Wrong # of PutFlowFiles", client.getFlowFilePuts().get("nifi").size(), 2);
    List<PutFlowFile> newPFF = client.getFlowFilePuts().get("nifi");
    for (PutFlowFile putFlowFile : old) {
        Assert.assertFalse("Duplication", newPFF.contains(putFlowFile));
    }
}