Examples with ProvenanceEventRecord org.apache.nifi.provenance.ProvenanceEventRecord used on opensource projects

Example 86 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestPutHBaseJSON method testELWithExtractedRowId.

@Test
public void testELWithExtractedRowId() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner("${hbase.table}", "${hbase.colFamily}", "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_FIELD_NAME, "${hbase.rowField}");
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("hbase.table", "myTable");
    attributes.put("hbase.colFamily", "myColFamily");
    attributes.put("hbase.rowField", "field1");
    final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes("UTF-8"), attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get("myTable");
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut("value1", "myColFamily", expectedColumns, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://myTable/value1", ProvenanceEventType.SEND);
}

Example 87 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestPutHBaseJSON method testSingleJsonDocAndProvidedRowIdwithNonString.

@Test
public void testSingleJsonDocAndProvidedRowIdwithNonString() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, "1");
    runner.setProperty(PutHBaseJSON.FIELD_ENCODING_STRATEGY, PutHBaseJSON.BYTES_ENCODING_VALUE);
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, DEFAULT_ROW);
    final String content = "{ \"field1\" : 1.23456, \"field2\" : 2345235, \"field3\" : false }";
    runner.enqueue(content.getBytes("UTF-8"));
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get(DEFAULT_TABLE_NAME);
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes(1.23456d));
    expectedColumns.put("field2", hBaseClient.toBytes(2345235l));
    expectedColumns.put("field3", hBaseClient.toBytes(false));
    HBaseTestUtil.verifyPut(DEFAULT_ROW, DEFAULT_COLUMN_FAMILY, expectedColumns, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    final ProvenanceEventRecord event = events.get(0);
    assertEquals("hbase://" + DEFAULT_TABLE_NAME + "/" + DEFAULT_ROW, event.getTransitUri());
}

Example 88 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestPutHBaseJSON method testELWithProvidedRowId.

@Test
public void testELWithProvidedRowId() throws IOException, InitializationException {
    final TestRunner runner = getTestRunner("${hbase.table}", "${hbase.colFamily}", "1");
    final MockHBaseClientService hBaseClient = getHBaseClientService(runner);
    runner.setProperty(PutHBaseJSON.ROW_ID, "${hbase.rowId}");
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("hbase.table", "myTable");
    attributes.put("hbase.colFamily", "myColFamily");
    attributes.put("hbase.rowId", "myRowId");
    final String content = "{ \"field1\" : \"value1\", \"field2\" : \"value2\" }";
    runner.enqueue(content.getBytes("UTF-8"), attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutHBaseCell.REL_SUCCESS);
    final MockFlowFile outFile = runner.getFlowFilesForRelationship(PutHBaseCell.REL_SUCCESS).get(0);
    outFile.assertContentEquals(content);
    assertNotNull(hBaseClient.getFlowFilePuts());
    assertEquals(1, hBaseClient.getFlowFilePuts().size());
    final List<PutFlowFile> puts = hBaseClient.getFlowFilePuts().get("myTable");
    assertEquals(1, puts.size());
    final Map<String, byte[]> expectedColumns = new HashMap<>();
    expectedColumns.put("field1", hBaseClient.toBytes("value1"));
    expectedColumns.put("field2", hBaseClient.toBytes("value2"));
    HBaseTestUtil.verifyPut("myRowId", "myColFamily", expectedColumns, puts);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    HBaseTestUtil.verifyEvent(runner.getProvenanceEvents(), "hbase://myTable/myRowId", ProvenanceEventType.SEND);
}

Example 89 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class GetHDFSTest method testDirectoryUsesValidEL.

@Test
public void testDirectoryUsesValidEL() throws IOException {
    GetHDFS proc = new TestableGetHDFS(kerberosProperties);
    TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(PutHDFS.DIRECTORY, "src/test/resources/${literal('testdata'):substring(0,8)}");
    runner.setProperty(GetHDFS.FILE_FILTER_REGEX, ".*.zip");
    runner.setProperty(GetHDFS.KEEP_SOURCE_FILE, "true");
    runner.setProperty(GetHDFS.COMPRESSION_CODEC, "AUTOMATIC");
    runner.run();
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetHDFS.REL_SUCCESS);
    assertEquals(1, flowFiles.size());
    MockFlowFile flowFile = flowFiles.get(0);
    assertTrue(flowFile.getAttribute(CoreAttributes.FILENAME.key()).equals("13545423550275052.zip"));
    InputStream expected = getClass().getResourceAsStream("/testdata/13545423550275052.zip");
    flowFile.assertContentEquals(expected);
    final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
    assertEquals(1, provenanceEvents.size());
    final ProvenanceEventRecord receiveEvent = provenanceEvents.get(0);
    assertEquals(ProvenanceEventType.RECEIVE, receiveEvent.getEventType());
    // If it runs with a real HDFS, the protocol will be "hdfs://", but with a local filesystem, just assert the filename.
    assertTrue(receiveEvent.getTransitUri().endsWith("13545423550275052.zip"));
}

Example 90 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class StandardProcessSession method handleContentNotFound.

private void handleContentNotFound(final ContentNotFoundException nfe, final StandardRepositoryRecord suspectRecord) {
    final ContentClaim registeredClaim = suspectRecord.getOriginalClaim();
    final ContentClaim transientClaim = suspectRecord.getWorkingClaim();
    final ContentClaim missingClaim = nfe.getMissingClaim();
    final ProvenanceEventRecord dropEvent = provenanceReporter.drop(suspectRecord.getCurrent(), nfe.getMessage() == null ? "Content Not Found" : nfe.getMessage());
    if (dropEvent != null) {
        context.getProvenanceRepository().registerEvent(dropEvent);
    }
    if (missingClaim == registeredClaim) {
        suspectRecord.markForAbort();
        rollback();
        throw new MissingFlowFileException("Unable to find content for FlowFile", nfe);
    }
    if (missingClaim == transientClaim) {
        rollback();
        throw new MissingFlowFileException("Unable to find content for FlowFile", nfe);
    }
}