Examples with Column org.apache.nifi.hbase.scan.Column used on opensource projects

Example 1 with Column

use of org.apache.nifi.hbase.scan.Column in project nifi by apache.

the class TestHBase_1_1_2_ClientService method testScan.

@Test
public void testScan() throws InitializationException, IOException {
    final String tableName = "nifi";
    final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
    // Mock an HBase Table so we can verify the put operations later
    final Table table = Mockito.mock(Table.class);
    when(table.getName()).thenReturn(TableName.valueOf(tableName));
    // create the controller service and link it to the test processor
    final MockHBaseClientService service = configureHBaseClientService(runner, table);
    runner.assertValid(service);
    // stage some results in the mock service...
    final long now = System.currentTimeMillis();
    final Map<String, String> cells = new HashMap<>();
    cells.put("greeting", "hello");
    cells.put("name", "nifi");
    service.addResult("row0", cells, now - 2);
    service.addResult("row1", cells, now - 1);
    service.addResult("row2", cells, now - 1);
    service.addResult("row3", cells, now);
    // perform a scan and verify the four rows were returned
    final CollectingResultHandler handler = new CollectingResultHandler();
    final HBaseClientService hBaseClientService = runner.getProcessContext().getProperty(TestProcessor.HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class);
    hBaseClientService.scan(tableName, new ArrayList<Column>(), null, now, handler);
    assertEquals(4, handler.results.size());
    // get row0 using the row id and verify it has 2 cells
    final ResultCell[] results = handler.results.get("row0");
    assertNotNull(results);
    assertEquals(2, results.length);
    verifyResultCell(results[0], COL_FAM, "greeting", "hello");
    verifyResultCell(results[1], COL_FAM, "name", "nifi");
}

Example 2 with Column

use of org.apache.nifi.hbase.scan.Column in project nifi by apache.

the class TestHBase_1_1_2_ClientService method testScanWithValidFilter.

@Test
public void testScanWithValidFilter() throws InitializationException, IOException {
    final String tableName = "nifi";
    final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class);
    // Mock an HBase Table so we can verify the put operations later
    final Table table = Mockito.mock(Table.class);
    when(table.getName()).thenReturn(TableName.valueOf(tableName));
    // create the controller service and link it to the test processor
    final MockHBaseClientService service = configureHBaseClientService(runner, table);
    runner.assertValid(service);
    // perform a scan and verify the four rows were returned
    final CollectingResultHandler handler = new CollectingResultHandler();
    final HBaseClientService hBaseClientService = runner.getProcessContext().getProperty(TestProcessor.HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class);
    // make sure we parse the filter expression without throwing an exception
    final String filter = "PrefixFilter ('Row') AND PageFilter (1) AND FirstKeyOnlyFilter ()";
    hBaseClientService.scan(tableName, new ArrayList<Column>(), filter, System.currentTimeMillis(), handler);
}

Example 3 with Column

use of org.apache.nifi.hbase.scan.Column in project nifi by apache.

the class FetchHBaseRow method getColumns.

/**
 * @param columnsValue a String in the form colFam:colQual,colFam:colQual
 * @return a list of Columns based on parsing the given String
 */
private List<Column> getColumns(final String columnsValue) {
    final String[] columns = (columnsValue == null || columnsValue.isEmpty() ? new String[0] : columnsValue.split(","));
    List<Column> columnsList = new ArrayList<>(columns.length);
    for (final String column : columns) {
        if (column.contains(":")) {
            final String[] parts = column.split(":");
            final byte[] cf = parts[0].getBytes(StandardCharsets.UTF_8);
            final byte[] cq = parts[1].getBytes(StandardCharsets.UTF_8);
            columnsList.add(new Column(cf, cq));
        } else {
            final byte[] cf = column.getBytes(StandardCharsets.UTF_8);
            columnsList.add(new Column(cf, null));
        }
    }
    return columnsList;
}

Example 4 with Column

use of org.apache.nifi.hbase.scan.Column in project nifi by apache.

the class FetchHBaseRow method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isBlank(tableName)) {
        getLogger().error("Table Name is blank or null for {}, transferring to failure", new Object[] { flowFile });
        session.transfer(session.penalize(flowFile), REL_FAILURE);
        return;
    }
    final String rowId = context.getProperty(ROW_ID).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isBlank(rowId)) {
        getLogger().error("Row Identifier is blank or null for {}, transferring to failure", new Object[] { flowFile });
        session.transfer(session.penalize(flowFile), REL_FAILURE);
        return;
    }
    final List<Column> columns = getColumns(context.getProperty(COLUMNS).evaluateAttributeExpressions(flowFile).getValue());
    final HBaseClientService hBaseClientService = context.getProperty(HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class);
    final String destination = context.getProperty(DESTINATION).getValue();
    final boolean base64Encode = context.getProperty(JSON_VALUE_ENCODING).getValue().equals(ENCODING_BASE64.getValue());
    final RowSerializer rowSerializer = base64Encode ? base64RowSerializer : regularRowSerializer;
    final FetchHBaseRowHandler handler = destination.equals(DESTINATION_CONTENT.getValue()) ? new FlowFileContentHandler(flowFile, session, rowSerializer) : new FlowFileAttributeHandler(flowFile, session, rowSerializer);
    final byte[] rowIdBytes = rowId.getBytes(StandardCharsets.UTF_8);
    try {
        hBaseClientService.scan(tableName, rowIdBytes, rowIdBytes, columns, handler);
    } catch (Exception e) {
        getLogger().error("Unable to fetch row {} from  {} due to {}", new Object[] { rowId, tableName, e });
        session.transfer(handler.getFlowFile(), REL_FAILURE);
        return;
    }
    FlowFile handlerFlowFile = handler.getFlowFile();
    if (!handler.handledRow()) {
        getLogger().debug("Row {} not found in {}, transferring to not found", new Object[] { rowId, tableName });
        session.transfer(handlerFlowFile, REL_NOT_FOUND);
        return;
    }
    if (getLogger().isDebugEnabled()) {
        getLogger().debug("Fetched {} from {} with row id {}", new Object[] { handlerFlowFile, tableName, rowId });
    }
    final Map<String, String> attributes = new HashMap<>();
    attributes.put(HBASE_TABLE_ATTR, tableName);
    if (destination.equals(DESTINATION_CONTENT.getValue())) {
        attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json");
    }
    handlerFlowFile = session.putAllAttributes(handlerFlowFile, attributes);
    final String transitUri = hBaseClientService.toTransitUri(tableName, rowId);
    // Regardless to where the result is written to, emit a fetch event.
    session.getProvenanceReporter().fetch(handlerFlowFile, transitUri);
    if (!destination.equals(DESTINATION_CONTENT.getValue())) {
        session.getProvenanceReporter().modifyAttributes(handlerFlowFile, "Added attributes to FlowFile from " + transitUri);
    }
    session.transfer(handlerFlowFile, REL_SUCCESS);
}

Example 5 with Column

use of org.apache.nifi.hbase.scan.Column in project nifi by apache.

the class GetHBase method parseColumns.

@OnScheduled
public void parseColumns(final ProcessContext context) throws IOException {
    final StateMap stateMap = context.getStateManager().getState(Scope.CLUSTER);
    if (stateMap.getVersion() < 0) {
        // no state has been stored in the State Manager - check if we have state stored in the
        // DistributedMapCacheClient service and migrate it if so
        final DistributedMapCacheClient client = context.getProperty(DISTRIBUTED_CACHE_SERVICE).asControllerService(DistributedMapCacheClient.class);
        final ScanResult scanResult = getState(client);
        if (scanResult != null) {
            storeState(scanResult, context.getStateManager());
        }
        clearState(client);
    }
    final String columnsValue = context.getProperty(COLUMNS).getValue();
    final String[] columns = (columnsValue == null || columnsValue.isEmpty() ? new String[0] : columnsValue.split(","));
    this.columns.clear();
    for (final String column : columns) {
        if (column.contains(":")) {
            final String[] parts = column.split(":");
            final byte[] cf = parts[0].getBytes(Charset.forName("UTF-8"));
            final byte[] cq = parts[1].getBytes(Charset.forName("UTF-8"));
            this.columns.add(new Column(cf, cq));
        } else {
            final byte[] cf = column.getBytes(Charset.forName("UTF-8"));
            this.columns.add(new Column(cf, null));
        }
    }
}