Examples with PutColumn org.apache.nifi.hbase.put.PutColumn used on opensource projects

Example 6 with PutColumn

use of org.apache.nifi.hbase.put.PutColumn in project nifi by apache.

the class PutHBaseCell method createPut.

@Override
protected PutFlowFile createPut(final ProcessSession session, final ProcessContext context, final FlowFile flowFile) {
    final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
    final String row = context.getProperty(ROW_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String columnFamily = context.getProperty(COLUMN_FAMILY).evaluateAttributeExpressions(flowFile).getValue();
    final String columnQualifier = context.getProperty(COLUMN_QUALIFIER).evaluateAttributeExpressions(flowFile).getValue();
    final String timestampValue = context.getProperty(TIMESTAMP).evaluateAttributeExpressions(flowFile).getValue();
    final Long timestamp;
    if (!StringUtils.isBlank(timestampValue)) {
        try {
            timestamp = Long.valueOf(timestampValue);
        } catch (Exception e) {
            getLogger().error("Invalid timestamp value: " + timestampValue, e);
            return null;
        }
    } else {
        timestamp = null;
    }
    final byte[] buffer = new byte[(int) flowFile.getSize()];
    session.read(flowFile, new InputStreamCallback() {

        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, buffer);
        }
    });
    final Collection<PutColumn> columns = Collections.singletonList(new PutColumn(columnFamily.getBytes(StandardCharsets.UTF_8), columnQualifier.getBytes(StandardCharsets.UTF_8), buffer, timestamp));
    byte[] rowKeyBytes = getRow(row, context.getProperty(ROW_ID_ENCODING_STRATEGY).getValue());
    return new PutFlowFile(tableName, rowKeyBytes, columns, flowFile);
}

Example 7 with PutColumn

use of org.apache.nifi.hbase.put.PutColumn in project nifi by apache.

the class PutHBaseRecord method createPut.

protected PutFlowFile createPut(ProcessContext context, Record record, RecordSchema schema, FlowFile flowFile, String rowFieldName, String columnFamily, String timestampFieldName, String fieldEncodingStrategy, String rowEncodingStrategy, String complexFieldStrategy) throws PutCreationFailedInvokedException {
    PutFlowFile retVal = null;
    final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
    final String nullStrategy = context.getProperty(NULL_FIELD_STRATEGY).getValue();
    boolean asString = STRING_ENCODING_VALUE.equals(fieldEncodingStrategy);
    final byte[] fam = clientService.toBytes(columnFamily);
    if (record != null) {
        final Long timestamp;
        if (!StringUtils.isBlank(timestampFieldName)) {
            try {
                timestamp = record.getAsLong(timestampFieldName);
            } catch (IllegalTypeConversionException e) {
                throw new PutCreationFailedInvokedException("Could not convert " + timestampFieldName + " to a long", e);
            }
            if (timestamp == null) {
                getLogger().warn("The value of timestamp field " + timestampFieldName + " was null, record will be inserted with latest timestamp");
            }
        } else {
            timestamp = null;
        }
        List<PutColumn> columns = new ArrayList<>();
        for (String name : schema.getFieldNames()) {
            if (name.equals(rowFieldName) || name.equals(timestampFieldName)) {
                continue;
            }
            Object val = record.getValue(name);
            final byte[] fieldValueBytes;
            if (val == null && nullStrategy.equals(NULL_FIELD_SKIP.getValue())) {
                continue;
            } else if (val == null && nullStrategy.equals(NULL_FIELD_EMPTY.getValue())) {
                fieldValueBytes = EMPTY;
            } else {
                fieldValueBytes = asBytes(name, schema.getField(name).get().getDataType().getFieldType(), record, asString, complexFieldStrategy);
            }
            if (fieldValueBytes != null) {
                columns.add(new PutColumn(fam, clientService.toBytes(name), fieldValueBytes, timestamp));
            }
        }
        String rowIdValue = record.getAsString(rowFieldName);
        if (rowIdValue == null) {
            throw new PutCreationFailedInvokedException(String.format("Row ID was null for flowfile with ID %s", flowFile.getAttribute("uuid")));
        }
        byte[] rowId = getRow(rowIdValue, rowEncodingStrategy);
        retVal = new PutFlowFile(tableName, rowId, columns, flowFile);
    }
    return retVal;
}

Example 8 with PutColumn

use of org.apache.nifi.hbase.put.PutColumn in project nifi by apache.

the class HBaseTestUtil method verifyPut.

public static void verifyPut(final String row, final String columnFamily, final Long timestamp, final Map<String, byte[]> columns, final List<PutFlowFile> puts) {
    boolean foundPut = false;
    for (final PutFlowFile put : puts) {
        if (!row.equals(new String(put.getRow(), StandardCharsets.UTF_8))) {
            continue;
        }
        if (put.getColumns() == null || put.getColumns().size() != columns.size()) {
            continue;
        }
        // start off assuming we have all the columns
        boolean foundAllColumns = true;
        for (Map.Entry<String, byte[]> entry : columns.entrySet()) {
            // determine if we have the current expected column
            boolean foundColumn = false;
            for (PutColumn putColumn : put.getColumns()) {
                if (columnFamily.equals(new String(putColumn.getColumnFamily(), StandardCharsets.UTF_8)) && entry.getKey().equals(new String(putColumn.getColumnQualifier(), StandardCharsets.UTF_8)) && Arrays.equals(entry.getValue(), putColumn.getBuffer()) && ((timestamp == null && putColumn.getTimestamp() == null) || (timestamp != null && timestamp.equals(putColumn.getTimestamp())))) {
                    foundColumn = true;
                    break;
                }
            }
            // if we didn't have the current expected column we know we don't have all expected columns
            if (!foundColumn) {
                foundAllColumns = false;
                break;
            }
        }
        // if we found all the expected columns this was a match so we can break
        if (foundAllColumns) {
            foundPut = true;
            break;
        }
    }
    assertTrue(foundPut);
}

Example 9 with PutColumn

use of org.apache.nifi.hbase.put.PutColumn in project nifi by apache.

the class TestPutHBaseCell method verifyPut.

private void verifyPut(byte[] row, byte[] columnFamily, byte[] columnQualifier, Long timestamp, String content, PutFlowFile put) {
    assertEquals(new String(row, StandardCharsets.UTF_8), new String(put.getRow(), StandardCharsets.UTF_8));
    assertNotNull(put.getColumns());
    assertEquals(1, put.getColumns().size());
    final PutColumn column = put.getColumns().iterator().next();
    assertEquals(new String(columnFamily, StandardCharsets.UTF_8), new String(column.getColumnFamily(), StandardCharsets.UTF_8));
    assertEquals(new String(columnQualifier, StandardCharsets.UTF_8), new String(column.getColumnQualifier(), StandardCharsets.UTF_8));
    assertEquals(content, new String(column.getBuffer(), StandardCharsets.UTF_8));
    assertEquals(timestamp, column.getTimestamp());
}

Example 10 with PutColumn

use of org.apache.nifi.hbase.put.PutColumn in project nifi by apache.

the class TestPutHBaseRecord method basicPutSetup.

private void basicPutSetup(String encodingStrategy, PutValidator validator, String batchSize, int expectedPuts) throws Exception {
    Assert.assertEquals(1L, 1L);
    TestRunner runner = getTestRunner(DEFAULT_TABLE_NAME, DEFAULT_COLUMN_FAMILY, batchSize);
    runner.setProperty(PutHBaseRecord.ROW_FIELD_NAME, "id");
    runner.setProperty(PutHBaseRecord.FIELD_ENCODING_STRATEGY, encodingStrategy);
    MockHBaseClientService client = getHBaseClientService(runner);
    generateTestData(runner);
    // This is to coax the processor into reading the data in the reader.l
    runner.enqueue("Test".getBytes("UTF-8"));
    runner.run();
    List<MockFlowFile> results = runner.getFlowFilesForRelationship(PutHBaseRecord.REL_SUCCESS);
    Assert.assertTrue("Wrong count", results.size() == 1);
    Assert.assertEquals("Wrong number of PutFlowFiles ", client.getFlowFilePuts().get("nifi").size(), expectedPuts);
    for (PutFlowFile putFlowFile : client.getFlowFilePuts().get("nifi")) {
        Iterator<PutColumn> columnIterator = putFlowFile.getColumns().iterator();
        PutColumn name = columnIterator.next();
        PutColumn code = columnIterator.next();
        Assert.assertNotNull("Name was null", name);
        Assert.assertNotNull("Code was null", code);
        String nFamName = new String(name.getColumnFamily());
        String cFamName = new String(code.getColumnFamily());
        String nQual = new String(name.getColumnQualifier());
        String cQual = new String(code.getColumnQualifier());
        Assert.assertEquals("Name column family didn't match", nFamName, DEFAULT_COLUMN_FAMILY);
        Assert.assertEquals("Code column family didn't match", cFamName, DEFAULT_COLUMN_FAMILY);
        Assert.assertEquals("Name qualifier didn't match", nQual, "name");
        Assert.assertEquals("Code qualifier didn't match", cQual, "code");
        validator.handle(name, code);
    }
}