Examples with KuduException org.apache.kudu.client.KuduException used on opensource projects

Example 1 with KuduException

use of org.apache.kudu.client.KuduException in project apex-malhar by apache.

the class AbstractKuduOutputOperator method performCommonProcessing.

/**
 * Sets the values from the Pojo into the Kudu mutation object.
 * @param currentOperation The operation instance that represents the current mutation. This will be applied to the
 *                         current session
 * @param kuduExecutionContext The tuple that contains the payload as well as other information like mutation type etc
 */
@SuppressWarnings(value = "unchecked")
private void performCommonProcessing(Operation currentOperation, KuduExecutionContext kuduExecutionContext) {
    currentOperation.setExternalConsistencyMode(kuduExecutionContext.getExternalConsistencyMode());
    Long propagatedTimeStamp = kuduExecutionContext.getPropagatedTimestamp();
    if (propagatedTimeStamp != null) {
        // set propagation timestamp only if enabled
        currentOperation.setPropagatedTimestamp(propagatedTimeStamp);
    }
    PartialRow partialRow = currentOperation.getRow();
    Object payload = kuduExecutionContext.getPayload();
    Set<String> doNotWriteColumns = kuduExecutionContext.getDoNotWriteColumns();
    if (doNotWriteColumns == null) {
        doNotWriteColumns = new HashSet<>();
    }
    for (String columnName : kuduColumnBasedGetters.keySet()) {
        if (doNotWriteColumns.contains(columnName)) {
            continue;
        }
        ColumnSchema columnSchema = allColumnDefs.get(columnName);
        Type dataType = columnSchema.getType();
        try {
            switch(dataType) {
                case STRING:
                    PojoUtils.Getter<Object, String> stringGetter = ((PojoUtils.Getter<Object, String>) kuduColumnBasedGetters.get(columnName));
                    if (stringGetter != null) {
                        final String stringValue = stringGetter.get(payload);
                        if (stringValue != null) {
                            partialRow.addString(columnName, stringValue);
                        }
                    }
                    break;
                case BINARY:
                    PojoUtils.Getter<Object, ByteBuffer> byteBufferGetter = ((PojoUtils.Getter<Object, ByteBuffer>) kuduColumnBasedGetters.get(columnName));
                    if (byteBufferGetter != null) {
                        final ByteBuffer byteBufferValue = byteBufferGetter.get(payload);
                        if (byteBufferValue != null) {
                            partialRow.addBinary(columnName, byteBufferValue);
                        }
                    }
                    break;
                case BOOL:
                    PojoUtils.GetterBoolean<Object> boolGetter = ((PojoUtils.GetterBoolean<Object>) kuduColumnBasedGetters.get(columnName));
                    if (boolGetter != null) {
                        final boolean boolValue = boolGetter.get(payload);
                        partialRow.addBoolean(columnName, boolValue);
                    }
                    break;
                case DOUBLE:
                    PojoUtils.GetterDouble<Object> doubleGetter = ((PojoUtils.GetterDouble<Object>) kuduColumnBasedGetters.get(columnName));
                    if (doubleGetter != null) {
                        final double doubleValue = doubleGetter.get(payload);
                        partialRow.addDouble(columnName, doubleValue);
                    }
                    break;
                case FLOAT:
                    PojoUtils.GetterFloat<Object> floatGetter = ((PojoUtils.GetterFloat<Object>) kuduColumnBasedGetters.get(columnName));
                    if (floatGetter != null) {
                        final float floatValue = floatGetter.get(payload);
                        partialRow.addFloat(columnName, floatValue);
                    }
                    break;
                case INT8:
                    PojoUtils.GetterByte<Object> byteGetter = ((PojoUtils.GetterByte<Object>) kuduColumnBasedGetters.get(columnName));
                    if (byteGetter != null) {
                        final byte byteValue = byteGetter.get(payload);
                        partialRow.addByte(columnName, byteValue);
                    }
                    break;
                case INT16:
                    PojoUtils.GetterShort<Object> shortGetter = ((PojoUtils.GetterShort<Object>) kuduColumnBasedGetters.get(columnName));
                    if (shortGetter != null) {
                        final short shortValue = shortGetter.get(payload);
                        partialRow.addShort(columnName, shortValue);
                    }
                    break;
                case INT32:
                    PojoUtils.GetterInt<Object> intGetter = ((PojoUtils.GetterInt<Object>) kuduColumnBasedGetters.get(columnName));
                    if (intGetter != null) {
                        final int intValue = intGetter.get(payload);
                        partialRow.addInt(columnName, intValue);
                    }
                    break;
                case INT64:
                case UNIXTIME_MICROS:
                    PojoUtils.GetterLong<Object> longGetter = ((PojoUtils.GetterLong<Object>) kuduColumnBasedGetters.get(columnName));
                    if (longGetter != null) {
                        final long longValue = longGetter.get(payload);
                        partialRow.addLong(columnName, longValue);
                    }
                    break;
                default:
                    LOG.error(columnName + " is not of the supported data type");
                    throw new UnsupportedOperationException("Kudu does not support data type for column " + columnName);
            }
        } catch (Exception ex) {
            LOG.error(" Exception while fetching the value of " + columnName + " because " + ex.getMessage());
            partialRow.setNull(columnName);
        }
    }
    try {
        kuduSession.apply(currentOperation);
    } catch (KuduException e) {
        throw new RuntimeException("Could not execute operation because " + e.getMessage(), e);
    }
}

Example 2 with KuduException

use of org.apache.kudu.client.KuduException in project nifi by apache.

the class AbstractKudu method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final FlowFile flowFile = session.get();
    try {
        if (flowFile == null)
            return;
        final Map<String, String> attributes = new HashMap<String, String>();
        final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>(null);
        final RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
        final KuduSession kuduSession = this.getKuduSession(kuduClient);
        session.read(flowFile, (final InputStream rawIn) -> {
            RecordReader recordReader = null;
            try (final BufferedInputStream in = new BufferedInputStream(rawIn)) {
                try {
                    recordReader = recordReaderFactory.createRecordReader(flowFile, in, getLogger());
                } catch (Exception ex) {
                    final RecordReaderFactoryException rrfe = new RecordReaderFactoryException("Unable to create RecordReader", ex);
                    exceptionHolder.set(rrfe);
                    return;
                }
                List<String> fieldNames = recordReader.getSchema().getFieldNames();
                final RecordSet recordSet = recordReader.createRecordSet();
                if (skipHeadLine)
                    recordSet.next();
                int numOfAddedRecord = 0;
                Record record = recordSet.next();
                while (record != null) {
                    org.apache.kudu.client.Operation oper = null;
                    if (operationType == OperationType.UPSERT) {
                        oper = upsertRecordToKudu(kuduTable, record, fieldNames);
                    } else {
                        oper = insertRecordToKudu(kuduTable, record, fieldNames);
                    }
                    kuduSession.apply(oper);
                    numOfAddedRecord++;
                    record = recordSet.next();
                }
                getLogger().info("KUDU: number of inserted records: " + numOfAddedRecord);
                attributes.put(RECORD_COUNT_ATTR, String.valueOf(numOfAddedRecord));
            } catch (KuduException ex) {
                getLogger().error("Exception occurred while interacting with Kudu due to " + ex.getMessage(), ex);
                exceptionHolder.set(ex);
            } catch (Exception e) {
                exceptionHolder.set(e);
            } finally {
                IOUtils.closeQuietly(recordReader);
            }
        });
        kuduSession.close();
        if (exceptionHolder.get() != null) {
            throw exceptionHolder.get();
        }
        // Update flow file's attributes after the ingestion
        session.putAllAttributes(flowFile, attributes);
        session.transfer(flowFile, REL_SUCCESS);
        session.getProvenanceReporter().send(flowFile, "Successfully added flowfile to kudu");
    } catch (IOException | FlowFileAccessException e) {
        getLogger().error("Failed to write due to {}", new Object[] { e });
        session.transfer(flowFile, REL_FAILURE);
    } catch (Throwable t) {
        getLogger().error("Failed to write due to {}", new Object[] { t });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 3 with KuduException

use of org.apache.kudu.client.KuduException in project apex-malhar by apache.

the class KuduClientTestCommons method createTestTable.

public static void createTestTable(String tableName, KuduClient client) throws Exception {
    List<String> rangeKeys = new ArrayList<>();
    rangeKeys.add("introwkey");
    List<String> hashPartitions = new ArrayList<>();
    hashPartitions.add("stringrowkey");
    hashPartitions.add("timestamprowkey");
    CreateTableOptions thisTableOptions = new CreateTableOptions().setNumReplicas(1).addHashPartitions(hashPartitions, HASH_BUCKETS_SIZE_FOR_ALL_HASH_COL).setRangePartitionColumns(rangeKeys);
    int stepsize = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
    int splitBoundary = stepsize;
    Schema schema = buildSchemaForUnitTestsTable();
    for (int i = 0; i < SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
        PartialRow splitRowBoundary = schema.newPartialRow();
        splitRowBoundary.addInt("introwkey", splitBoundary);
        thisTableOptions = thisTableOptions.addSplitRow(splitRowBoundary);
        splitBoundary += stepsize;
    }
    try {
        client.createTable(tableName, schema, thisTableOptions);
    } catch (KuduException e) {
        LOG.error("Error while creating table for unit tests " + e.getMessage(), e);
        throw e;
    }
}