Examples with KuduScanner org.apache.kudu.client.KuduScanner used on opensource projects

Example 11 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project apex-malhar by apache.

the class KuduClientTestCommons method lookUpAndPopulateRecord.

protected void lookUpAndPopulateRecord(UnitTestTablePojo keyInfo) throws Exception {
    KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("introwkey"), KuduPredicate.ComparisonOp.EQUAL, keyInfo.getIntrowkey())).addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("stringrowkey"), KuduPredicate.ComparisonOp.EQUAL, keyInfo.getStringrowkey())).addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("timestamprowkey"), KuduPredicate.ComparisonOp.EQUAL, keyInfo.getTimestamprowkey())).build();
    RowResultIterator rowResultItr = scanner.nextRows();
    while (rowResultItr.hasNext()) {
        RowResult thisRow = rowResultItr.next();
        keyInfo.setFloatdata(thisRow.getFloat("floatdata"));
        keyInfo.setBooldata(thisRow.getBoolean("booldata"));
        keyInfo.setBinarydata(thisRow.getBinary("binarydata"));
        keyInfo.setLongdata(thisRow.getLong("longdata"));
        keyInfo.setTimestampdata(thisRow.getLong("timestampdata"));
        keyInfo.setStringdata("stringdata");
        break;
    }
}

Example 12 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project apex-malhar by apache.

the class KuduInputOperatorCommons method countNumRowsInTable.

public long countNumRowsInTable() throws Exception {
    List<String> allProjectedCols = new ArrayList<>(unitTestStepwiseScanInputOperator.getKuduColNameToSchemaMapping().keySet());
    KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).setProjectedColumnNames(allProjectedCols).build();
    long counter = 0;
    while (scanner.hasMoreRows()) {
        RowResultIterator rowResultItr = scanner.nextRows();
        while (rowResultItr.hasNext()) {
            RowResult thisRow = rowResultItr.next();
            counter += 1;
        }
    }
    return counter;
}

Example 13 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project apex-malhar by apache.

the class KuduPartitionScannerCallable method call.

@Override
public Long call() throws Exception {
    long numRowsScanned = 0;
    KuduScanner aPartitionSpecificScanner = KuduScanToken.deserializeIntoScanner(kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle);
    LOG.info("Scanning the following tablet " + KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle));
    KuduRecordWithMeta<T> beginScanRecord = new KuduRecordWithMeta<>();
    beginScanRecord.setBeginScanMarker(true);
    beginScanRecord.setTabletMetadata(kuduPartitionScanAssignmentMeta);
    // Add a record entry that denotes the end of this scan.
    bufferForTransmittingRecords.add(beginScanRecord);
    while (aPartitionSpecificScanner.hasMoreRows()) {
        LOG.debug("Number of columns being returned for this read " + aPartitionSpecificScanner.getProjectionSchema().getColumnCount());
        RowResultIterator resultIterator = aPartitionSpecificScanner.nextRows();
        if (resultIterator == null) {
            break;
        } else {
            while (resultIterator.hasNext()) {
                KuduRecordWithMeta<T> recordWithMeta = new KuduRecordWithMeta<>();
                RowResult aRow = resultIterator.next();
                recordWithMeta.setPositionInScan(numRowsScanned);
                T payload = clazzForResultObject.newInstance();
                recordWithMeta.setThePayload(payload);
                recordWithMeta.setEndOfScanMarker(false);
                recordWithMeta.setTabletMetadata(kuduPartitionScanAssignmentMeta);
                setValuesInPOJO(aRow, payload);
                bufferForTransmittingRecords.add(recordWithMeta);
                numRowsScanned += 1;
            }
        }
    }
    aPartitionSpecificScanner.close();
    KuduRecordWithMeta<T> endScanRecord = new KuduRecordWithMeta<>();
    endScanRecord.setEndOfScanMarker(true);
    endScanRecord.setTabletMetadata(kuduPartitionScanAssignmentMeta);
    // Add a record entry that denotes the end of this scan.
    bufferForTransmittingRecords.add(endScanRecord);
    LOG.info(" Scanned a total of " + numRowsScanned + " for this scanner thread @tablet " + KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle));
    return numRowsScanned;
}

Example 14 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project presto by prestodb.

the class SchemaEmulationByTableNameConvention method listSchemaNames.

@Override
public List<String> listSchemaNames(KuduClient client) {
    try {
        if (rawSchemasTable == null) {
            if (!client.tableExists(rawSchemasTableName)) {
                createAndFillSchemasTable(client);
            }
            rawSchemasTable = getSchemasTable(client);
        }
        KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build();
        RowResultIterator iterator = scanner.nextRows();
        ArrayList<String> result = new ArrayList<>();
        while (iterator != null) {
            for (RowResult row : iterator) {
                result.add(row.getString(0));
            }
            iterator = scanner.nextRows();
        }
        return result;
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}

Example 15 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project canal by alibaba.

the class KuduTemplate method countRow.

/**
 * 统计kudu表数据
 *
 * @param tableName
 * @return
 */
public long countRow(String tableName) {
    this.checkClient();
    long rowCount = 0L;
    try {
        KuduTable kuduTable = kuduClient.openTable(tableName);
        // 创建scanner扫描
        KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
        // 遍历数据
        while (scanner.hasMoreRows()) {
            while (scanner.nextRows().hasNext()) {
                rowCount++;
            }
        }
    } catch (KuduException e) {
        e.printStackTrace();
    }
    return rowCount;
}