Examples with Scan io.cdap.cdap.api.dataset.table.Scan used on opensource projects

Example 1 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class MetadataStoreDataset method listKV.

/**
 * returns mapping of all that has first id parts in range of startId and stopId for default COLUMN
 *
 * @param startId start row key
 * @param stopId stop row key
 * @param typeOfT the type of the result
 * @param limit limit number of result
 * @param keyFilter filter of the key
 * @param valueFilter filter for the result
 * @return map of row key to result
 */
public <T> Map<MDSKey, T> listKV(MDSKey startId, @Nullable MDSKey stopId, Type typeOfT, int limit, Predicate<MDSKey> keyFilter, Predicate<T> valueFilter) {
    byte[] startKey = startId.getKey();
    byte[] stopKey = stopId == null ? Bytes.stopKeyForPrefix(startKey) : stopId.getKey();
    Scan scan = new Scan(startKey, stopKey);
    return listKV(scan, typeOfT, limit, keyFilter, valueFilter);
}

Example 2 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class HBaseTableTest method testCachedEncodedTransaction.

@Test
public void testCachedEncodedTransaction() throws Exception {
    String tableName = "testEncodedTxTable";
    DatasetProperties props = DatasetProperties.EMPTY;
    getTableAdmin(CONTEXT1, tableName, props).create();
    DatasetSpecification tableSpec = DatasetSpecification.builder(tableName, HBaseTable.class.getName()).build();
    // use a transaction codec that counts the number of times encode() is called
    final AtomicInteger encodeCount = new AtomicInteger();
    final TransactionCodec codec = new TransactionCodec() {

        @Override
        public byte[] encode(Transaction tx) throws IOException {
            encodeCount.incrementAndGet();
            return super.encode(tx);
        }
    };
    // use a table util that creates an HTable that validates the encoded tx on each get
    final AtomicReference<Transaction> txRef = new AtomicReference<>();
    HBaseTableUtil util = new DelegatingHBaseTableUtil(hBaseTableUtil) {

        @Override
        public org.apache.hadoop.hbase.client.Table createTable(Configuration conf, TableId tableId) throws IOException {
            org.apache.hadoop.hbase.client.Table table = super.createTable(conf, tableId);
            return new DelegatingTable(table) {

                @Override
                public Result get(org.apache.hadoop.hbase.client.Get get) throws IOException {
                    Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    return super.get(get);
                }

                @Override
                public Result[] get(List<org.apache.hadoop.hbase.client.Get> gets) throws IOException {
                    for (org.apache.hadoop.hbase.client.Get get : gets) {
                        Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    }
                    return super.get(gets);
                }

                @Override
                public ResultScanner getScanner(org.apache.hadoop.hbase.client.Scan scan) throws IOException {
                    Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(scan.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    return super.getScanner(scan);
                }
            };
        }
    };
    HBaseTable table = new HBaseTable(CONTEXT1, tableSpec, Collections.<String, String>emptyMap(), cConf, TEST_HBASE.getConfiguration(), util, codec);
    DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
    // test all operations: only the first one encodes
    Transaction tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.put(b("row1"), b("col1"), b("val1"));
    Assert.assertEquals(0, encodeCount.get());
    table.get(b("row"));
    Assert.assertEquals(1, encodeCount.get());
    table.get(ImmutableList.of(new Get("a"), new Get("b")));
    Assert.assertEquals(1, encodeCount.get());
    Scanner scanner = table.scan(new Scan(null, null));
    Assert.assertEquals(1, encodeCount.get());
    scanner.close();
    table.increment(b("z"), b("z"), 0L);
    Assert.assertEquals(1, encodeCount.get());
    table.commitTx();
    table.postTxCommit();
    // test that for the next tx, we encode again
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(2, encodeCount.get());
    table.commitTx();
    // test that we encode again, even of postTxCommit was not called
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(3, encodeCount.get());
    table.commitTx();
    table.rollbackTx();
    // test that rollback does not encode the tx
    Assert.assertEquals(3, encodeCount.get());
    // test that we encode again if the previous tx rolled back
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(4, encodeCount.get());
    table.commitTx();
    table.close();
    Assert.assertEquals(4, encodeCount.get());
}

Example 3 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class HBaseTableTest method testScannerCache.

private void testScannerCache(int rowsExpected, String tableName, @Nullable String property, @Nullable String argument, @Nullable String scanArgument) throws Exception {
    // Now scan and sleep for a while after each result
    Transaction tx = txClient.startShort();
    DatasetProperties props = property == null ? DatasetProperties.EMPTY : DatasetProperties.of(ImmutableMap.of(HConstants.HBASE_CLIENT_SCANNER_CACHING, property));
    Map<String, String> arguments = argument == null ? Collections.<String, String>emptyMap() : ImmutableMap.of(HConstants.HBASE_CLIENT_SCANNER_CACHING, argument);
    Scan scan = new Scan(null, null);
    if (scanArgument != null) {
        scan.setProperty(HConstants.HBASE_CLIENT_SCANNER_CACHING, scanArgument);
    }
    try (Table table = getTable(CONTEXT1, tableName, props, arguments)) {
        ((TransactionAware) table).startTx(tx);
        Scanner scanner = table.scan(scan);
        int scanCount = 0;
        try {
            while (scanner.next() != null) {
                scanCount++;
                TimeUnit.MILLISECONDS.sleep(10);
            }
            scanner.close();
        } finally {
            LOG.info("Scanned {} rows.", scanCount);
            txClient.abort(tx);
        }
        Assert.assertEquals(rowsExpected, scanCount);
    }
}

Example 4 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class TableTest method testMetrics.

private void testMetrics(boolean readless) throws Exception {
    final String tableName = "survive";
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(readless).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, tableName, props);
    admin.create();
    try (Table table = getTable(CONTEXT1, tableName, props)) {
        final Map<String, Long> metrics = Maps.newHashMap();
        ((MeteredDataset) table).setMetricsCollector(new MetricsCollector() {

            @Override
            public void increment(String metricName, long value) {
                Long old = metrics.get(metricName);
                metrics.put(metricName, old == null ? value : old + value);
            }

            @Override
            public void gauge(String metricName, long value) {
                metrics.put(metricName, value);
            }
        });
        // Note that we don't need to finish tx for metrics to be reported
        Transaction tx0 = txClient.startShort();
        ((TransactionAware) table).startTx(tx0);
        int writes = 0;
        int reads = 0;
        table.put(new Put(R1, C1, V1));
        verifyDatasetMetrics(metrics, ++writes, reads);
        table.compareAndSwap(R1, C1, V1, V2);
        verifyDatasetMetrics(metrics, ++writes, ++reads);
        // note: will not write anything as expected value will not match
        table.compareAndSwap(R1, C1, V1, V2);
        verifyDatasetMetrics(metrics, writes, ++reads);
        table.increment(new Increment(R2, C2, 1L));
        if (readless) {
            verifyDatasetMetrics(metrics, ++writes, reads);
        } else {
            verifyDatasetMetrics(metrics, ++writes, ++reads);
        }
        table.incrementAndGet(new Increment(R2, C2, 1L));
        verifyDatasetMetrics(metrics, ++writes, ++reads);
        table.get(new Get(R1, C1, V1));
        verifyDatasetMetrics(metrics, writes, ++reads);
        Scanner scanner = table.scan(new Scan(null, null));
        while (scanner.next() != null) {
            verifyDatasetMetrics(metrics, writes, ++reads);
        }
        table.delete(new Delete(R1, C1, V1));
        verifyDatasetMetrics(metrics, ++writes, reads);
    } finally {
        // drop table
        admin.drop();
    }
}

Example 5 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class BufferingTableTest method testScanWithBuffering.

/**
 * Tests that writes being buffered in memory by the client are still visible during scans.
 */
@Test
public void testScanWithBuffering() throws Exception {
    String testScanWithBuffering = "testScanWithBuffering";
    DatasetAdmin admin = getTableAdmin(CONTEXT1, testScanWithBuffering);
    admin.create();
    try (Table table1 = getTable(CONTEXT1, testScanWithBuffering)) {
        // 
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx1);
        table1.put(Bytes.toBytes("1_01"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_02"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_03"), a(C1), a(V1));
        // written values should not yet be persisted
        TableAssert.assertScan(new byte[0][], new byte[0][][], ((BufferingTable) table1).scanPersisted(new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_"))));
        // buffered values should be visible in a scan
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) table1).getTxChanges());
        Assert.assertTrue(((TransactionAware) table1).commitTx());
        txClient.commitOrThrow(tx1);
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx2);
        // written values should be visible after commit
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.commitOrThrow(tx2);
        Transaction tx3 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx3);
        // test merging of buffered writes on existing rows
        table1.put(Bytes.toBytes("1_01"), a(C2), a(V2));
        table1.put(Bytes.toBytes("1_02"), a(C1), a(V2));
        table1.put(Bytes.toBytes("1_02a"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_02b"), a(C1), a(V1));
        table1.put(Bytes.toBytes("1_04"), a(C2), a(V2));
        // persisted values should be the same
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_03")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), ((BufferingTable) table1).scanPersisted(new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_"))));
        // all values should be visible in buffered scan
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_02a"), Bytes.toBytes("1_02b"), Bytes.toBytes("1_03"), Bytes.toBytes("1_04")), aa(// 1_01
        a(C1, V1, C2, V2), // 1_02
        a(C1, V2), // 1_02a
        a(C1, V1), // 1_02b
        a(C1, V1), // 1_03
        a(C1, V1), // 1_04
        a(C2, V2)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) table1).getTxChanges());
        Assert.assertTrue(((TransactionAware) table1).commitTx());
        txClient.commitOrThrow(tx3);
        Transaction tx4 = txClient.startShort();
        ((TransactionAware) table1).startTx(tx4);
        // all values should be visible after commit
        TableAssert.assertScan(a(Bytes.toBytes("1_01"), Bytes.toBytes("1_02"), Bytes.toBytes("1_02a"), Bytes.toBytes("1_02b"), Bytes.toBytes("1_03"), Bytes.toBytes("1_04")), aa(// 1_01
        a(C1, V1, C2, V2), // 1_02
        a(C1, V2), // 1_02a
        a(C1, V1), // 1_02b
        a(C1, V1), // 1_03
        a(C1, V1), // 1_04
        a(C2, V2)), table1.scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        txClient.commitOrThrow(tx4);
    } finally {
        admin.drop();
    }
}