Examples with Scan io.cdap.cdap.api.dataset.table.Scan used on opensource projects

Example 6 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class HBaseTableTest method testEnableIncrements.

@Test
public void testEnableIncrements() throws Exception {
    // setup a table with increments disabled and with it enabled
    String disableTableName = "incr-disable";
    String enabledTableName = "incr-enable";
    TableId disabledTableId = hBaseTableUtil.createHTableId(NAMESPACE1, disableTableName);
    TableId enabledTableId = hBaseTableUtil.createHTableId(NAMESPACE1, enabledTableName);
    DatasetProperties propsDisabled = TableProperties.builder().setReadlessIncrementSupport(false).setConflictDetection(ConflictDetection.COLUMN).build();
    HBaseTableAdmin disabledAdmin = getTableAdmin(CONTEXT1, disableTableName, propsDisabled);
    disabledAdmin.create();
    HBaseAdmin admin = TEST_HBASE.getHBaseAdmin();
    DatasetProperties propsEnabled = TableProperties.builder().setReadlessIncrementSupport(true).setConflictDetection(ConflictDetection.COLUMN).build();
    HBaseTableAdmin enabledAdmin = getTableAdmin(CONTEXT1, enabledTableName, propsEnabled);
    enabledAdmin.create();
    try {
        try {
            HTableDescriptor htd = hBaseTableUtil.getHTableDescriptor(admin, disabledTableId);
            List<String> cps = htd.getCoprocessors();
            assertFalse(cps.contains(IncrementHandler.class.getName()));
            htd = hBaseTableUtil.getHTableDescriptor(admin, enabledTableId);
            cps = htd.getCoprocessors();
            assertTrue(cps.contains(IncrementHandler.class.getName()));
        } finally {
            admin.close();
        }
        try (BufferingTable table = getTable(CONTEXT1, enabledTableName, propsEnabled)) {
            byte[] row = Bytes.toBytes("row1");
            byte[] col = Bytes.toBytes("col1");
            DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
            Transaction tx = txSystemClient.startShort();
            table.startTx(tx);
            table.increment(row, col, 10);
            table.commitTx();
            // verify that value was written as a delta value
            final byte[] expectedValue = Bytes.add(IncrementHandlerState.DELTA_MAGIC_PREFIX, Bytes.toBytes(10L));
            final AtomicBoolean foundValue = new AtomicBoolean();
            byte[] enabledTableNameBytes = hBaseTableUtil.getHTableDescriptor(admin, enabledTableId).getName();
            TEST_HBASE.forEachRegion(enabledTableNameBytes, new Function<HRegion, Object>() {

                @Override
                public Object apply(HRegion hRegion) {
                    org.apache.hadoop.hbase.client.Scan scan = hBaseTableUtil.buildScan().build();
                    try {
                        RegionScanner scanner = hRegion.getScanner(scan);
                        List<Cell> results = Lists.newArrayList();
                        boolean hasMore;
                        do {
                            hasMore = scanner.next(results);
                            for (Cell cell : results) {
                                if (CellUtil.matchingValue(cell, expectedValue)) {
                                    foundValue.set(true);
                                }
                            }
                        } while (hasMore);
                    } catch (IOException ioe) {
                        fail("IOException scanning region: " + ioe.getMessage());
                    }
                    return null;
                }
            });
            assertTrue("Should have seen the expected encoded delta value in the " + enabledTableName + " table region", foundValue.get());
        }
    } finally {
        disabledAdmin.drop();
        enabledAdmin.drop();
    }
}

Example 7 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class ObjectMappedTableDatasetTest method testScan.

@Test
public void testScan() throws Exception {
    dsFrameworkUtil.createInstance(ObjectMappedTable.class.getName(), RECORDS_ID, ObjectMappedTableProperties.builder().setType(Record.class).build());
    try {
        final ObjectMappedTableDataset<Record> records = dsFrameworkUtil.getInstance(RECORDS_ID);
        TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) records);
        Record record1 = new Record(Integer.MAX_VALUE, Long.MAX_VALUE, Float.MAX_VALUE, Double.MAX_VALUE, "foobar", Bytes.toBytes("foobar"), ByteBuffer.wrap(Bytes.toBytes("foobar")), UUID.randomUUID());
        Record record2 = new Record(Integer.MIN_VALUE, Long.MIN_VALUE, Float.MIN_VALUE, Double.MIN_VALUE, "baz", Bytes.toBytes("baz"), ByteBuffer.wrap(Bytes.toBytes("baz")), UUID.randomUUID());
        Record record3 = new Record(1, 0L, 3.14f, 3.14159265358979323846, "hello", Bytes.toBytes("world"), ByteBuffer.wrap(Bytes.toBytes("yo")), UUID.randomUUID());
        final List<KeyValue<byte[], Record>> recordList = Lists.newArrayList();
        recordList.add(new KeyValue<>(Bytes.toBytes("123"), record1));
        recordList.add(new KeyValue<>(Bytes.toBytes("456"), record2));
        recordList.add(new KeyValue<>(Bytes.toBytes("789"), record3));
        for (final KeyValue<byte[], Record> record : recordList) {
            txnl.execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    records.write(record.getKey(), record.getValue());
                }
            });
        }
        final List<KeyValue<byte[], Record>> actualList = Lists.newArrayList();
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan((String) null, null);
                while (results.hasNext()) {
                    actualList.add(results.next());
                }
                results.close();
            }
        });
        Assert.assertEquals(recordList.size(), actualList.size());
        for (int i = 0; i < actualList.size(); i++) {
            KeyValue<byte[], Record> expected = recordList.get(i);
            KeyValue<byte[], Record> actual = actualList.get(i);
            Assert.assertArrayEquals(expected.getKey(), actual.getKey());
            Assert.assertEquals(expected.getValue(), actual.getValue());
        }
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan("789", null);
                KeyValue<byte[], Record> actualRecord = results.next();
                Assert.assertFalse(results.hasNext());
                Assert.assertArrayEquals(actualRecord.getKey(), recordList.get(2).getKey());
                Assert.assertEquals(actualRecord.getValue(), recordList.get(2).getValue());
                results.close();
                results = records.scan(null, "124");
                actualRecord = results.next();
                Assert.assertFalse(results.hasNext());
                Assert.assertArrayEquals(actualRecord.getKey(), recordList.get(0).getKey());
                Assert.assertEquals(actualRecord.getValue(), recordList.get(0).getValue());
                results.close();
                results = records.scan(null, "123");
                Assert.assertFalse(results.hasNext());
                results.close();
            }
        });
        actualList.clear();
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Scan scan = new Scan(null, null);
                CloseableIterator<KeyValue<byte[], Record>> results = records.scan(scan);
                while (results.hasNext()) {
                    actualList.add(results.next());
                }
            }
        });
        Assert.assertEquals(recordList.size(), actualList.size());
    } finally {
        dsFrameworkUtil.deleteInstance(RECORDS_ID);
    }
}

Example 8 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class TableTest method verifyScanWithFuzzyRowFilter.

private static void verifyScanWithFuzzyRowFilter(Table table) {
    FuzzyRowFilter filter = new FuzzyRowFilter(ImmutableList.of(ImmutablePair.of(new byte[] { '*', 'b', '*', 'b' }, new byte[] { 0x01, 0x00, 0x01, 0x00 })));
    Scanner scanner = table.scan(new Scan(null, null, filter));
    int count = 0;
    while (true) {
        Row entry = scanner.next();
        if (entry == null) {
            break;
        }
        Assert.assertTrue(entry.getRow()[1] == 'b' && entry.getRow()[3] == 'b');
        Assert.assertEquals(1, entry.getColumns().size());
        Assert.assertTrue(entry.getColumns().containsKey(C1));
        Assert.assertArrayEquals(V1, entry.get(C1));
        count++;
    }
    Assert.assertEquals(9, count);
}

Example 9 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class TableTest method testBasicScanWithTx.

@Test
public void testBasicScanWithTx() throws Exception {
    // todo: make work with tx well (merge with buffer, conflicts) and add tests for that
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable1 = getTable(CONTEXT1, MY_TABLE);
        Table myTable2 = getTable(CONTEXT1, MY_TABLE)) {
        // write r1...r5 and commit
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx1);
        myTable1.put(R1, a(C1), a(V1));
        myTable1.put(R2, a(C2), a(V2));
        myTable1.put(R3, a(C3, C4), a(V3, V4));
        myTable1.put(R4, a(C4), a(V4));
        myTable1.put(R5, a(C5), a(V5));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx1);
        // Now, we will test scans
        // currently not testing races/conflicts/etc as this logic is not there for scans yet; so using one same tx
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) myTable2).startTx(tx2);
        // bounded scan
        TableAssert.assertScan(a(R2, R3, R4), aa(a(C2, V2), a(C3, V3, C4, V4), a(C4, V4)), myTable2, new Scan(R2, R5));
        // open start scan
        TableAssert.assertScan(a(R1, R2, R3), aa(a(C1, V1), a(C2, V2), a(C3, V3, C4, V4)), myTable2, new Scan(null, R4));
        // open end scan
        TableAssert.assertScan(a(R3, R4, R5), aa(a(C3, V3, C4, V4), a(C4, V4), a(C5, V5)), myTable2, new Scan(R3, null));
        // open ends scan
        TableAssert.assertScan(a(R1, R2, R3, R4, R5), aa(a(C1, V1), a(C2, V2), a(C3, V3, C4, V4), a(C4, V4), a(C5, V5)), myTable2, new Scan(null, null));
        // adding/changing/removing some columns
        myTable2.put(R2, a(C1, C2, C3), a(V4, V3, V2));
        myTable2.delete(R3, a(C4));
        txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable2).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable2).commitTx());
        txClient.commitOrThrow(tx2);
        // Checking that changes are reflected in new scans in new tx
        Transaction tx3 = txClient.startShort();
        // NOTE: table can be re-used betweet tx
        ((TransactionAware) myTable1).startTx(tx3);
        TableAssert.assertScan(a(R2, R3, R4), aa(a(C1, V4, C2, V3, C3, V2), a(C3, V3), a(C4, V4)), myTable1, new Scan(R2, R5));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx3);
    } finally {
        admin.drop();
    }
}

Example 10 with Scan

use of io.cdap.cdap.api.dataset.table.Scan in project cdap by caskdata.

the class TableTest method testScanAndDelete.

@Test
public void testScanAndDelete() throws Exception {
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE);
    admin.create();
    try (Table myTable1 = getTable(CONTEXT1, MY_TABLE)) {
        Transaction tx1 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx1);
        myTable1.put(Bytes.toBytes("1_09a"), a(C1), a(V1));
        txClient.canCommitOrThrow(tx1, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx1);
        // 
        Transaction tx2 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx2);
        myTable1.delete(Bytes.toBytes("1_09a"));
        myTable1.put(Bytes.toBytes("1_08a"), a(C1), a(V1));
        myTable1.put(Bytes.toBytes("1_09b"), a(C1), a(V1));
        txClient.canCommitOrThrow(tx2, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx2);
        // 
        Transaction tx3 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx3);
        TableAssert.assertScan(a(Bytes.toBytes("1_08a"), Bytes.toBytes("1_09b")), aa(a(C1, V1), a(C1, V1)), myTable1, new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
        myTable1.delete(Bytes.toBytes("1_08a"));
        myTable1.put(Bytes.toBytes("1_07a"), a(C1), a(V1));
        myTable1.delete(Bytes.toBytes("1_09b"));
        myTable1.put(Bytes.toBytes("1_08b"), a(C1), a(V1));
        myTable1.put(Bytes.toBytes("1_09c"), a(C1), a(V1));
        txClient.canCommitOrThrow(tx3, ((TransactionAware) myTable1).getTxChanges());
        Assert.assertTrue(((TransactionAware) myTable1).commitTx());
        txClient.commitOrThrow(tx3);
        // Now, we will test scans
        Transaction tx4 = txClient.startShort();
        ((TransactionAware) myTable1).startTx(tx4);
        TableAssert.assertScan(a(Bytes.toBytes("1_07a"), Bytes.toBytes("1_08b"), Bytes.toBytes("1_09c")), aa(a(C1, V1), a(C1, V1), a(C1, V1)), myTable1, new Scan(Bytes.toBytes("1_"), Bytes.toBytes("2_")));
    } finally {
        admin.drop();
    }
}