Examples with Scanner io.cdap.cdap.api.dataset.table.Scanner used on opensource projects

Example 1 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class IndexedTable method readByIndex.

/**
 * Reads table rows by the given secondary index key.  If no rows are indexed by the given key, then a
 * {@link io.cdap.cdap.api.dataset.table.Scanner} with no results will be returned.
 *
 * @return a Scanner returning rows from the data table, whose stored value for the given column matches the
 * given value.
 * @throws java.lang.IllegalArgumentException if the given column is not configured for indexing.
 */
@ReadOnly
public Scanner readByIndex(byte[] column, byte[] value) {
    assertIndexedColumn(column);
    byte[] rowKeyPrefix = Bytes.concat(keyPrefix, column, KEY_DELIMITER, value, KEY_DELIMITER);
    byte[] stopRow = Bytes.stopKeyForPrefix(rowKeyPrefix);
    Scanner indexScan = index.scan(rowKeyPrefix, stopRow);
    return new IndexScanner(indexScan, column, value);
}

Example 2 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class IndexedTable method scanByIndex.

/**
 * Reads table rows within the given secondary index key range. If no rows are indexed, falling within the given
 * range, then a {@link io.cdap.cdap.api.dataset.table.Scanner} with no results will be returned.
 *
 * @param column the column to use for the index lookup
 * @param startValue the inclusive start of the range for which rows must fall within to be returned in the scan.
 *                   {@code null} means start from first row of the table
 * @param endValue the exclusive end of the range for which rows must fall within to be returned in the scan
 *                 {@code null} means end with the last row of the table
 * @return a Scanner returning rows from the data table, whose stored value for the given column is within the the
 *         given range.
 * @throws java.lang.IllegalArgumentException if the given column is not configured for indexing.
 */
@ReadOnly
public Scanner scanByIndex(byte[] column, @Nullable byte[] startValue, @Nullable byte[] endValue) {
    assertIndexedColumn(column);
    // KEY_DELIMITER is not used at the end of the rowKeys, because they are used for a range scan,
    // instead of a fixed-match lookup
    byte[] startRow = startValue == null ? Bytes.concat(keyPrefix, column, KEY_DELIMITER) : Bytes.concat(keyPrefix, column, KEY_DELIMITER, startValue);
    byte[] stopRow = endValue == null ? Bytes.stopKeyForPrefix(Bytes.concat(keyPrefix, column, KEY_DELIMITER)) : Bytes.concat(keyPrefix, column, KEY_DELIMITER, endValue);
    Scanner indexScan = index.scan(startRow, stopRow);
    return new IndexRangeScanner(indexScan, column, startValue, endValue);
}

Example 3 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class KeyValueTable method scan.

/**
 * Scans table.
 * @param startRow start row inclusive. {@code null} means start from first row of the table
 * @param stopRow stop row exclusive. {@code null} means scan all rows to the end of the table
 * @return {@link io.cdap.cdap.api.dataset.lib.CloseableIterator} of
 * {@link KeyValue KeyValue<byte[], byte[]>}
 */
public CloseableIterator<KeyValue<byte[], byte[]>> scan(byte[] startRow, byte[] stopRow) {
    final Scanner scanner = table.scan(startRow, stopRow);
    return new AbstractCloseableIterator<KeyValue<byte[], byte[]>>() {

        private boolean closed = false;

        @Override
        protected KeyValue<byte[], byte[]> computeNext() {
            if (closed) {
                return endOfData();
            }
            Row next = scanner.next();
            if (next != null) {
                return new KeyValue<>(next.getRow(), next.get(KEY_COLUMN));
            }
            close();
            return null;
        }

        @Override
        public void close() {
            scanner.close();
            endOfData();
            closed = true;
        }
    };
}

Example 4 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class IndexedTableTest method testIncrementIndexing.

@Test
public void testIncrementIndexing() throws Exception {
    DatasetId incrTabInstance = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("incrtab");
    dsFrameworkUtil.createInstance("indexedTable", incrTabInstance, DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "idx1,idx2,idx3").build());
    final IndexedTable iTable = dsFrameworkUtil.getInstance(incrTabInstance);
    final byte[] idxCol1 = Bytes.toBytes("idx1");
    final byte[] idxCol2 = Bytes.toBytes("idx2");
    final byte[] idxCol3 = Bytes.toBytes("idx3");
    final byte[] row1 = Bytes.toBytes("row1");
    try {
        TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor(iTable);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                long result = iTable.incrementAndGet(row1, idxCol1, 1);
                assertEquals(1L, result);
            }
        });
        final byte[] oneBytes = Bytes.toBytes(1L);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                try (Scanner scanner = iTable.readByIndex(idxCol1, oneBytes)) {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, new byte[][] { idxCol1 }, new byte[][] { oneBytes });
                    assertEmpty(scanner);
                }
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                long result = iTable.incrementAndGet(row1, idxCol1, 1);
                assertEquals(2L, result);
            }
        });
        final byte[] twoBytes = Bytes.toBytes(2L);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // previous index by value 1 should be gone
                Scanner scanner = iTable.readByIndex(idxCol1, oneBytes);
                try {
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                // should now be indexed by value 2
                scanner = iTable.readByIndex(idxCol1, twoBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, new byte[][] { idxCol1 }, new byte[][] { twoBytes });
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
            }
        });
        final byte[] threeBytes = Bytes.toBytes(3L);
        final byte[][] idxCols = new byte[][] { idxCol1, idxCol2, idxCol3 };
        final byte[][] expectedValues = new byte[][] { threeBytes, oneBytes, oneBytes };
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Row result = iTable.incrementAndGet(row1, idxCols, new long[] { 1, 1, 1 });
                assertNotNull(result);
                TableAssert.assertColumns(result, idxCols, expectedValues);
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Scanner scanner = iTable.readByIndex(idxCol1, threeBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, idxCols, expectedValues);
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.readByIndex(idxCol2, oneBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, idxCols, expectedValues);
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.readByIndex(idxCol3, oneBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, idxCols, expectedValues);
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
            }
        });
        final byte[] row2 = Bytes.toBytes("row2");
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // read-less increment on an indexed column should throw an exception
                try {
                    iTable.increment(row2, idxCol1, 1L);
                    fail("Expected IllegalArgumentException performing increment on indexed column");
                } catch (IllegalArgumentException iae) {
                // expected
                }
                // read-less increment on a non-indexed column should succeed
                iTable.increment(row2, valCol, 1L);
                byte[] result = iTable.get(row2, valCol);
                assertArrayEquals(oneBytes, result);
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                iTable.put(row2, valCol, valA);
            }
        });
        // increment against a column with non-long value should fail
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                try {
                    iTable.incrementAndGet(row2, valCol, 1L);
                    fail("Expected NumberFormatException from increment on a column with non-long value");
                } catch (NumberFormatException nfe) {
                // expected
                }
            }
        });
    } finally {
        dsFrameworkUtil.deleteInstance(incrTabInstance);
    }
}

Example 5 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class IndexedTableTest method testIndexKeyDelimiterAmbiguity.

@Test
public void testIndexKeyDelimiterAmbiguity() throws Exception {
    final byte[] a = { 'a' };
    final byte[] ab = { 'a', 0, 'b' };
    final byte[] abc = { 'a', 0, 'b', 0, 'c' };
    final byte[] bc = { 'b', 0, 'c' };
    final byte[] bcd = { 'b', 0, 'c', 'd' };
    final byte[] c = { 'c' };
    final byte[] d = { 'd' };
    final byte[] w = { 'w' };
    final byte[] x = { 'x' };
    final byte[] y = { 'y' };
    final byte[] z = { 'z' };
    DatasetId delimTabInstance = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("delimtab");
    dsFrameworkUtil.createInstance("indexedTable", delimTabInstance, DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, Bytes.toString(a) + "," + Bytes.toString(ab)).build());
    final IndexedTable iTable = dsFrameworkUtil.getInstance(delimTabInstance);
    try {
        TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor(iTable);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                iTable.put(x, a, bc);
                iTable.put(y, ab, c);
                iTable.put(w, a, bcd);
                iTable.put(z, abc, d);
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // ensure that readByIndex filters teh false positive rows in index
                Scanner scanner = iTable.readByIndex(a, bc);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(x, row.getRow());
                    Assert.assertArrayEquals(bc, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.readByIndex(ab, c);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(y, row.getRow());
                    Assert.assertArrayEquals(c, row.get(ab));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                // ensure that scanByIndex filters the false positive rows in index
                scanner = iTable.scanByIndex(a, bcd, null);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(w, row.getRow());
                    Assert.assertArrayEquals(bcd, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.scanByIndex(a, null, bcd);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(x, row.getRow());
                    Assert.assertArrayEquals(bc, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
            }
        });
    } finally {
        dsFrameworkUtil.deleteInstance(delimTabInstance);
    }
}