Examples with Scanner co.cask.cdap.api.dataset.table.Scanner used on opensource projects

Example 41 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MetricsDataMigrator method migrateMetricsData.

private void migrateMetricsData(EntityTable entityTable, MetricsTable metricsTable, String scope, Version version) {
    MetricsEntityCodec codec = getEntityCodec(entityTable);
    int idSize = getIdSize(version);
    Row row;
    long rowCount = 0;
    try {
        Scanner scanner = metricsTable.scan(null, null, null);
        while ((row = scanner.next()) != null) {
            byte[] rowKey = row.getRow();
            int offset = 0;
            String context = codec.decode(MetricsEntityType.CONTEXT, rowKey, offset, idSize);
            context = getContextBasedOnVersion(context, version);
            offset += codec.getEncodedSize(MetricsEntityType.CONTEXT, idSize);
            String metricName = codec.decode(MetricsEntityType.METRIC, rowKey, offset, idSize);
            offset += codec.getEncodedSize(MetricsEntityType.METRIC, idSize);
            scope = getScopeBasedOnVersion(scope, metricName, version);
            metricName = getMetricNameBasedOnVersion(metricName, version);
            String runId = codec.decode(MetricsEntityType.RUN, rowKey, offset, idSize);
            parseAndAddNewMetricValue(scope, context, metricName, runId, row.getColumns());
            rowCount++;
            printStatus(rowCount);
        }
        System.out.println("Migrated " + rowCount + " records");
    } catch (Exception e) {
        LOG.warn("Exception during data-transfer in aggregates table", e);
    //no-op
    }
}

Example 42 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MetadataStoreDataset method deleteAll.

public void deleteAll(MDSKey id, @Nullable Predicate<MDSKey> filter) {
    byte[] prefix = id.getKey();
    byte[] stopKey = Bytes.stopKeyForPrefix(prefix);
    try {
        try (Scanner scan = table.scan(prefix, stopKey)) {
            Row next;
            while ((next = scan.next()) != null) {
                String columnValue = next.getString(COLUMN);
                if (columnValue == null) {
                    continue;
                }
                MDSKey key = new MDSKey(next.getRow());
                if (filter != null && !filter.apply(key)) {
                    continue;
                }
                table.delete(new Delete(next.getRow()).add(COLUMN));
            }
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 43 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MetadataStoreDataset method listKV.

private <T> Map<MDSKey, T> listKV(Scan runScan, Type typeOfT, int limit, @Nullable Predicate<MDSKey> keyFilter, @Nullable Predicate<T> valueFilter) {
    try {
        Map<MDSKey, T> map = Maps.newLinkedHashMap();
        try (Scanner scan = table.scan(runScan)) {
            Row next;
            while ((limit > 0) && (next = scan.next()) != null) {
                MDSKey key = new MDSKey(next.getRow());
                byte[] columnValue = next.get(COLUMN);
                if (columnValue == null) {
                    continue;
                }
                T value = deserialize(columnValue, typeOfT);
                // Key Filter doesn't pass
                if (keyFilter != null && !keyFilter.apply(key)) {
                    continue;
                }
                // If Value Filter doesn't pass
                if (valueFilter != null && !valueFilter.apply(value)) {
                    continue;
                }
                map.put(key, value);
                limit--;
            }
            return map;
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 44 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MetadataStoreDataset method listCombinedFilterKV.

private <T> Map<MDSKey, T> listCombinedFilterKV(Scan runScan, Type typeOfT, int limit, @Nullable Predicate<KeyValue<T>> combinedFilter) {
    try {
        Map<MDSKey, T> map = Maps.newLinkedHashMap();
        try (Scanner scan = table.scan(runScan)) {
            Row next;
            while ((limit > 0) && (next = scan.next()) != null) {
                MDSKey key = new MDSKey(next.getRow());
                byte[] columnValue = next.get(COLUMN);
                if (columnValue == null) {
                    continue;
                }
                T value = deserialize(columnValue, typeOfT);
                KeyValue<T> kv = new KeyValue<>(key, value);
                // Combined Filter doesn't pass
                if (combinedFilter != null && !combinedFilter.apply(kv)) {
                    continue;
                }
                map.put(kv.getKey(), kv.getValue());
                limit--;
            }
            return map;
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 45 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class IndexedTableTest method testIncrementIndexing.

@Test
public void testIncrementIndexing() throws Exception {
    DatasetId incrTabInstance = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("incrtab");
    dsFrameworkUtil.createInstance("indexedTable", incrTabInstance, DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "idx1,idx2,idx3").build());
    final IndexedTable iTable = dsFrameworkUtil.getInstance(incrTabInstance);
    final byte[] idxCol1 = Bytes.toBytes("idx1");
    final byte[] idxCol2 = Bytes.toBytes("idx2");
    final byte[] idxCol3 = Bytes.toBytes("idx3");
    final byte[] row1 = Bytes.toBytes("row1");
    try {
        TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor(iTable);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                long result = iTable.incrementAndGet(row1, idxCol1, 1);
                assertEquals(1L, result);
            }
        });
        final byte[] oneBytes = Bytes.toBytes(1L);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                try (Scanner scanner = iTable.readByIndex(idxCol1, oneBytes)) {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, new byte[][] { idxCol1 }, new byte[][] { oneBytes });
                    assertEmpty(scanner);
                }
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                long result = iTable.incrementAndGet(row1, idxCol1, 1);
                assertEquals(2L, result);
            }
        });
        final byte[] twoBytes = Bytes.toBytes(2L);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // previous index by value 1 should be gone
                Scanner scanner = iTable.readByIndex(idxCol1, oneBytes);
                try {
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                // should now be indexed by value 2
                scanner = iTable.readByIndex(idxCol1, twoBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, new byte[][] { idxCol1 }, new byte[][] { twoBytes });
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
            }
        });
        final byte[] threeBytes = Bytes.toBytes(3L);
        final byte[][] idxCols = new byte[][] { idxCol1, idxCol2, idxCol3 };
        final byte[][] expectedValues = new byte[][] { threeBytes, oneBytes, oneBytes };
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Row result = iTable.incrementAndGet(row1, idxCols, new long[] { 1, 1, 1 });
                assertNotNull(result);
                TableAssert.assertColumns(result, idxCols, expectedValues);
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                Scanner scanner = iTable.readByIndex(idxCol1, threeBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, idxCols, expectedValues);
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.readByIndex(idxCol2, oneBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, idxCols, expectedValues);
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.readByIndex(idxCol3, oneBytes);
                try {
                    Row row = scanner.next();
                    TableAssert.assertRow(row, row1, idxCols, expectedValues);
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
            }
        });
        final byte[] row2 = Bytes.toBytes("row2");
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // read-less increment on an indexed column should throw an exception
                try {
                    iTable.increment(row2, idxCol1, 1L);
                    fail("Expected IllegalArgumentException performing increment on indexed column");
                } catch (IllegalArgumentException iae) {
                // expected
                }
                // read-less increment on a non-indexed column should succeed
                iTable.increment(row2, valCol, 1L);
                byte[] result = iTable.get(row2, valCol);
                assertArrayEquals(oneBytes, result);
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                iTable.put(row2, valCol, valA);
            }
        });
        // increment against a column with non-long value should fail
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                try {
                    iTable.incrementAndGet(row2, valCol, 1L);
                    fail("Expected NumberFormatException from increment on a column with non-long value");
                } catch (NumberFormatException nfe) {
                // expected
                }
            }
        });
    } finally {
        dsFrameworkUtil.deleteInstance(incrTabInstance);
    }
}