Examples with Scanner io.cdap.cdap.api.dataset.table.Scanner used on opensource projects

Example 51 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by cdapio.

the class DefaultPreviewStore method get.

@Override
public Map<String, List<JsonElement>> get(ApplicationId applicationId, String tracerName) {
    // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe.
    Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create();
    byte[] startRowKey = getPreviewRowKeyBuilder(DATA_ROW_KEY_PREFIX, applicationId).add(tracerName).build().getKey();
    byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey();
    Map<String, List<JsonElement>> result = new HashMap<>();
    try (Scanner scanner = previewTable.scan(startRowKey, stopRowKey, null, null, null)) {
        Row indexRow;
        while ((indexRow = scanner.next()) != null) {
            Map<byte[], byte[]> columns = indexRow.getColumns();
            String propertyName = Bytes.toString(columns.get(PROPERTY));
            JsonElement value = gson.fromJson(Bytes.toString(columns.get(VALUE)), JsonElement.class);
            List<JsonElement> values = result.computeIfAbsent(propertyName, k -> new ArrayList<>());
            values.add(value);
        }
    } catch (IOException e) {
        String message = String.format("Error while reading preview data for application '%s' and tracer '%s'.", applicationId, tracerName);
        throw new RuntimeException(message, e);
    }
    return result;
}

Example 52 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by cdapio.

the class DefaultPreviewStore method deleteExpiredData.

@Override
public void deleteExpiredData(long ttlInSeconds) {
    Gson gson = new GsonBuilder().registerTypeAdapter(EntityId.class, new EntityIdTypeAdapter()).create();
    byte[] startRowKey = new MDSKey.Builder().add(META_ROW_KEY_PREFIX).build().getKey();
    byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey();
    long currentTimeInSeconds = System.currentTimeMillis() / 1000;
    try (Scanner scanner = previewTable.scan(startRowKey, stopRowKey, null, null, null)) {
        Row indexRow;
        while ((indexRow = scanner.next()) != null) {
            Map<byte[], byte[]> columns = indexRow.getColumns();
            String applicationIdGson = Bytes.toString(columns.get(APPID));
            if (applicationIdGson == null) {
                continue;
            }
            ApplicationId applicationId = gson.fromJson(applicationIdGson, ApplicationId.class);
            long applicationSubmitTime = RunIds.getTime(applicationId.getApplication(), TimeUnit.SECONDS);
            if ((currentTimeInSeconds - applicationSubmitTime) > ttlInSeconds) {
                remove(applicationId);
            }
        }
    } catch (IOException e) {
        throw new RuntimeException("Error while scanning the preview requests for deletion.", e);
    }
}

Example 53 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by cdapio.

the class HBaseTableTest method testCachedEncodedTransaction.

@Test
public void testCachedEncodedTransaction() throws Exception {
    String tableName = "testEncodedTxTable";
    DatasetProperties props = DatasetProperties.EMPTY;
    getTableAdmin(CONTEXT1, tableName, props).create();
    DatasetSpecification tableSpec = DatasetSpecification.builder(tableName, HBaseTable.class.getName()).build();
    // use a transaction codec that counts the number of times encode() is called
    final AtomicInteger encodeCount = new AtomicInteger();
    final TransactionCodec codec = new TransactionCodec() {

        @Override
        public byte[] encode(Transaction tx) throws IOException {
            encodeCount.incrementAndGet();
            return super.encode(tx);
        }
    };
    // use a table util that creates an HTable that validates the encoded tx on each get
    final AtomicReference<Transaction> txRef = new AtomicReference<>();
    HBaseTableUtil util = new DelegatingHBaseTableUtil(hBaseTableUtil) {

        @Override
        public org.apache.hadoop.hbase.client.Table createTable(Configuration conf, TableId tableId) throws IOException {
            org.apache.hadoop.hbase.client.Table table = super.createTable(conf, tableId);
            return new DelegatingTable(table) {

                @Override
                public Result get(org.apache.hadoop.hbase.client.Get get) throws IOException {
                    Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    return super.get(get);
                }

                @Override
                public Result[] get(List<org.apache.hadoop.hbase.client.Get> gets) throws IOException {
                    for (org.apache.hadoop.hbase.client.Get get : gets) {
                        Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(get.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    }
                    return super.get(gets);
                }

                @Override
                public ResultScanner getScanner(org.apache.hadoop.hbase.client.Scan scan) throws IOException {
                    Assert.assertEquals(txRef.get().getTransactionId(), codec.decode(scan.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY)).getTransactionId());
                    return super.getScanner(scan);
                }
            };
        }
    };
    HBaseTable table = new HBaseTable(CONTEXT1, tableSpec, Collections.<String, String>emptyMap(), cConf, TEST_HBASE.getConfiguration(), util, codec);
    DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
    // test all operations: only the first one encodes
    Transaction tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.put(b("row1"), b("col1"), b("val1"));
    Assert.assertEquals(0, encodeCount.get());
    table.get(b("row"));
    Assert.assertEquals(1, encodeCount.get());
    table.get(ImmutableList.of(new Get("a"), new Get("b")));
    Assert.assertEquals(1, encodeCount.get());
    Scanner scanner = table.scan(new Scan(null, null));
    Assert.assertEquals(1, encodeCount.get());
    scanner.close();
    table.increment(b("z"), b("z"), 0L);
    Assert.assertEquals(1, encodeCount.get());
    table.commitTx();
    table.postTxCommit();
    // test that for the next tx, we encode again
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(2, encodeCount.get());
    table.commitTx();
    // test that we encode again, even of postTxCommit was not called
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(3, encodeCount.get());
    table.commitTx();
    table.rollbackTx();
    // test that rollback does not encode the tx
    Assert.assertEquals(3, encodeCount.get());
    // test that we encode again if the previous tx rolled back
    tx = txSystemClient.startShort();
    txRef.set(tx);
    table.startTx(tx);
    table.get(b("row"));
    Assert.assertEquals(4, encodeCount.get());
    table.commitTx();
    table.close();
    Assert.assertEquals(4, encodeCount.get());
}

Example 54 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by cdapio.

the class IndexedTableTest method testIndexKeyDelimiterAmbiguity.

@Test
public void testIndexKeyDelimiterAmbiguity() throws Exception {
    final byte[] a = { 'a' };
    final byte[] ab = { 'a', 0, 'b' };
    final byte[] abc = { 'a', 0, 'b', 0, 'c' };
    final byte[] bc = { 'b', 0, 'c' };
    final byte[] bcd = { 'b', 0, 'c', 'd' };
    final byte[] c = { 'c' };
    final byte[] d = { 'd' };
    final byte[] w = { 'w' };
    final byte[] x = { 'x' };
    final byte[] y = { 'y' };
    final byte[] z = { 'z' };
    DatasetId delimTabInstance = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("delimtab");
    dsFrameworkUtil.createInstance("indexedTable", delimTabInstance, DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, Bytes.toString(a) + "," + Bytes.toString(ab)).build());
    final IndexedTable iTable = dsFrameworkUtil.getInstance(delimTabInstance);
    try {
        TransactionExecutor tx = dsFrameworkUtil.newTransactionExecutor(iTable);
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                iTable.put(x, a, bc);
                iTable.put(y, ab, c);
                iTable.put(w, a, bcd);
                iTable.put(z, abc, d);
            }
        });
        tx.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // ensure that readByIndex filters teh false positive rows in index
                Scanner scanner = iTable.readByIndex(a, bc);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(x, row.getRow());
                    Assert.assertArrayEquals(bc, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.readByIndex(ab, c);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(y, row.getRow());
                    Assert.assertArrayEquals(c, row.get(ab));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                // ensure that scanByIndex filters the false positive rows in index
                scanner = iTable.scanByIndex(a, bcd, null);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(w, row.getRow());
                    Assert.assertArrayEquals(bcd, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
                scanner = iTable.scanByIndex(a, null, bcd);
                try {
                    Row row = scanner.next();
                    Assert.assertNotNull(row);
                    Assert.assertArrayEquals(x, row.getRow());
                    Assert.assertArrayEquals(bc, row.get(a));
                    assertEmpty(scanner);
                } finally {
                    scanner.close();
                }
            }
        });
    } finally {
        dsFrameworkUtil.deleteInstance(delimTabInstance);
    }
}

Example 55 with Scanner

use of io.cdap.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class AbstractMockSink method clear.

/**
 * Clear any records written to this sink.
 *
 * @param tableManager dataset manager used to get the sink dataset
 */
public static void clear(DataSetManager<Table> tableManager) {
    tableManager.flush();
    Table table = tableManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        Row row;
        while ((row = scanner.next()) != null) {
            table.delete(row.getRow());
        }
    }
    tableManager.flush();
}