Examples with Scanner co.cask.cdap.api.dataset.table.Scanner used on opensource projects

Example 21 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MockSink method readOutput.

/**
   * Used to read the records written by this sink.
   *
   * @param tableManager dataset manager used to get the sink dataset to read from
   */
public static List<StructuredRecord> readOutput(DataSetManager<Table> tableManager) throws Exception {
    tableManager.flush();
    Table table = tableManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        List<StructuredRecord> records = new ArrayList<>();
        Row row;
        while ((row = scanner.next()) != null) {
            Schema schema = Schema.parseJson(row.getString(SCHEMA_COL));
            String recordStr = row.getString(RECORD_COL);
            records.add(StructuredRecordStringConverter.fromJsonString(recordStr, schema));
        }
        return records;
    }
}

Example 22 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class HBaseConsumerStateStore method configureGroups.

@Override
public void configureGroups(Iterable<? extends ConsumerGroupConfig> groupConfigs) {
    com.google.common.collect.Table<Long, Integer, byte[]> startRows = fetchAllStartRows();
    // Writes a new barrier info for all the groups
    byte[] startRow = QueueEntryRow.getQueueEntryRowKey(queueName, transaction.getWritePointer(), 0);
    Put put = new Put(Bytes.add(queueName.toBytes(), startRow));
    Set<Long> groupsIds = Sets.newHashSet();
    for (ConsumerGroupConfig groupConfig : groupConfigs) {
        long groupId = groupConfig.getGroupId();
        if (!groupsIds.add(groupId)) {
            throw new IllegalArgumentException("Same consumer group is provided multiple times");
        }
        put.add(Bytes.toBytes(groupId), GSON.toJson(groupConfig));
        // For new instance, set the start row to barrier start row
        for (int instanceId = 0; instanceId < groupConfig.getGroupSize(); instanceId++) {
            if (!startRows.contains(groupId, instanceId)) {
                table.put(queueName.toBytes(), getConsumerStateColumn(groupId, instanceId), startRow);
            }
        }
    }
    // Remove all states for groups that are removed.
    deleteRemovedGroups(table.get(queueName.toBytes()), groupsIds);
    // Remove all barriers for groups that are removed.
    // Also remove barriers that have all consumers consumed pass that barrier
    // Multimap from groupId to barrier start rows. Ordering need to be maintained as the scan order.
    Multimap<Long, byte[]> deletes = LinkedHashMultimap.create();
    try (Scanner scanner = table.scan(barrierScanStartRow, barrierScanEndRow)) {
        Row row = scanner.next();
        while (row != null) {
            deleteRemovedGroups(row, groupsIds);
            // Check all instances in all groups
            for (Map.Entry<byte[], byte[]> entry : row.getColumns().entrySet()) {
                QueueBarrier barrier = decodeBarrierInfo(row.getRow(), entry.getValue());
                if (barrier == null) {
                    continue;
                }
                long groupId = barrier.getGroupConfig().getGroupId();
                boolean delete = true;
                // Check if all instances in a group has consumed passed the current barrier
                for (int instanceId = 0; instanceId < barrier.getGroupConfig().getGroupSize(); instanceId++) {
                    byte[] consumerStartRow = startRows.get(groupId, instanceId);
                    if (consumerStartRow == null || Bytes.compareTo(consumerStartRow, barrier.getStartRow()) < 0) {
                        delete = false;
                        break;
                    }
                }
                if (delete) {
                    deletes.put(groupId, row.getRow());
                }
            }
            row = scanner.next();
        }
    }
    // Remove barries that have all consumers consumed passed it
    for (Map.Entry<Long, Collection<byte[]>> entry : deletes.asMap().entrySet()) {
        // Retains the last barrier info
        if (entry.getValue().size() <= 1) {
            continue;
        }
        Deque<byte[]> rows = Lists.newLinkedList(entry.getValue());
        rows.removeLast();
        byte[] groupColumn = Bytes.toBytes(entry.getKey());
        for (byte[] rowKey : rows) {
            table.delete(rowKey, groupColumn);
        }
    }
    table.put(put);
}

Example 23 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class HBaseConsumerStateStore method getLatestConsumerGroups.

void getLatestConsumerGroups(Collection<? super ConsumerGroupConfig> result) {
    try (Scanner scanner = table.scan(barrierScanStartRow, barrierScanEndRow)) {
        // Get the last row
        Row lastRow = null;
        Row row = scanner.next();
        while (row != null) {
            lastRow = row;
            row = scanner.next();
        }
        if (lastRow == null) {
            throw new IllegalStateException("No consumer group information. Queue: " + queueName);
        }
        for (Map.Entry<byte[], byte[]> entry : lastRow.getColumns().entrySet()) {
            result.add(GSON.fromJson(new String(entry.getValue(), Charsets.UTF_8), ConsumerGroupConfig.class));
        }
    }
}

Example 24 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class HBaseConsumerStateStore method clear.

/**
   * Remove all states related to the queue that this state store is representing.
   */
void clear() {
    // Scan and delete all barrier rows
    try (Scanner scanner = table.scan(barrierScanStartRow, barrierScanEndRow)) {
        Row row = scanner.next();
        while (row != null) {
            table.delete(row.getRow());
            row = scanner.next();
        }
        // Also delete the consumer state rows
        table.delete(queueName.toBytes());
    }
}

Example 25 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class BufferingTableTest method testChangingParamsAndReturnValues.

@Test
public void testChangingParamsAndReturnValues() throws Exception {
    // The test verifies that one can re-use byte arrays passed as parameters to write methods of a table without
    // affecting the stored data.
    // Also, one can re-use (modify) returned data from the table without affecting the stored data.
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(isReadlessIncrementSupported()).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE, props);
    admin.create();
    try {
        // writing some data: we'll need it to test delete later
        Transaction tx = txClient.startShort();
        BufferingTable table = getTable(CONTEXT1, MY_TABLE, props);
        table.startTx(tx);
        table.put(new byte[] { 0 }, new byte[] { 9 }, new byte[] { 8 });
        table.commitTx();
        txClient.commit(tx);
        // start new for in-mem buffer behavior testing
        tx = txClient.startShort();
        table.startTx(tx);
        // write some data but not commit
        byte[] rowParam = new byte[] { 1 };
        byte[] colParam = new byte[] { 2 };
        byte[] valParam = Bytes.toBytes(3L);
        table.put(rowParam, colParam, valParam);
        verify123(table);
        // change passed earlier byte arrays in place, this should not affect stored previously values
        rowParam[0]++;
        colParam[0]++;
        valParam[0]++;
        verify123(table);
        // try get row and change returned values in place, which should not affect the data stored
        Row getRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getRowResult = getRow.getColumns();
        Assert.assertEquals(1, getRowResult.size());
        byte[] colFromGetRow = getRowResult.keySet().iterator().next();
        byte[] valFromGetRow = getRowResult.get(colFromGetRow);
        getRowResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetRow);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetRow);
        colFromGetRow[0]++;
        valFromGetRow[0]++;
        verify123(table);
        // try get set of columns in a row and change returned values in place, which should not affect the data stored
        Row getColumnSetRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getColumnSetResult = getColumnSetRow.getColumns();
        Assert.assertEquals(1, getColumnSetResult.size());
        byte[] colFromGetColumnSet = getColumnSetResult.keySet().iterator().next();
        byte[] valFromGetColumnSet = getColumnSetResult.values().iterator().next();
        getColumnSetResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetColumnSet);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumnSet);
        colFromGetColumnSet[0]++;
        valFromGetColumnSet[0]++;
        verify123(table);
        // try get column and change returned value in place, which should not affect the data stored
        byte[] valFromGetColumn = table.get(new byte[] { 1 }, new byte[] { 2 });
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumn);
        valFromGetColumn[0]++;
        verify123(table);
        // try scan and change returned value in place, which should not affect the data stored
        Scanner scan = table.scan(new byte[] { 1 }, null);
        Row next = scan.next();
        Assert.assertNotNull(next);
        byte[] rowFromScan = next.getRow();
        Assert.assertArrayEquals(new byte[] { 1 }, rowFromScan);
        Map<byte[], byte[]> cols = next.getColumns();
        Assert.assertEquals(1, cols.size());
        byte[] colFromScan = cols.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromScan);
        byte[] valFromScan = next.get(new byte[] { 2 });
        Assert.assertNotNull(valFromScan);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromScan);
        Assert.assertNull(scan.next());
        cols.remove(new byte[] { 2 });
        rowFromScan[0]++;
        colFromScan[0]++;
        valFromScan[0]++;
        verify123(table);
        // try delete and change params in place: this should not affect stored data
        rowParam = new byte[] { 1 };
        colParam = new byte[] { 2 };
        table.delete(rowParam, colParam);
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        rowParam[0] = 0;
        colParam[0] = 9;
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        // try increment column and change params in place: this should not affect stored data
        byte[] rowIncParam = new byte[] { 1 };
        byte[] colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, 3);
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment set of columns and change params in place, try also to change values in returned map: this all
        // should not affect stored data.
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment and change returned values: should not affect the stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        Row countersRow = table.incrementAndGet(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        Map<byte[], byte[]> counters = countersRow.getColumns();
        Assert.assertEquals(1, counters.size());
        byte[] colFromInc = counters.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromInc);
        Assert.assertEquals(3, Bytes.toLong(counters.get(colFromInc)));
        counters.remove(new byte[] { 2 });
        colFromInc[0]++;
        verify123(table);
        // try increment write and change params in place: this should not affect stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try compareAndSwap and change params in place: this should not affect stored data
        byte[] rowSwapParam = new byte[] { 1 };
        byte[] colSwapParam = new byte[] { 2 };
        byte[] valSwapParam = Bytes.toBytes(3L);
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(3L), Bytes.toBytes(4L));
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(4L), valSwapParam);
        verify123(table);
        rowSwapParam[0]++;
        colSwapParam[0]++;
        valSwapParam[0]++;
        verify123(table);
    // We don't care to persist changes and commit tx here: we tested what we wanted
    } finally {
        admin.drop();
    }
}