Examples with Scanner co.cask.cdap.api.dataset.table.Scanner used on opensource projects

Example 31 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class DefaultPreviewStore method get.

@Override
public Map<String, List<JsonElement>> get(ApplicationId applicationId, String tracerName) {
    // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe.
    Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create();
    byte[] startRowKey = new MDSKey.Builder().add(applicationId.getNamespace()).add(applicationId.getApplication()).add(tracerName).build().getKey();
    byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey();
    Map<String, List<JsonElement>> result = new HashMap<>();
    try (Scanner scanner = table.scan(startRowKey, stopRowKey, null, null, null)) {
        Row indexRow;
        while ((indexRow = scanner.next()) != null) {
            Map<byte[], byte[]> columns = indexRow.getColumns();
            String propertyName = Bytes.toString(columns.get(PROPERTY));
            JsonElement value = gson.fromJson(Bytes.toString(columns.get(VALUE)), JsonElement.class);
            List<JsonElement> values = result.get(propertyName);
            if (values == null) {
                values = new ArrayList<>();
                result.put(propertyName, values);
            }
            values.add(value);
        }
    } catch (IOException e) {
        String message = String.format("Error while reading preview data for application '%s' and tracer '%s'.", applicationId, tracerName);
        throw new RuntimeException(message, e);
    }
    return result;
}

Example 32 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class LevelDBTableCore method deleteRange.

public void deleteRange(byte[] startRow, byte[] stopRow, @Nullable FuzzyRowFilter filter, @Nullable byte[][] columns) throws IOException {
    if (columns != null) {
        if (columns.length == 0) {
            return;
        }
        columns = Arrays.copyOf(columns, columns.length);
        Arrays.sort(columns, Bytes.BYTES_COMPARATOR);
    }
    DB db = getDB();
    DBIterator iterator = db.iterator();
    seekToStart(iterator, startRow);
    byte[] endKey = stopRow == null ? null : createEndKey(stopRow);
    DBIterator deleteIterator = db.iterator();
    seekToStart(deleteIterator, startRow);
    // todo make configurable
    final int deletesPerRound = 1024;
    try (Scanner scanner = new LevelDBScanner(iterator, endKey, filter, columns, null)) {
        Row rowValues;
        WriteBatch batch = db.createWriteBatch();
        int deletesInBatch = 0;
        // go through all matching cells and delete them in batches.
        while ((rowValues = scanner.next()) != null) {
            byte[] row = rowValues.getRow();
            for (byte[] column : rowValues.getColumns().keySet()) {
                addToDeleteBatch(batch, deleteIterator, row, column);
                deletesInBatch++;
                // perform the deletes when we have built up a batch.
                if (deletesInBatch >= deletesPerRound) {
                    // delete all the entries that were found
                    db.write(batch, getWriteOptions());
                    batch = db.createWriteBatch();
                    deletesInBatch = 0;
                }
            }
        }
        // perform any outstanding deletes
        if (deletesInBatch > 0) {
            db.write(batch, getWriteOptions());
        }
    } finally {
        deleteIterator.close();
    }
}

Example 33 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MetadataStoreDataset method scan.

/**
   * Run a scan on MDS.
   *
   * @param startId  scan start key
   * @param stopId   scan stop key
   * @param typeOfT  type of value
   * @param function function to process each element returned from scan.
   *                 If function.apply returns false then the scan is stopped.
   *                 Also, function.apply should not return null.
   * @param <T>      type of value
   */
public <T> void scan(MDSKey startId, @Nullable MDSKey stopId, Type typeOfT, Function<KeyValue<T>, Boolean> function) {
    byte[] startKey = startId.getKey();
    byte[] stopKey = stopId == null ? Bytes.stopKeyForPrefix(startKey) : stopId.getKey();
    try (Scanner scan = table.scan(startKey, stopKey)) {
        Row next;
        while ((next = scan.next()) != null) {
            byte[] columnValue = next.get(COLUMN);
            if (columnValue == null) {
                continue;
            }
            T value = deserialize(columnValue, typeOfT);
            MDSKey key = new MDSKey(next.getRow());
            //noinspection ConstantConditions
            if (!function.apply(new KeyValue<>(key, value))) {
                break;
            }
        }
    }
}

Example 34 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MetadataStoreDataset method getFirst.

// returns first that matches
@Nullable
public <T> T getFirst(MDSKey id, Type typeOfT) {
    try {
        Scanner scan = table.scan(id.getKey(), Bytes.stopKeyForPrefix(id.getKey()));
        try {
            Row row = scan.next();
            if (row == null || row.isEmpty()) {
                return null;
            }
            byte[] value = row.get(COLUMN);
            if (value == null) {
                return null;
            }
            return deserialize(value, typeOfT);
        } finally {
            scan.close();
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 35 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class LevelDBQueueEvictor method doEvict.

private synchronized int doEvict(Transaction transaction) throws IOException {
    final byte[] stopRow = QueueEntryRow.getStopRowForTransaction(queueRowPrefix, transaction);
    Row row;
    List<byte[]> rowsToDelete = Lists.newArrayList();
    // the scan must be non-transactional in order to see the state columns (which have latest timestamp)
    try (Scanner scanner = core.scan(queueRowPrefix, stopRow, null, null, Transaction.ALL_VISIBLE_LATEST)) {
        while ((row = scanner.next()) != null) {
            int processed = 0;
            for (Map.Entry<byte[], byte[]> entry : row.getColumns().entrySet()) {
                // is it a state column for a consumer instance?
                if (!QueueEntryRow.isStateColumn(entry.getKey())) {
                    continue;
                }
                // is the write pointer of this state committed w.r.t. the current transaction, and is it processed?
                if (QueueEntryRow.isCommittedProcessed(entry.getValue(), transaction)) {
                    ++processed;
                }
            }
            if (processed >= numGroups) {
                rowsToDelete.add(row.getRow());
            }
        }
    }
    if (!rowsToDelete.isEmpty()) {
        core.deleteRows(rowsToDelete);
        LOG.trace("Evicted {} entries from queue {}", rowsToDelete.size(), name);
    } else {
        LOG.trace("Nothing to evict from queue {}", name);
    }
    return rowsToDelete.size();
}