Examples with Scanner co.cask.cdap.api.dataset.table.Scanner used on opensource projects

Example 36 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class HBaseQueueAdmin method deleteFlowConfigs.

private void deleteFlowConfigs(FlowId flowId) throws Exception {
    // It's a bit hacky here since we know how the HBaseConsumerStateStore works.
    // Maybe we need another Dataset set that works across all queues.
    final QueueName prefixName = QueueName.from(URI.create(QueueName.prefixForFlow(flowId)));
    DatasetId stateStoreId = getStateStoreId(flowId.getNamespace());
    Map<String, String> args = ImmutableMap.of(HBaseQueueDatasetModule.PROPERTY_QUEUE_NAME, prefixName.toString());
    HBaseConsumerStateStore stateStore = datasetFramework.getDataset(stateStoreId, args, null);
    if (stateStore == null) {
        // If the state store doesn't exists, meaning there is no queue, hence nothing to do.
        return;
    }
    try {
        final Table table = stateStore.getInternalTable();
        Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) table).execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // Prefix name is "/" terminated ("queue:///namespace/app/flow/"), hence the scan is unique for the flow
                byte[] startRow = Bytes.toBytes(prefixName.toString());
                try (Scanner scanner = table.scan(startRow, Bytes.stopKeyForPrefix(startRow))) {
                    Row row = scanner.next();
                    while (row != null) {
                        table.delete(row.getRow());
                        row = scanner.next();
                    }
                }
            }
        });
    } finally {
        stateStore.close();
    }
}

Example 37 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class HBaseConsumerStateStore method completed.

/**
   * Called by consumer to signal process completion up to the current barrier that the consumer is in.
   */
void completed(long groupId, int instanceId) {
    // Get the current consumer state to get the end barrier info
    ConsumerState consumerState = getConsumerState(groupId, instanceId);
    QueueBarrier nextBarrier = consumerState.getNextBarrier();
    if (nextBarrier == null) {
        // End row shouldn't be null if this method is called
        throw new IllegalArgumentException(String.format("No end barrier information for consumer. Queue: %s, GroupId: %d, InstanceId: %d", queueName, groupId, instanceId));
    }
    byte[] stateColumn = getConsumerStateColumn(groupId, instanceId);
    // If the instance exists in the next barrier, set the start row to the barrier start
    if (instanceId < nextBarrier.getGroupConfig().getGroupSize()) {
        table.put(queueName.toBytes(), stateColumn, nextBarrier.getStartRow());
        return;
    }
    // find the next start barrier that this instance needs to consume from
    try (Scanner scanner = table.scan(Bytes.add(queueName.toBytes(), nextBarrier.getStartRow()), barrierScanEndRow)) {
        Row row;
        boolean found = false;
        while (!found && (row = scanner.next()) != null) {
            QueueBarrier queueBarrier = decodeBarrierInfo(row, groupId);
            if (queueBarrier == null || instanceId >= queueBarrier.getGroupConfig().getGroupSize()) {
                continue;
            }
            table.put(queueName.toBytes(), stateColumn, queueBarrier.getStartRow());
            found = true;
        }
        if (!found) {
            // Remove the state since this consumer instance is not longer active
            table.delete(queueName.toBytes(), stateColumn);
        }
    }
}

Example 38 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class WorkflowDataset method delete.

public void delete(ApplicationId id) {
    MDSKey mdsKey = new MDSKey.Builder().add(id.getNamespace()).add(id.getApplication()).build();
    Scanner scanner = table.scan(mdsKey.getKey(), Bytes.stopKeyForPrefix(mdsKey.getKey()));
    Row row;
    try {
        while ((row = scanner.next()) != null) {
            table.delete(row.getRow());
        }
    } finally {
        scanner.close();
    }
}

Example 39 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class WorkflowDataset method scan.

/**
   * This function scans the workflow.stats dataset for a list of workflow runs in a time range.
   *
   * @param id The workflow id
   * @param timeRangeStart Start of the time range that the scan should begin from
   * @param timeRangeEnd End of the time range that the scan should end at
   * @return List of WorkflowRunRecords
   */
private List<WorkflowRunRecord> scan(WorkflowId id, long timeRangeStart, long timeRangeEnd) {
    byte[] startRowKey = getRowKeyBuilder(id, timeRangeStart).build().getKey();
    byte[] endRowKey = getRowKeyBuilder(id, timeRangeEnd).build().getKey();
    Scan scan = new Scan(startRowKey, endRowKey);
    Scanner scanner = table.scan(scan);
    Row indexRow;
    List<WorkflowRunRecord> workflowRunRecordList = new ArrayList<>();
    while ((indexRow = scanner.next()) != null) {
        Map<byte[], byte[]> columns = indexRow.getColumns();
        String workflowRunId = Bytes.toString(columns.get(RUNID));
        long timeTaken = Bytes.toLong(columns.get(TIME_TAKEN));
        List<ProgramRun> programRunList = GSON.fromJson(Bytes.toString(columns.get(NODES)), PROGRAM_RUNS_TYPE);
        WorkflowRunRecord workflowRunRecord = new WorkflowRunRecord(workflowRunId, timeTaken, programRunList);
        workflowRunRecordList.add(workflowRunRecord);
    }
    return workflowRunRecordList;
}

Example 40 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class MockSink method clear.

/**
   * Clear any records written to this sink.
   *
   * @param tableManager dataset manager used to get the sink dataset
   */
public static void clear(DataSetManager<Table> tableManager) {
    tableManager.flush();
    Table table = tableManager.get();
    try (Scanner scanner = table.scan(null, null)) {
        Row row;
        while ((row = scanner.next()) != null) {
            table.delete(row.getRow());
        }
    }
    tableManager.flush();
}