Examples with Scanner co.cask.cdap.api.dataset.table.Scanner used on opensource projects

Example 76 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class WorkflowDataset method scan.

/**
 * This function scans the workflow.stats dataset for a list of workflow runs in a time range.
 *
 * @param id The workflow id
 * @param timeRangeStart Start of the time range that the scan should begin from
 * @param timeRangeEnd End of the time range that the scan should end at
 * @return List of WorkflowRunRecords
 */
private List<WorkflowRunRecord> scan(WorkflowId id, long timeRangeStart, long timeRangeEnd) {
    byte[] startRowKey = getRowKeyBuilder(id, timeRangeStart).build().getKey();
    byte[] endRowKey = getRowKeyBuilder(id, timeRangeEnd).build().getKey();
    Scan scan = new Scan(startRowKey, endRowKey);
    Scanner scanner = table.scan(scan);
    Row indexRow;
    List<WorkflowRunRecord> workflowRunRecordList = new ArrayList<>();
    while ((indexRow = scanner.next()) != null) {
        Map<byte[], byte[]> columns = indexRow.getColumns();
        String workflowRunId = Bytes.toString(columns.get(RUNID));
        long timeTaken = Bytes.toLong(columns.get(TIME_TAKEN));
        List<ProgramRun> programRunList = GSON.fromJson(Bytes.toString(columns.get(NODES)), PROGRAM_RUNS_TYPE);
        WorkflowRunRecord workflowRunRecord = new WorkflowRunRecord(workflowRunId, timeTaken, programRunList);
        workflowRunRecordList.add(workflowRunRecord);
    }
    return workflowRunRecordList;
}

Example 77 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class WorkflowDataset method delete.

public void delete(ApplicationId id) {
    MDSKey mdsKey = new MDSKey.Builder().add(id.getNamespace()).add(id.getApplication()).build();
    Scanner scanner = table.scan(mdsKey.getKey(), Bytes.stopKeyForPrefix(mdsKey.getKey()));
    Row row;
    try {
        while ((row = scanner.next()) != null) {
            table.delete(row.getRow());
        }
    } finally {
        scanner.close();
    }
}

Example 78 with Scanner

use of co.cask.cdap.api.dataset.table.Scanner in project cdap by caskdata.

the class DataQualityAppTest method testDefaultConfig.

@Test
public void testDefaultConfig() throws Exception {
    Map<String, Set<String>> testMap = new HashMap<>();
    Set<String> testSet = new HashSet<>();
    testSet.add("DiscreteValuesHistogram");
    testMap.put("content_length", testSet);
    DataQualityApp.DataQualityConfig config = new DataQualityApp.DataQualityConfig(WORKFLOW_SCHEDULE_MINUTES, getStreamSource(), "dataQuality", testMap);
    ApplicationId appId = NamespaceId.DEFAULT.app("newApp");
    AppRequest<DataQualityApp.DataQualityConfig> appRequest = new AppRequest<>(new ArtifactSummary(appArtifact.getArtifact(), appArtifact.getVersion()), config);
    ApplicationManager applicationManager = deployApplication(appId, appRequest);
    MapReduceManager mrManager = applicationManager.getMapReduceManager("FieldAggregator").start();
    mrManager.waitForRun(ProgramRunStatus.COMPLETED, 180, TimeUnit.SECONDS);
    Table logDataStore = (Table) getDataset("dataQuality").get();
    DiscreteValuesHistogram discreteValuesHistogramAggregationFunction = new DiscreteValuesHistogram();
    Row row;
    try (Scanner scanner = logDataStore.scan(null, null)) {
        while ((row = scanner.next()) != null) {
            if (Bytes.toString(row.getRow()).contains("content_length")) {
                Map<byte[], byte[]> columnsMapBytes = row.getColumns();
                byte[] output = columnsMapBytes.get(Bytes.toBytes("DiscreteValuesHistogram"));
                if (output != null) {
                    discreteValuesHistogramAggregationFunction.combine(output);
                }
            }
        }
    }
    Map<String, Integer> outputMap = discreteValuesHistogramAggregationFunction.retrieveAggregation();
    Map<String, Integer> expectedMap = Maps.newHashMap();
    expectedMap.put("256", 3);
    Assert.assertEquals(expectedMap, outputMap);
}