Examples with RowIterator org.apache.accumulo.core.client.RowIterator used on opensource projects

Example 11 with RowIterator

use of org.apache.accumulo.core.client.RowIterator in project accumulo by apache.

the class AccumuloRowInputFormat method getRecordReader.

@Override
public RecordReader<Text, PeekingIterator<Entry<Key, Value>>> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
    log.setLevel(getLogLevel(job));
    RecordReaderBase<Text, PeekingIterator<Entry<Key, Value>>> recordReader = new RecordReaderBase<>() {

        RowIterator rowIterator;

        @Override
        public void initialize(InputSplit inSplit, JobConf job) throws IOException {
            super.initialize(inSplit, job);
            rowIterator = new RowIterator(scannerIterator);
        }

        @Override
        public boolean next(Text key, PeekingIterator<Entry<Key, Value>> value) throws IOException {
            if (!rowIterator.hasNext())
                return false;
            value.initialize(rowIterator.next());
            numKeysRead = rowIterator.getKVCount();
            key.set((currentKey = value.peek().getKey()).getRow());
            return true;
        }

        @Override
        public Text createKey() {
            return new Text();
        }

        @Override
        public PeekingIterator<Entry<Key, Value>> createValue() {
            return new PeekingIterator<>();
        }
    };
    recordReader.initialize(split, job);
    return recordReader;
}

Example 12 with RowIterator

use of org.apache.accumulo.core.client.RowIterator in project accumulo by apache.

the class AccumuloRowInputFormat method createRecordReader.

@Override
public RecordReader<Text, PeekingIterator<Entry<Key, Value>>> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    log.setLevel(getLogLevel(context));
    return new RecordReaderBase<>() {

        RowIterator rowIterator;

        @Override
        public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
            super.initialize(inSplit, attempt);
            rowIterator = new RowIterator(scannerIterator);
            currentK = new Text();
            currentV = null;
        }

        @Override
        public boolean nextKeyValue() throws IOException, InterruptedException {
            if (!rowIterator.hasNext())
                return false;
            currentV = new PeekingIterator<>(rowIterator.next());
            numKeysRead = rowIterator.getKVCount();
            currentKey = currentV.peek().getKey();
            currentK = new Text(currentKey.getRow());
            return true;
        }
    };
}

Example 13 with RowIterator

use of org.apache.accumulo.core.client.RowIterator in project accumulo by apache.

the class InputConfigurator method binOffline.

public static Map<String, Map<KeyExtent, List<Range>>> binOffline(TableId tableId, List<Range> ranges, ClientContext context) throws AccumuloException, TableNotFoundException {
    Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
    if (context.getTableState(tableId) != TableState.OFFLINE) {
        context.clearTableListCache();
        if (context.getTableState(tableId) != TableState.OFFLINE) {
            throw new AccumuloException("Table is online tableId:" + tableId + " cannot scan table in offline mode ");
        }
    }
    for (Range range : ranges) {
        Text startRow;
        if (range.getStartKey() != null)
            startRow = range.getStartKey().getRow();
        else
            startRow = new Text();
        Range metadataRange = new Range(new KeyExtent(tableId, startRow, null).toMetaRow(), true, null, false);
        Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
        TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(LastLocationColumnFamily.NAME);
        scanner.fetchColumnFamily(CurrentLocationColumnFamily.NAME);
        scanner.fetchColumnFamily(FutureLocationColumnFamily.NAME);
        scanner.setRange(metadataRange);
        RowIterator rowIter = new RowIterator(scanner);
        KeyExtent lastExtent = null;
        while (rowIter.hasNext()) {
            Iterator<Map.Entry<Key, Value>> row = rowIter.next();
            String last = "";
            KeyExtent extent = null;
            String location = null;
            while (row.hasNext()) {
                Map.Entry<Key, Value> entry = row.next();
                Key key = entry.getKey();
                if (key.getColumnFamily().equals(LastLocationColumnFamily.NAME)) {
                    last = entry.getValue().toString();
                }
                if (key.getColumnFamily().equals(CurrentLocationColumnFamily.NAME) || key.getColumnFamily().equals(FutureLocationColumnFamily.NAME)) {
                    location = entry.getValue().toString();
                }
                if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
                    extent = KeyExtent.fromMetaPrevRow(entry);
                }
            }
            if (location != null)
                return null;
            if (!extent.tableId().equals(tableId)) {
                throw new AccumuloException("Saw unexpected table Id " + tableId + " " + extent);
            }
            if (lastExtent != null && !extent.isPreviousExtent(lastExtent)) {
                throw new AccumuloException(" " + lastExtent + " is not previous extent " + extent);
            }
            binnedRanges.computeIfAbsent(last, k -> new HashMap<>()).computeIfAbsent(extent, k -> new ArrayList<>()).add(range);
            if (extent.endRow() == null || range.afterEndKey(new Key(extent.endRow()).followingKey(PartialKey.ROW))) {
                break;
            }
            lastExtent = extent;
        }
    }
    return binnedRanges;
}

Example 14 with RowIterator

use of org.apache.accumulo.core.client.RowIterator in project accumulo by apache.

the class AccumuloRowInputFormat method getRecordReader.

@Override
public RecordReader<Text, PeekingIterator<Entry<Key, Value>>> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
    AccumuloRecordReader<Text, PeekingIterator<Entry<Key, Value>>> recordReader = new AccumuloRecordReader<>(CLASS) {

        RowIterator rowIterator;

        @Override
        public void initialize(InputSplit inSplit, JobConf job) throws IOException {
            super.initialize(inSplit, job);
            rowIterator = new RowIterator(scannerIterator);
        }

        @Override
        public boolean next(Text key, PeekingIterator<Entry<Key, Value>> value) {
            if (!rowIterator.hasNext())
                return false;
            value.initialize(rowIterator.next());
            numKeysRead = rowIterator.getKVCount();
            key.set((currentKey = value.peek().getKey()).getRow());
            return true;
        }

        @Override
        public Text createKey() {
            return new Text();
        }

        @Override
        public PeekingIterator<Entry<Key, Value>> createValue() {
            return new PeekingIterator<>();
        }
    };
    recordReader.initialize(split, job);
    return recordReader;
}

Example 15 with RowIterator

use of org.apache.accumulo.core.client.RowIterator in project accumulo by apache.

the class ConditionalWriterIT method testThreads.

@Test
public void testThreads() throws Exception {
    // test multiple threads using a single conditional writer
    String tableName = getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        NewTableConfiguration ntc = new NewTableConfiguration();
        switch(random.nextInt(3)) {
            case 1:
                ntc = ntc.withSplits(nss("4"));
                break;
            case 2:
                ntc = ntc.withSplits(nss("3", "5"));
                break;
        }
        client.tableOperations().create(tableName, ntc);
        try (ConditionalWriter cw = client.createConditionalWriter(tableName)) {
            ArrayList<ByteSequence> rows = new ArrayList<>();
            for (int i = 0; i < 1000; i++) {
                rows.add(new ArrayByteSequence(FastFormat.toZeroPaddedString(abs(random.nextLong()), 16, 16, new byte[0])));
            }
            ArrayList<ConditionalMutation> mutations = new ArrayList<>();
            for (ByteSequence row : rows) mutations.add(new Stats(row).toMutation());
            ArrayList<ByteSequence> rows2 = new ArrayList<>();
            Iterator<Result> results = cw.write(mutations.iterator());
            while (results.hasNext()) {
                Result result = results.next();
                assertEquals(Status.ACCEPTED, result.getStatus());
                rows2.add(new ArrayByteSequence(result.getMutation().getRow()));
            }
            Collections.sort(rows);
            Collections.sort(rows2);
            assertEquals(rows, rows2);
            AtomicBoolean failed = new AtomicBoolean(false);
            ExecutorService tp = Executors.newFixedThreadPool(5);
            for (int i = 0; i < 5; i++) {
                tp.submit(new MutatorTask(tableName, client, rows, cw, failed));
            }
            tp.shutdown();
            while (!tp.isTerminated()) {
                tp.awaitTermination(1, TimeUnit.MINUTES);
            }
            assertFalse("A MutatorTask failed with an exception", failed.get());
        }
        try (Scanner scanner = client.createScanner(tableName, Authorizations.EMPTY)) {
            RowIterator rowIter = new RowIterator(scanner);
            while (rowIter.hasNext()) {
                Iterator<Entry<Key, Value>> row = rowIter.next();
                new Stats(row);
            }
        }
    }
}