Examples with CellScanner org.apache.hadoop.hbase.CellScanner used on opensource projects

Example 11 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class TestPrefixTree method testHBASE11728.

@Test
public void testHBASE11728() throws Exception {
    Put put = new Put(Bytes.toBytes("a-b-0-0"));
    put.addColumn(fam, qual1, Bytes.toBytes("c1-value"));
    region.put(put);
    put = new Put(row1_bytes);
    put.addColumn(fam, qual1, Bytes.toBytes("c1-value"));
    region.put(put);
    put = new Put(row2_bytes);
    put.addColumn(fam, qual2, Bytes.toBytes("c2-value"));
    region.put(put);
    put = new Put(row3_bytes);
    put.addColumn(fam, qual2, Bytes.toBytes("c2-value-2"));
    region.put(put);
    put = new Put(row4_bytes);
    put.addColumn(fam, qual2, Bytes.toBytes("c2-value-3"));
    region.put(put);
    region.flush(true);
    String[] rows = new String[3];
    rows[0] = row1;
    rows[1] = row2;
    rows[2] = row3;
    byte[][] val = new byte[3][];
    val[0] = Bytes.toBytes("c1-value");
    val[1] = Bytes.toBytes("c2-value");
    val[2] = Bytes.toBytes("c2-value-2");
    Scan scan = new Scan();
    scan.setStartRow(row1_bytes);
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    RegionScanner scanner = region.getScanner(scan);
    List<Cell> cells = new ArrayList<>();
    for (int i = 0; i < 3; i++) {
        assertEquals(i < 2, scanner.next(cells));
        CellScanner cellScanner = Result.create(cells).cellScanner();
        while (cellScanner.advance()) {
            assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner.current().getRowOffset(), cellScanner.current().getRowLength()));
            assertEquals(Bytes.toString(val[i]), Bytes.toString(cellScanner.current().getValueArray(), cellScanner.current().getValueOffset(), cellScanner.current().getValueLength()));
        }
        cells.clear();
    }
    scanner.close();
    // Add column
    scan = new Scan();
    scan.addColumn(fam, qual2);
    scan.setStartRow(row1_bytes);
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    scanner = region.getScanner(scan);
    for (int i = 1; i < 3; i++) {
        assertEquals(i < 2, scanner.next(cells));
        CellScanner cellScanner = Result.create(cells).cellScanner();
        while (cellScanner.advance()) {
            assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner.current().getRowOffset(), cellScanner.current().getRowLength()));
        }
        cells.clear();
    }
    scanner.close();
    scan = new Scan();
    scan.addColumn(fam, qual2);
    scan.setStartRow(Bytes.toBytes("a-b-A-1-"));
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    scanner = region.getScanner(scan);
    for (int i = 1; i < 3; i++) {
        assertEquals(i < 2, scanner.next(cells));
        CellScanner cellScanner = Result.create(cells).cellScanner();
        while (cellScanner.advance()) {
            assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner.current().getRowOffset(), cellScanner.current().getRowLength()));
        }
        cells.clear();
    }
    scanner.close();
    scan = new Scan();
    scan.addColumn(fam, qual2);
    scan.setStartRow(Bytes.toBytes("a-b-A-1-140239"));
    scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
    scanner = region.getScanner(scan);
    assertFalse(scanner.next(cells));
    assertFalse(cells.isEmpty());
    scanner.close();
}

Example 12 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class LoadTestDataGeneratorWithTags method beforeMutate.

@Override
public Mutation beforeMutate(long rowkeyBase, Mutation m) throws IOException {
    if (m instanceof Put) {
        List<Cell> updatedCells = new ArrayList<>();
        int numTags;
        if (minNumTags == maxNumTags) {
            numTags = minNumTags;
        } else {
            numTags = minNumTags + random.nextInt(maxNumTags - minNumTags);
        }
        List<Tag> tags;
        for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance(); ) {
            Cell cell = cellScanner.current();
            byte[] tag = LoadTestTool.generateData(random, minTagLength + random.nextInt(maxTagLength - minTagLength));
            tags = new ArrayList<>();
            for (int n = 0; n < numTags; n++) {
                tags.add(new ArrayBackedTag((byte) 127, tag));
            }
            Cell updatedCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), cell.getTimestamp(), Type.codeToType(cell.getTypeByte()), cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(), tags);
            updatedCells.add(updatedCell);
        }
        m.getFamilyCellMap().clear();
        // Clear and add new Cells to the Mutation.
        for (Cell cell : updatedCells) {
            ((Put) m).add(cell);
        }
    }
    return m;
}

Example 13 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hive by apache.

the class PutWritable method write.

@Override
public void write(final DataOutput out) throws IOException {
    ProtobufUtil.toMutationNoData(MutationType.PUT, put).writeDelimitedTo(DataOutputOutputStream.from(out));
    out.writeInt(put.size());
    CellScanner scanner = put.cellScanner();
    while (scanner.advance()) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(scanner.current());
        KeyValue.write(kv, out);
    }
}

Example 14 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class TestTableSnapshotScanner method verifyRow.

private static void verifyRow(Result result) throws IOException {
    byte[] row = result.getRow();
    CellScanner scanner = result.cellScanner();
    while (scanner.advance()) {
        Cell cell = scanner.current();
        //assert that all Cells in the Result have the same key
        Assert.assertEquals(0, Bytes.compareTo(row, 0, row.length, cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
    }
    for (int j = 0; j < FAMILIES.length; j++) {
        byte[] actual = result.getValue(FAMILIES[j], FAMILIES[j]);
        Assert.assertArrayEquals("Row in snapshot does not match, expected:" + Bytes.toString(row) + " ,actual:" + Bytes.toString(actual), row, actual);
    }
}

Example 15 with CellScanner

use of org.apache.hadoop.hbase.CellScanner in project hbase by apache.

the class TestWALLockup method testLockupWhenSyncInMiddleOfZigZagSetup.

/**
   * Reproduce locking up that happens when we get an inopportune sync during setup for
   * zigzaglatch wait. See HBASE-14317. If below is broken, we will see this test timeout because
   * it is locked up.
   * <p>First I need to set up some mocks for Server and RegionServerServices. I also need to
   * set up a dodgy WAL that will throw an exception when we go to append to it.
   */
@Test(timeout = 20000)
public void testLockupWhenSyncInMiddleOfZigZagSetup() throws IOException {
    // A WAL that we can have throw exceptions when a flag is set.
    class DodgyFSLog extends FSHLog {

        // Set this when want the WAL to start throwing exceptions.
        volatile boolean throwException = false;

        // Latch to hold up processing until after another operation has had time to run.
        CountDownLatch latch = new CountDownLatch(1);

        public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf) throws IOException {
            super(fs, root, logDir, conf);
        }

        @Override
        protected void afterCreatingZigZagLatch() {
            // the lock up we've seen in production.
            if (throwException) {
                try {
                    LOG.info("LATCHED");
                    // because all WALs have rolled. In this case, just give up on test.
                    if (!this.latch.await(5, TimeUnit.SECONDS)) {
                        LOG.warn("GIVE UP! Failed waiting on latch...Test is ABORTED!");
                    }
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }
        }

        @Override
        protected void beforeWaitOnSafePoint() {
            if (throwException) {
                LOG.info("COUNTDOWN");
                // be stuck; test won't go down
                while (this.latch.getCount() <= 0) Threads.sleep(1);
                this.latch.countDown();
            }
        }

        @Override
        protected Writer createWriterInstance(Path path) throws IOException {
            final Writer w = super.createWriterInstance(path);
            return new Writer() {

                @Override
                public void close() throws IOException {
                    w.close();
                }

                @Override
                public void sync() throws IOException {
                    if (throwException) {
                        throw new IOException("FAKE! Failed to replace a bad datanode...SYNC");
                    }
                    w.sync();
                }

                @Override
                public void append(Entry entry) throws IOException {
                    if (throwException) {
                        throw new IOException("FAKE! Failed to replace a bad datanode...APPEND");
                    }
                    w.append(entry);
                }

                @Override
                public long getLength() {
                    return w.getLength();
                }
            };
        }
    }
    // Mocked up server and regionserver services. Needed below.
    Server server = Mockito.mock(Server.class);
    Mockito.when(server.getConfiguration()).thenReturn(CONF);
    Mockito.when(server.isStopped()).thenReturn(false);
    Mockito.when(server.isAborted()).thenReturn(false);
    RegionServerServices services = Mockito.mock(RegionServerServices.class);
    // OK. Now I have my mocked up Server & RegionServerServices and dodgy WAL, go ahead with test.
    FileSystem fs = FileSystem.get(CONF);
    Path rootDir = new Path(dir + getName());
    DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF);
    Path originalWAL = dodgyWAL.getCurrentFileName();
    // I need a log roller running.
    LogRoller logRoller = new LogRoller(server, services);
    logRoller.addWAL(dodgyWAL);
    // There is no 'stop' once a logRoller is running.. it just dies.
    logRoller.start();
    // Now get a region and start adding in edits.
    HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
    final HRegion region = initHRegion(tableName, null, null, dodgyWAL);
    byte[] bytes = Bytes.toBytes(getName());
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    scopes.put(COLUMN_FAMILY_BYTES, 0);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    try {
        // First get something into memstore. Make a Put and then pull the Cell out of it. Will
        // manage append and sync carefully in below to manufacture hang. We keep adding same
        // edit. WAL subsystem doesn't care.
        Put put = new Put(bytes);
        put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
        WALKey key = new WALKey(region.getRegionInfo().getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(), mvcc, scopes);
        WALEdit edit = new WALEdit();
        CellScanner CellScanner = put.cellScanner();
        assertTrue(CellScanner.advance());
        edit.add(CellScanner.current());
        // out other side of the ringbuffer. If small numbers, stuff doesn't make it to WAL
        for (int i = 0; i < 1000; i++) {
            region.put(put);
        }
        // Set it so we start throwing exceptions.
        LOG.info("SET throwing of exception on append");
        dodgyWAL.throwException = true;
        // This append provokes a WAL roll request
        dodgyWAL.append(region.getRegionInfo(), key, edit, true);
        boolean exception = false;
        try {
            dodgyWAL.sync();
        } catch (Exception e) {
            exception = true;
        }
        assertTrue("Did not get sync exception", exception);
        // Get a memstore flush going too so we have same hung profile as up in the issue over
        // in HBASE-14317. Flush hangs trying to get sequenceid because the ringbuffer is held up
        // by the zigzaglatch waiting on syncs to come home.
        Thread t = new Thread("Flusher") {

            public void run() {
                try {
                    if (region.getMemstoreSize() <= 0) {
                        throw new IOException("memstore size=" + region.getMemstoreSize());
                    }
                    region.flush(false);
                } catch (IOException e) {
                    // Can fail trying to flush in middle of a roll. Not a failure. Will succeed later
                    // when roll completes.
                    LOG.info("In flush", e);
                }
                LOG.info("Exiting");
            }

            ;
        };
        t.setDaemon(true);
        t.start();
        // Wait until 
        while (dodgyWAL.latch.getCount() > 0) Threads.sleep(1);
        // Now assert I got a new WAL file put in place even though loads of errors above.
        assertTrue(originalWAL != dodgyWAL.getCurrentFileName());
        // Can I append to it?
        dodgyWAL.throwException = false;
        try {
            region.put(put);
        } catch (Exception e) {
            LOG.info("In the put", e);
        }
    } finally {
        // To stop logRoller, its server has to say it is stopped.
        Mockito.when(server.isStopped()).thenReturn(true);
        if (logRoller != null)
            logRoller.close();
        try {
            if (region != null)
                region.close();
            if (dodgyWAL != null)
                dodgyWAL.close();
        } catch (Exception e) {
            LOG.info("On way out", e);
        }
    }
}