Examples with HFileScanner org.apache.hadoop.hbase.io.hfile.HFileScanner used on opensource projects

Example 6 with HFileScanner

use of org.apache.hadoop.hbase.io.hfile.HFileScanner in project hbase by apache.

the class TestStoreFile method checkHalfHFile.

private void checkHalfHFile(final HRegionFileSystem regionFs, final StoreFile f) throws IOException {
    Cell midkey = f.createReader().midkey();
    KeyValue midKV = (KeyValue) midkey;
    byte[] midRow = CellUtil.cloneRow(midKV);
    // Create top split.
    HRegionInfo topHri = new HRegionInfo(regionFs.getRegionInfo().getTable(), null, midRow);
    Path topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, midRow, true);
    // Create bottom split.
    HRegionInfo bottomHri = new HRegionInfo(regionFs.getRegionInfo().getTable(), midRow, null);
    Path bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, midRow, false);
    // Make readers on top and bottom.
    StoreFileReader top = new StoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE).createReader();
    StoreFileReader bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE).createReader();
    ByteBuffer previous = null;
    LOG.info("Midkey: " + midKV.toString());
    ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midKV.getKey());
    try {
        // Now make two HalfMapFiles and assert they can read the full backing
        // file, one from the top and the other from the bottom.
        // Test bottom half first.
        // Now test reading from the top.
        boolean first = true;
        ByteBuffer key = null;
        HFileScanner topScanner = top.getScanner(false, false);
        while ((!topScanner.isSeeked() && topScanner.seekTo()) || (topScanner.isSeeked() && topScanner.next())) {
            key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
            if ((topScanner.getReader().getComparator().compare(midKV, key.array(), key.arrayOffset(), key.limit())) > 0) {
                fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + midkey);
            }
            if (first) {
                first = false;
                LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key)));
            }
        }
        LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key)));
        first = true;
        HFileScanner bottomScanner = bottom.getScanner(false, false);
        while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
            previous = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
            key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
            if (first) {
                first = false;
                LOG.info("First in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
            }
            assertTrue(key.compareTo(bbMidkeyBytes) < 0);
        }
        if (previous != null) {
            LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
        }
        // Remove references.
        regionFs.cleanupDaughterRegion(topHri);
        regionFs.cleanupDaughterRegion(bottomHri);
        // Next test using a midkey that does not exist in the file.
        // First, do a key that is < than first key. Ensure splits behave
        // properly.
        byte[] badmidkey = Bytes.toBytes("  .");
        assertTrue(fs.exists(f.getPath()));
        topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true);
        bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false);
        assertNull(bottomPath);
        top = new StoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE).createReader();
        // Now read from the top.
        first = true;
        topScanner = top.getScanner(false, false);
        KeyValue.KeyOnlyKeyValue keyOnlyKV = new KeyValue.KeyOnlyKeyValue();
        while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) {
            key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
            keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit());
            assertTrue(topScanner.getReader().getComparator().compare(keyOnlyKV, badmidkey, 0, badmidkey.length) >= 0);
            if (first) {
                first = false;
                KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
                LOG.info("First top when key < bottom: " + keyKV);
                String tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
                for (int i = 0; i < tmp.length(); i++) {
                    assertTrue(tmp.charAt(i) == 'a');
                }
            }
        }
        KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
        LOG.info("Last top when key < bottom: " + keyKV);
        String tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
        for (int i = 0; i < tmp.length(); i++) {
            assertTrue(tmp.charAt(i) == 'z');
        }
        // Remove references.
        regionFs.cleanupDaughterRegion(topHri);
        regionFs.cleanupDaughterRegion(bottomHri);
        // Test when badkey is > than last key in file ('||' > 'zz').
        badmidkey = Bytes.toBytes("|||");
        topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true);
        bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false);
        assertNull(topPath);
        bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE).createReader();
        first = true;
        bottomScanner = bottom.getScanner(false, false);
        while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
            key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
            if (first) {
                first = false;
                keyKV = KeyValueUtil.createKeyValueFromKey(key);
                LOG.info("First bottom when key > top: " + keyKV);
                tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
                for (int i = 0; i < tmp.length(); i++) {
                    assertTrue(tmp.charAt(i) == 'a');
                }
            }
        }
        keyKV = KeyValueUtil.createKeyValueFromKey(key);
        LOG.info("Last bottom when key > top: " + keyKV);
        for (int i = 0; i < tmp.length(); i++) {
            assertTrue(Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength()).charAt(i) == 'z');
        }
    } finally {
        if (top != null) {
            // evict since we are about to delete the file
            top.close(true);
        }
        if (bottom != null) {
            // evict since we are about to delete the file
            bottom.close(true);
        }
        fs.delete(f.getPath(), true);
    }
}

Example 7 with HFileScanner

use of org.apache.hadoop.hbase.io.hfile.HFileScanner in project hbase by apache.

the class TestLoadIncrementalHFiles method verifyHFile.

private int verifyHFile(Path p) throws IOException {
    Configuration conf = util.getConfiguration();
    HFile.Reader reader = HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), conf);
    reader.loadFileInfo();
    HFileScanner scanner = reader.getScanner(false, false);
    scanner.seekTo();
    int count = 0;
    do {
        count++;
    } while (scanner.next());
    assertTrue(count > 0);
    reader.close();
    return count;
}

Example 8 with HFileScanner

use of org.apache.hadoop.hbase.io.hfile.HFileScanner in project hbase by apache.

the class TestImportTSVWithVisibilityLabels method getKVCountFromHfile.

/**
   * Method returns the total KVs in given hfile
   * @param fs File System
   * @param p HFile path
   * @return KV count in the given hfile
   * @throws IOException
   */
private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
    Configuration conf = util.getConfiguration();
    HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
    reader.loadFileInfo();
    HFileScanner scanner = reader.getScanner(false, false);
    scanner.seekTo();
    int count = 0;
    do {
        count++;
    } while (scanner.next());
    reader.close();
    return count;
}

Example 9 with HFileScanner

use of org.apache.hadoop.hbase.io.hfile.HFileScanner in project hbase by apache.

the class TestImportTsv method getKVCountFromHfile.

/**
   * Method returns the total KVs in given hfile
   * @param fs File System
   * @param p HFile path
   * @return KV count in the given hfile
   * @throws IOException
   */
private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
    Configuration conf = util.getConfiguration();
    HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
    reader.loadFileInfo();
    HFileScanner scanner = reader.getScanner(false, false);
    scanner.seekTo();
    int count = 0;
    do {
        count++;
    } while (scanner.next());
    reader.close();
    return count;
}

Example 10 with HFileScanner

use of org.apache.hadoop.hbase.io.hfile.HFileScanner in project hbase by apache.

the class TestRegionReplicas method testVerifySecondaryAbilityToReadWithOnFiles.

@Test(timeout = 300000)
public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception {
    // disable the store file refresh chore (we do this by hand)
    HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
    restartRegionServer();
    try {
        LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
        openRegion(HTU, getRS(), hriSecondary);
        // load some data to primary
        LOG.info("Loading data to primary region");
        for (int i = 0; i < 3; ++i) {
            HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000);
            Region region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
            region.flush(true);
        }
        Region primaryRegion = getRS().getFromOnlineRegions(hriPrimary.getEncodedName());
        Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount());
        // Refresh store files on the secondary
        Region secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
        secondaryRegion.getStore(f).refreshStoreFiles();
        Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());
        // force compaction
        LOG.info("Force Major compaction on primary region " + hriPrimary);
        primaryRegion.compact(true);
        Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());
        List<RegionServerThread> regionServerThreads = HTU.getMiniHBaseCluster().getRegionServerThreads();
        HRegionServer hrs = null;
        for (RegionServerThread rs : regionServerThreads) {
            if (rs.getRegionServer().getOnlineRegion(primaryRegion.getRegionInfo().getRegionName()) != null) {
                hrs = rs.getRegionServer();
                break;
            }
        }
        CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, hrs, false);
        cleaner.chore();
        // scan all the hfiles on the secondary.
        // since there are no read on the secondary when we ask locations to
        // the NN a FileNotFound exception will be returned and the FileLink
        // should be able to deal with it giving us all the result we expect.
        int keys = 0;
        int sum = 0;
        for (StoreFile sf : secondaryRegion.getStore(f).getStorefiles()) {
            // Our file does not exist anymore. was moved by the compaction above.
            LOG.debug(getRS().getFileSystem().exists(sf.getPath()));
            Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));
            HFileScanner scanner = sf.getReader().getScanner(false, false);
            scanner.seekTo();
            do {
                keys++;
                Cell cell = scanner.getCell();
                sum += Integer.parseInt(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
            } while (scanner.next());
        }
        Assert.assertEquals(3000, keys);
        Assert.assertEquals(4498500, sum);
    } finally {
        HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
        closeRegion(HTU, getRS(), hriSecondary);
    }
}