Examples with HFileContextBuilder org.apache.hadoop.hbase.io.hfile.HFileContextBuilder used on opensource projects

Example 46 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestStoreFile method testBasicHalfMapFile.

/**
   * Write a file and then assert that we can read from top and bottom halves
   * using two HalfMapFiles.
   * @throws Exception
   */
@Test
public void testBasicHalfMapFile() throws Exception {
    final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testBasicHalfMapFileTb"));
    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri);
    HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();
    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(regionFs.createTempName()).withFileContext(meta).build();
    writeStoreFile(writer);
    Path sfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
    StoreFile sf = new StoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE);
    checkHalfHFile(regionFs, sf);
}

Example 47 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestStoreFile method testBloomFilter.

@Test
public void testBloomFilter() throws Exception {
    FileSystem fs = FileSystem.getLocal(conf);
    conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);
    conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
    // write the file
    Path f = new Path(ROOT_DIR, getName());
    HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();
    // Make a store file and write data to it.
    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f).withBloomType(BloomType.ROW).withMaxKeyCount(2000).withFileContext(meta).build();
    bloomWriteRead(writer, fs);
}

Example 48 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestStoreFile method testBloomTypes.

@Test
public void testBloomTypes() throws Exception {
    float err = (float) 0.01;
    FileSystem fs = FileSystem.getLocal(conf);
    conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err);
    conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
    int rowCount = 50;
    int colCount = 10;
    int versions = 2;
    // run once using columns and once using rows
    BloomType[] bt = { BloomType.ROWCOL, BloomType.ROW };
    int[] expKeys = { rowCount * colCount, rowCount };
    // below line deserves commentary.  it is expected bloom false positives
    //  column = rowCount*2*colCount inserts
    //  row-level = only rowCount*2 inserts, but failures will be magnified by
    //              2nd for loop for every column (2*colCount)
    float[] expErr = { 2 * rowCount * colCount * err, 2 * rowCount * 2 * colCount * err };
    for (int x : new int[] { 0, 1 }) {
        // write the file
        Path f = new Path(ROOT_DIR, getName() + x);
        HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();
        // Make a store file and write data to it.
        StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f).withBloomType(bt[x]).withMaxKeyCount(expKeys[x]).withFileContext(meta).build();
        long now = System.currentTimeMillis();
        for (int i = 0; i < rowCount * 2; i += 2) {
            // rows
            for (int j = 0; j < colCount * 2; j += 2) {
                // column qualifiers
                String row = String.format(localFormatter, i);
                String col = String.format(localFormatter, j);
                for (int k = 0; k < versions; ++k) {
                    // versions
                    KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(), ("col" + col).getBytes(), now - k, Bytes.toBytes((long) -1));
                    writer.append(kv);
                }
            }
        }
        writer.close();
        StoreFileReader reader = new StoreFileReader(fs, f, cacheConf, conf);
        reader.loadFileInfo();
        reader.loadBloomfilter();
        StoreFileScanner scanner = getStoreFileScanner(reader, false, false);
        assertEquals(expKeys[x], reader.generalBloomFilter.getKeyCount());
        Store store = mock(Store.class);
        HColumnDescriptor hcd = mock(HColumnDescriptor.class);
        when(hcd.getName()).thenReturn(Bytes.toBytes("family"));
        when(store.getFamily()).thenReturn(hcd);
        // check false positives rate
        int falsePos = 0;
        int falseNeg = 0;
        for (int i = 0; i < rowCount * 2; ++i) {
            // rows
            for (int j = 0; j < colCount * 2; ++j) {
                // column qualifiers
                String row = String.format(localFormatter, i);
                String col = String.format(localFormatter, j);
                TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
                columns.add(("col" + col).getBytes());
                Scan scan = new Scan(row.getBytes(), row.getBytes());
                scan.addColumn("family".getBytes(), ("col" + col).getBytes());
                boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
                boolean shouldRowExist = i % 2 == 0;
                boolean shouldColExist = j % 2 == 0;
                shouldColExist = shouldColExist || bt[x] == BloomType.ROW;
                if (shouldRowExist && shouldColExist) {
                    if (!exists)
                        falseNeg++;
                } else {
                    if (exists)
                        falsePos++;
                }
            }
        }
        // evict because we are about to delete the file
        reader.close(true);
        fs.delete(f, true);
        System.out.println(bt[x].toString());
        System.out.println("  False negatives: " + falseNeg);
        System.out.println("  False positives: " + falsePos);
        assertEquals(0, falseNeg);
        assertTrue(falsePos < 2 * expErr[x]);
    }
}

Example 49 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestStoreFileScannerWithTagCompression method testReseek.

@Test
public void testReseek() throws Exception {
    // write the file
    Path f = new Path(ROOT_DIR, "testReseek");
    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).withIncludesTags(true).withCompressTags(true).withDataBlockEncoding(DataBlockEncoding.PREFIX).build();
    // Make a store file and write data to it.
    StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(f).withFileContext(meta).build();
    writeStoreFile(writer);
    writer.close();
    StoreFileReader reader = new StoreFileReader(fs, f, cacheConf, conf);
    StoreFileScanner s = reader.getStoreFileScanner(false, false, false, 0, 0, false);
    try {
        // Now do reseek with empty KV to position to the beginning of the file
        KeyValue k = KeyValueUtil.createFirstOnRow(Bytes.toBytes("k2"));
        s.reseek(k);
        Cell kv = s.next();
        kv = s.next();
        kv = s.next();
        byte[] key5 = Bytes.toBytes("k5");
        assertTrue(Bytes.equals(key5, 0, key5.length, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()));
        List<Tag> tags = KeyValueUtil.ensureKeyValue(kv).getTags();
        assertEquals(1, tags.size());
        assertEquals("tag3", Bytes.toString(TagUtil.cloneValue(tags.get(0))));
    } finally {
        s.close();
    }
}

Example 50 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestReversibleScanners method testReversibleKeyValueHeap.

@Test
public void testReversibleKeyValueHeap() throws IOException {
    // write data to one memstore and two store files
    FileSystem fs = TEST_UTIL.getTestFileSystem();
    Path hfilePath = new Path(new Path(TEST_UTIL.getDataTestDir("testReversibleKeyValueHeap"), "regionname"), "familyname");
    CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
    HFileContextBuilder hcBuilder = new HFileContextBuilder();
    hcBuilder.withBlockSize(2 * 1024);
    HFileContext hFileContext = hcBuilder.build();
    StoreFileWriter writer1 = new StoreFileWriter.Builder(TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(hfilePath).withFileContext(hFileContext).build();
    StoreFileWriter writer2 = new StoreFileWriter.Builder(TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(hfilePath).withFileContext(hFileContext).build();
    MemStore memstore = new DefaultMemStore();
    writeMemstoreAndStoreFiles(memstore, new StoreFileWriter[] { writer1, writer2 });
    StoreFile sf1 = new StoreFile(fs, writer1.getPath(), TEST_UTIL.getConfiguration(), cacheConf, BloomType.NONE);
    StoreFile sf2 = new StoreFile(fs, writer2.getPath(), TEST_UTIL.getConfiguration(), cacheConf, BloomType.NONE);
    /**
     * Test without MVCC
     */
    int startRowNum = ROWSIZE / 2;
    ReversedKeyValueHeap kvHeap = getReversibleKeyValueHeap(memstore, sf1, sf2, ROWS[startRowNum], MAXMVCC);
    internalTestSeekAndNextForReversibleKeyValueHeap(kvHeap, startRowNum);
    startRowNum = ROWSIZE - 1;
    kvHeap = getReversibleKeyValueHeap(memstore, sf1, sf2, HConstants.EMPTY_START_ROW, MAXMVCC);
    internalTestSeekAndNextForReversibleKeyValueHeap(kvHeap, startRowNum);
    /**
     * Test with MVCC
     */
    for (int readPoint = 0; readPoint < MAXMVCC; readPoint++) {
        LOG.info("Setting read point to " + readPoint);
        startRowNum = ROWSIZE - 1;
        kvHeap = getReversibleKeyValueHeap(memstore, sf1, sf2, HConstants.EMPTY_START_ROW, readPoint);
        for (int i = startRowNum; i >= 0; i--) {
            if (i - 2 < 0)
                break;
            i = i - 2;
            kvHeap.seekToPreviousRow(KeyValueUtil.createFirstOnRow(ROWS[i + 1]));
            Pair<Integer, Integer> nextReadableNum = getNextReadableNumWithBackwardScan(i, 0, readPoint);
            if (nextReadableNum == null)
                break;
            KeyValue expecedKey = makeKV(nextReadableNum.getFirst(), nextReadableNum.getSecond());
            assertEquals(expecedKey, kvHeap.peek());
            i = nextReadableNum.getFirst();
            int qualNum = nextReadableNum.getSecond();
            if (qualNum + 1 < QUALSIZE) {
                kvHeap.backwardSeek(makeKV(i, qualNum + 1));
                nextReadableNum = getNextReadableNumWithBackwardScan(i, qualNum + 1, readPoint);
                if (nextReadableNum == null)
                    break;
                expecedKey = makeKV(nextReadableNum.getFirst(), nextReadableNum.getSecond());
                assertEquals(expecedKey, kvHeap.peek());
                i = nextReadableNum.getFirst();
                qualNum = nextReadableNum.getSecond();
            }
            kvHeap.next();
            if (qualNum + 1 >= QUALSIZE) {
                nextReadableNum = getNextReadableNumWithBackwardScan(i - 1, 0, readPoint);
            } else {
                nextReadableNum = getNextReadableNumWithBackwardScan(i, qualNum + 1, readPoint);
            }
            if (nextReadableNum == null)
                break;
            expecedKey = makeKV(nextReadableNum.getFirst(), nextReadableNum.getSecond());
            assertEquals(expecedKey, kvHeap.peek());
            i = nextReadableNum.getFirst();
        }
    }
}