Examples with HStoreFile org.apache.hadoop.hbase.regionserver.HStoreFile used on opensource projects

Example 21 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class TestVisibilityLabels method testFlushedFileWithVisibilityTags.

@Test
public void testFlushedFileWithVisibilityTags() throws Exception {
    final byte[] qual2 = Bytes.toBytes("qual2");
    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam)).build();
    TEST_UTIL.getAdmin().createTable(tableDescriptor);
    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
        Put p1 = new Put(row1);
        p1.addColumn(fam, qual, value);
        p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
        Put p2 = new Put(row1);
        p2.addColumn(fam, qual2, value);
        p2.setCellVisibility(new CellVisibility(SECRET));
        RowMutations rm = new RowMutations(row1);
        rm.add(p1);
        rm.add(p2);
        table.mutateRow(rm);
    }
    TEST_UTIL.getAdmin().flush(tableName);
    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
    HStore store = regions.get(0).getStore(fam);
    Collection<HStoreFile> storefiles = store.getStorefiles();
    assertTrue(storefiles.size() > 0);
    for (HStoreFile storeFile : storefiles) {
        assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
    }
}

Example 22 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class SnapshotManifest method addRegion.

protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException {
    // 1. dump region meta info into the snapshot directory
    final String snapshotName = desc.getName();
    LOG.debug("Storing '" + region + "' region-info for snapshot=" + snapshotName);
    Object regionData = visitor.regionOpen(region.getRegionInfo());
    monitor.rethrowException();
    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");
    for (HStore store : region.getStores()) {
        // 2.1. build the snapshot reference for the store
        Object familyData = visitor.familyOpen(regionData, store.getColumnFamilyDescriptor().getName());
        monitor.rethrowException();
        List<HStoreFile> storeFiles = new ArrayList<>(store.getStorefiles());
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
        }
        // 2.2. iterate through all the store's files and create "references".
        for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
            HStoreFile storeFile = storeFiles.get(i);
            monitor.rethrowException();
            // create "reference" to this store file.
            LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): " + storeFile.getPath() + " for snapshot=" + snapshotName);
            visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
        }
        visitor.familyClose(regionData, familyData);
    }
    visitor.regionClose(regionData);
}

Example 23 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class TestAdmin1 method testHFileReplication.

/**
 * Test DFS replication for column families, where one CF has default replication(3) and the other
 * is set to 1.
 */
@Test
public void testHFileReplication() throws Exception {
    final TableName tableName = TableName.valueOf(this.name.getMethodName());
    String fn1 = "rep1";
    String fn = "defaultRep";
    TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.of(fn)).setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(fn1)).setDFSReplication((short) 1).build()).build();
    Table table = TEST_UTIL.createTable(htd, null);
    TEST_UTIL.waitTableAvailable(tableName);
    Put p = new Put(Bytes.toBytes("defaultRep_rk"));
    byte[] q1 = Bytes.toBytes("q1");
    byte[] v1 = Bytes.toBytes("v1");
    p.addColumn(Bytes.toBytes(fn), q1, v1);
    List<Put> puts = new ArrayList<>(2);
    puts.add(p);
    p = new Put(Bytes.toBytes("rep1_rk"));
    p.addColumn(Bytes.toBytes(fn1), q1, v1);
    puts.add(p);
    try {
        table.put(puts);
        ADMIN.flush(tableName);
        List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(tableName);
        for (HRegion r : regions) {
            HStore store = r.getStore(Bytes.toBytes(fn));
            for (HStoreFile sf : store.getStorefiles()) {
                assertTrue(sf.toString().contains(fn));
                assertTrue("Column family " + fn + " should have 3 copies", CommonFSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf.getFileInfo().getFileStatus().getReplication()));
            }
            store = r.getStore(Bytes.toBytes(fn1));
            for (HStoreFile sf : store.getStorefiles()) {
                assertTrue(sf.toString().contains(fn1));
                assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo().getFileStatus().getReplication());
            }
        }
    } finally {
        if (ADMIN.isTableEnabled(tableName)) {
            ADMIN.disableTable(tableName);
            ADMIN.deleteTable(tableName);
        }
    }
}

Example 24 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class SplitTableRegionProcedure method splitStoreFiles.

/**
 * Create Split directory
 * @param env MasterProcedureEnv
 */
private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs) throws IOException {
    final Configuration conf = env.getMasterConfiguration();
    TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
    // The following code sets up a thread pool executor with as many slots as
    // there's files to split. It then fires up everything, waits for
    // completion and finally checks for any exception
    // 
    // Note: From HBASE-26187, splitStoreFiles now creates daughter region dirs straight under the
    // table dir. In case of failure, the proc would go through this again, already existing
    // region dirs and split files would just be ignored, new split files should get created.
    int nbFiles = 0;
    final Map<String, Collection<StoreFileInfo>> files = new HashMap<String, Collection<StoreFileInfo>>(htd.getColumnFamilyCount());
    for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
        String family = cfd.getNameAsString();
        StoreFileTracker tracker = StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, cfd, regionFs);
        Collection<StoreFileInfo> sfis = tracker.load();
        if (sfis == null) {
            continue;
        }
        Collection<StoreFileInfo> filteredSfis = null;
        for (StoreFileInfo sfi : sfis) {
            // splitable.
            if (sfi.isReference()) {
                LOG.info("Skipping split of " + sfi + "; presuming ready for archiving.");
                continue;
            }
            if (filteredSfis == null) {
                filteredSfis = new ArrayList<StoreFileInfo>(sfis.size());
                files.put(family, filteredSfis);
            }
            filteredSfis.add(sfi);
            nbFiles++;
        }
    }
    if (nbFiles == 0) {
        // no file needs to be splitted.
        return new Pair<>(Collections.emptyList(), Collections.emptyList());
    }
    // Max #threads is the smaller of the number of storefiles or the default max determined above.
    int maxThreads = Math.min(conf.getInt(HConstants.REGION_SPLIT_THREADS_MAX, conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT)), nbFiles);
    LOG.info("pid=" + getProcId() + " splitting " + nbFiles + " storefiles, region=" + getParentRegion().getShortNameToLog() + ", threads=" + maxThreads);
    final ExecutorService threadPool = Executors.newFixedThreadPool(maxThreads, new ThreadFactoryBuilder().setNameFormat("StoreFileSplitter-pool-%d").setDaemon(true).setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
    final List<Future<Pair<Path, Path>>> futures = new ArrayList<Future<Pair<Path, Path>>>(nbFiles);
    // Split each store file.
    for (Map.Entry<String, Collection<StoreFileInfo>> e : files.entrySet()) {
        byte[] familyName = Bytes.toBytes(e.getKey());
        final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName);
        final Collection<StoreFileInfo> storeFiles = e.getValue();
        if (storeFiles != null && storeFiles.size() > 0) {
            final Configuration storeConfiguration = StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
            for (StoreFileInfo storeFileInfo : storeFiles) {
                // As this procedure is running on master, use CacheConfig.DISABLED means
                // don't cache any block.
                // We also need to pass through a suitable CompoundConfiguration as if this
                // is running in a regionserver's Store context, or we might not be able
                // to read the hfiles.
                storeFileInfo.setConf(storeConfiguration);
                StoreFileSplitter sfs = new StoreFileSplitter(regionFs, familyName, new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
                futures.add(threadPool.submit(sfs));
            }
        }
    }
    // Shutdown the pool
    threadPool.shutdown();
    // Wait for all the tasks to finish.
    // When splits ran on the RegionServer, how-long-to-wait-configuration was named
    // hbase.regionserver.fileSplitTimeout. If set, use its value.
    long fileSplitTimeout = conf.getLong("hbase.master.fileSplitTimeout", conf.getLong("hbase.regionserver.fileSplitTimeout", 600000));
    try {
        boolean stillRunning = !threadPool.awaitTermination(fileSplitTimeout, TimeUnit.MILLISECONDS);
        if (stillRunning) {
            threadPool.shutdownNow();
            // wait for the thread to shutdown completely.
            while (!threadPool.isTerminated()) {
                Thread.sleep(50);
            }
            throw new IOException("Took too long to split the" + " files and create the references, aborting split");
        }
    } catch (InterruptedException e) {
        throw (InterruptedIOException) new InterruptedIOException().initCause(e);
    }
    List<Path> daughterA = new ArrayList<>();
    List<Path> daughterB = new ArrayList<>();
    // Look for any exception
    for (Future<Pair<Path, Path>> future : futures) {
        try {
            Pair<Path, Path> p = future.get();
            if (p.getFirst() != null) {
                daughterA.add(p.getFirst());
            }
            if (p.getSecond() != null) {
                daughterB.add(p.getSecond());
            }
        } catch (InterruptedException e) {
            throw (InterruptedIOException) new InterruptedIOException().initCause(e);
        } catch (ExecutionException e) {
            throw new IOException(e);
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("pid=" + getProcId() + " split storefiles for region " + getParentRegion().getShortNameToLog() + " Daughter A: " + daughterA + " storefiles, Daughter B: " + daughterB + " storefiles.");
    }
    return new Pair<>(daughterA, daughterB);
}

Example 25 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class MergeTableRegionsProcedure method mergeStoreFiles.

private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs, HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException {
    final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(mergedRegion.getTable());
    List<Path> mergedFiles = new ArrayList<>();
    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
        String family = hcd.getNameAsString();
        StoreFileTracker tracker = StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, hcd, regionFs);
        final Collection<StoreFileInfo> storeFiles = tracker.load();
        if (storeFiles != null && storeFiles.size() > 0) {
            final Configuration storeConfiguration = StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
            for (StoreFileInfo storeFileInfo : storeFiles) {
                // Create reference file(s) to parent region file here in mergedDir.
                // As this procedure is running on master, use CacheConfig.DISABLED means
                // don't cache any block.
                // We also need to pass through a suitable CompoundConfiguration as if this
                // is running in a regionserver's Store context, or we might not be able
                // to read the hfiles.
                storeFileInfo.setConf(storeConfiguration);
                Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
                mergedFiles.add(refFile);
            }
        }
    }
    return mergedFiles;
}