Examples with StoreFileTracker org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker used on opensource projects

Example 1 with StoreFileTracker

use of org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker in project hbase by apache.

the class HRegionFileSystem method insertRegionFilesIntoStoreTracker.

private void insertRegionFilesIntoStoreTracker(List<Path> allFiles, MasterProcedureEnv env, HRegionFileSystem regionFs) throws IOException {
    TableDescriptor tblDesc = env.getMasterServices().getTableDescriptors().get(regionInfo.getTable());
    // we need to map trackers per store
    Map<String, StoreFileTracker> trackerMap = new HashMap<>();
    // we need to map store files per store
    Map<String, List<StoreFileInfo>> fileInfoMap = new HashMap<>();
    for (Path file : allFiles) {
        String familyName = file.getParent().getName();
        trackerMap.computeIfAbsent(familyName, t -> StoreFileTrackerFactory.create(conf, tblDesc, tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs));
        fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>());
        List<StoreFileInfo> infos = fileInfoMap.get(familyName);
        infos.add(new StoreFileInfo(conf, fs, file, true));
    }
    for (Map.Entry<String, StoreFileTracker> entry : trackerMap.entrySet()) {
        entry.getValue().add(fileInfoMap.get(entry.getKey()));
    }
}

Example 2 with StoreFileTracker

use of org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker in project hbase by apache.

the class MasterRegion method tryMigrate.

private static void tryMigrate(Configuration conf, FileSystem fs, Path tableDir, RegionInfo regionInfo, TableDescriptor oldTd, TableDescriptor newTd) throws IOException {
    Class<? extends StoreFileTracker> oldSft = StoreFileTrackerFactory.getTrackerClass(oldTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
    Class<? extends StoreFileTracker> newSft = StoreFileTrackerFactory.getTrackerClass(newTd.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
    if (oldSft.equals(newSft)) {
        LOG.debug("old store file tracker {} is the same with new store file tracker, skip migration", StoreFileTrackerFactory.getStoreFileTrackerName(oldSft));
        if (!oldTd.equals(newTd)) {
            // we may change other things such as adding a new family, so here we still need to persist
            // the new table descriptor
            LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
            FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
        }
        return;
    }
    LOG.info("Migrate store file tracker from {} to {}", oldSft.getSimpleName(), newSft.getSimpleName());
    HRegionFileSystem hfs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false);
    for (ColumnFamilyDescriptor oldCfd : oldTd.getColumnFamilies()) {
        StoreFileTracker oldTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
        StoreFileTracker newTracker = StoreFileTrackerFactory.create(conf, oldTd, oldCfd, hfs);
        List<StoreFileInfo> files = oldTracker.load();
        LOG.debug("Store file list for {}: {}", oldCfd.getNameAsString(), files);
        newTracker.set(oldTracker.load());
    }
    // persist the new table descriptor after migration
    LOG.info("Update table descriptor from {} to {}", oldTd, newTd);
    FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, newTd, true);
}

Example 3 with StoreFileTracker

use of org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker in project hbase by apache.

the class SnapshotManifest method addRegion.

protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor visitor) throws IOException {
    boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
    try {
        Path baseDir = tableDir;
        // Open the RegionFS
        if (isMobRegion) {
            baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
        }
        HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs, baseDir, regionInfo, true);
        monitor.rethrowException();
        // 1. dump region meta info into the snapshot directory
        LOG.debug("Storing region-info for snapshot.");
        Object regionData = visitor.regionOpen(regionInfo);
        monitor.rethrowException();
        // 2. iterate through all the stores in the region
        LOG.debug("Creating references for hfiles");
        // 1000 files/batch, far more than the number of store files under a single column family.
        for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
            Object familyData = visitor.familyOpen(regionData, cfd.getName());
            monitor.rethrowException();
            StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, htd, cfd, regionFs);
            List<StoreFileInfo> storeFiles = tracker.load();
            if (storeFiles.isEmpty()) {
                LOG.debug("No files under family: {}", cfd.getNameAsString());
                continue;
            }
            // 2.1. build the snapshot reference for the store
            // iterate through all the store's files and create "references".
            addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
            visitor.familyClose(regionData, familyData);
        }
        visitor.regionClose(regionData);
    } catch (IOException e) {
        // the mob directory might not be created yet, so do nothing when it is a mob region
        if (!isMobRegion) {
            throw e;
        }
    }
}

Example 4 with StoreFileTracker

use of org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker in project hbase by apache.

the class RestoreSnapshotHelper method restoreRegion.

/**
 * Restore region by removing files not in the snapshot
 * and adding the missing ones from the snapshot.
 */
private void restoreRegion(final RegionInfo regionInfo, final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
    Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles = getRegionHFileReferences(regionManifest);
    String tableName = tableDesc.getTableName().getNameAsString();
    final String snapshotName = snapshotDesc.getName();
    Path regionPath = new Path(tableDir, regionInfo.getEncodedName());
    HRegionFileSystem regionFS = (fs.exists(regionPath)) ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, false) : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);
    // Restore families present in the table
    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
        byte[] family = Bytes.toBytes(familyDir.getName());
        Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
        List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
        List<StoreFileInfo> filesToTrack = new ArrayList<>();
        if (snapshotFamilyFiles != null) {
            List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>();
            for (SnapshotRegionManifest.StoreFile storeFile : snapshotFamilyFiles) {
                if (familyFiles.contains(storeFile.getName())) {
                    // HFile already present
                    familyFiles.remove(storeFile.getName());
                    // no need to restore already present files, but we need to add those to tracker
                    filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true));
                } else {
                    // HFile missing
                    hfilesToAdd.add(storeFile);
                }
            }
            // Remove hfiles not present in the snapshot
            for (String hfileName : familyFiles) {
                Path hfile = new Path(familyDir, hfileName);
                if (!fs.getFileStatus(hfile).isDirectory()) {
                    LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
                    HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
                }
            }
            // Restore Missing files
            for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) {
                LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
                String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
                // mark the reference file to be added to tracker
                filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
            }
        } else {
            // Family doesn't exists in the snapshot
            LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
            HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
            fs.delete(familyDir, true);
        }
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        // simply reset list of tracked files with the matching files
        // and the extra one present in the snapshot
        tracker.set(filesToTrack);
    }
    // Add families not present in the table
    for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry : snapshotFiles.entrySet()) {
        Path familyDir = new Path(regionDir, familyEntry.getKey());
        StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder().withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build());
        List<StoreFileInfo> files = new ArrayList<>();
        if (!fs.mkdirs(familyDir)) {
            throw new IOException("Unable to create familyDir=" + familyDir);
        }
        for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) {
            LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName);
            String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
            files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true));
        }
        tracker.set(files);
    }
}

Example 5 with StoreFileTracker

use of org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker in project hbase by apache.

the class SplitTableRegionProcedure method splitStoreFiles.

/**
 * Create Split directory
 * @param env MasterProcedureEnv
 */
private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs) throws IOException {
    final Configuration conf = env.getMasterConfiguration();
    TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
    // The following code sets up a thread pool executor with as many slots as
    // there's files to split. It then fires up everything, waits for
    // completion and finally checks for any exception
    // 
    // Note: From HBASE-26187, splitStoreFiles now creates daughter region dirs straight under the
    // table dir. In case of failure, the proc would go through this again, already existing
    // region dirs and split files would just be ignored, new split files should get created.
    int nbFiles = 0;
    final Map<String, Collection<StoreFileInfo>> files = new HashMap<String, Collection<StoreFileInfo>>(htd.getColumnFamilyCount());
    for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
        String family = cfd.getNameAsString();
        StoreFileTracker tracker = StoreFileTrackerFactory.create(env.getMasterConfiguration(), htd, cfd, regionFs);
        Collection<StoreFileInfo> sfis = tracker.load();
        if (sfis == null) {
            continue;
        }
        Collection<StoreFileInfo> filteredSfis = null;
        for (StoreFileInfo sfi : sfis) {
            // splitable.
            if (sfi.isReference()) {
                LOG.info("Skipping split of " + sfi + "; presuming ready for archiving.");
                continue;
            }
            if (filteredSfis == null) {
                filteredSfis = new ArrayList<StoreFileInfo>(sfis.size());
                files.put(family, filteredSfis);
            }
            filteredSfis.add(sfi);
            nbFiles++;
        }
    }
    if (nbFiles == 0) {
        // no file needs to be splitted.
        return new Pair<>(Collections.emptyList(), Collections.emptyList());
    }
    // Max #threads is the smaller of the number of storefiles or the default max determined above.
    int maxThreads = Math.min(conf.getInt(HConstants.REGION_SPLIT_THREADS_MAX, conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT)), nbFiles);
    LOG.info("pid=" + getProcId() + " splitting " + nbFiles + " storefiles, region=" + getParentRegion().getShortNameToLog() + ", threads=" + maxThreads);
    final ExecutorService threadPool = Executors.newFixedThreadPool(maxThreads, new ThreadFactoryBuilder().setNameFormat("StoreFileSplitter-pool-%d").setDaemon(true).setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
    final List<Future<Pair<Path, Path>>> futures = new ArrayList<Future<Pair<Path, Path>>>(nbFiles);
    // Split each store file.
    for (Map.Entry<String, Collection<StoreFileInfo>> e : files.entrySet()) {
        byte[] familyName = Bytes.toBytes(e.getKey());
        final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName);
        final Collection<StoreFileInfo> storeFiles = e.getValue();
        if (storeFiles != null && storeFiles.size() > 0) {
            final Configuration storeConfiguration = StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
            for (StoreFileInfo storeFileInfo : storeFiles) {
                // As this procedure is running on master, use CacheConfig.DISABLED means
                // don't cache any block.
                // We also need to pass through a suitable CompoundConfiguration as if this
                // is running in a regionserver's Store context, or we might not be able
                // to read the hfiles.
                storeFileInfo.setConf(storeConfiguration);
                StoreFileSplitter sfs = new StoreFileSplitter(regionFs, familyName, new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
                futures.add(threadPool.submit(sfs));
            }
        }
    }
    // Shutdown the pool
    threadPool.shutdown();
    // Wait for all the tasks to finish.
    // When splits ran on the RegionServer, how-long-to-wait-configuration was named
    // hbase.regionserver.fileSplitTimeout. If set, use its value.
    long fileSplitTimeout = conf.getLong("hbase.master.fileSplitTimeout", conf.getLong("hbase.regionserver.fileSplitTimeout", 600000));
    try {
        boolean stillRunning = !threadPool.awaitTermination(fileSplitTimeout, TimeUnit.MILLISECONDS);
        if (stillRunning) {
            threadPool.shutdownNow();
            // wait for the thread to shutdown completely.
            while (!threadPool.isTerminated()) {
                Thread.sleep(50);
            }
            throw new IOException("Took too long to split the" + " files and create the references, aborting split");
        }
    } catch (InterruptedException e) {
        throw (InterruptedIOException) new InterruptedIOException().initCause(e);
    }
    List<Path> daughterA = new ArrayList<>();
    List<Path> daughterB = new ArrayList<>();
    // Look for any exception
    for (Future<Pair<Path, Path>> future : futures) {
        try {
            Pair<Path, Path> p = future.get();
            if (p.getFirst() != null) {
                daughterA.add(p.getFirst());
            }
            if (p.getSecond() != null) {
                daughterB.add(p.getSecond());
            }
        } catch (InterruptedException e) {
            throw (InterruptedIOException) new InterruptedIOException().initCause(e);
        } catch (ExecutionException e) {
            throw new IOException(e);
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("pid=" + getProcId() + " split storefiles for region " + getParentRegion().getShortNameToLog() + " Daughter A: " + daughterA + " storefiles, Daughter B: " + daughterB + " storefiles.");
    }
    return new Pair<>(daughterA, daughterB);
}