Examples with StoreFile org.apache.hadoop.hbase.regionserver.StoreFile used on opensource projects

Example 1 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class HFileArchiver method archiveStoreFile.

/**
   * Archive the store file
   * @param fs the filesystem where the store files live
   * @param regionInfo region hosting the store files
   * @param conf {@link Configuration} to examine to determine the archive directory
   * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
   * @param family the family hosting the store files
   * @param storeFile file to be archived
   * @throws IOException if the files could not be correctly disposed.
   */
public static void archiveStoreFile(Configuration conf, FileSystem fs, HRegionInfo regionInfo, Path tableDir, byte[] family, Path storeFile) throws IOException {
    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
    // make sure we don't archive if we can't and that the archive dir exists
    if (!fs.mkdirs(storeArchiveDir)) {
        throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + Bytes.toString(family) + ", deleting compacted files instead.");
    }
    // do the actual archive
    long start = EnvironmentEdgeManager.currentTime();
    File file = new FileablePath(fs, storeFile);
    if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) {
        throw new IOException("Failed to archive/delete the file for region:" + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
    }
}

Example 2 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class HFileArchiver method archiveStoreFiles.

/**
   * Remove the store files, either by archiving them or outright deletion
   * @param conf {@link Configuration} to examine to determine the archive directory
   * @param fs the filesystem where the store files live
   * @param regionInfo {@link HRegionInfo} of the region hosting the store files
   * @param family the family hosting the store files
   * @param compactedFiles files to be disposed of. No further reading of these files should be
   *          attempted; otherwise likely to cause an {@link IOException}
   * @throws IOException if the files could not be correctly disposed.
   */
public static void archiveStoreFiles(Configuration conf, FileSystem fs, HRegionInfo regionInfo, Path tableDir, byte[] family, Collection<StoreFile> compactedFiles) throws IOException, FailedArchiveException {
    // sometimes in testing, we don't have rss, so we need to check for that
    if (fs == null) {
        LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:" + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family));
        deleteStoreFilesWithoutArchiving(compactedFiles);
        return;
    }
    // short circuit if we don't have any files to delete
    if (compactedFiles.isEmpty()) {
        LOG.debug("No store files to dispose, done!");
        return;
    }
    // build the archive path
    if (regionInfo == null || family == null)
        throw new IOException("Need to have a region and a family to archive from.");
    Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
    // make sure we don't archive if we can't and that the archive dir exists
    if (!fs.mkdirs(storeArchiveDir)) {
        throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:" + Bytes.toString(family) + ", deleting compacted files instead.");
    }
    // otherwise we attempt to archive the store files
    if (LOG.isDebugEnabled())
        LOG.debug("Archiving compacted store files.");
    // Wrap the storefile into a File
    StoreToFile getStorePath = new StoreToFile(fs);
    Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath);
    // do the actual archive
    List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, storeFiles, EnvironmentEdgeManager.currentTime());
    if (!failedArchive.isEmpty()) {
        throw new FailedArchiveException("Failed to archive/delete all the files for region:" + Bytes.toString(regionInfo.getRegionName()) + ", family:" + Bytes.toString(family) + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.", Collections2.transform(failedArchive, FUNC_FILE_TO_PATH));
    }
}

Example 3 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class MobUtils method cleanExpiredMobFiles.

/**
   * Cleans the expired mob files.
   * Cleans the files whose creation date is older than (current - columnFamily.ttl), and
   * the minVersions of that column family is 0.
   * @param fs The current file system.
   * @param conf The current configuration.
   * @param tableName The current table name.
   * @param columnDescriptor The descriptor of the current column family.
   * @param cacheConfig The cacheConfig that disables the block cache.
   * @param current The current time.
   * @throws IOException
   */
public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName, HColumnDescriptor columnDescriptor, CacheConfig cacheConfig, long current) throws IOException {
    long timeToLive = columnDescriptor.getTimeToLive();
    if (Integer.MAX_VALUE == timeToLive) {
        // no need to clean, because the TTL is not set.
        return;
    }
    Calendar calendar = Calendar.getInstance();
    calendar.setTimeInMillis(current - timeToLive * 1000);
    calendar.set(Calendar.HOUR_OF_DAY, 0);
    calendar.set(Calendar.MINUTE, 0);
    calendar.set(Calendar.SECOND, 0);
    Date expireDate = calendar.getTime();
    LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!");
    FileStatus[] stats = null;
    Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName);
    Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString());
    try {
        stats = fs.listStatus(path);
    } catch (FileNotFoundException e) {
        LOG.warn("Failed to find the mob file " + path, e);
    }
    if (null == stats) {
        // no file found
        return;
    }
    List<StoreFile> filesToClean = new ArrayList<>();
    int deletedFileCount = 0;
    for (FileStatus file : stats) {
        String fileName = file.getPath().getName();
        try {
            if (HFileLink.isHFileLink(file.getPath())) {
                HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
                fileName = hfileLink.getOriginPath().getName();
            }
            Date fileDate = parseDate(MobFileName.getDateFromName(fileName));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Checking file " + fileName);
            }
            if (fileDate.getTime() < expireDate.getTime()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(fileName + " is an expired file");
                }
                filesToClean.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
            }
        } catch (Exception e) {
            LOG.error("Cannot parse the fileName " + fileName, e);
        }
    }
    if (!filesToClean.isEmpty()) {
        try {
            removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(), filesToClean);
            deletedFileCount = filesToClean.size();
        } catch (IOException e) {
            LOG.error("Failed to delete the mob files " + filesToClean, e);
        }
    }
    LOG.info(deletedFileCount + " expired mob files are deleted");
}

Example 4 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class MobUtils method validateMobFile.

/**
   * Validates a mob file by opening and closing it.
   * @param conf The current configuration.
   * @param fs The current file system.
   * @param path The path where the mob file is saved.
   * @param cacheConfig The current cache config.
   */
private static void validateMobFile(Configuration conf, FileSystem fs, Path path, CacheConfig cacheConfig) throws IOException {
    StoreFile storeFile = null;
    try {
        storeFile = new StoreFile(fs, path, conf, cacheConfig, BloomType.NONE);
        storeFile.createReader();
    } catch (IOException e) {
        LOG.error("Failed to open mob file[" + path + "], keep it in temp directory.", e);
        throw e;
    } finally {
        if (storeFile != null) {
            storeFile.closeReader(false);
        }
    }
}

Example 5 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class PartitionedMobCompactor method compactMobFiles.

/**
   * Compacts the selected small mob files and all the del files.
   * @param request The compaction request.
   * @return The paths of new mob files after compactions.
   * @throws IOException if IO failure is encountered
   */
protected List<Path> compactMobFiles(final PartitionedMobCompactionRequest request) throws IOException {
    Collection<CompactionPartition> partitions = request.compactionPartitions;
    if (partitions == null || partitions.isEmpty()) {
        LOG.info("No partitions of mob files");
        return Collections.emptyList();
    }
    List<Path> paths = new ArrayList<>();
    final Connection c = ConnectionFactory.createConnection(conf);
    final Table table = c.getTable(tableName);
    try {
        Map<CompactionPartitionId, Future<List<Path>>> results = new HashMap<>();
        // compact the mob files by partitions in parallel.
        for (final CompactionPartition partition : partitions) {
            // How to efficiently come up a list of delFiles for one partition?
            // Search the delPartitions and collect all the delFiles for the partition
            // One optimization can do is that if there is no del file, we do not need to
            // come up with startKey/endKey.
            List<StoreFile> delFiles = getListOfDelFilesForPartition(partition, request.getDelPartitions());
            results.put(partition.getPartitionId(), pool.submit(new Callable<List<Path>>() {

                @Override
                public List<Path> call() throws Exception {
                    LOG.info("Compacting mob files for partition " + partition.getPartitionId());
                    return compactMobFilePartition(request, partition, delFiles, c, table);
                }
            }));
        }
        // compact the partitions in parallel.
        List<CompactionPartitionId> failedPartitions = new ArrayList<>();
        for (Entry<CompactionPartitionId, Future<List<Path>>> result : results.entrySet()) {
            try {
                paths.addAll(result.getValue().get());
            } catch (Exception e) {
                // just log the error
                LOG.error("Failed to compact the partition " + result.getKey(), e);
                failedPartitions.add(result.getKey());
            }
        }
        if (!failedPartitions.isEmpty()) {
            // if any partition fails in the compaction, directly throw an exception.
            throw new IOException("Failed to compact the partitions " + failedPartitions);
        }
    } finally {
        try {
            table.close();
        } catch (IOException e) {
            LOG.error("Failed to close the Table", e);
        }
    }
    return paths;
}