Examples with Path org.apache.hadoop.fs.Path used on opensource projects

Example 56 with Path

use of org.apache.hadoop.fs.Path in project hbase by apache.

the class SnapshotFileCache method getSnapshotsInProgress.

@VisibleForTesting
List<String> getSnapshotsInProgress(final SnapshotManager snapshotManager) throws IOException {
    List<String> snapshotInProgress = Lists.newArrayList();
    // only add those files to the cache, but not to the known snapshots
    Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
    // only add those files to the cache, but not to the known snapshots
    FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
    if (running != null) {
        for (FileStatus run : running) {
            ReentrantLock lock = null;
            if (snapshotManager != null) {
                lock = snapshotManager.getLocks().acquireLock(run.getPath().getName());
            }
            try {
                snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
            } catch (CorruptedSnapshotException e) {
                // See HBASE-16464
                if (e.getCause() instanceof FileNotFoundException) {
                    // If the snapshot is corrupt, we will delete it
                    fs.delete(run.getPath(), true);
                    LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause());
                } else {
                    throw e;
                }
            } finally {
                if (lock != null) {
                    lock.unlock();
                }
            }
        }
    }
    return snapshotInProgress;
}

Example 57 with Path

use of org.apache.hadoop.fs.Path in project hbase by apache.

the class MobUtils method cleanExpiredMobFiles.

/**
   * Cleans the expired mob files.
   * Cleans the files whose creation date is older than (current - columnFamily.ttl), and
   * the minVersions of that column family is 0.
   * @param fs The current file system.
   * @param conf The current configuration.
   * @param tableName The current table name.
   * @param columnDescriptor The descriptor of the current column family.
   * @param cacheConfig The cacheConfig that disables the block cache.
   * @param current The current time.
   * @throws IOException
   */
public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName, HColumnDescriptor columnDescriptor, CacheConfig cacheConfig, long current) throws IOException {
    long timeToLive = columnDescriptor.getTimeToLive();
    if (Integer.MAX_VALUE == timeToLive) {
        // no need to clean, because the TTL is not set.
        return;
    }
    Calendar calendar = Calendar.getInstance();
    calendar.setTimeInMillis(current - timeToLive * 1000);
    calendar.set(Calendar.HOUR_OF_DAY, 0);
    calendar.set(Calendar.MINUTE, 0);
    calendar.set(Calendar.SECOND, 0);
    Date expireDate = calendar.getTime();
    LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!");
    FileStatus[] stats = null;
    Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName);
    Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString());
    try {
        stats = fs.listStatus(path);
    } catch (FileNotFoundException e) {
        LOG.warn("Failed to find the mob file " + path, e);
    }
    if (null == stats) {
        // no file found
        return;
    }
    List<StoreFile> filesToClean = new ArrayList<>();
    int deletedFileCount = 0;
    for (FileStatus file : stats) {
        String fileName = file.getPath().getName();
        try {
            if (HFileLink.isHFileLink(file.getPath())) {
                HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
                fileName = hfileLink.getOriginPath().getName();
            }
            Date fileDate = parseDate(MobFileName.getDateFromName(fileName));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Checking file " + fileName);
            }
            if (fileDate.getTime() < expireDate.getTime()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(fileName + " is an expired file");
                }
                filesToClean.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
            }
        } catch (Exception e) {
            LOG.error("Cannot parse the fileName " + fileName, e);
        }
    }
    if (!filesToClean.isEmpty()) {
        try {
            removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(), filesToClean);
            deletedFileCount = filesToClean.size();
        } catch (IOException e) {
            LOG.error("Failed to delete the mob files " + filesToClean, e);
        }
    }
    LOG.info(deletedFileCount + " expired mob files are deleted");
}

Example 58 with Path

use of org.apache.hadoop.fs.Path in project hbase by apache.

the class MobUtils method getMobRegionPath.

/**
   * Gets the region dir of the mob files.
   * It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}.
   * @param conf The current configuration.
   * @param tableName The current table name.
   * @return The region dir of the mob files.
   */
public static Path getMobRegionPath(Configuration conf, TableName tableName) {
    Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName);
    HRegionInfo regionInfo = getMobRegionInfo(tableName);
    return new Path(tablePath, regionInfo.getEncodedName());
}

Example 59 with Path

use of org.apache.hadoop.fs.Path in project hbase by apache.

the class PartitionedMobCompactor method compactMobFiles.

/**
   * Compacts the selected small mob files and all the del files.
   * @param request The compaction request.
   * @return The paths of new mob files after compactions.
   * @throws IOException if IO failure is encountered
   */
protected List<Path> compactMobFiles(final PartitionedMobCompactionRequest request) throws IOException {
    Collection<CompactionPartition> partitions = request.compactionPartitions;
    if (partitions == null || partitions.isEmpty()) {
        LOG.info("No partitions of mob files");
        return Collections.emptyList();
    }
    List<Path> paths = new ArrayList<>();
    final Connection c = ConnectionFactory.createConnection(conf);
    final Table table = c.getTable(tableName);
    try {
        Map<CompactionPartitionId, Future<List<Path>>> results = new HashMap<>();
        // compact the mob files by partitions in parallel.
        for (final CompactionPartition partition : partitions) {
            // How to efficiently come up a list of delFiles for one partition?
            // Search the delPartitions and collect all the delFiles for the partition
            // One optimization can do is that if there is no del file, we do not need to
            // come up with startKey/endKey.
            List<StoreFile> delFiles = getListOfDelFilesForPartition(partition, request.getDelPartitions());
            results.put(partition.getPartitionId(), pool.submit(new Callable<List<Path>>() {

                @Override
                public List<Path> call() throws Exception {
                    LOG.info("Compacting mob files for partition " + partition.getPartitionId());
                    return compactMobFilePartition(request, partition, delFiles, c, table);
                }
            }));
        }
        // compact the partitions in parallel.
        List<CompactionPartitionId> failedPartitions = new ArrayList<>();
        for (Entry<CompactionPartitionId, Future<List<Path>>> result : results.entrySet()) {
            try {
                paths.addAll(result.getValue().get());
            } catch (Exception e) {
                // just log the error
                LOG.error("Failed to compact the partition " + result.getKey(), e);
                failedPartitions.add(result.getKey());
            }
        }
        if (!failedPartitions.isEmpty()) {
            // if any partition fails in the compaction, directly throw an exception.
            throw new IOException("Failed to compact the partitions " + failedPartitions);
        }
    } finally {
        try {
            table.close();
        } catch (IOException e) {
            LOG.error("Failed to close the Table", e);
        }
    }
    return paths;
}

Example 60 with Path

use of org.apache.hadoop.fs.Path in project hbase by apache.

the class PartitionedMobCompactor method performCompaction.

/**
   * Performs the compaction on the selected files.
   * <ol>
   * <li>Compacts the del files.</li>
   * <li>Compacts the selected small mob files and all the del files.</li>
   * <li>If all the candidates are selected, delete the del files.</li>
   * </ol>
   * @param request The compaction request.
   * @return The paths of new mob files generated in the compaction.
   * @throws IOException if IO failure is encountered
   */
protected List<Path> performCompaction(PartitionedMobCompactionRequest request) throws IOException {
    // merge the del files, it is per del partition
    for (CompactionDelPartition delPartition : request.getDelPartitions()) {
        if (delPartition.getDelFileCount() <= 1)
            continue;
        List<Path> newDelPaths = compactDelFiles(request, delPartition.listDelFiles());
        delPartition.cleanDelFiles();
        delPartition.addDelFileList(newDelPaths);
    }
    List<Path> paths = null;
    int totalDelFileCount = 0;
    try {
        for (CompactionDelPartition delPartition : request.getDelPartitions()) {
            for (Path newDelPath : delPartition.listDelFiles()) {
                StoreFile sf = new StoreFile(fs, newDelPath, conf, compactionCacheConfig, BloomType.NONE);
                // pre-create reader of a del file to avoid race condition when opening the reader in each
                // partition.
                sf.createReader();
                delPartition.addStoreFile(sf);
                totalDelFileCount++;
            }
        }
        LOG.info("After merging, there are " + totalDelFileCount + " del files");
        // compact the mob files by partitions.
        paths = compactMobFiles(request);
        LOG.info("After compaction, there are " + paths.size() + " mob files");
    } finally {
        for (CompactionDelPartition delPartition : request.getDelPartitions()) {
            closeStoreFileReaders(delPartition.getStoreFiles());
        }
    }
    // archive the del files if all the mob files are selected.
    if (request.type == CompactionType.ALL_FILES && !request.getDelPartitions().isEmpty()) {
        LOG.info("After a mob compaction with all files selected, archiving the del files ");
        for (CompactionDelPartition delPartition : request.getDelPartitions()) {
            LOG.info(delPartition.listDelFiles());
            try {
                MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), delPartition.getStoreFiles());
            } catch (IOException e) {
                LOG.error("Failed to archive the del files " + delPartition.getStoreFiles(), e);
            }
        }
    }
    return paths;
}