Examples with HFileLink org.apache.hadoop.hbase.io.HFileLink used on opensource projects

Example 1 with HFileLink

use of org.apache.hadoop.hbase.io.HFileLink in project hbase by apache.

the class MobUtils method cleanExpiredMobFiles.

/**
   * Cleans the expired mob files.
   * Cleans the files whose creation date is older than (current - columnFamily.ttl), and
   * the minVersions of that column family is 0.
   * @param fs The current file system.
   * @param conf The current configuration.
   * @param tableName The current table name.
   * @param columnDescriptor The descriptor of the current column family.
   * @param cacheConfig The cacheConfig that disables the block cache.
   * @param current The current time.
   * @throws IOException
   */
public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName, HColumnDescriptor columnDescriptor, CacheConfig cacheConfig, long current) throws IOException {
    long timeToLive = columnDescriptor.getTimeToLive();
    if (Integer.MAX_VALUE == timeToLive) {
        // no need to clean, because the TTL is not set.
        return;
    }
    Calendar calendar = Calendar.getInstance();
    calendar.setTimeInMillis(current - timeToLive * 1000);
    calendar.set(Calendar.HOUR_OF_DAY, 0);
    calendar.set(Calendar.MINUTE, 0);
    calendar.set(Calendar.SECOND, 0);
    Date expireDate = calendar.getTime();
    LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!");
    FileStatus[] stats = null;
    Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName);
    Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString());
    try {
        stats = fs.listStatus(path);
    } catch (FileNotFoundException e) {
        LOG.warn("Failed to find the mob file " + path, e);
    }
    if (null == stats) {
        // no file found
        return;
    }
    List<StoreFile> filesToClean = new ArrayList<>();
    int deletedFileCount = 0;
    for (FileStatus file : stats) {
        String fileName = file.getPath().getName();
        try {
            if (HFileLink.isHFileLink(file.getPath())) {
                HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
                fileName = hfileLink.getOriginPath().getName();
            }
            Date fileDate = parseDate(MobFileName.getDateFromName(fileName));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Checking file " + fileName);
            }
            if (fileDate.getTime() < expireDate.getTime()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(fileName + " is an expired file");
                }
                filesToClean.add(new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE));
            }
        } catch (Exception e) {
            LOG.error("Cannot parse the fileName " + fileName, e);
        }
    }
    if (!filesToClean.isEmpty()) {
        try {
            removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(), filesToClean);
            deletedFileCount = filesToClean.size();
        } catch (IOException e) {
            LOG.error("Failed to delete the mob files " + filesToClean, e);
        }
    }
    LOG.info(deletedFileCount + " expired mob files are deleted");
}

Example 2 with HFileLink

use of org.apache.hadoop.hbase.io.HFileLink in project hbase by apache.

the class SnapshotInfo method getSnapshotFilesMap.

/**
   * Gets the store files map for snapshot
   * @param conf the {@link Configuration} to use
   * @param snapshot {@link SnapshotDescription} to get stats from
   * @param exec the {@link ExecutorService} to use
   * @param filesMap {@link Map} the map to put the mapping entries
   * @param uniqueHFilesArchiveSize {@link AtomicLong} the accumulated store file size in archive
   * @param uniqueHFilesSize {@link AtomicLong} the accumulated store file size shared
   * @param uniqueHFilesMobSize {@link AtomicLong} the accumulated mob store file size shared
   * @return the snapshot stats
   */
private static void getSnapshotFilesMap(final Configuration conf, final SnapshotDescription snapshot, final ExecutorService exec, final ConcurrentHashMap<Path, Integer> filesMap, final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize, final AtomicLong uniqueHFilesMobSize) throws IOException {
    HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
    Path rootDir = FSUtils.getRootDir(conf);
    final FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
    SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec, new SnapshotReferenceUtil.SnapshotVisitor() {

        @Override
        public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
            if (!storeFile.hasReference()) {
                HFileLink link = HFileLink.build(conf, snapshot.getTableName(), regionInfo.getEncodedName(), family, storeFile.getName());
                long size;
                Integer count;
                Path p;
                AtomicLong al;
                int c = 0;
                if (fs.exists(link.getArchivePath())) {
                    p = link.getArchivePath();
                    al = uniqueHFilesArchiveSize;
                    size = fs.getFileStatus(p).getLen();
                } else if (fs.exists(link.getMobPath())) {
                    p = link.getMobPath();
                    al = uniqueHFilesMobSize;
                    size = fs.getFileStatus(p).getLen();
                } else {
                    p = link.getOriginPath();
                    al = uniqueHFilesSize;
                    size = link.getFileStatus(fs).getLen();
                }
                // If it has been counted, do not double count
                count = filesMap.get(p);
                if (count != null) {
                    c = count.intValue();
                } else {
                    al.addAndGet(size);
                }
                filesMap.put(p, ++c);
            }
        }
    });
}

Example 3 with HFileLink

use of org.apache.hadoop.hbase.io.HFileLink in project hbase by apache.

the class SnapshotReferenceUtil method verifyStoreFile.

/**
   * Verify the validity of the snapshot store file
   *
   * @param conf The current {@link Configuration} instance.
   * @param fs {@link FileSystem}
   * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
   * @param snapshot the {@link SnapshotDescription} of the snapshot to verify
   * @param regionInfo {@link HRegionInfo} of the region that contains the store file
   * @param family family that contains the store file
   * @param storeFile the store file to verify
   * @throws CorruptedSnapshotException if the snapshot is corrupted
   * @throws IOException if an error occurred while scanning the directory
   */
private static void verifyStoreFile(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription snapshot, final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
    TableName table = TableName.valueOf(snapshot.getTable());
    String fileName = storeFile.getName();
    Path refPath = null;
    if (StoreFileInfo.isReference(fileName)) {
        // If is a reference file check if the parent file is present in the snapshot
        refPath = new Path(new Path(regionInfo.getEncodedName(), family), fileName);
        refPath = StoreFileInfo.getReferredToFile(refPath);
        String refRegion = refPath.getParent().getParent().getName();
        refPath = HFileLink.createPath(table, refRegion, family, refPath.getName());
        if (!HFileLink.buildFromHFileLinkPattern(conf, refPath).exists(fs)) {
            throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName + " path=" + refPath, ProtobufUtil.createSnapshotDesc(snapshot));
        }
        if (storeFile.hasReference()) {
            // we already have the Reference information embedded here.
            return;
        }
    }
    Path linkPath;
    if (refPath != null && HFileLink.isHFileLink(refPath)) {
        linkPath = new Path(family, refPath.getName());
    } else if (HFileLink.isHFileLink(fileName)) {
        linkPath = new Path(family, fileName);
    } else {
        linkPath = new Path(family, HFileLink.createHFileLinkName(table, regionInfo.getEncodedName(), fileName));
    }
    // check if the linked file exists (in the archive, or in the table dir)
    HFileLink link = null;
    if (MobUtils.isMobRegionInfo(regionInfo)) {
        // for mob region
        link = HFileLink.buildFromHFileLinkPattern(MobUtils.getQualifiedMobRootDir(conf), HFileArchiveUtil.getArchivePath(conf), linkPath);
    } else {
        // not mob region
        link = HFileLink.buildFromHFileLinkPattern(conf, linkPath);
    }
    try {
        FileStatus fstat = link.getFileStatus(fs);
        if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) {
            String msg = "hfile: " + fileName + " size does not match with the expected one. " + " found=" + fstat.getLen() + " expected=" + storeFile.getFileSize();
            LOG.error(msg);
            throw new CorruptedSnapshotException(msg, ProtobufUtil.createSnapshotDesc(snapshot));
        }
    } catch (FileNotFoundException e) {
        String msg = "Can't find hfile: " + fileName + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath() + ") directory for the primary table.";
        LOG.error(msg);
        throw new CorruptedSnapshotException(msg, ProtobufUtil.createSnapshotDesc(snapshot));
    }
}

Example 4 with HFileLink

use of org.apache.hadoop.hbase.io.HFileLink in project hbase by apache.

the class TestMobFileLink method testMobFilePath.

@Test
public void testMobFilePath() throws IOException {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    Configuration conf = HBaseConfiguration.create();
    FileSystem fs = FileSystem.get(conf);
    Path rootDir = FSUtils.getRootDir(conf);
    Path tableDir = FSUtils.getTableDir(rootDir, tableName);
    Path archiveDir = FSUtils.getTableDir(HFileArchiveUtil.getArchivePath(conf), tableName);
    String fileName = "mobFile";
    String encodedRegionName = MobUtils.getMobRegionInfo(tableName).getEncodedName();
    String columnFamily = "columnFamily";
    Path regionDir = new Path(tableDir, encodedRegionName);
    Path archivedRegionDir = new Path(archiveDir, encodedRegionName);
    Path expectedMobFilePath = new Path(MobUtils.getMobFamilyPath(conf, tableName, columnFamily), fileName).makeQualified(fs.getUri(), fs.getWorkingDirectory());
    Path expectedOriginPath = new Path(new Path(regionDir, columnFamily), fileName).makeQualified(fs.getUri(), fs.getWorkingDirectory());
    Path expectedArchivePath = new Path(new Path(archivedRegionDir, columnFamily), fileName).makeQualified(fs.getUri(), fs.getWorkingDirectory());
    String hfileLinkName = tableName.getNameAsString() + "=" + encodedRegionName + "-" + fileName;
    Path hfileLinkPath = new Path(columnFamily, hfileLinkName);
    HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, hfileLinkPath);
    Assert.assertEquals(expectedMobFilePath, hfileLink.getMobPath());
    Assert.assertEquals(expectedOriginPath, hfileLink.getOriginPath());
    Assert.assertEquals(expectedArchivePath, hfileLink.getArchivePath());
}

Example 5 with HFileLink

use of org.apache.hadoop.hbase.io.HFileLink in project hbase by apache.

the class TestCorruptedRegionStoreFile method setupTable.

private void setupTable(final TableName tableName) throws IOException {
    // load the table
    Table table = UTIL.createTable(tableName, FAMILY_NAME);
    try {
        rowCount = 0;
        byte[] value = new byte[1024];
        byte[] q = Bytes.toBytes("q");
        while (rowCount < NUM_ROWS) {
            Put put = new Put(Bytes.toBytes(String.format("%010d", rowCount)));
            put.setDurability(Durability.SKIP_WAL);
            put.addColumn(FAMILY_NAME, q, value);
            table.put(put);
            if ((rowCount++ % ROW_PER_FILE) == 0) {
                UTIL.getAdmin().flush(tableName);
            }
        }
    } finally {
        UTIL.getAdmin().flush(tableName);
        table.close();
    }
    assertEquals(NUM_ROWS, rowCount);
    // get the store file paths
    storeFiles.clear();
    tableDir = FSUtils.getTableDir(getRootDir(), tableName);
    FSVisitor.visitTableStoreFiles(getFileSystem(), tableDir, new FSVisitor.StoreFileVisitor() {

        @Override
        public void storeFile(final String region, final String family, final String hfile) throws IOException {
            HFileLink link = HFileLink.build(UTIL.getConfiguration(), tableName, region, family, hfile);
            storeFiles.add(link.getOriginPath());
        }
    });
    assertTrue("Expected at least " + NUM_FILES + " store files", storeFiles.size() >= NUM_FILES);
    LOG.info("Store files: " + storeFiles);
}