Examples with StoreFile org.apache.hadoop.hbase.regionserver.StoreFile used on opensource projects

Example 6 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class PartitionedMobCompactor method performCompaction.

/**
   * Performs the compaction on the selected files.
   * <ol>
   * <li>Compacts the del files.</li>
   * <li>Compacts the selected small mob files and all the del files.</li>
   * <li>If all the candidates are selected, delete the del files.</li>
   * </ol>
   * @param request The compaction request.
   * @return The paths of new mob files generated in the compaction.
   * @throws IOException if IO failure is encountered
   */
protected List<Path> performCompaction(PartitionedMobCompactionRequest request) throws IOException {
    // merge the del files, it is per del partition
    for (CompactionDelPartition delPartition : request.getDelPartitions()) {
        if (delPartition.getDelFileCount() <= 1)
            continue;
        List<Path> newDelPaths = compactDelFiles(request, delPartition.listDelFiles());
        delPartition.cleanDelFiles();
        delPartition.addDelFileList(newDelPaths);
    }
    List<Path> paths = null;
    int totalDelFileCount = 0;
    try {
        for (CompactionDelPartition delPartition : request.getDelPartitions()) {
            for (Path newDelPath : delPartition.listDelFiles()) {
                StoreFile sf = new StoreFile(fs, newDelPath, conf, compactionCacheConfig, BloomType.NONE);
                // pre-create reader of a del file to avoid race condition when opening the reader in each
                // partition.
                sf.createReader();
                delPartition.addStoreFile(sf);
                totalDelFileCount++;
            }
        }
        LOG.info("After merging, there are " + totalDelFileCount + " del files");
        // compact the mob files by partitions.
        paths = compactMobFiles(request);
        LOG.info("After compaction, there are " + paths.size() + " mob files");
    } finally {
        for (CompactionDelPartition delPartition : request.getDelPartitions()) {
            closeStoreFileReaders(delPartition.getStoreFiles());
        }
    }
    // archive the del files if all the mob files are selected.
    if (request.type == CompactionType.ALL_FILES && !request.getDelPartitions().isEmpty()) {
        LOG.info("After a mob compaction with all files selected, archiving the del files ");
        for (CompactionDelPartition delPartition : request.getDelPartitions()) {
            LOG.info(delPartition.listDelFiles());
            try {
                MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), delPartition.getStoreFiles());
            } catch (IOException e) {
                LOG.error("Failed to archive the del files " + delPartition.getStoreFiles(), e);
            }
        }
    }
    return paths;
}

Example 7 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class PartitionedMobCompactor method getFileInfo.

/**
   * Gets the max seqId and number of cells of the store files.
   * @param storeFiles The store files.
   * @return The pair of the max seqId and number of cells of the store files.
   * @throws IOException if IO failure is encountered
   */
private Pair<Long, Long> getFileInfo(List<StoreFile> storeFiles) throws IOException {
    long maxSeqId = 0;
    long maxKeyCount = 0;
    for (StoreFile sf : storeFiles) {
        // the readers will be closed later after the merge.
        maxSeqId = Math.max(maxSeqId, sf.getMaxSequenceId());
        byte[] count = sf.createReader().loadFileInfo().get(StoreFile.MOB_CELLS_COUNT);
        if (count != null) {
            maxKeyCount += Bytes.toLong(count);
        }
    }
    return new Pair<>(maxSeqId, maxKeyCount);
}

Example 8 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class PartitionedMobCompactor method compactDelFiles.

/**
   * Compacts the del files in batches which avoids opening too many files.
   * @param request The compaction request.
   * @param delFilePaths Del file paths to compact
   * @return The paths of new del files after merging or the original files if no merging
   *         is necessary.
   * @throws IOException if IO failure is encountered
   */
protected List<Path> compactDelFiles(PartitionedMobCompactionRequest request, List<Path> delFilePaths) throws IOException {
    if (delFilePaths.size() <= delFileMaxCount) {
        return delFilePaths;
    }
    // when there are more del files than the number that is allowed, merge it firstly.
    int offset = 0;
    List<Path> paths = new ArrayList<>();
    while (offset < delFilePaths.size()) {
        // get the batch
        int batch = compactionBatchSize;
        if (delFilePaths.size() - offset < compactionBatchSize) {
            batch = delFilePaths.size() - offset;
        }
        List<StoreFile> batchedDelFiles = new ArrayList<>();
        if (batch == 1) {
            // only one file left, do not compact it, directly add it to the new files.
            paths.add(delFilePaths.get(offset));
            offset++;
            continue;
        }
        for (int i = offset; i < batch + offset; i++) {
            batchedDelFiles.add(new StoreFile(fs, delFilePaths.get(i), conf, compactionCacheConfig, BloomType.NONE));
        }
        // compact the del files in a batch.
        paths.add(compactDelFilesInBatch(request, batchedDelFiles));
        // move to the next batch.
        offset += batch;
    }
    return compactDelFiles(request, paths);
}

Example 9 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class SnapshotManifest method addRegion.

/**
   * Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
   * This is used by the "online snapshot" when the table is enabled.
   */
public void addRegion(final HRegion region) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);
    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing '" + region + "' region-info for snapshot.");
    Object regionData = visitor.regionOpen(region.getRegionInfo());
    monitor.rethrowException();
    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");
    for (Store store : region.getStores()) {
        // 2.1. build the snapshot reference for the store
        Object familyData = visitor.familyOpen(regionData, store.getFamily().getName());
        monitor.rethrowException();
        List<StoreFile> storeFiles = new ArrayList<>(store.getStorefiles());
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
        }
        // 2.2. iterate through all the store's files and create "references".
        for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
            StoreFile storeFile = storeFiles.get(i);
            monitor.rethrowException();
            // create "reference" to this store file.
            LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): " + storeFile.getPath());
            visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
        }
        visitor.familyClose(regionData, familyData);
    }
    visitor.regionClose(regionData);
}

Example 10 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class TestAdmin1 method testHFileReplication.

/*
   * Test DFS replication for column families, where one CF has default replication(3) and the other
   * is set to 1.
   */
@Test(timeout = 300000)
public void testHFileReplication() throws Exception {
    final TableName tableName = TableName.valueOf(this.name.getMethodName());
    String fn1 = "rep1";
    HColumnDescriptor hcd1 = new HColumnDescriptor(fn1);
    hcd1.setDFSReplication((short) 1);
    String fn = "defaultRep";
    HColumnDescriptor hcd = new HColumnDescriptor(fn);
    HTableDescriptor htd = new HTableDescriptor(tableName);
    htd.addFamily(hcd);
    htd.addFamily(hcd1);
    Table table = TEST_UTIL.createTable(htd, null);
    TEST_UTIL.waitTableAvailable(tableName);
    Put p = new Put(Bytes.toBytes("defaultRep_rk"));
    byte[] q1 = Bytes.toBytes("q1");
    byte[] v1 = Bytes.toBytes("v1");
    p.addColumn(Bytes.toBytes(fn), q1, v1);
    List<Put> puts = new ArrayList<>(2);
    puts.add(p);
    p = new Put(Bytes.toBytes("rep1_rk"));
    p.addColumn(Bytes.toBytes(fn1), q1, v1);
    puts.add(p);
    try {
        table.put(puts);
        admin.flush(tableName);
        List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(tableName);
        for (HRegion r : regions) {
            Store store = r.getStore(Bytes.toBytes(fn));
            for (StoreFile sf : store.getStorefiles()) {
                assertTrue(sf.toString().contains(fn));
                assertTrue("Column family " + fn + " should have 3 copies", FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf.getFileInfo().getFileStatus().getReplication()));
            }
            store = r.getStore(Bytes.toBytes(fn1));
            for (StoreFile sf : store.getStorefiles()) {
                assertTrue(sf.toString().contains(fn1));
                assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo().getFileStatus().getReplication());
            }
        }
    } finally {
        if (admin.isTableEnabled(tableName)) {
            this.admin.disableTable(tableName);
            this.admin.deleteTable(tableName);
        }
    }
}