Examples with Reference org.apache.hadoop.hbase.io.Reference used on opensource projects

Example 1 with Reference

use of org.apache.hadoop.hbase.io.Reference in project hbase by apache.

the class RestoreSnapshotHelper method restoreReferenceFile.

/**
   * Create a new {@link Reference} as copy of the source one.
   * <p><blockquote><pre>
   * The source table looks like:
   *    1234/abc      (original file)
   *    5678/abc.1234 (reference file)
   *
   * After the clone operation looks like:
   *   wxyz/table=1234-abc
   *   stuv/table=1234-abc.wxyz
   *
   * NOTE that the region name in the clone changes (md5 of regioninfo)
   * and the reference should reflect that change.
   * </pre></blockquote>
   * @param familyDir destination directory for the store file
   * @param regionInfo destination region info for the table
   * @param storeFile reference file name
   */
private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
    String hfileName = storeFile.getName();
    // Extract the referred information (hfile name and parent region)
    Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(new Path(snapshotTable.getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo.getEncodedName()), familyDir.getName()), hfileName));
    String snapshotRegionName = refPath.getParent().getParent().getName();
    String fileName = refPath.getName();
    // The new reference should have the cloned region name as parent, if it is a clone.
    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
    if (clonedRegionName == null)
        clonedRegionName = snapshotRegionName;
    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
    Path linkPath = null;
    String refLink = fileName;
    if (!HFileLink.isHFileLink(fileName)) {
        refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
        linkPath = new Path(familyDir, HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
    }
    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
    // Create the new reference
    if (storeFile.hasReference()) {
        Reference reference = Reference.convert(storeFile.getReference());
        reference.write(fs, outPath);
    } else {
        InputStream in;
        if (linkPath != null) {
            in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
        } else {
            linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(), regionInfo.getEncodedName()), familyDir.getName()), hfileName);
            in = fs.open(linkPath);
        }
        OutputStream out = fs.create(outPath);
        IOUtils.copyBytes(in, out, conf);
    }
    // Add the daughter region to the map
    String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
    LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
    synchronized (parentsMap) {
        Pair<String, String> daughters = parentsMap.get(clonedRegionName);
        if (daughters == null) {
            daughters = new Pair<>(regionName, null);
            parentsMap.put(clonedRegionName, daughters);
        } else if (!regionName.equals(daughters.getFirst())) {
            daughters.setSecond(regionName);
        }
    }
}

Example 2 with Reference

use of org.apache.hadoop.hbase.io.Reference in project hbase by apache.

the class TestCatalogJanitor method createReferences.

/**
   * @param services Master services instance.
   * @param htd
   * @param parent
   * @param daughter
   * @param midkey
   * @param top True if we are to write a 'top' reference.
   * @return Path to reference we created.
   * @throws IOException
   */
private Path createReferences(final MasterServices services, final HTableDescriptor htd, final HRegionInfo parent, final HRegionInfo daughter, final byte[] midkey, final boolean top) throws IOException {
    Path rootdir = services.getMasterFileSystem().getRootDir();
    Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
    Path storedir = HStore.getStoreHomedir(tabledir, daughter, htd.getColumnFamilies()[0].getName());
    Reference ref = top ? Reference.createTopReference(midkey) : Reference.createBottomReference(midkey);
    long now = System.currentTimeMillis();
    // Reference name has this format: StoreFile#REF_NAME_PARSER
    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
    FileSystem fs = services.getMasterFileSystem().getFileSystem();
    ref.write(fs, p);
    return p;
}

Example 3 with Reference

use of org.apache.hadoop.hbase.io.Reference in project hbase by apache.

the class TestCatalogJanitor method testCleanParent.

@Test
public void testCleanParent() throws IOException, InterruptedException {
    HBaseTestingUtility htu = new HBaseTestingUtility();
    setRootDirAndCleanIt(htu, "testCleanParent");
    MasterServices services = new MockMasterServices(htu);
    try {
        CatalogJanitor janitor = new CatalogJanitor(services);
        // Create regions.
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
        htd.addFamily(new HColumnDescriptor("f"));
        HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
        HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
        HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
        // Test that when both daughter regions are in place, that we do not
        // remove the parent.
        Result r = createResult(parent, splita, splitb);
        // Add a reference under splitA directory so we don't clear out the parent.
        Path rootdir = services.getMasterFileSystem().getRootDir();
        Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
        Path storedir = HStore.getStoreHomedir(tabledir, splita, htd.getColumnFamilies()[0].getName());
        Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
        long now = System.currentTimeMillis();
        // Reference name has this format: StoreFile#REF_NAME_PARSER
        Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
        FileSystem fs = services.getMasterFileSystem().getFileSystem();
        Path path = ref.write(fs, p);
        assertTrue(fs.exists(path));
        assertFalse(janitor.cleanParent(parent, r));
        // Remove the reference file and try again.
        assertTrue(fs.delete(p, true));
        assertTrue(janitor.cleanParent(parent, r));
    } finally {
        services.stop("shutdown");
    }
}

Example 4 with Reference

use of org.apache.hadoop.hbase.io.Reference in project hbase by apache.

the class HRegionFileSystem method mergeStoreFile.

/**
   * Write out a merge reference under the given merges directory. Package local
   * so it doesnt leak out of regionserver.
   * @param mergedRegion {@link HRegionInfo} of the merged region
   * @param familyName Column Family Name
   * @param f File to create reference.
   * @param mergedDir
   * @return Path to created reference.
   * @throws IOException
   */
public Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName, final StoreFile f, final Path mergedDir) throws IOException {
    Path referenceDir = new Path(new Path(mergedDir, mergedRegion.getEncodedName()), familyName);
    // A whole reference to the store file.
    Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above. The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String mergingRegionName = regionInfoForFs.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName);
    return r.write(fs, p);
}

Example 5 with Reference

use of org.apache.hadoop.hbase.io.Reference in project hbase by apache.

the class HRegionFileSystem method splitStoreFile.

/**
   * Write out a split reference. Package local so it doesnt leak out of
   * regionserver.
   * @param hri {@link HRegionInfo} of the destination
   * @param familyName Column Family Name
   * @param f File to split.
   * @param splitRow Split Row
   * @param top True if we are referring to the top half of the hfile.
   * @param splitPolicy
   * @return Path to created reference.
   * @throws IOException
   */
public Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f, final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy) throws IOException {
    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
        // If it is outside the range, return directly.
        if (f.getReader() == null) {
            f.createReader();
        }
        try {
            if (top) {
                //check if larger than last key.
                Cell splitKey = CellUtil.createFirstOnRow(splitRow);
                Cell lastKey = f.getLastKey();
                // If lastKey is null means storefile is empty.
                if (lastKey == null) {
                    return null;
                }
                if (f.getComparator().compare(splitKey, lastKey) > 0) {
                    return null;
                }
            } else {
                //check if smaller than first key
                Cell splitKey = CellUtil.createLastOnRow(splitRow);
                Cell firstKey = f.getFirstKey();
                // If firstKey is null means storefile is empty.
                if (firstKey == null) {
                    return null;
                }
                if (f.getComparator().compare(splitKey, firstKey) < 0) {
                    return null;
                }
            }
        } finally {
            f.closeReader(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true);
        }
    }
    Path splitDir = new Path(getSplitsDir(hri), familyName);
    // A reference to the bottom half of the hsf store file.
    Reference r = top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = regionInfoForFs.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
}