Examples with HBaseTestingUtil org.apache.hadoop.hbase.HBaseTestingUtil used on opensource projects

Example 31 with HBaseTestingUtil

use of org.apache.hadoop.hbase.HBaseTestingUtil in project hbase by apache.

the class TestFileLink method testHDFSLinkReadDuringRename.

/**
 * Test, on HDFS, that the FileLink is still readable
 * even when the current file gets renamed.
 */
@Test
public void testHDFSLinkReadDuringRename() throws Exception {
    HBaseTestingUtil testUtil = new HBaseTestingUtil();
    Configuration conf = testUtil.getConfiguration();
    conf.setInt("dfs.blocksize", 1024 * 1024);
    conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024);
    testUtil.startMiniDFSCluster(1);
    MiniDFSCluster cluster = testUtil.getDFSCluster();
    FileSystem fs = cluster.getFileSystem();
    assertEquals("hdfs", fs.getUri().getScheme());
    try {
        testLinkReadDuringRename(fs, testUtil.getDefaultRootDirPath());
    } finally {
        testUtil.shutdownMiniCluster();
    }
}

Example 32 with HBaseTestingUtil

use of org.apache.hadoop.hbase.HBaseTestingUtil in project hbase by apache.

the class TestFileLink method testHDFSLinkReadDuringDelete.

/**
 * Test that link is still readable even when the current file gets deleted.
 *
 * NOTE: This test is valid only on HDFS.
 * When a file is deleted from a local file-system, it is simply 'unlinked'.
 * The inode, which contains the file's data, is not deleted until all
 * processes have finished with it.
 * In HDFS when the request exceed the cached block locations,
 * a query to the namenode is performed, using the filename,
 * and the deleted file doesn't exists anymore (FileNotFoundException).
 */
@Test
public void testHDFSLinkReadDuringDelete() throws Exception {
    HBaseTestingUtil testUtil = new HBaseTestingUtil();
    Configuration conf = testUtil.getConfiguration();
    conf.setInt("dfs.blocksize", 1024 * 1024);
    conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024);
    testUtil.startMiniDFSCluster(1);
    MiniDFSCluster cluster = testUtil.getDFSCluster();
    FileSystem fs = cluster.getFileSystem();
    assertEquals("hdfs", fs.getUri().getScheme());
    try {
        List<Path> files = new ArrayList<>();
        for (int i = 0; i < 3; i++) {
            Path path = new Path(String.format("test-data-%d", i));
            writeSomeData(fs, path, 1 << 20, (byte) i);
            files.add(path);
        }
        FileLink link = new FileLink(files);
        FSDataInputStream in = link.open(fs);
        try {
            byte[] data = new byte[8192];
            int n;
            // Switch to file 1
            n = in.read(data);
            dataVerify(data, n, (byte) 0);
            fs.delete(files.get(0), true);
            skipBuffer(in, (byte) 0);
            // Switch to file 2
            n = in.read(data);
            dataVerify(data, n, (byte) 1);
            fs.delete(files.get(1), true);
            skipBuffer(in, (byte) 1);
            // Switch to file 3
            n = in.read(data);
            dataVerify(data, n, (byte) 2);
            fs.delete(files.get(2), true);
            skipBuffer(in, (byte) 2);
            // No more files available
            try {
                n = in.read(data);
                assert (n <= 0);
            } catch (FileNotFoundException e) {
                assertTrue(true);
            }
        } finally {
            in.close();
        }
    } finally {
        testUtil.shutdownMiniCluster();
    }
}

Example 33 with HBaseTestingUtil

use of org.apache.hadoop.hbase.HBaseTestingUtil in project hbase by apache.

the class TestCacheConfig method testFileBucketCacheConfig.

@Test
public void testFileBucketCacheConfig() throws IOException {
    HBaseTestingUtil htu = new HBaseTestingUtil(this.conf);
    try {
        Path p = new Path(htu.getDataTestDir(), "bc.txt");
        FileSystem fs = FileSystem.get(this.conf);
        fs.create(p).close();
        this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "file:" + p);
        doBucketCacheConfigTest();
    } finally {
        htu.cleanupTestDir();
    }
}

Example 34 with HBaseTestingUtil

use of org.apache.hadoop.hbase.HBaseTestingUtil in project hbase by apache.

the class TestMobCompactionWithDefaults method htuStart.

@BeforeClass
public static void htuStart() throws Exception {
    HTU = new HBaseTestingUtil();
    conf = HTU.getConfiguration();
    conf.setInt("hfile.format.version", 3);
    // Disable automatic MOB compaction
    conf.setLong(MobConstants.MOB_COMPACTION_CHORE_PERIOD, 0);
    // Disable automatic MOB file cleaner chore
    conf.setLong(MobConstants.MOB_CLEANER_PERIOD, 0);
    // Set minimum age to archive to 10 sec
    conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, minAgeToArchive);
    // Set compacted file discharger interval to a half minAgeToArchive
    conf.setLong("hbase.hfile.compaction.discharger.interval", minAgeToArchive / 2);
    conf.setBoolean("hbase.regionserver.compaction.enabled", false);
    HTU.startMiniCluster();
}

Example 35 with HBaseTestingUtil

use of org.apache.hadoop.hbase.HBaseTestingUtil in project hbase by apache.

the class TestMobFileCleanerChore method setUp.

@Before
public void setUp() throws Exception {
    HTU = new HBaseTestingUtil();
    conf = HTU.getConfiguration();
    initConf();
    HTU.startMiniCluster();
    admin = HTU.getAdmin();
    chore = new MobFileCleanerChore();
    familyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(fam).setMobEnabled(true).setMobThreshold(mobLen).setMaxVersions(1).build();
    tableDescriptor = HTU.createModifyableTableDescriptor("testMobCompactTable").setColumnFamily(familyDescriptor).build();
    table = HTU.createTable(tableDescriptor, null);
}