Examples with NamenodeStats com.facebook.presto.hive.NamenodeStats used on opensource projects

Example 1 with NamenodeStats

use of com.facebook.presto.hive.NamenodeStats in project presto by prestodb.

the class TestHiveFileIterator method testPathFilterWithRecursion.

@Test
public void testPathFilterWithRecursion() throws IOException {
    // set up
    File rootDir = createTempDir();
    String basePath = rootDir.getAbsolutePath();
    // create 8 files in root directory - 3 pathFilter matched and 5 non matched files.
    createFiles(basePath, 3, true);
    createFiles(basePath, 5, false);
    // create two directories
    List<File> subDirs = createDirs(basePath, 2);
    // create 5 files in dir1 - 3 pathFilter matched and 2 non matched files.
    String dir1 = subDirs.get(0).getAbsolutePath();
    createFiles(dir1, 3, true);
    createFiles(dir1, 2, false);
    // create 7 files in dir2 - 3 pathFilter matched and 4 non matched files.
    String dir2 = subDirs.get(1).getAbsolutePath();
    createFiles(dir2, 3, true);
    createFiles(dir2, 4, false);
    Path rootPath = new Path("file://" + basePath + File.separator);
    PathFilter pathFilter = path -> path.getName().contains(PATH_FILTER_MATCHED_PREFIX);
    HiveFileIterator hiveFileIterator = new HiveFileIterator(rootPath, listDirectoryOperation, new NamenodeStats(), RECURSE, pathFilter);
    int actualCount = Iterators.size(hiveFileIterator);
    assertEquals(actualCount, 9);
    // cleanup
    deleteTestDir(rootDir);
}

Example 2 with NamenodeStats

use of com.facebook.presto.hive.NamenodeStats in project presto by prestodb.

the class TestHiveFileIterator method testDefaultPathFilterWithRecursion.

@Test
public void testDefaultPathFilterWithRecursion() throws IOException {
    // set up
    File rootDir = createTempDir();
    String basePath = rootDir.getAbsolutePath();
    // create 8 files in root directory - 3 pathFilter matched and 5 non matched files.
    createFiles(basePath, 3, true);
    createFiles(basePath, 5, false);
    // create two directories
    List<File> subDirs = createDirs(basePath, 2);
    // create 5 files in dir1 - 3 pathFilter matched and 2 non matched files.
    String dir1 = subDirs.get(0).getAbsolutePath();
    createFiles(dir1, 3, true);
    createFiles(dir1, 2, false);
    // create 7 files in dir2 - 3 pathFilter matched and 4 non matched files.
    String dir2 = subDirs.get(1).getAbsolutePath();
    createFiles(dir2, 3, true);
    createFiles(dir2, 4, false);
    Path rootPath = new Path("file://" + basePath + File.separator);
    PathFilter pathFilter = path -> true;
    HiveFileIterator hiveFileIterator = new HiveFileIterator(rootPath, listDirectoryOperation, new NamenodeStats(), RECURSE, pathFilter);
    int actualCount = Iterators.size(hiveFileIterator);
    assertEquals(actualCount, 20);
    // cleanup
    deleteTestDir(rootDir);
}

Example 3 with NamenodeStats

use of com.facebook.presto.hive.NamenodeStats in project presto by prestodb.

the class TestHiveFileIterator method testDefaultPathFilterNoRecursion.

@Test
public void testDefaultPathFilterNoRecursion() throws IOException {
    // set up
    File rootDir = createTempDir();
    String basePath = rootDir.getAbsolutePath();
    // create 8 files in root directory - 3 pathFilter matched and 5 non matched files.
    createFiles(basePath, 3, true);
    createFiles(basePath, 5, false);
    Path rootPath = new Path("file://" + basePath + File.separator);
    PathFilter pathFilter = path -> true;
    HiveFileIterator hiveFileIterator = new HiveFileIterator(rootPath, listDirectoryOperation, new NamenodeStats(), IGNORED, pathFilter);
    int actualCount = Iterators.size(hiveFileIterator);
    assertEquals(actualCount, 8);
    // cleanup
    deleteTestDir(rootDir);
}

Example 4 with NamenodeStats

use of com.facebook.presto.hive.NamenodeStats in project presto by prestodb.

the class TestHiveFileIterator method testPathFilterWithNoRecursion.

@Test
public void testPathFilterWithNoRecursion() throws IOException {
    // set up
    File rootDir = createTempDir();
    String basePath = rootDir.getAbsolutePath();
    // create 8 files in root directory - 3 pathFilter matched and 5 non matched files.
    createFiles(basePath, 3, true);
    createFiles(basePath, 5, false);
    Path rootPath = new Path("file://" + basePath + File.separator);
    PathFilter pathFilter = path -> path.getName().contains(PATH_FILTER_MATCHED_PREFIX);
    HiveFileIterator hiveFileIterator = new HiveFileIterator(rootPath, listDirectoryOperation, new NamenodeStats(), IGNORED, pathFilter);
    int actualCount = Iterators.size(hiveFileIterator);
    assertEquals(actualCount, 3);
    // cleanup
    deleteTestDir(rootDir);
}