Examples with PathFilter org.apache.hadoop.fs.PathFilter used on opensource projects

Example 76 with PathFilter

use of org.apache.hadoop.fs.PathFilter in project hive by apache.

the class MapReduceCompiler method decideExecMode.

@Override
protected void decideExecMode(List<Task<? extends Serializable>> rootTasks, Context ctx, GlobalLimitCtx globalLimitCtx) throws SemanticException {
    // bypass for explain queries for now
    if (ctx.isExplainSkipExecution()) {
        return;
    }
    // user has told us to run in local mode or doesn't want auto-local mode
    if (ctx.isLocalOnlyExecutionMode() || !conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) {
        return;
    }
    final Context lCtx = ctx;
    PathFilter p = new PathFilter() {

        @Override
        public boolean accept(Path file) {
            return !lCtx.isMRTmpFileURI(file.toUri().getPath());
        }
    };
    List<ExecDriver> mrtasks = Utilities.getMRTasks(rootTasks);
    // map-reduce jobs will be run locally based on data size
    // first find out if any of the jobs needs to run non-locally
    boolean hasNonLocalJob = false;
    for (ExecDriver mrtask : mrtasks) {
        try {
            ContentSummary inputSummary = Utilities.getInputSummary(ctx, mrtask.getWork().getMapWork(), p);
            int numReducers = getNumberOfReducers(mrtask.getWork(), conf);
            long estimatedInput;
            if (globalLimitCtx != null && globalLimitCtx.isEnable()) {
                // If the global limit optimization is triggered, we will
                // estimate input data actually needed based on limit rows.
                // estimated Input = (num_limit * max_size_per_row) * (estimated_map + 2)
                // 
                long sizePerRow = HiveConf.getLongVar(conf, HiveConf.ConfVars.HIVELIMITMAXROWSIZE);
                estimatedInput = (globalLimitCtx.getGlobalOffset() + globalLimitCtx.getGlobalLimit()) * sizePerRow;
                long minSplitSize = HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE);
                long estimatedNumMap = inputSummary.getLength() / minSplitSize + 1;
                estimatedInput = estimatedInput * (estimatedNumMap + 1);
            } else {
                estimatedInput = inputSummary.getLength();
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Task: " + mrtask.getId() + ", Summary: " + inputSummary.getLength() + "," + inputSummary.getFileCount() + "," + numReducers + ", estimated Input: " + estimatedInput);
            }
            if (MapRedTask.isEligibleForLocalMode(conf, numReducers, estimatedInput, inputSummary.getFileCount()) != null) {
                hasNonLocalJob = true;
                break;
            } else {
                mrtask.setLocalMode(true);
            }
        } catch (IOException e) {
            throw new SemanticException(e);
        }
    }
    if (!hasNonLocalJob) {
        // Entire query can be run locally.
        // Save the current tracker value and restore it when done.
        ctx.setOriginalTracker(ShimLoader.getHadoopShims().getJobLauncherRpcAddress(conf));
        ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, "local");
        console.printInfo("Automatically selecting local only mode for query");
    }
}

Example 77 with PathFilter

use of org.apache.hadoop.fs.PathFilter in project nifi by apache.

the class TestNotificationEventPathFilter method acceptPathShouldProperlyMatchAllSubdirectoriesThatMatchWatchDirectoryAndFileFilter.

@Test
public void acceptPathShouldProperlyMatchAllSubdirectoriesThatMatchWatchDirectoryAndFileFilter() throws Exception {
    PathFilter filter = new NotificationEventPathFilter(Pattern.compile("/root(/.*)?"), true);
    assertTrue(filter.accept(new Path("/root/sometest.txt")));
}

Example 78 with PathFilter

use of org.apache.hadoop.fs.PathFilter in project nifi by apache.

the class TestNotificationEventPathFilter method acceptPathShouldProperlyAcceptPathsWhereTheNonLastComponentStartsWithADot.

@Test
public void acceptPathShouldProperlyAcceptPathsWhereTheNonLastComponentStartsWithADot() throws Exception {
    PathFilter filter = new NotificationEventPathFilter(Pattern.compile(".*"), true);
    assertTrue(filter.accept(new Path("/some/long/path/.some_hidden_file/should/work")));
    assertTrue(filter.accept(new Path("/.some_hidden_file/should/still/accept")));
}

Example 79 with PathFilter

use of org.apache.hadoop.fs.PathFilter in project nifi by apache.

the class TestNotificationEventPathFilter method acceptPathShouldProperlyMatchWhenWatchDirectoryMatchesPath.

@Test
public void acceptPathShouldProperlyMatchWhenWatchDirectoryMatchesPath() throws Exception {
    PathFilter filter = new NotificationEventPathFilter(Pattern.compile("/root(/.*)?"), false);
    assertTrue(filter.accept(new Path("/root")));
}

Example 80 with PathFilter

use of org.apache.hadoop.fs.PathFilter in project drill by axbaretto.

the class DrillFileSystemUtilTest method testListDirectoriesWithFilter.

@Test
public void testListDirectoriesWithFilter() throws IOException {
    List<FileStatus> statuses = DrillFileSystemUtil.listDirectories(fs, base, false, new PathFilter() {

        @Override
        public boolean accept(Path path) {
            return path.getName().endsWith("a");
        }
    });
    assertEquals("Directory count should match", 1, statuses.size());
    assertEquals("Directory name should match", "a", statuses.get(0).getPath().getName());
}