Examples with FilenameFilter java.io.FilenameFilter used on opensource projects

Example 31 with FilenameFilter

use of java.io.FilenameFilter in project flink by apache.

the class TestBaseUtils method getAllInvolvedFiles.

private static File[] getAllInvolvedFiles(String resultPath, final String[] excludePrefixes) {
    final File result = asFile(resultPath);
    assertTrue("Result file was not written", result.exists());
    if (result.isDirectory()) {
        return result.listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                for (String p : excludePrefixes) {
                    if (name.startsWith(p)) {
                        return false;
                    }
                }
                return true;
            }
        });
    } else {
        return new File[] { result };
    }
}

Example 32 with FilenameFilter

use of java.io.FilenameFilter in project flink by apache.

the class YARNSessionCapacitySchedulerITCase method testDetachedPerJobYarnClusterInternal.

private void testDetachedPerJobYarnClusterInternal(String job) {
    YarnClient yc = YarnClient.createYarnClient();
    yc.init(yarnConfiguration);
    yc.start();
    // get temporary folder for writing output of wordcount example
    File tmpOutFolder = null;
    try {
        tmpOutFolder = tmp.newFolder();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    // get temporary file for reading input data for wordcount example
    File tmpInFile;
    try {
        tmpInFile = tmp.newFile();
        FileUtils.writeStringToFile(tmpInFile, WordCountData.TEXT);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    Runner runner = startWithArgs(new String[] { "run", "-m", "yarn-cluster", "-yj", flinkUberjar.getAbsolutePath(), "-yt", flinkLibFolder.getAbsolutePath(), "-yn", "1", "-yjm", "768", // test if the cutoff is passed correctly
    "-yD", // test if the cutoff is passed correctly
    "yarn.heap-cutoff-ratio=0.5", "-yD", "yarn.tags=test-tag", "-ytm", "1024", // test requesting slots from YARN.
    "-ys", // test requesting slots from YARN.
    "2", "--yarndetached", job, "--input", tmpInFile.getAbsoluteFile().toString(), "--output", tmpOutFolder.getAbsoluteFile().toString() }, "Job has been submitted with JobID", RunTypes.CLI_FRONTEND);
    // it should usually be 2, but on slow machines, the number varies
    Assert.assertTrue("There should be at most 2 containers running", getRunningContainers() <= 2);
    // give the runner some time to detach
    for (int attempt = 0; runner.isAlive() && attempt < 5; attempt++) {
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
        }
    }
    Assert.assertFalse("The runner should detach.", runner.isAlive());
    LOG.info("CLI Frontend has returned, so the job is running");
    // find out the application id and wait until it has finished.
    try {
        List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
        ApplicationId tmpAppId;
        if (apps.size() == 1) {
            // Better method to find the right appId. But sometimes the app is shutting down very fast
            // Only one running
            tmpAppId = apps.get(0).getApplicationId();
            LOG.info("waiting for the job with appId {} to finish", tmpAppId);
            // wait until the app has finished
            while (yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING)).size() > 0) {
                sleep(500);
            }
        } else {
            // get appId by finding the latest finished appid
            apps = yc.getApplications();
            Collections.sort(apps, new Comparator<ApplicationReport>() {

                @Override
                public int compare(ApplicationReport o1, ApplicationReport o2) {
                    return o1.getApplicationId().compareTo(o2.getApplicationId()) * -1;
                }
            });
            tmpAppId = apps.get(0).getApplicationId();
            LOG.info("Selected {} as the last appId from {}", tmpAppId, Arrays.toString(apps.toArray()));
        }
        final ApplicationId id = tmpAppId;
        // now it has finished.
        // check the output files.
        File[] listOfOutputFiles = tmpOutFolder.listFiles();
        Assert.assertNotNull("Taskmanager output not found", listOfOutputFiles);
        LOG.info("The job has finished. TaskManager output files found in {}", tmpOutFolder);
        // read all output files in output folder to one output string
        String content = "";
        for (File f : listOfOutputFiles) {
            if (f.isFile()) {
                content += FileUtils.readFileToString(f) + "\n";
            }
        }
        //String content = FileUtils.readFileToString(taskmanagerOut);
        // check for some of the wordcount outputs.
        Assert.assertTrue("Expected string 'da 5' or '(all,2)' not found in string '" + content + "'", content.contains("da 5") || content.contains("(da,5)") || content.contains("(all,2)"));
        Assert.assertTrue("Expected string 'der 29' or '(mind,1)' not found in string'" + content + "'", content.contains("der 29") || content.contains("(der,29)") || content.contains("(mind,1)"));
        // check if the heap size for the TaskManager was set correctly
        File jobmanagerLog = YarnTestBase.findFile("..", new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                return name.contains("jobmanager.log") && dir.getAbsolutePath().contains(id.toString());
            }
        });
        Assert.assertNotNull("Unable to locate JobManager log", jobmanagerLog);
        content = FileUtils.readFileToString(jobmanagerLog);
        // TM was started with 1024 but we cut off 50% (NOT THE DEFAULT VALUE)
        String expected = "Starting TaskManagers with command: $JAVA_HOME/bin/java -Xms424m -Xmx424m";
        Assert.assertTrue("Expected string '" + expected + "' not found in JobManager log: '" + jobmanagerLog + "'", content.contains(expected));
        expected = " (2/2) (attempt #0) to ";
        Assert.assertTrue("Expected string '" + expected + "' not found in JobManager log." + "This string checks that the job has been started with a parallelism of 2. Log contents: '" + jobmanagerLog + "'", content.contains(expected));
        // make sure the detached app is really finished.
        LOG.info("Checking again that app has finished");
        ApplicationReport rep;
        do {
            sleep(500);
            rep = yc.getApplicationReport(id);
            LOG.info("Got report {}", rep);
        } while (rep.getYarnApplicationState() == YarnApplicationState.RUNNING);
        verifyApplicationTags(rep);
    } catch (Throwable t) {
        LOG.warn("Error while detached yarn session was running", t);
        Assert.fail(t.getMessage());
    } finally {
        //cleanup the yarn-properties file
        String confDirPath = System.getenv("FLINK_CONF_DIR");
        File configDirectory = new File(confDirPath);
        LOG.info("testDetachedPerJobYarnClusterInternal: Using configuration directory " + configDirectory.getAbsolutePath());
        // load the configuration
        LOG.info("testDetachedPerJobYarnClusterInternal: Trying to load configuration file");
        GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());
        try {
            File yarnPropertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(GlobalConfiguration.loadConfiguration());
            if (yarnPropertiesFile.exists()) {
                LOG.info("testDetachedPerJobYarnClusterInternal: Cleaning up temporary Yarn address reference: {}", yarnPropertiesFile.getAbsolutePath());
                yarnPropertiesFile.delete();
            }
        } catch (Exception e) {
            LOG.warn("testDetachedPerJobYarnClusterInternal: Exception while deleting the JobManager address file", e);
        }
    }
}

Example 33 with FilenameFilter

use of java.io.FilenameFilter in project groovy by apache.

the class Groovydoc method parsePackages.

/**
     * Add the directories matched by the nested dirsets to the resulting
     * packages list and the base directories of the dirsets to the Path.
     * It also handles the packages and excludepackages attributes and
     * elements.
     *
     * @param resultantPackages a list to which we add the packages found
     * @param sourcePath a path to which we add each basedir found
     * @since 1.5
     */
private void parsePackages(List<String> resultantPackages, Path sourcePath) {
    List<String> addedPackages = new ArrayList<String>();
    List<DirSet> dirSets = new ArrayList<DirSet>(packageSets);
    // and nested excludepackage elements
    if (this.sourcePath != null) {
        PatternSet ps = new PatternSet();
        if (!packageNames.isEmpty()) {
            for (String pn : packageNames) {
                String pkg = pn.replace('.', '/');
                if (pkg.endsWith("*")) {
                    pkg += "*";
                }
                ps.createInclude().setName(pkg);
            }
        } else {
            ps.createInclude().setName("**");
        }
        for (String epn : excludePackageNames) {
            String pkg = epn.replace('.', '/');
            if (pkg.endsWith("*")) {
                pkg += "*";
            }
            ps.createExclude().setName(pkg);
        }
        String[] pathElements = this.sourcePath.list();
        for (String pathElement : pathElements) {
            File dir = new File(pathElement);
            if (dir.isDirectory()) {
                DirSet ds = new DirSet();
                ds.setDefaultexcludes(useDefaultExcludes);
                ds.setDir(dir);
                ds.createPatternSet().addConfiguredPatternset(ps);
                dirSets.add(ds);
            } else {
                log.warn("Skipping " + pathElement + " since it is no directory.");
            }
        }
    }
    for (DirSet ds : dirSets) {
        File baseDir = ds.getDir(getProject());
        log.debug("scanning " + baseDir + " for packages.");
        DirectoryScanner dsc = ds.getDirectoryScanner(getProject());
        String[] dirs = dsc.getIncludedDirectories();
        boolean containsPackages = false;
        for (String dir : dirs) {
            // are there any groovy or java files in this directory?
            File pd = new File(baseDir, dir);
            String[] files = pd.list(new FilenameFilter() {

                public boolean accept(File dir1, String name) {
                    if (!includeNoSourcePackages && name.equals("package.html"))
                        return true;
                    final StringTokenizer tokenizer = new StringTokenizer(extensions, ":");
                    while (tokenizer.hasMoreTokens()) {
                        String ext = tokenizer.nextToken();
                        if (name.endsWith(ext))
                            return true;
                    }
                    return false;
                }
            });
            for (String filename : Arrays.asList(files)) {
                sourceFilesToDoc.add(dir + File.separator + filename);
            }
            if (files.length > 0) {
                if ("".equals(dir)) {
                    log.warn(baseDir + " contains source files in the default package," + " you must specify them as source files not packages.");
                } else {
                    containsPackages = true;
                    String pn = dir.replace(File.separatorChar, '.');
                    if (!addedPackages.contains(pn)) {
                        addedPackages.add(pn);
                        resultantPackages.add(pn);
                    }
                }
            }
        }
        if (containsPackages) {
            // We don't need to care for duplicates here,
            // Path.list does it for us.
            sourcePath.createPathElement().setLocation(baseDir);
        } else {
            log.verbose(baseDir + " doesn't contain any packages, dropping it.");
        }
    }
}

Example 34 with FilenameFilter

use of java.io.FilenameFilter in project realm-java by realm.

the class RealmTests method namedPipeDirForExternalStorage.

@Test
public void namedPipeDirForExternalStorage() {
    // Test for https://github.com/realm/realm-java/issues/3140
    realm.close();
    realm = null;
    final File namedPipeDir = SharedRealm.getTemporaryDirectory();
    assertTrue(namedPipeDir.isDirectory());
    TestHelper.deleteRecursively(namedPipeDir);
    //noinspection ResultOfMethodCallIgnored
    namedPipeDir.mkdirs();
    final File externalFilesDir = context.getExternalFilesDir(null);
    final RealmConfiguration config = new RealmConfiguration.Builder().directory(externalFilesDir).name("external.realm").build();
    Realm.deleteRealm(config);
    // Test if it works when the namedPipeDir is empty.
    Realm realmOnExternalStorage = Realm.getInstance(config);
    realmOnExternalStorage.close();
    assertTrue(namedPipeDir.isDirectory());
    Assume.assumeTrue("SELinux is not enforced on this device.", TestHelper.isSelinuxEnforcing());
    // Only checks the fifo file created by call, since all Realm instances share the same fifo created by
    // external_commit_helper which might not be created in the newly created dir if there are Realm instances
    // are not deleted when TestHelper.deleteRecursively(namedPipeDir) called.
    File[] files = namedPipeDir.listFiles(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.matches("realm_.*cv");
        }
    });
    assertEquals(2, files.length);
    // Tests if it works when the namedPipeDir and the named pipe files already exist.
    realmOnExternalStorage = Realm.getInstance(config);
    realmOnExternalStorage.close();
}

Example 35 with FilenameFilter

use of java.io.FilenameFilter in project jetty.project by eclipse.

the class FileSessionDataStore method doGetExpired.

/** 
     * @see org.eclipse.jetty.server.session.SessionDataStore#getExpired(Set)
     */
@Override
public Set<String> doGetExpired(final Set<String> candidates) {
    final long now = System.currentTimeMillis();
    HashSet<String> expired = new HashSet<String>();
    File[] files = _storeDir.listFiles(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            if (dir != _storeDir)
                return false;
            //dir may contain files that don't match our naming pattern
            int index = name.indexOf('_');
            if (index < 0)
                return false;
            try {
                long expiry = Long.parseLong(name.substring(0, index));
                return expiry > 0 && expiry < now;
            } catch (NumberFormatException e) {
                return false;
            }
        }
    });
    if (files != null) {
        for (File f : files) {
            expired.add(getIdFromFile(f));
        }
    }
    //longer exist and they should be expired
    for (String c : candidates) {
        if (!expired.contains(c)) {
            //check if the file exists
            File f = getFile(_storeDir, c);
            if (f == null || !f.exists())
                expired.add(c);
        }
    }
    return expired;
}