Examples with Path java.nio.file.Path used on opensource projects

Example 11 with Path

use of java.nio.file.Path in project storm by apache.

the class DirectoryCleaner method deleteOldestWhileTooLarge.

/**
     * If totalSize of files exceeds the either the per-worker quota or global quota,
     * Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs.
     * We use the parameter for_per_dir to switch between the two deletion modes.
     * @param dirs the list of directories to be scanned for deletion
     * @param quota the per-dir quota or the total quota for the all directories
     * @param for_per_dir if true, deletion happens for a single dir; otherwise, for all directories globally
     * @param active_dirs only for global deletion, we want to skip the active logs in active_dirs
     * @return number of files deleted
     */
public int deleteOldestWhileTooLarge(List<File> dirs, long quota, boolean for_per_dir, Set<String> active_dirs) throws IOException {
    // max number of files to delete for every round
    final int PQ_SIZE = 1024;
    // max rounds of scanning the dirs
    final int MAX_ROUNDS = 512;
    long totalSize = 0;
    int deletedFiles = 0;
    for (File dir : dirs) {
        try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
            for (Path path : stream) {
                File file = path.toFile();
                totalSize += file.length();
            }
        }
    }
    long toDeleteSize = totalSize - quota;
    if (toDeleteSize <= 0) {
        return deletedFiles;
    }
    Comparator<File> comparator = new Comparator<File>() {

        public int compare(File f1, File f2) {
            if (f1.lastModified() > f2.lastModified()) {
                return -1;
            } else {
                return 1;
            }
        }
    };
    // the oldest pq_size files in this directory will be placed in PQ, with the newest at the root
    PriorityQueue<File> pq = new PriorityQueue<File>(PQ_SIZE, comparator);
    int round = 0;
    while (toDeleteSize > 0) {
        LOG.debug("To delete size is {}, start a new round of deletion, round: {}", toDeleteSize, round);
        for (File dir : dirs) {
            try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
                for (Path path : stream) {
                    File file = path.toFile();
                    if (for_per_dir) {
                        if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
                            // skip active log files
                            continue;
                        }
                    } else {
                        // for global cleanup
                        if (active_dirs.contains(dir.getCanonicalPath())) {
                            // for an active worker's dir, make sure for the last "/"
                            if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
                                // skip active log files
                                continue;
                            }
                        } else {
                            if (META_LOG_PATTERN.matcher(file.getName()).matches()) {
                                // skip yaml and pid files
                                continue;
                            }
                        }
                    }
                    if (pq.size() < PQ_SIZE) {
                        pq.offer(file);
                    } else {
                        if (file.lastModified() < pq.peek().lastModified()) {
                            pq.poll();
                            pq.offer(file);
                        }
                    }
                }
            }
        }
        // need to reverse the order of elements in PQ to delete files from oldest to newest
        Stack<File> stack = new Stack<File>();
        while (!pq.isEmpty()) {
            File file = pq.poll();
            stack.push(file);
        }
        while (!stack.isEmpty() && toDeleteSize > 0) {
            File file = stack.pop();
            toDeleteSize -= file.length();
            LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getCanonicalPath(), file.length(), file.lastModified());
            file.delete();
            deletedFiles++;
        }
        pq.clear();
        round++;
        if (round >= MAX_ROUNDS) {
            if (for_per_dir) {
                LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " + "a single directory : {}, will delete the rest files in next interval.", MAX_ROUNDS, dirs.get(0).getCanonicalPath());
            } else {
                LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " + "will delete the rest files in next interval.", MAX_ROUNDS);
            }
            break;
        }
    }
    return deletedFiles;
}

Example 12 with Path

use of java.nio.file.Path in project storm by apache.

the class DirectoryCleaner method getFilesForDir.

// Note that to avoid memory problem, we only return the first 1024 files in a directory
public static List<File> getFilesForDir(File dir) throws IOException {
    List<File> files = new ArrayList<File>();
    final int MAX_NUM = 1024;
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath())) {
        for (Path path : stream) {
            files.add(path.toFile());
            if (files.size() >= MAX_NUM) {
                break;
            }
        }
    }
    return files;
}

Example 13 with Path

use of java.nio.file.Path in project storm by apache.

the class AdvancedFSOps method deleteIfExists.

/**
     * Delete a file or a directory and all of the children. If it exists.
     * @param path what to delete
     * @throws IOException on any error.
     */
public void deleteIfExists(File path) throws IOException {
    LOG.info("Deleting path {}", path);
    Path p = path.toPath();
    if (Files.exists(p)) {
        try {
            FileUtils.forceDelete(path);
        } catch (FileNotFoundException ignored) {
        }
    }
}

Example 14 with Path

use of java.nio.file.Path in project storm by apache.

the class AdvancedFSOps method createSymlink.

/**
     * Create a symbolic link pointing at target
     * @param link the link to create
     * @param target where it should point to
     * @throws IOException on any error.
     */
public void createSymlink(File link, File target) throws IOException {
    Path plink = link.toPath().toAbsolutePath();
    Path ptarget = target.toPath().toAbsolutePath();
    LOG.debug("Creating symlink [{}] to [{}]", plink, ptarget);
    if (Files.exists(plink)) {
        if (Files.isSameFile(plink, ptarget)) {
            //It already points where we want it to
            return;
        }
        FileUtils.forceDelete(link);
    }
    Files.createSymbolicLink(plink, ptarget);
}

Example 15 with Path

use of java.nio.file.Path in project hbase by apache.

the class TestUpdateConfiguration method testAllCustomOnlineConfigChange.

@Test
public void testAllCustomOnlineConfigChange() throws IOException {
    LOG.debug("Starting the test");
    Path cnfPath = FileSystems.getDefault().getPath("target/test-classes/hbase-site.xml");
    Path cnf2Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site2.xml");
    Path cnf3Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site3.xml");
    // make a backup of hbase-site.xml
    Files.copy(cnfPath, cnf3Path, StandardCopyOption.REPLACE_EXISTING);
    // update hbase-site.xml by overwriting it
    Files.copy(cnf2Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
    Admin admin = TEST_UTIL.getAdmin();
    admin.updateConfiguration();
    // Check the configuration of the Masters
    Configuration masterConfiguration = TEST_UTIL.getMiniHBaseCluster().getMaster(0).getConfiguration();
    int custom = masterConfiguration.getInt("hbase.custom.config", 0);
    assertEquals(custom, 1000);
    Configuration backupMasterConfiguration = TEST_UTIL.getMiniHBaseCluster().getMaster(1).getConfiguration();
    custom = backupMasterConfiguration.getInt("hbase.custom.config", 0);
    assertEquals(custom, 1000);
    // Check the configuration of the RegionServer
    Configuration regionServerConfiguration = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getConfiguration();
    custom = regionServerConfiguration.getInt("hbase.custom.config", 0);
    assertEquals(custom, 1000);
    // restore hbase-site.xml
    Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
}