Examples with FileDepthSelector org.apache.commons.vfs2.FileDepthSelector used on opensource projects

Example 1 with FileDepthSelector

use of org.apache.commons.vfs2.FileDepthSelector in project pentaho-metaverse by pentaho.

the class VfsLineageCollector method listArtifacts.

@Override
public List<String> listArtifacts(final String startingDate, final String endingDate) throws IllegalArgumentException {
    List<String> paths = new ArrayList<>();
    try {
        FileSystemOptions opts = new FileSystemOptions();
        FileObject lineageRootFolder = KettleVFS.getFileObject(getOutputFolder(), opts);
        FileSelector dateRangeFilter = new VfsDateRangeFilter(format, startingDate, endingDate);
        FileSelector depthFilter = new FileDepthSelector(1, 256);
        if (lineageRootFolder.exists() && lineageRootFolder.getType() == FileType.FOLDER) {
            // get the folders that come on or after the startingDate
            FileObject[] dayFolders = lineageRootFolder.findFiles(dateRangeFilter);
            for (FileObject dayFolder : dayFolders) {
                FileObject[] listThisFolder = dayFolder.findFiles(depthFilter);
                for (FileObject currentFile : listThisFolder) {
                    if (currentFile.getType() == FileType.FILE) {
                        paths.add(currentFile.getName().getPath());
                    }
                }
            }
        }
        return paths;
    } catch (Exception e) {
        throw new IllegalArgumentException(e);
    }
}

Example 2 with FileDepthSelector

use of org.apache.commons.vfs2.FileDepthSelector in project pentaho-metaverse by pentaho.

the class VfsLineageCollector method listArtifactsForFile.

@Override
public List<String> listArtifactsForFile(String pathToArtifact, String startingDate, String endingDate) throws IllegalArgumentException {
    List<String> paths = new ArrayList<>();
    try {
        FileSystemOptions opts = new FileSystemOptions();
        FileObject lineageRootFolder = KettleVFS.getFileObject(getOutputFolder(), opts);
        FileSelector dateRangeFilter = new VfsDateRangeFilter(format, startingDate, endingDate);
        FileSelector depthFilter = new FileDepthSelector(1, 256);
        if (lineageRootFolder.exists() && lineageRootFolder.getType() == FileType.FOLDER) {
            // get all of the date folders of lineage we have
            FileObject[] dayFolders = lineageRootFolder.findFiles(dateRangeFilter);
            for (FileObject dayFolder : dayFolders) {
                FileObject[] listThisFolder = dayFolder.findFiles(depthFilter);
                for (FileObject currentFile : listThisFolder) {
                    FileObject requested = currentFile.resolveFile(pathToArtifact);
                    if (requested.exists() && requested.getType() == FileType.FOLDER) {
                        FileObject[] requestedChildren = requested.getChildren();
                        for (FileObject requestedChild : requestedChildren) {
                            if (requestedChild.getType() == FileType.FILE) {
                                paths.add(requestedChild.getName().getPath());
                            }
                        }
                    }
                }
            }
        }
        return paths;
    } catch (Exception e) {
        throw new IllegalArgumentException(e);
    }
}

Example 3 with FileDepthSelector

use of org.apache.commons.vfs2.FileDepthSelector in project pentaho-hadoop-shims by pentaho.

the class DistributedCacheUtilImpl method stageBigDataPlugin.

/**
 * Move files from the source folder to the destination folder, overwriting any files that may already exist there.
 *
 * @param fs           File system to write to
 * @param dest         Destination to move source file/folder into
 * @param pluginFolder Big Data plugin folder
 * @throws KettleFileException
 * @throws IOException
 */
private void stageBigDataPlugin(FileSystem fs, Path dest, FileObject pluginFolder, String shimIdentifier) throws KettleFileException, IOException {
    Path pluginsDir = new Path(dest, PATH_PLUGINS);
    Path bigDataPluginDir = new Path(pluginsDir, pluginFolder.getName().getBaseName());
    // Stage everything except the hadoop-configurations and pmr libraries
    for (FileObject f : pluginFolder.findFiles(new FileDepthSelector(1, 1))) {
        if (!"hadoop-configurations".equals(f.getName().getBaseName()) && !"pentaho-mapreduce-libraries.zip".equals(f.getName().getBaseName())) {
            stageForCache(f, fs, new Path(bigDataPluginDir, f.getName().getBaseName()), "", true, false);
        }
    }
    FileObject pmrLibsDir = null;
    FileObject hadoopConfigurationsDir = pluginFolder.getChild("hadoop-configurations");
    if (hadoopConfigurationsDir != null) {
        FileObject shimDir = hadoopConfigurationsDir.getChild(shimIdentifier);
        if (shimDir != null) {
            FileObject shimLibsDir = shimDir.getChild("lib");
            if (shimLibsDir != null) {
                pmrLibsDir = shimLibsDir.getChild("pmr");
            }
        }
    }
    Path pdiLib = new Path(dest, "lib");
    if (pmrLibsDir != null) {
        for (FileObject f : pmrLibsDir.getChildren()) {
            stageForCache(f, fs, new Path(pdiLib, f.getName().getBaseName()), "", true, false);
        }
    }
}