Examples with AllFileSelector org.apache.commons.vfs2.AllFileSelector used on opensource projects

Example 1 with AllFileSelector

use of org.apache.commons.vfs2.AllFileSelector in project pentaho-kettle by pentaho.

the class JobEntryUnZip method unzipFile.

private boolean unzipFile(FileObject sourceFileObject, String realTargetdirectory, String realWildcard, String realWildcardExclude, Result result, Job parentJob, FileObject movetodir, String realMovetodirectory) {
    boolean retval = false;
    String unzipToFolder = realTargetdirectory;
    try {
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.ProcessingFile", sourceFileObject.toString()));
        }
        // 
        if (rootzip) {
            String shortSourceFilename = sourceFileObject.getName().getBaseName();
            int lenstring = shortSourceFilename.length();
            int lastindexOfDot = shortSourceFilename.lastIndexOf('.');
            if (lastindexOfDot == -1) {
                lastindexOfDot = lenstring;
            }
            String foldername = realTargetdirectory + "/" + shortSourceFilename.substring(0, lastindexOfDot);
            FileObject rootfolder = KettleVFS.getFileObject(foldername, this);
            if (!rootfolder.exists()) {
                try {
                    rootfolder.createFolder();
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.RootFolderCreated", foldername));
                    }
                } catch (Exception e) {
                    throw new Exception(BaseMessages.getString(PKG, "JobUnZip.Error.CanNotCreateRootFolder", foldername), e);
                }
            }
            unzipToFolder = foldername;
        }
        // Try to read the entries from the VFS object...
        // 
        String zipFilename = "zip:" + sourceFileObject.getName().getFriendlyURI();
        FileObject zipFile = KettleVFS.getFileObject(zipFilename, this);
        FileObject[] items = zipFile.findFiles(new AllFileSelector() {

            public boolean traverseDescendents(FileSelectInfo info) {
                return true;
            }

            public boolean includeFile(FileSelectInfo info) {
                // Never return the parent directory of a file list.
                if (info.getDepth() == 0) {
                    return false;
                }
                FileObject fileObject = info.getFile();
                return fileObject != null;
            }
        });
        Pattern pattern = null;
        if (!Utils.isEmpty(realWildcard)) {
            pattern = Pattern.compile(realWildcard);
        }
        Pattern patternexclude = null;
        if (!Utils.isEmpty(realWildcardExclude)) {
            patternexclude = Pattern.compile(realWildcardExclude);
        }
        for (FileObject item : items) {
            if (successConditionBroken) {
                if (!successConditionBrokenExit) {
                    logError(BaseMessages.getString(PKG, "JobUnZip.Error.SuccessConditionbroken", "" + NrErrors));
                    successConditionBrokenExit = true;
                }
                return false;
            }
            synchronized (KettleVFS.getInstance().getFileSystemManager()) {
                FileObject newFileObject = null;
                try {
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.ProcessingZipEntry", item.getName().getURI(), sourceFileObject.toString()));
                    }
                    // get real destination filename
                    // 
                    String newFileName = unzipToFolder + Const.FILE_SEPARATOR + getTargetFilename(item);
                    newFileObject = KettleVFS.getFileObject(newFileName, this);
                    if (item.getType().equals(FileType.FOLDER)) {
                        // 
                        if (log.isDetailed()) {
                            logDetailed(BaseMessages.getString(PKG, "JobUnZip.CreatingDirectory.Label", newFileName));
                        }
                        // 
                        if (!newFileObject.exists()) {
                            newFileObject.createFolder();
                        }
                    } else {
                        // File
                        // 
                        boolean getIt = true;
                        boolean getItexclude = false;
                        // 
                        if (pattern != null) {
                            Matcher matcher = pattern.matcher(item.getName().getURI());
                            getIt = matcher.matches();
                        }
                        if (patternexclude != null) {
                            Matcher matcherexclude = patternexclude.matcher(item.getName().getURI());
                            getItexclude = matcherexclude.matches();
                        }
                        boolean take = takeThisFile(item, newFileName);
                        if (getIt && !getItexclude && take) {
                            if (log.isDetailed()) {
                                logDetailed(BaseMessages.getString(PKG, "JobUnZip.ExtractingEntry.Label", item.getName().getURI(), newFileName));
                            }
                            if (iffileexist == IF_FILE_EXISTS_UNIQ) {
                                // Create file with unique name
                                int lenstring = newFileName.length();
                                int lastindexOfDot = newFileName.lastIndexOf('.');
                                if (lastindexOfDot == -1) {
                                    lastindexOfDot = lenstring;
                                }
                                newFileName = newFileName.substring(0, lastindexOfDot) + StringUtil.getFormattedDateTimeNow(true) + newFileName.substring(lastindexOfDot, lenstring);
                                if (log.isDebug()) {
                                    logDebug(BaseMessages.getString(PKG, "JobUnZip.Log.CreatingUniqFile", newFileName));
                                }
                            }
                            // 
                            if (!newFileObject.getParent().exists()) {
                                // creates the whole path.
                                newFileObject.getParent().createFolder();
                            }
                            InputStream is = null;
                            OutputStream os = null;
                            try {
                                is = KettleVFS.getInputStream(item);
                                os = KettleVFS.getOutputStream(newFileObject, false);
                                if (is != null) {
                                    byte[] buff = new byte[2048];
                                    int len;
                                    while ((len = is.read(buff)) > 0) {
                                        os.write(buff, 0, len);
                                    }
                                    // Add filename to result filenames
                                    addFilenameToResultFilenames(result, parentJob, newFileName);
                                }
                            } finally {
                                if (is != null) {
                                    is.close();
                                }
                                if (os != null) {
                                    os.close();
                                }
                            }
                        }
                    // end if take
                    }
                } catch (Exception e) {
                    updateErrors();
                    logError(BaseMessages.getString(PKG, "JobUnZip.Error.CanNotProcessZipEntry", item.getName().getURI(), sourceFileObject.toString()), e);
                } finally {
                    if (newFileObject != null) {
                        try {
                            newFileObject.close();
                            if (setOriginalModificationDate) {
                                // Change last modification date
                                newFileObject.getContent().setLastModifiedTime(item.getContent().getLastModifiedTime());
                            }
                        } catch (Exception e) {
                        /* Ignore */
                        }
                    // ignore this
                    }
                    // Close file object
                    // close() does not release resources!
                    KettleVFS.getInstance().getFileSystemManager().closeFileSystem(item.getFileSystem());
                    if (items != null) {
                        items = null;
                    }
                }
            }
        // Synchronized block on KettleVFS.getInstance().getFileSystemManager()
        }
        // Unzip done...
        if (afterunzip > 0) {
            doUnzipPostProcessing(sourceFileObject, movetodir, realMovetodirectory);
        }
        retval = true;
    } catch (Exception e) {
        updateErrors();
        log.logError(BaseMessages.getString(PKG, "JobUnZip.ErrorUnzip.Label", sourceFileObject.toString(), e.getMessage()), e);
    }
    return retval;
}

Example 2 with AllFileSelector

use of org.apache.commons.vfs2.AllFileSelector in project pentaho-kettle by pentaho.

the class JobEntryDosToUnix method processFileFolder.

private boolean processFileFolder(String sourcefilefoldername, String wildcard, int convertion, Job parentJob, Result result) {
    boolean entrystatus = false;
    FileObject sourcefilefolder = null;
    FileObject CurrentFile = null;
    // Get real source file and wilcard
    String realSourceFilefoldername = environmentSubstitute(sourcefilefoldername);
    if (Utils.isEmpty(realSourceFilefoldername)) {
        logError(BaseMessages.getString(PKG, "JobDosToUnix.log.FileFolderEmpty", sourcefilefoldername));
        // Update Errors
        updateErrors();
        return entrystatus;
    }
    String realWildcard = environmentSubstitute(wildcard);
    try {
        sourcefilefolder = KettleVFS.getFileObject(realSourceFilefoldername);
        if (sourcefilefolder.exists()) {
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobDosToUnix.Log.FileExists", sourcefilefolder.toString()));
            }
            if (sourcefilefolder.getType() == FileType.FILE) {
                entrystatus = convertOneFile(sourcefilefolder, convertion, result, parentJob);
            } else if (sourcefilefolder.getType() == FileType.FOLDER) {
                FileObject[] fileObjects = sourcefilefolder.findFiles(new AllFileSelector() {

                    public boolean traverseDescendents(FileSelectInfo info) {
                        return info.getDepth() == 0 || include_subfolders;
                    }

                    public boolean includeFile(FileSelectInfo info) {
                        FileObject fileObject = info.getFile();
                        try {
                            if (fileObject == null) {
                                return false;
                            }
                            if (fileObject.getType() != FileType.FILE) {
                                return false;
                            }
                        } catch (Exception ex) {
                            // Upon error don't process the file.
                            return false;
                        } finally {
                            if (fileObject != null) {
                                try {
                                    fileObject.close();
                                } catch (IOException ex) {
                                /* Ignore */
                                }
                            }
                        }
                        return true;
                    }
                });
                if (fileObjects != null) {
                    for (int j = 0; j < fileObjects.length && !parentJob.isStopped(); j++) {
                        if (successConditionBroken) {
                            if (!successConditionBrokenExit) {
                                logError(BaseMessages.getString(PKG, "JobDosToUnix.Error.SuccessConditionbroken", "" + nrAllErrors));
                                successConditionBrokenExit = true;
                            }
                            return false;
                        }
                        // Fetch files in list one after one ...
                        CurrentFile = fileObjects[j];
                        if (!CurrentFile.getParent().toString().equals(sourcefilefolder.toString())) {
                            // Not in the Base Folder..Only if include sub folders
                            if (include_subfolders) {
                                if (GetFileWildcard(CurrentFile.toString(), realWildcard)) {
                                    convertOneFile(CurrentFile, convertion, result, parentJob);
                                }
                            }
                        } else {
                            // In the base folder
                            if (GetFileWildcard(CurrentFile.toString(), realWildcard)) {
                                convertOneFile(CurrentFile, convertion, result, parentJob);
                            }
                        }
                    }
                }
            } else {
                logError(BaseMessages.getString(PKG, "JobDosToUnix.Error.UnknowFileFormat", sourcefilefolder.toString()));
                // Update Errors
                updateErrors();
            }
        } else {
            logError(BaseMessages.getString(PKG, "JobDosToUnix.Error.SourceFileNotExists", realSourceFilefoldername));
            // Update Errors
            updateErrors();
        }
    } catch (Exception e) {
        logError(BaseMessages.getString(PKG, "JobDosToUnix.Error.Exception.Processing", realSourceFilefoldername.toString(), e.getMessage()));
        // Update Errors
        updateErrors();
    } finally {
        if (sourcefilefolder != null) {
            try {
                sourcefilefolder.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
        if (CurrentFile != null) {
            try {
                CurrentFile.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
    }
    return entrystatus;
}

Example 3 with AllFileSelector

use of org.apache.commons.vfs2.AllFileSelector in project pentaho-kettle by pentaho.

the class JobEntryPGPEncryptFiles method ProcessFileFolder.

private boolean ProcessFileFolder(int actionType, String sourcefilefoldername, String userID, String destinationfilefoldername, String wildcard, Job parentJob, Result result, String MoveToFolder) {
    boolean entrystatus = false;
    FileObject sourcefilefolder = null;
    FileObject destinationfilefolder = null;
    FileObject movetofolderfolder = null;
    FileObject Currentfile = null;
    // Get real source, destination file and wildcard
    String realSourceFilefoldername = environmentSubstitute(sourcefilefoldername);
    String realuserID = environmentSubstitute(userID);
    String realDestinationFilefoldername = environmentSubstitute(destinationfilefoldername);
    String realWildcard = environmentSubstitute(wildcard);
    try {
        sourcefilefolder = KettleVFS.getFileObject(realSourceFilefoldername);
        destinationfilefolder = KettleVFS.getFileObject(realDestinationFilefoldername);
        if (!Utils.isEmpty(MoveToFolder)) {
            movetofolderfolder = KettleVFS.getFileObject(MoveToFolder);
        }
        if (sourcefilefolder.exists()) {
            // PDI will create it
            if (CreateDestinationFolder(destinationfilefolder)) {
                // Basic Tests
                if (sourcefilefolder.getType().equals(FileType.FOLDER) && destination_is_a_file) {
                    // Source is a folder, destination is a file
                    // WARNING !!! CAN NOT MOVE FOLDER TO FILE !!!
                    logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.Forbidden"), BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.CanNotMoveFolderToFile", realSourceFilefoldername, realDestinationFilefoldername));
                    // Update Errors
                    updateErrors();
                } else {
                    if (destinationfilefolder.getType().equals(FileType.FOLDER) && sourcefilefolder.getType().equals(FileType.FILE)) {
                        // Source is a file, destination is a folder
                        // return destination short filename
                        String shortfilename = sourcefilefolder.getName().getBaseName();
                        try {
                            shortfilename = getDestinationFilename(sourcefilefolder.getName().getBaseName());
                        } catch (Exception e) {
                            logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.GettingFilename", sourcefilefolder.getName().getBaseName(), e.toString()));
                            return entrystatus;
                        }
                        // Move the file to the destination folder
                        String destinationfilenamefull = destinationfilefolder.toString() + Const.FILE_SEPARATOR + shortfilename;
                        FileObject destinationfile = KettleVFS.getFileObject(destinationfilenamefull);
                        entrystatus = EncryptFile(actionType, shortfilename, sourcefilefolder, realuserID, destinationfile, movetofolderfolder, parentJob, result);
                    } else if (sourcefilefolder.getType().equals(FileType.FILE) && destination_is_a_file) {
                        // Source is a file, destination is a file
                        FileObject destinationfile = KettleVFS.getFileObject(realDestinationFilefoldername);
                        // return destination short filename
                        String shortfilename = destinationfile.getName().getBaseName();
                        try {
                            shortfilename = getDestinationFilename(destinationfile.getName().getBaseName());
                        } catch (Exception e) {
                            logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.GettingFilename", sourcefilefolder.getName().getBaseName(), e.toString()));
                            return entrystatus;
                        }
                        String destinationfilenamefull = destinationfilefolder.getParent().toString() + Const.FILE_SEPARATOR + shortfilename;
                        destinationfile = KettleVFS.getFileObject(destinationfilenamefull);
                        entrystatus = EncryptFile(actionType, shortfilename, sourcefilefolder, realuserID, destinationfile, movetofolderfolder, parentJob, result);
                    } else {
                        // Both source and destination are folders
                        if (isDetailed()) {
                            logDetailed("  ");
                            logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FetchFolder", sourcefilefolder.toString()));
                        }
                        FileObject[] fileObjects = sourcefilefolder.findFiles(new AllFileSelector() {

                            public boolean traverseDescendents(FileSelectInfo info) {
                                return info.getDepth() == 0 || include_subfolders;
                            }

                            public boolean includeFile(FileSelectInfo info) {
                                FileObject fileObject = info.getFile();
                                try {
                                    if (fileObject == null) {
                                        return false;
                                    }
                                } catch (Exception ex) {
                                    // Upon error don't process the file.
                                    return false;
                                } finally {
                                    if (fileObject != null) {
                                        try {
                                            fileObject.close();
                                            fileObject = null;
                                        } catch (IOException ex) {
                                        /* Ignore */
                                        }
                                    }
                                }
                                return true;
                            }
                        });
                        if (fileObjects != null) {
                            for (int j = 0; j < fileObjects.length && !parentJob.isStopped(); j++) {
                                // Success condition broken?
                                if (successConditionBroken) {
                                    if (!successConditionBrokenExit) {
                                        logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.SuccessConditionbroken", "" + NrErrors));
                                        successConditionBrokenExit = true;
                                    }
                                    return false;
                                }
                                // Fetch files in list one after one ...
                                Currentfile = fileObjects[j];
                                if (!EncryptOneFile(actionType, Currentfile, sourcefilefolder, realuserID, realDestinationFilefoldername, realWildcard, parentJob, result, movetofolderfolder)) {
                                    // Update Errors
                                    updateErrors();
                                }
                            }
                        }
                    }
                }
                entrystatus = true;
            } else {
                // Destination Folder or Parent folder is missing
                logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.DestinationFolderNotFound", realDestinationFilefoldername));
            }
        } else {
            logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.SourceFileNotExists", realSourceFilefoldername));
        }
    } catch (Exception e) {
        logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.Exception.MoveProcess", realSourceFilefoldername.toString(), destinationfilefolder.toString(), e.getMessage()));
        // Update Errors
        updateErrors();
    } finally {
        if (sourcefilefolder != null) {
            try {
                sourcefilefolder.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
        if (destinationfilefolder != null) {
            try {
                destinationfilefolder.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
        if (Currentfile != null) {
            try {
                Currentfile.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
        if (movetofolderfolder != null) {
            try {
                movetofolderfolder.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
    }
    return entrystatus;
}

Example 4 with AllFileSelector

use of org.apache.commons.vfs2.AllFileSelector in project pentaho-kettle by pentaho.

the class JobEntryTalendJobExec method prepareJarFiles.

private URL[] prepareJarFiles(FileObject zipFile) throws Exception {
    // zip:file:///tmp/foo.zip
    FileInputList fileList = FileInputList.createFileList(this, new String[] { "zip:" + zipFile.toString() }, // Include mask: only jar files
    new String[] { ".*\\.jar$" }, // Exclude mask: only jar files
    new String[] { ".*classpath\\.jar$" }, // File required
    new String[] { "Y" }, // Search sub-directories
    new boolean[] { true });
    List<URL> files = new ArrayList<URL>();
    // 
    for (FileObject file : fileList.getFiles()) {
        FileObject jarfilecopy = KettleVFS.createTempFile(file.getName().getBaseName(), ".jar", environmentSubstitute("${java.io.tmpdir}"));
        jarfilecopy.copyFrom(file, new AllFileSelector());
        files.add(jarfilecopy.getURL());
    }
    return files.toArray(new URL[files.size()]);
}

Example 5 with AllFileSelector

use of org.apache.commons.vfs2.AllFileSelector in project pentaho-kettle by pentaho.

the class JobEntryEvalFilesMetrics method ProcessFileFolder.

private void ProcessFileFolder(String sourcefilefoldername, String wildcard, String includeSubfolders, Job parentJob, Result result) {
    FileObject sourcefilefolder = null;
    FileObject CurrentFile = null;
    // Get real source file and wildcard
    String realSourceFilefoldername = environmentSubstitute(sourcefilefoldername);
    if (Utils.isEmpty(realSourceFilefoldername)) {
        // Filename is empty!
        logError(BaseMessages.getString(PKG, "JobEvalFilesMetrics.log.FileFolderEmpty"));
        incrementErrors();
        return;
    }
    String realWildcard = environmentSubstitute(wildcard);
    final boolean include_subfolders = YES.equalsIgnoreCase(includeSubfolders);
    try {
        sourcefilefolder = KettleVFS.getFileObject(realSourceFilefoldername, this);
        if (sourcefilefolder.exists()) {
            // File exists
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobEvalFilesMetrics.Log.FileExists", sourcefilefolder.toString()));
            }
            if (sourcefilefolder.getType() == FileType.FILE) {
                // We deals here with a file
                // let's get file size
                getFileSize(sourcefilefolder, result, parentJob);
            } else if (sourcefilefolder.getType() == FileType.FOLDER) {
                // We have a folder
                // we will fetch and extract files
                FileObject[] fileObjects = sourcefilefolder.findFiles(new AllFileSelector() {

                    public boolean traverseDescendents(FileSelectInfo info) {
                        return info.getDepth() == 0 || include_subfolders;
                    }

                    public boolean includeFile(FileSelectInfo info) {
                        FileObject fileObject = info.getFile();
                        try {
                            if (fileObject == null) {
                                return false;
                            }
                            if (fileObject.getType() != FileType.FILE) {
                                return false;
                            }
                        } catch (Exception ex) {
                            // Upon error don't process the file.
                            return false;
                        } finally {
                            if (fileObject != null) {
                                try {
                                    fileObject.close();
                                } catch (IOException ex) {
                                /* Ignore */
                                }
                            }
                        }
                        return true;
                    }
                });
                if (fileObjects != null) {
                    for (int j = 0; j < fileObjects.length && !parentJob.isStopped(); j++) {
                        // Fetch files in list one after one ...
                        CurrentFile = fileObjects[j];
                        if (!CurrentFile.getParent().toString().equals(sourcefilefolder.toString())) {
                            // Not in the Base Folder..Only if include sub folders
                            if (include_subfolders) {
                                if (GetFileWildcard(CurrentFile.getName().getBaseName(), realWildcard)) {
                                    getFileSize(CurrentFile, result, parentJob);
                                }
                            }
                        } else {
                            // In the base folder
                            if (GetFileWildcard(CurrentFile.getName().getBaseName(), realWildcard)) {
                                getFileSize(CurrentFile, result, parentJob);
                            }
                        }
                    }
                }
            } else {
                incrementErrors();
                logError(BaseMessages.getString(PKG, "JobEvalFilesMetrics.Error.UnknowFileFormat", sourcefilefolder.toString()));
            }
        } else {
            incrementErrors();
            logError(BaseMessages.getString(PKG, "JobEvalFilesMetrics.Error.SourceFileNotExists", realSourceFilefoldername));
        }
    } catch (Exception e) {
        incrementErrors();
        logError(BaseMessages.getString(PKG, "JobEvalFilesMetrics.Error.Exception.Processing", realSourceFilefoldername.toString(), e.getMessage()));
    } finally {
        if (sourcefilefolder != null) {
            try {
                sourcefilefolder.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
        if (CurrentFile != null) {
            try {
                CurrentFile.close();
            } catch (IOException ex) {
            /* Ignore */
            }
        }
    }
}