Examples with Type org.apache.commons.vfs2.UserAuthenticationData.Type used on opensource projects

Example 41 with Type

use of org.apache.commons.vfs2.UserAuthenticationData.Type in project pentaho-kettle by pentaho.

the class ValueDataUtil method createChecksum.

/**
 * @param metaA
 *   The ValueMetaInterface
 * @param dataA
 *   Filename
 * @param type
 *   Algorithm to be used when computing the checksum (MD5 or SHA-1)
 * @param failIfNoFile
 *   Indicates if the transformation should fail if no file is found
 * @return File's checksum
 * @throws KettleFileNotFoundException
 */
public static String createChecksum(ValueMetaInterface metaA, Object dataA, String type, boolean failIfNoFile) throws KettleFileNotFoundException {
    if (dataA == null) {
        return null;
    }
    String md5Hash = null;
    FileObject file = null;
    InputStream in = null;
    try {
        file = KettleVFS.getFileObject(dataA.toString());
        throwsErrorOnFileNotFound(file);
        in = KettleVFS.getInputStream(file);
        int bytes = in.available();
        byte[] buffer = new byte[bytes];
        in.read(buffer);
        StringBuffer md5HashBuff = new StringBuffer(32);
        byte[] b = MessageDigest.getInstance(type).digest(buffer);
        int len = b.length;
        for (int x = 0; x < len; x++) {
            md5HashBuff.append(String.format("%02x", b[x]));
        }
        md5Hash = md5HashBuff.toString();
    } catch (KettleFileNotFoundException e) {
        if (failIfNoFile) {
            throw e;
        }
        log.debug(e.getMessage());
    } catch (Exception e) {
        log.debug(e.getMessage());
    } finally {
        IOUtils.closeQuietly(file);
        IOUtils.closeQuietly(in);
    }
    return md5Hash;
}

Example 42 with Type

use of org.apache.commons.vfs2.UserAuthenticationData.Type in project pentaho-kettle by pentaho.

the class JobEntryFoldersCompare method execute.

public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);
    boolean ok = true;
    String realFilename1 = getRealFilename1();
    String realFilename2 = getRealFilename2();
    FileObject folder1 = null;
    FileObject folder2 = null;
    FileObject filefolder1 = null;
    FileObject filefolder2 = null;
    try {
        if (filename1 != null && filename2 != null) {
            // Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS
            if (parentJobMeta.getNamedClusterEmbedManager() != null) {
                parentJobMeta.getNamedClusterEmbedManager().passEmbeddedMetastoreKey(this, parentJobMeta.getEmbeddedMetastoreProviderKey());
            }
            // Get Folders/Files to compare
            folder1 = KettleVFS.getFileObject(realFilename1, this);
            folder2 = KettleVFS.getFileObject(realFilename2, this);
            if (folder1.exists() && folder2.exists()) {
                if (!folder1.getType().equals(folder2.getType())) {
                    // pb...we try to compare file with folder !!!
                    logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.CanNotCompareFilesFolders"));
                    if (folder1.getType() == FileType.FILE) {
                        logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", realFilename1));
                    } else if (folder1.getType() == FileType.FOLDER) {
                        logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", realFilename1));
                    } else {
                        logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", realFilename1));
                    }
                    if (folder2.getType() == FileType.FILE) {
                        logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", realFilename2));
                    } else if (folder2.getType() == FileType.FOLDER) {
                        logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", realFilename2));
                    } else {
                        logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", realFilename2));
                    }
                } else {
                    if (folder1.getType() == FileType.FILE) {
                        // simply compare 2 files ..
                        if (equalFileContents(folder1, folder2)) {
                            result.setResult(true);
                        } else {
                            result.setResult(false);
                        }
                    } else if (folder1.getType() == FileType.FOLDER) {
                        // We compare 2 folders ...
                        FileObject[] list1 = folder1.findFiles(new TextFileSelector(folder1.toString()));
                        FileObject[] list2 = folder2.findFiles(new TextFileSelector(folder2.toString()));
                        int lenList1 = list1.length;
                        int lenList2 = list2.length;
                        if (log.isDetailed()) {
                            logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FolderContains", realFilename1, "" + lenList1));
                            logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FolderContains", realFilename2, "" + lenList2));
                        }
                        if (lenList1 == lenList2) {
                            HashMap<String, String> collection1 = new HashMap<String, String>();
                            HashMap<String, String> collection2 = new HashMap<String, String>();
                            for (int i = 0; i < list1.length; i++) {
                                // Put files list1 in TreeMap collection1
                                collection1.put(list1[i].getName().getBaseName(), list1[i].toString());
                            }
                            for (int i = 0; i < list2.length; i++) {
                                // Put files list2 in TreeMap collection2
                                collection2.put(list2[i].getName().getBaseName(), list2[i].toString());
                            }
                            // Let's now fetch Folder1
                            // and for each entry, we will search it in Folder2
                            // if the entry exists..we will compare file entry (file or folder?)
                            // if the 2 entry are file (not folder), we will compare content
                            Set<Map.Entry<String, String>> entrees = collection1.entrySet();
                            Iterator<Map.Entry<String, String>> iterateur = entrees.iterator();
                            while (iterateur.hasNext()) {
                                Map.Entry<String, String> entree = iterateur.next();
                                if (!collection2.containsKey(entree.getKey())) {
                                    ok = false;
                                    if (log.isDetailed()) {
                                        logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FileCanNotBeFoundIn", entree.getKey().toString(), realFilename2));
                                    }
                                } else {
                                    if (log.isDebug()) {
                                        logDebug(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FileIsFoundIn", entree.getKey().toString(), realFilename2));
                                    }
                                    filefolder1 = KettleVFS.getFileObject(entree.getValue().toString(), this);
                                    filefolder2 = KettleVFS.getFileObject(collection2.get(entree.getKey()).toString(), this);
                                    if (!filefolder2.getType().equals(filefolder1.getType())) {
                                        // The file1 exist in the folder2..but they don't have the same type
                                        ok = false;
                                        if (log.isDetailed()) {
                                            logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FilesNotSameType", filefolder1.toString(), filefolder2.toString()));
                                        }
                                        if (filefolder1.getType() == FileType.FILE) {
                                            logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", filefolder1.toString()));
                                        } else if (filefolder1.getType() == FileType.FOLDER) {
                                            logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", filefolder1.toString()));
                                        } else {
                                            logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", filefolder1.toString()));
                                        }
                                        if (filefolder2.getType() == FileType.FILE) {
                                            logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", filefolder2.toString()));
                                        } else if (filefolder2.getType() == FileType.FOLDER) {
                                            logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", filefolder2.toString()));
                                        } else {
                                            logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", filefolder2.toString()));
                                        }
                                    } else {
                                        // Files are the same type ...
                                        if (filefolder2.getType() == FileType.FILE) {
                                            // Let's compare file size
                                            if (comparefilesize) {
                                                long filefolder1_size = filefolder1.getContent().getSize();
                                                long filefolder2_size = filefolder2.getContent().getSize();
                                                if (filefolder1_size != filefolder2_size) {
                                                    ok = false;
                                                    if (log.isDetailed()) {
                                                        logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FilesNotSameSize", filefolder1.toString(), filefolder2.toString()));
                                                        logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.SizeFileIs", filefolder1.toString(), "" + filefolder1_size));
                                                        logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.SizeFileIs", filefolder2.toString(), "" + filefolder2_size));
                                                    }
                                                }
                                            }
                                            if (ok) {
                                                // Let's compare files content..
                                                if (comparefilecontent) {
                                                    if (!equalFileContents(filefolder1, filefolder2)) {
                                                        ok = false;
                                                        if (log.isDetailed()) {
                                                            logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FilesNotSameContent", filefolder1.toString(), filefolder2.toString()));
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            // logBasic(entree.getKey() + " - " + entree.getValue());
                            }
                            result.setResult(ok);
                        } else {
                            // The 2 folders don't have the same files number
                            if (log.isDetailed()) {
                                logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FoldersDifferentFiles", realFilename1.toString(), realFilename2.toString()));
                            }
                        }
                    }
                // else: File type unknown !!
                }
            } else {
                if (!folder1.exists()) {
                    logError(BaseMessages.getString(PKG, "JobFileCompare.Log.FileNotExist", realFilename1));
                }
                if (!folder2.exists()) {
                    logError(BaseMessages.getString(PKG, "JobFileCompare.Log.FileNotExist", realFilename2));
                }
                result.setResult(false);
                result.setNrErrors(1);
            }
        } else {
            logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.Need2Files"));
        }
    } catch (Exception e) {
        result.setResult(false);
        result.setNrErrors(1);
        logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.ErrorComparing", realFilename2, realFilename2, e.getMessage()));
    } finally {
        try {
            if (folder1 != null) {
                folder1.close();
                folder1 = null;
            }
            if (folder2 != null) {
                folder2.close();
                folder2 = null;
            }
            if (filefolder1 != null) {
                filefolder1.close();
                filefolder1 = null;
            }
            if (filefolder2 != null) {
                filefolder2.close();
                filefolder2 = null;
            }
        } catch (IOException e) {
        // Ignore errors
        }
    }
    return result;
}

Example 43 with Type

use of org.apache.commons.vfs2.UserAuthenticationData.Type in project pentaho-kettle by pentaho.

the class GetFileNames method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    if (!meta.isFileField()) {
        if (data.filenr >= data.filessize) {
            setOutputDone();
            return false;
        }
    } else {
        if (data.filenr >= data.filessize) {
            // Grab one row from previous step ...
            data.readrow = getRow();
        }
        if (data.readrow == null) {
            setOutputDone();
            return false;
        }
        if (first) {
            first = false;
            data.inputRowMeta = getInputRowMeta();
            data.outputRowMeta = data.inputRowMeta.clone();
            meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
            // Get total previous fields
            data.totalpreviousfields = data.inputRowMeta.size();
            // Check is filename field is provided
            if (Utils.isEmpty(meta.getDynamicFilenameField())) {
                logError(BaseMessages.getString(PKG, "GetFileNames.Log.NoField"));
                throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Log.NoField"));
            }
            // cache the position of the field
            if (data.indexOfFilenameField < 0) {
                data.indexOfFilenameField = data.inputRowMeta.indexOfValue(meta.getDynamicFilenameField());
                if (data.indexOfFilenameField < 0) {
                    // The field is unreachable !
                    logError(BaseMessages.getString(PKG, "GetFileNames.Log.ErrorFindingField", meta.getDynamicFilenameField()));
                    throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Exception.CouldnotFindField", meta.getDynamicFilenameField()));
                }
            }
            // If wildcard field is specified, Check if field exists
            if (!Utils.isEmpty(meta.getDynamicWildcardField())) {
                if (data.indexOfWildcardField < 0) {
                    data.indexOfWildcardField = data.inputRowMeta.indexOfValue(meta.getDynamicWildcardField());
                    if (data.indexOfWildcardField < 0) {
                        // The field is unreachable !
                        logError(BaseMessages.getString(PKG, "GetFileNames.Log.ErrorFindingField") + "[" + meta.getDynamicWildcardField() + "]");
                        throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Exception.CouldnotFindField", meta.getDynamicWildcardField()));
                    }
                }
            }
            // If ExcludeWildcard field is specified, Check if field exists
            if (!Utils.isEmpty(meta.getDynamicExcludeWildcardField())) {
                if (data.indexOfExcludeWildcardField < 0) {
                    data.indexOfExcludeWildcardField = data.inputRowMeta.indexOfValue(meta.getDynamicExcludeWildcardField());
                    if (data.indexOfExcludeWildcardField < 0) {
                        // The field is unreachable !
                        logError(BaseMessages.getString(PKG, "GetFileNames.Log.ErrorFindingField") + "[" + meta.getDynamicExcludeWildcardField() + "]");
                        throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Exception.CouldnotFindField", meta.getDynamicExcludeWildcardField()));
                    }
                }
            }
        }
    }
    try {
        Object[] outputRow = buildEmptyRow();
        int outputIndex = 0;
        Object[] extraData = new Object[data.nrStepFields];
        if (meta.isFileField()) {
            if (data.filenr >= data.filessize) {
                // Get value of dynamic filename field ...
                String filename = getInputRowMeta().getString(data.readrow, data.indexOfFilenameField);
                String wildcard = "";
                if (data.indexOfWildcardField >= 0) {
                    wildcard = getInputRowMeta().getString(data.readrow, data.indexOfWildcardField);
                }
                String excludewildcard = "";
                if (data.indexOfExcludeWildcardField >= 0) {
                    excludewildcard = getInputRowMeta().getString(data.readrow, data.indexOfExcludeWildcardField);
                }
                String[] filesname = { filename };
                String[] filesmask = { wildcard };
                String[] excludefilesmask = { excludewildcard };
                String[] filesrequired = { "N" };
                boolean[] includesubfolders = { meta.isDynamicIncludeSubFolders() };
                // Get files list
                data.files = meta.getDynamicFileList(this, filesname, filesmask, excludefilesmask, filesrequired, includesubfolders);
                data.filessize = data.files.nrOfFiles();
                data.filenr = 0;
            }
            // Clone current input row
            outputRow = data.readrow.clone();
        }
        if (data.filessize > 0) {
            data.file = data.files.getFile(data.filenr);
            if (meta.isAddResultFile()) {
                // Add this to the result file names...
                ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
                resultFile.setComment(BaseMessages.getString(PKG, "GetFileNames.Log.FileReadByStep"));
                addResultFile(resultFile);
            }
            // filename
            extraData[outputIndex++] = KettleVFS.getFilename(data.file);
            // short_filename
            extraData[outputIndex++] = data.file.getName().getBaseName();
            try {
                // Path
                extraData[outputIndex++] = KettleVFS.getFilename(data.file.getParent());
                // type
                extraData[outputIndex++] = data.file.getType().toString();
                // exists
                extraData[outputIndex++] = Boolean.valueOf(data.file.exists());
                // ishidden
                extraData[outputIndex++] = Boolean.valueOf(data.file.isHidden());
                // isreadable
                extraData[outputIndex++] = Boolean.valueOf(data.file.isReadable());
                // iswriteable
                extraData[outputIndex++] = Boolean.valueOf(data.file.isWriteable());
                // lastmodifiedtime
                extraData[outputIndex++] = new Date(data.file.getContent().getLastModifiedTime());
                // size
                Long size = null;
                if (data.file.getType().equals(FileType.FILE)) {
                    size = new Long(data.file.getContent().getSize());
                }
                extraData[outputIndex++] = size;
            } catch (IOException e) {
                throw new KettleException(e);
            }
            // extension
            extraData[outputIndex++] = data.file.getName().getExtension();
            // uri
            extraData[outputIndex++] = Const.optionallyDecodeUriString(data.file.getName().getURI());
            // rooturi
            extraData[outputIndex++] = data.file.getName().getRootURI();
            // See if we need to add the row number to the row...
            if (meta.includeRowNumber() && !Utils.isEmpty(meta.getRowNumberField())) {
                extraData[outputIndex++] = new Long(data.rownr);
            }
            data.rownr++;
            // Add row data
            outputRow = RowDataUtil.addRowData(outputRow, data.totalpreviousfields, extraData);
            // Send row
            putRow(data.outputRowMeta, outputRow);
            if (meta.getRowLimit() > 0 && data.rownr >= meta.getRowLimit()) {
                // limit has been reached: stop now.
                setOutputDone();
                return false;
            }
        }
    } catch (Exception e) {
        throw new KettleStepException(e);
    }
    data.filenr++;
    if (checkFeedback(getLinesInput())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "GetFileNames.Log.NrLine", "" + getLinesInput()));
        }
    }
    return true;
}

Example 44 with Type

use of org.apache.commons.vfs2.UserAuthenticationData.Type in project pentaho-kettle by pentaho.

the class LoadFileInput method getOneRow.

Object[] getOneRow() throws KettleException {
    if (!openNextFile()) {
        return null;
    }
    // Build an empty row based on the meta-data
    Object[] outputRowData = buildEmptyRow();
    try {
        // Create new row or clone
        if (meta.getIsInFields()) {
            outputRowData = copyOrCloneArrayFromLoadFile(outputRowData, data.readrow);
        }
        // Read fields...
        for (int i = 0; i < data.nrInputFields; i++) {
            // Get field
            LoadFileInputField loadFileInputField = meta.getInputFields()[i];
            Object o = null;
            int indexField = data.totalpreviousfields + i;
            ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(indexField);
            ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(indexField);
            switch(loadFileInputField.getElementType()) {
                case LoadFileInputField.ELEMENT_TYPE_FILECONTENT:
                    // DO Trimming!
                    switch(loadFileInputField.getTrimType()) {
                        case LoadFileInputField.TYPE_TRIM_LEFT:
                            if (meta.getEncoding() != null) {
                                data.filecontent = Const.ltrim(new String(data.filecontent, meta.getEncoding())).getBytes();
                            } else {
                                data.filecontent = Const.ltrim(new String(data.filecontent)).getBytes();
                            }
                            break;
                        case LoadFileInputField.TYPE_TRIM_RIGHT:
                            if (meta.getEncoding() != null) {
                                data.filecontent = Const.rtrim(new String(data.filecontent, meta.getEncoding())).getBytes();
                            } else {
                                data.filecontent = Const.rtrim(new String(data.filecontent)).getBytes();
                            }
                            break;
                        case LoadFileInputField.TYPE_TRIM_BOTH:
                            if (meta.getEncoding() != null) {
                                data.filecontent = Const.trim(new String(data.filecontent, meta.getEncoding())).getBytes();
                            } else {
                                data.filecontent = Const.trim(new String(data.filecontent)).getBytes();
                            }
                            break;
                        default:
                            break;
                    }
                    if (targetValueMeta.getType() != ValueMetaInterface.TYPE_BINARY) {
                        // handle as a String
                        if (meta.getEncoding() != null) {
                            o = new String(data.filecontent, meta.getEncoding());
                        } else {
                            o = new String(data.filecontent);
                        }
                    } else {
                        // save as byte[] without any conversion
                        o = data.filecontent;
                    }
                    break;
                case LoadFileInputField.ELEMENT_TYPE_FILESIZE:
                    o = String.valueOf(data.fileSize);
                    break;
                default:
                    break;
            }
            if (targetValueMeta.getType() == ValueMetaInterface.TYPE_BINARY) {
                // save as byte[] without any conversion
                outputRowData[indexField] = o;
            } else {
                // convert string (processing type) to the target type
                outputRowData[indexField] = targetValueMeta.convertData(sourceValueMeta, o);
            }
            // Do we need to repeat this field if it is null?
            if (loadFileInputField.isRepeated()) {
                if (data.previousRow != null && o == null) {
                    outputRowData[indexField] = data.previousRow[indexField];
                }
            }
        }
        // End of loop over fields...
        int rowIndex = data.totalpreviousfields + data.nrInputFields;
        // See if we need to add the filename to the row...
        if (meta.includeFilename() && meta.getFilenameField() != null && meta.getFilenameField().length() > 0) {
            outputRowData[rowIndex++] = data.filename;
        }
        // See if we need to add the row number to the row...
        if (meta.includeRowNumber() && meta.getRowNumberField() != null && meta.getRowNumberField().length() > 0) {
            outputRowData[rowIndex++] = new Long(data.rownr);
        }
        // Possibly add short filename...
        if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
            outputRowData[rowIndex++] = data.shortFilename;
        }
        // Add Extension
        if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
            outputRowData[rowIndex++] = data.extension;
        }
        // add path
        if (meta.getPathField() != null && meta.getPathField().length() > 0) {
            outputRowData[rowIndex++] = data.path;
        }
        // add Hidden
        if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
            outputRowData[rowIndex++] = new Boolean(data.hidden);
        }
        // Add modification date
        if (meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0) {
            outputRowData[rowIndex++] = data.lastModificationDateTime;
        }
        // Add Uri
        if (meta.getUriField() != null && meta.getUriField().length() > 0) {
            outputRowData[rowIndex++] = data.uriName;
        }
        // Add RootUri
        if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
            outputRowData[rowIndex++] = data.rootUriName;
        }
        RowMetaInterface irow = getInputRowMeta();
        // copy it to make
        data.previousRow = irow == null ? outputRowData : irow.cloneRow(outputRowData);
        // surely the next step doesn't change it in between...
        incrementLinesInput();
        data.rownr++;
    } catch (Exception e) {
        throw new KettleException("Error during processing a row", e);
    }
    return outputRowData;
}