Examples with KettleFileException org.pentaho.di.core.exception.KettleFileException used on opensource projects

Example 56 with KettleFileException

use of org.pentaho.di.core.exception.KettleFileException in project pentaho-kettle by pentaho.

the class JobEntryFoldersCompare method equalFileContents.

/**
 * Check whether 2 files have the same contents.
 *
 * @param file1
 *          first file to compare
 * @param file2
 *          second file to compare
 * @return true if files are equal, false if they are not
 *
 * @throws IOException
 *           upon IO problems
 */
protected boolean equalFileContents(FileObject file1, FileObject file2) throws KettleFileException {
    // Really read the contents and do comparisons
    DataInputStream in1 = null;
    DataInputStream in2 = null;
    try {
        // Really read the contents and do comparisons
        in1 = new DataInputStream(new BufferedInputStream(KettleVFS.getInputStream(KettleVFS.getFilename(file1), this)));
        in2 = new DataInputStream(new BufferedInputStream(KettleVFS.getInputStream(KettleVFS.getFilename(file2), this)));
        char ch1, ch2;
        while (in1.available() != 0 && in2.available() != 0) {
            ch1 = (char) in1.readByte();
            ch2 = (char) in2.readByte();
            if (ch1 != ch2) {
                return false;
            }
        }
        if (in1.available() != in2.available()) {
            return false;
        } else {
            return true;
        }
    } catch (IOException e) {
        throw new KettleFileException(e);
    } finally {
        if (in1 != null) {
            try {
                in1.close();
            } catch (IOException ignored) {
            // Nothing to see here...
            }
        }
        if (in2 != null) {
            try {
                in2.close();
            } catch (Exception ignored) {
            // We can't do anything else here...
            }
        }
    }
}

Example 57 with KettleFileException

use of org.pentaho.di.core.exception.KettleFileException in project pentaho-kettle by pentaho.

the class Trans method setInternalKettleVariables.

/**
 * Sets the internal kettle variables.
 *
 * @param var
 *          the new internal kettle variables
 */
public void setInternalKettleVariables(VariableSpace var) {
    if (transMeta != null && !Utils.isEmpty(transMeta.getFilename())) {
        // we have a finename that's defined.
        try {
            FileObject fileObject = KettleVFS.getFileObject(transMeta.getFilename(), var);
            FileName fileName = fileObject.getName();
            // The filename of the transformation
            variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_NAME, fileName.getBaseName());
            // The directory of the transformation
            FileName fileDir = fileName.getParent();
            variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY, fileDir.getURI());
        } catch (KettleFileException e) {
            variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY, "");
            variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_NAME, "");
        }
    } else {
        variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY, "");
        variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_NAME, "");
    }
    boolean hasRepoDir = transMeta.getRepositoryDirectory() != null && transMeta.getRepository() != null;
    // The name of the transformation
    variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_NAME, Const.NVL(transMeta.getName(), ""));
    // setup fallbacks
    if (hasRepoDir) {
        variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY, variables.getVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_REPOSITORY_DIRECTORY));
    } else {
        variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_REPOSITORY_DIRECTORY, variables.getVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY));
    }
    // TODO PUT THIS INSIDE OF THE "IF"
    // The name of the directory in the repository
    variables.setVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_REPOSITORY_DIRECTORY, transMeta.getRepositoryDirectory() != null ? transMeta.getRepositoryDirectory().getPath() : "");
    if (hasRepoDir) {
        variables.setVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, variables.getVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_REPOSITORY_DIRECTORY));
        if ("/".equals(variables.getVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY))) {
            variables.setVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, "");
        }
    } else {
        variables.setVariable(Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, variables.getVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY));
    }
}

Example 58 with KettleFileException

use of org.pentaho.di.core.exception.KettleFileException in project pentaho-kettle by pentaho.

the class Translator2 method readFiles.

public void readFiles() throws KettleFileException {
    log.logBasic(BaseMessages.getString(PKG, "i18n.Log.ScanningSourceDirectories"));
    try {
        // crawl through the source directories...
        // 
        crawler = new MessagesSourceCrawler(log, rootDirectories, singleMessagesFile, xmlFolders);
        crawler.setScanPhrases(scanPhrases);
        crawler.setFilesToAvoid(filesToAvoid);
        crawler.crawl();
        store = new TranslationsStore(log, localeList, referenceLocale, crawler.getSourcePackageOccurrences());
        store.read(rootDirectories);
        // What are the statistics?
        // 
        Map<String, int[]> folderKeyCounts = new HashMap<String, int[]>();
        Map<String, Integer> nrKeysPerFolder = new HashMap<String, Integer>();
        for (String sourceFolder : rootDirectories) {
            folderKeyCounts.put(sourceFolder, new int[localeList.size()]);
        }
        for (String sourceFolder : rootDirectories) {
            int[] keyCounts = folderKeyCounts.get(sourceFolder);
            int nrKeys = 0;
            for (int i = 0; i < localeList.size(); i++) {
                String locale = localeList.get(i);
                // 
                for (KeyOccurrence keyOccurrence : crawler.getKeyOccurrences(sourceFolder)) {
                    // 
                    if (showKey(keyOccurrence.getKey(), keyOccurrence.getMessagesPackage())) {
                        String value = store.lookupKeyValue(locale, keyOccurrence.getMessagesPackage(), keyOccurrence.getKey());
                        if (!Utils.isEmpty(value)) {
                            keyCounts[i]++;
                        }
                        if (locale.equals(referenceLocale)) {
                            nrKeys++;
                        }
                    }
                }
            }
            nrKeysPerFolder.put(sourceFolder, new Integer(nrKeys));
        }
        DecimalFormat pctFormat = new DecimalFormat("#00.00");
        DecimalFormat nrFormat = new DecimalFormat("00");
        for (String sourceFolder : rootDirectories) {
            System.out.println("-------------------------------------");
            System.out.println("Folder: " + sourceFolder);
            System.out.println("-------------------------------------");
            int nrKeys = nrKeysPerFolder.get(sourceFolder);
            System.out.println(BaseMessages.getString(PKG, "i18n.Log.NumberOfKeysFound", "" + nrKeys));
            int[] keyCounts = folderKeyCounts.get(sourceFolder);
            String[] locales = localeList.toArray(new String[localeList.size()]);
            for (int i = 0; i < locales.length; i++) {
                for (int j = 0; j < locales.length - 1; j++) {
                    if (keyCounts[j] < keyCounts[j + 1]) {
                        int c = keyCounts[j];
                        keyCounts[j] = keyCounts[j + 1];
                        keyCounts[j + 1] = c;
                        String l = locales[j];
                        locales[j] = locales[j + 1];
                        locales[j + 1] = l;
                    }
                }
            }
            for (int i = 0; i < locales.length; i++) {
                double donePct = 100 * (double) keyCounts[i] / nrKeys;
                int missingKeys = nrKeys - keyCounts[i];
                String statusKeys = "# " + nrFormat.format(i + 1) + " : " + locales[i] + " : " + pctFormat.format(donePct) + "% " + BaseMessages.getString(PKG, "i18n.Log.CompleteKeys", keyCounts[i]) + (missingKeys != 0 ? BaseMessages.getString(PKG, "i18n.Log.MissingKeys", missingKeys) : "");
                System.out.println(statusKeys);
            }
        }
    } catch (Exception e) {
        throw new KettleFileException(BaseMessages.getString(PKG, "i18n.Log.UnableToGetFiles", rootDirectories.toString()), e);
    }
}

Example 59 with KettleFileException

use of org.pentaho.di.core.exception.KettleFileException in project pentaho-kettle by pentaho.

the class JobEntrySSH2PUT method getFiles.

private List<FileObject> getFiles(String localfolder) throws KettleFileException {
    try {
        List<FileObject> myFileList = new ArrayList<FileObject>();
        // Get all the files in the local directory...
        FileObject localFiles = KettleVFS.getFileObject(localfolder, this);
        FileObject[] children = localFiles.getChildren();
        if (children != null) {
            for (int i = 0; i < children.length; i++) {
                // Get filename of file or directory
                if (children[i].getType().equals(FileType.FILE)) {
                    myFileList.add(children[i]);
                }
            }
        // end for
        }
        return myFileList;
    } catch (IOException e) {
        throw new KettleFileException(e);
    }
}

Example 60 with KettleFileException

use of org.pentaho.di.core.exception.KettleFileException in project pentaho-kettle by pentaho.

the class S3CsvInput method readOneRow.

/**
 * Read a single row of data from the file...
 *
 * @param doConversions if you want to do conversions, set to false for the header row.
 * @return a row of data...
 * @throws KettleException
 */
private Object[] readOneRow(boolean doConversions) throws KettleException {
    try {
        Object[] outputRowData = RowDataUtil.allocateRowData(data.outputRowMeta.size());
        int outputIndex = 0;
        boolean newLineFound = false;
        int newLines = 0;
        // 
        while (!newLineFound && outputIndex < data.convertRowMeta.size()) {
            if (data.endBuffer >= data.bufferSize) {
                // Oops, we need to read more data...
                // Better resize this before we read other things in it...
                // 
                data.resizeByteBuffer();
                // Also read another chunk of data, now that we have the space for it...
                if (!data.readBufferFromFile()) {
                    // TODO handle EOF properly for EOF in the middle of the row, etc.
                    return null;
                }
            }
            // OK, at this point we should have data in the byteBuffer and we should be able to scan for the next
            // delimiter (;)
            // So let's look for a delimiter.
            // Also skip over the enclosures ("), it is NOT taking into account escaped enclosures.
            // Later we can add an option for having escaped or double enclosures in the file. <sigh>
            // 
            boolean delimiterFound = false;
            boolean enclosureFound = false;
            int escapedEnclosureFound = 0;
            while (!delimiterFound) {
                // 
                if (data.byteBuffer[data.endBuffer] == data.delimiter[0]) {
                    delimiterFound = true;
                } else if (data.byteBuffer[data.endBuffer] == '\n' || data.byteBuffer[data.endBuffer] == '\r') {
                    // Perhaps we found a new line?
                    // 
                    // 
                    data.endBuffer++;
                    data.totalBytesRead++;
                    newLines = 1;
                    if (data.endBuffer >= data.bufferSize) {
                        // Oops, we need to read more data...
                        // Better resize this before we read other things in it...
                        // 
                        data.resizeByteBuffer();
                        // Also read another chunk of data, now that we have the space for it...
                        // Ignore EOF, there might be other stuff in the buffer.
                        // 
                        data.readBufferFromFile();
                    }
                    // re-check for double delimiters...
                    if (data.byteBuffer[data.endBuffer] == '\n' || data.byteBuffer[data.endBuffer] == '\r') {
                        data.endBuffer++;
                        data.totalBytesRead++;
                        newLines = 2;
                        if (data.endBuffer >= data.bufferSize) {
                            // Oops, we need to read more data...
                            // Better resize this before we read other things in it...
                            // 
                            data.resizeByteBuffer();
                            // Also read another chunk of data, now that we have the space for it...
                            // Ignore EOF, there might be other stuff in the buffer.
                            // 
                            data.readBufferFromFile();
                        }
                    }
                    newLineFound = true;
                    delimiterFound = true;
                } else if (data.enclosure != null && data.byteBuffer[data.endBuffer] == data.enclosure[0]) {
                    // Perhaps we need to skip over an enclosed part?
                    // We always expect exactly one enclosure character
                    // If we find the enclosure doubled, we consider it escaped.
                    // --> "" is converted to " later on.
                    // 
                    enclosureFound = true;
                    boolean keepGoing;
                    do {
                        if (data.increaseEndBuffer()) {
                            enclosureFound = false;
                            break;
                        }
                        keepGoing = data.byteBuffer[data.endBuffer] != data.enclosure[0];
                        if (!keepGoing) {
                            // Read another byte...
                            if (data.increaseEndBuffer()) {
                                enclosureFound = false;
                                break;
                            }
                            // If this character is also an enclosure, we can consider the enclosure "escaped".
                            // As such, if this is an enclosure, we keep going...
                            // 
                            keepGoing = data.byteBuffer[data.endBuffer] == data.enclosure[0];
                            if (keepGoing) {
                                escapedEnclosureFound++;
                            }
                        }
                    } while (keepGoing);
                    // 
                    if (data.endBuffer >= data.bufferSize) {
                        // consider it a newline to break out of the upper while loop
                        newLineFound = true;
                        // to remove the enclosures in case of missing newline on last line.
                        newLines += 2;
                        break;
                    }
                } else {
                    data.endBuffer++;
                    data.totalBytesRead++;
                    if (data.endBuffer >= data.bufferSize) {
                        // Oops, we need to read more data...
                        // Better resize this before we read other things in it...
                        // 
                        data.resizeByteBuffer();
                        // Also read another chunk of data, now that we have the space for it...
                        if (!data.readBufferFromFile()) {
                            // 
                            if (data.endBuffer >= data.bufferSize) {
                                // consider it a newline to break out of the upper while loop
                                newLineFound = true;
                                break;
                            }
                        }
                    }
                }
            }
            // If we're still here, we found a delimiter..
            // Since the starting point never changed really, we just can grab range:
            // 
            // [startBuffer-endBuffer[
            // 
            // This is the part we want.
            // 
            int length = data.endBuffer - data.startBuffer;
            if (newLineFound) {
                length -= newLines;
                if (length <= 0) {
                    length = 0;
                }
            }
            if (enclosureFound) {
                data.startBuffer++;
                length -= 2;
                if (length <= 0) {
                    length = 0;
                }
            }
            if (length <= 0) {
                length = 0;
            }
            byte[] field = new byte[length];
            System.arraycopy(data.byteBuffer, data.startBuffer, field, 0, length);
            // 
            if (escapedEnclosureFound > 0) {
                if (log.isRowLevel()) {
                    logRowlevel("Escaped enclosures found in " + new String(field));
                }
                field = data.removeEscapedEnclosures(field, escapedEnclosureFound);
            }
            if (doConversions) {
                if (meta.isLazyConversionActive()) {
                    outputRowData[outputIndex++] = field;
                } else {
                    // We're not lazy so we convert the data right here and now.
                    // The convert object uses binary storage as such we just have to ask the native type from it.
                    // That will do the actual conversion.
                    // 
                    ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(outputIndex);
                    outputRowData[outputIndex++] = sourceValueMeta.convertBinaryStringToNativeType(field);
                }
            } else {
                // nothing for the header, no conversions here.
                outputRowData[outputIndex++] = null;
            }
            // OK, move on to the next field...
            if (!newLineFound) {
                data.endBuffer++;
                data.totalBytesRead++;
            }
            data.startBuffer = data.endBuffer;
        }
        // 
        if (meta.isIncludingFilename() && !Utils.isEmpty(meta.getFilenameField())) {
            if (meta.isLazyConversionActive()) {
                outputRowData[outputIndex++] = data.binaryFilename;
            } else {
                outputRowData[outputIndex++] = data.filenames[data.filenr - 1];
            }
        }
        if (data.isAddingRowNumber) {
            outputRowData[outputIndex++] = new Long(data.rowNumber++);
        }
        incrementLinesInput();
        return outputRowData;
    } catch (Exception e) {
        throw new KettleFileException("Exception reading line using NIO", e);
    }
}