Examples with ResultFile org.pentaho.di.core.ResultFile used on opensource projects

Example 51 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class AbstractFileErrorHandler method getWriter.

/**
 * returns the OutputWiter if exists. Otherwhise it will create a new one.
 *
 * @return
 * @throws KettleException
 */
Writer getWriter(Object source) throws KettleException {
    try {
        Writer outputStreamWriter = writers.get(source);
        if (outputStreamWriter != null) {
            return outputStreamWriter;
        }
        FileObject file = getReplayFilename(destinationDirectory, processingFilename, dateString, fileExtension, source);
        ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, file, baseStep.getTransMeta().getName(), baseStep.getStepname());
        baseStep.addResultFile(resultFile);
        try {
            if (encoding == null) {
                outputStreamWriter = new OutputStreamWriter(KettleVFS.getOutputStream(file, false));
            } else {
                outputStreamWriter = new OutputStreamWriter(KettleVFS.getOutputStream(file, false), encoding);
            }
        } catch (Exception e) {
            throw new KettleException(BaseMessages.getString(PKG, "AbstractFileErrorHandler.Exception.CouldNotCreateFileErrorHandlerForFile") + file.getName().getURI(), e);
        }
        writers.put(source, outputStreamWriter);
        return outputStreamWriter;
    } catch (KettleFileException e) {
        throw new KettleException(BaseMessages.getString(PKG, "AbstractFileErrorHandler.Exception.CouldNotCreateFileErrorHandlerForFile"), e);
    }
}

Example 52 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class ChangeFileEncoding method changeEncoding.

private void changeEncoding(String sourceFilename, String targetFilename) throws KettleException {
    BufferedWriter buffWriter = null;
    BufferedReader buffReader = null;
    try {
        buffWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(targetFilename, false), data.targetEncoding));
        if (Utils.isEmpty(data.sourceEncoding)) {
            buffReader = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFilename)));
        } else {
            buffReader = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFilename), data.sourceEncoding));
        }
        char[] cBuf = new char[8192];
        int readSize = 0;
        while ((readSize = buffReader.read(cBuf)) != -1) {
            buffWriter.write(cBuf, 0, readSize);
        }
        // add filename to result filenames?
        if (meta.addSourceResultFilenames()) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.sourceFile, getTransMeta().getName(), getStepname());
            resultFile.setComment(BaseMessages.getString(PKG, "ChangeFileEncoding.Log.FileAddedResult"));
            addResultFile(resultFile);
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "ChangeFileEncoding.Log.FilenameAddResult", data.sourceFile.toString()));
            }
        }
        // add filename to result filenames?
        if (meta.addTargetResultFilenames()) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(targetFilename), getTransMeta().getName(), getStepname());
            resultFile.setComment(BaseMessages.getString(PKG, "ChangeFileEncoding.Log.FileAddedResult"));
            addResultFile(resultFile);
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "ChangeFileEncoding.Log.FilenameAddResult", targetFilename));
            }
        }
    } catch (Exception e) {
        throw new KettleException(BaseMessages.getString(PKG, "ChangeFileEncoding.Error.CreatingFile"), e);
    } finally {
        try {
            if (buffWriter != null) {
                buffWriter.flush();
                buffWriter.close();
            }
            if (buffReader != null) {
                buffReader.close();
            }
        } catch (Exception e) {
        // Ignore
        }
    }
}

Example 53 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class GetFileNames method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    if (!meta.isFileField()) {
        if (data.filenr >= data.filessize) {
            setOutputDone();
            return false;
        }
    } else {
        if (data.filenr >= data.filessize) {
            // Grab one row from previous step ...
            data.readrow = getRow();
        }
        if (data.readrow == null) {
            setOutputDone();
            return false;
        }
        if (first) {
            first = false;
            data.inputRowMeta = getInputRowMeta();
            data.outputRowMeta = data.inputRowMeta.clone();
            meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
            // Get total previous fields
            data.totalpreviousfields = data.inputRowMeta.size();
            // Check is filename field is provided
            if (Utils.isEmpty(meta.getDynamicFilenameField())) {
                logError(BaseMessages.getString(PKG, "GetFileNames.Log.NoField"));
                throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Log.NoField"));
            }
            // cache the position of the field
            if (data.indexOfFilenameField < 0) {
                data.indexOfFilenameField = data.inputRowMeta.indexOfValue(meta.getDynamicFilenameField());
                if (data.indexOfFilenameField < 0) {
                    // The field is unreachable !
                    logError(BaseMessages.getString(PKG, "GetFileNames.Log.ErrorFindingField", meta.getDynamicFilenameField()));
                    throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Exception.CouldnotFindField", meta.getDynamicFilenameField()));
                }
            }
            // If wildcard field is specified, Check if field exists
            if (!Utils.isEmpty(meta.getDynamicWildcardField())) {
                if (data.indexOfWildcardField < 0) {
                    data.indexOfWildcardField = data.inputRowMeta.indexOfValue(meta.getDynamicWildcardField());
                    if (data.indexOfWildcardField < 0) {
                        // The field is unreachable !
                        logError(BaseMessages.getString(PKG, "GetFileNames.Log.ErrorFindingField") + "[" + meta.getDynamicWildcardField() + "]");
                        throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Exception.CouldnotFindField", meta.getDynamicWildcardField()));
                    }
                }
            }
            // If ExcludeWildcard field is specified, Check if field exists
            if (!Utils.isEmpty(meta.getDynamicExcludeWildcardField())) {
                if (data.indexOfExcludeWildcardField < 0) {
                    data.indexOfExcludeWildcardField = data.inputRowMeta.indexOfValue(meta.getDynamicExcludeWildcardField());
                    if (data.indexOfExcludeWildcardField < 0) {
                        // The field is unreachable !
                        logError(BaseMessages.getString(PKG, "GetFileNames.Log.ErrorFindingField") + "[" + meta.getDynamicExcludeWildcardField() + "]");
                        throw new KettleException(BaseMessages.getString(PKG, "GetFileNames.Exception.CouldnotFindField", meta.getDynamicExcludeWildcardField()));
                    }
                }
            }
        }
    }
    try {
        Object[] outputRow = buildEmptyRow();
        int outputIndex = 0;
        Object[] extraData = new Object[data.nrStepFields];
        if (meta.isFileField()) {
            if (data.filenr >= data.filessize) {
                // Get value of dynamic filename field ...
                String filename = getInputRowMeta().getString(data.readrow, data.indexOfFilenameField);
                String wildcard = "";
                if (data.indexOfWildcardField >= 0) {
                    wildcard = getInputRowMeta().getString(data.readrow, data.indexOfWildcardField);
                }
                String excludewildcard = "";
                if (data.indexOfExcludeWildcardField >= 0) {
                    excludewildcard = getInputRowMeta().getString(data.readrow, data.indexOfExcludeWildcardField);
                }
                String[] filesname = { filename };
                String[] filesmask = { wildcard };
                String[] excludefilesmask = { excludewildcard };
                String[] filesrequired = { "N" };
                boolean[] includesubfolders = { meta.isDynamicIncludeSubFolders() };
                // Get files list
                data.files = meta.getDynamicFileList(this, filesname, filesmask, excludefilesmask, filesrequired, includesubfolders);
                data.filessize = data.files.nrOfFiles();
                data.filenr = 0;
            }
            // Clone current input row
            outputRow = data.readrow.clone();
        }
        if (data.filessize > 0) {
            data.file = data.files.getFile(data.filenr);
            if (meta.isAddResultFile()) {
                // Add this to the result file names...
                ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
                resultFile.setComment(BaseMessages.getString(PKG, "GetFileNames.Log.FileReadByStep"));
                addResultFile(resultFile);
            }
            // filename
            extraData[outputIndex++] = KettleVFS.getFilename(data.file);
            // short_filename
            extraData[outputIndex++] = data.file.getName().getBaseName();
            try {
                // Path
                extraData[outputIndex++] = KettleVFS.getFilename(data.file.getParent());
                // type
                extraData[outputIndex++] = data.file.getType().toString();
                // exists
                extraData[outputIndex++] = Boolean.valueOf(data.file.exists());
                // ishidden
                extraData[outputIndex++] = Boolean.valueOf(data.file.isHidden());
                // isreadable
                extraData[outputIndex++] = Boolean.valueOf(data.file.isReadable());
                // iswriteable
                extraData[outputIndex++] = Boolean.valueOf(data.file.isWriteable());
                // lastmodifiedtime
                extraData[outputIndex++] = new Date(data.file.getContent().getLastModifiedTime());
                // size
                Long size = null;
                if (data.file.getType().equals(FileType.FILE)) {
                    size = new Long(data.file.getContent().getSize());
                }
                extraData[outputIndex++] = size;
            } catch (IOException e) {
                throw new KettleException(e);
            }
            // extension
            extraData[outputIndex++] = data.file.getName().getExtension();
            // uri
            extraData[outputIndex++] = data.file.getName().getURI();
            // rooturi
            extraData[outputIndex++] = data.file.getName().getRootURI();
            // See if we need to add the row number to the row...
            if (meta.includeRowNumber() && !Utils.isEmpty(meta.getRowNumberField())) {
                extraData[outputIndex++] = new Long(data.rownr);
            }
            data.rownr++;
            // Add row data
            outputRow = RowDataUtil.addRowData(outputRow, data.totalpreviousfields, extraData);
            // Send row
            putRow(data.outputRowMeta, outputRow);
            if (meta.getRowLimit() > 0 && data.rownr >= meta.getRowLimit()) {
                // limit has been reached: stop now.
                setOutputDone();
                return false;
            }
        }
    } catch (Exception e) {
        throw new KettleStepException(e);
    }
    data.filenr++;
    if (checkFeedback(getLinesInput())) {
        if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "GetFileNames.Log.NrLine", "" + getLinesInput()));
        }
    }
    return true;
}

Example 54 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class GetFilesRowsCount method openNextFile.

private boolean openNextFile() {
    if (data.last_file) {
        // Done!
        return false;
    }
    try {
        if (!meta.isFileField()) {
            if (data.filenr >= data.files.nrOfFiles()) {
                if (log.isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.FinishedProcessing"));
                }
                return false;
            }
            // Is this the last file?
            data.last_file = (data.filenr == data.files.nrOfFiles() - 1);
            data.file = data.files.getFile((int) data.filenr);
        } else {
            // Get row from input rowset & set row busy!
            data.readrow = getRow();
            if (data.readrow == null) {
                if (log.isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.FinishedProcessing"));
                }
                return false;
            }
            if (first) {
                first = false;
                data.inputRowMeta = getInputRowMeta();
                data.outputRowMeta = data.inputRowMeta.clone();
                meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
                // Get total previous fields
                data.totalpreviousfields = data.inputRowMeta.size();
                // Check is filename field is provided
                if (Utils.isEmpty(meta.setOutputFilenameField())) {
                    logError(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.NoField"));
                    throw new KettleException(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.NoField"));
                }
                // cache the position of the field
                if (data.indexOfFilenameField < 0) {
                    data.indexOfFilenameField = getInputRowMeta().indexOfValue(meta.setOutputFilenameField());
                    if (data.indexOfFilenameField < 0) {
                        // The field is unreachable !
                        logError(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.ErrorFindingField", meta.setOutputFilenameField()));
                        throw new KettleException(BaseMessages.getString(PKG, "GetFilesRowsCount.Exception.CouldnotFindField", meta.setOutputFilenameField()));
                    }
                }
            }
            // End if first
            String filename = getInputRowMeta().getString(data.readrow, data.indexOfFilenameField);
            if (log.isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.FilenameInStream", meta.setOutputFilenameField(), filename));
            }
            data.file = KettleVFS.getFileObject(filename, getTransMeta());
            // Init Row number
            if (meta.isFileField()) {
                data.rownr = 0;
            }
        }
        // Move file pointer ahead!
        data.filenr++;
        if (meta.isAddResultFile()) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
            resultFile.setComment(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.FileAddedResult"));
            addResultFile(resultFile);
        }
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.OpeningFile", data.file.toString()));
        }
        getRowNumber();
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.FileOpened", data.file.toString()));
        }
    } catch (Exception e) {
        logError(BaseMessages.getString(PKG, "GetFilesRowsCount.Log.UnableToOpenFile", "" + data.filenr, data.file.toString(), e.toString()));
        stopAll();
        setErrors(1);
        return false;
    }
    return true;
}

Example 55 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class ExcelWriterStep method prepareNextOutputFile.

public void prepareNextOutputFile() throws KettleException {
    try {
        // sheet name shouldn't exceed 31 character
        if (data.realSheetname != null && data.realSheetname.length() > 31) {
            throw new KettleException(BaseMessages.getString(PKG, "ExcelWriterStep.Exception.MaxSheetName", data.realSheetname));
        }
        // clear style cache
        int numOfFields = meta.getOutputFields() != null && meta.getOutputFields().length > 0 ? meta.getOutputFields().length : 0;
        if (numOfFields == 0) {
            numOfFields = data.inputRowMeta != null ? data.inputRowMeta.size() : 0;
        }
        data.clearStyleCache(numOfFields);
        // build new filename
        String buildFilename = buildFilename(data.splitnr);
        data.file = KettleVFS.getFileObject(buildFilename, getTransMeta());
        if (log.isDebug()) {
            logDebug(BaseMessages.getString(PKG, "ExcelWriterStep.Log.OpeningFile", buildFilename));
        }
        // determine whether existing file must be deleted
        if (data.file.exists() && data.createNewFile) {
            if (!data.file.delete()) {
                if (log.isBasic()) {
                    logBasic(BaseMessages.getString(PKG, "ExcelWriterStep.Log.CouldNotDeleteStaleFile", buildFilename));
                }
                setErrors(1);
                throw new KettleException("Could not delete stale file " + buildFilename);
            }
        }
        // adding filename to result
        if (meta.isAddToResultFiles()) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
            resultFile.setComment("This file was created with an Excel writer step by Pentaho Data Integration");
            addResultFile(resultFile);
        }
        boolean appendingToSheet = true;
        // if now no file exists we must create it as indicated by user
        if (!data.file.exists()) {
            // if template file is enabled
            if (meta.isTemplateEnabled()) {
                // handle template case (must have same format)
                // ensure extensions match
                String templateExt = KettleVFS.getFileObject(data.realTemplateFileName).getName().getExtension();
                if (!meta.getExtension().equalsIgnoreCase(templateExt)) {
                    throw new KettleException("Template Format Mismatch: Template has extension: " + templateExt + ", but output file has extension: " + meta.getExtension() + ". Template and output file must share the same format!");
                }
                if (KettleVFS.getFileObject(data.realTemplateFileName).exists()) {
                    // if the template exists just copy the template in place
                    copyFile(KettleVFS.getFileObject(data.realTemplateFileName, getTransMeta()), data.file);
                } else {
                    // template is missing, log it and get out
                    if (log.isBasic()) {
                        logBasic(BaseMessages.getString(PKG, "ExcelWriterStep.Log.TemplateMissing", data.realTemplateFileName));
                    }
                    setErrors(1);
                    throw new KettleException("Template file missing: " + data.realTemplateFileName);
                }
            } else {
                // handle fresh file case, just create a fresh workbook
                Workbook wb = meta.getExtension().equalsIgnoreCase("xlsx") ? new XSSFWorkbook() : new HSSFWorkbook();
                BufferedOutputStreamWithCloseDetection out = new BufferedOutputStreamWithCloseDetection(KettleVFS.getOutputStream(data.file, false));
                wb.createSheet(data.realSheetname);
                wb.write(out);
                out.close();
                wb.close();
            }
            appendingToSheet = false;
        }
        // file is guaranteed to be in place now
        if (meta.getExtension().equalsIgnoreCase("xlsx")) {
            XSSFWorkbook xssfWorkbook = new XSSFWorkbook(KettleVFS.getInputStream(data.file));
            if (meta.isStreamingData() && !meta.isTemplateEnabled()) {
                data.wb = new SXSSFWorkbook(xssfWorkbook, 100);
            } else {
                // Initialize it later after writing header/template because SXSSFWorkbook can't read/rewrite existing data,
                // only append.
                data.wb = xssfWorkbook;
            }
        } else {
            data.wb = new HSSFWorkbook(KettleVFS.getInputStream(data.file));
        }
        int existingActiveSheetIndex = data.wb.getActiveSheetIndex();
        int replacingSheetAt = -1;
        if (data.wb.getSheet(data.realSheetname) != null) {
            // sheet exists, replace or reuse as indicated by user
            if (data.createNewSheet) {
                replacingSheetAt = data.wb.getSheetIndex(data.wb.getSheet(data.realSheetname));
                data.wb.removeSheetAt(replacingSheetAt);
            }
        }
        // if sheet is now missing, we need to create a new one
        if (data.wb.getSheet(data.realSheetname) == null) {
            if (meta.isTemplateSheetEnabled()) {
                Sheet ts = data.wb.getSheet(data.realTemplateSheetName);
                // if template sheet is missing, break
                if (ts == null) {
                    throw new KettleException(BaseMessages.getString(PKG, "ExcelWriterStep.Exception.TemplateNotFound", data.realTemplateSheetName));
                }
                data.sheet = data.wb.cloneSheet(data.wb.getSheetIndex(ts));
                data.wb.setSheetName(data.wb.getSheetIndex(data.sheet), data.realSheetname);
                // unhide sheet in case it was hidden
                data.wb.setSheetHidden(data.wb.getSheetIndex(data.sheet), false);
                if (meta.isTemplateSheetHidden()) {
                    data.wb.setSheetHidden(data.wb.getSheetIndex(ts), true);
                }
            } else {
                // no template to use, simply create a new sheet
                data.sheet = data.wb.createSheet(data.realSheetname);
            }
            if (replacingSheetAt > -1) {
                data.wb.setSheetOrder(data.sheet.getSheetName(), replacingSheetAt);
            }
            // preserves active sheet selection in workbook
            data.wb.setActiveSheet(existingActiveSheetIndex);
            data.wb.setSelectedTab(existingActiveSheetIndex);
            appendingToSheet = false;
        } else {
            // sheet is there and should be reused
            data.sheet = data.wb.getSheet(data.realSheetname);
        }
        // if use chose to make the current sheet active, do so
        if (meta.isMakeSheetActive()) {
            int sheetIndex = data.wb.getSheetIndex(data.sheet);
            data.wb.setActiveSheet(sheetIndex);
            data.wb.setSelectedTab(sheetIndex);
        }
        // handle write protection
        if (meta.isSheetProtected()) {
            protectSheet(data.sheet, data.realPassword);
        }
        // starting cell support
        data.startingRow = 0;
        data.startingCol = 0;
        if (!Utils.isEmpty(data.realStartingCell)) {
            CellReference cellRef = new CellReference(data.realStartingCell);
            data.startingRow = cellRef.getRow();
            data.startingCol = cellRef.getCol();
        }
        data.posX = data.startingCol;
        data.posY = data.startingRow;
        // Find last row and append accordingly
        if (!data.createNewSheet && meta.isAppendLines() && appendingToSheet) {
            data.posY = 0;
            if (data.sheet.getPhysicalNumberOfRows() > 0) {
                data.posY = data.sheet.getLastRowNum();
                data.posY++;
            }
        }
        // Find last row and append accordingly
        if (!data.createNewSheet && meta.getAppendOffset() != 0 && appendingToSheet) {
            data.posY += meta.getAppendOffset();
        }
        // may have to write a few empty lines
        if (!data.createNewSheet && meta.getAppendEmpty() > 0 && appendingToSheet) {
            for (int i = 0; i < meta.getAppendEmpty(); i++) {
                openLine();
                if (!data.shiftExistingCells || meta.isAppendLines()) {
                    data.posY++;
                }
            }
        }
        // may have to write a header here
        if (meta.isHeaderEnabled() && !(!data.createNewSheet && meta.isAppendOmitHeader() && appendingToSheet)) {
            writeHeader();
        }
        if (meta.isStreamingData() && meta.isTemplateEnabled()) {
            data.wb = new SXSSFWorkbook((XSSFWorkbook) data.wb, 100);
            data.sheet = data.wb.getSheet(data.realSheetname);
        }
        if (log.isDebug()) {
            logDebug(BaseMessages.getString(PKG, "ExcelWriterStep.Log.FileOpened", buildFilename));
        }
        // this is the number of the new output file
        data.splitnr++;
    } catch (Exception e) {
        logError("Error opening new file", e);
        setErrors(1);
        throw new KettleException(e);
    }
}