Examples with ResultFile org.pentaho.di.core.ResultFile used on opensource projects

Example 21 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class FilesFromResultMeta method getFields.

public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    // Add the fields from a ResultFile
    try {
        ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject("foo.bar", space), "parentOrigin", "origin");
        RowMetaAndData add = resultFile.getRow();
        // Set the origin on the fields...
        for (int i = 0; i < add.size(); i++) {
            add.getValueMeta(i).setOrigin(name);
        }
        r.addRowMeta(add.getRowMeta());
    } catch (KettleFileException e) {
        throw new KettleStepException(e);
    }
}

Example 22 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class FixedInput method init.

public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (FixedInputMeta) smi;
    data = (FixedInputData) sdi;
    if (super.init(smi, sdi)) {
        try {
            data.preferredBufferSize = Integer.parseInt(environmentSubstitute(meta.getBufferSize()));
            data.lineWidth = Integer.parseInt(environmentSubstitute(meta.getLineWidth()));
            data.filename = environmentSubstitute(meta.getFilename());
            if (Utils.isEmpty(data.filename)) {
                logError(BaseMessages.getString(PKG, "FixedInput.MissingFilename.Message"));
                return false;
            }
            FileObject fileObject = KettleVFS.getFileObject(data.filename, getTransMeta());
            try {
                data.fis = getFileInputStream(fileObject.getURL());
                data.fc = data.fis.getChannel();
                data.bb = ByteBuffer.allocateDirect(data.preferredBufferSize);
            } catch (IOException e) {
                logError(e.toString());
                return false;
            }
            // Add filename to result filenames ?
            if (meta.isAddResultFile()) {
                ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, getTransMeta().getName(), toString());
                resultFile.setComment("File was read by a Fixed input step");
                addResultFile(resultFile);
            }
            logBasic("Opened file with name [" + data.filename + "]");
            data.stopReading = false;
            if (meta.isRunningInParallel()) {
                data.stepNumber = getUniqueStepNrAcrossSlaves();
                data.totalNumberOfSteps = getUniqueStepCountAcrossSlaves();
                data.fileSize = fileObject.getContent().getSize();
            }
            // 
            if (meta.isRunningInParallel()) {
                // including line separator bytes
                int totalLineWidth = data.lineWidth + meta.getLineSeparatorLength();
                // 100.000 / 100 = 1000 rows
                long nrRows = data.fileSize / totalLineWidth;
                // 0, 333, 667
                long rowsToSkip = Math.round(data.stepNumber * nrRows / (double) data.totalNumberOfSteps);
                // 333, 667, 1000
                long nextRowsToSkip = Math.round((data.stepNumber + 1) * nrRows / (double) data.totalNumberOfSteps);
                data.rowsToRead = nextRowsToSkip - rowsToSkip;
                long bytesToSkip = rowsToSkip * totalLineWidth;
                logBasic("Step #" + data.stepNumber + " is skipping " + bytesToSkip + " to position in file, then it's reading " + data.rowsToRead + " rows.");
                data.fc.position(bytesToSkip);
            }
            return true;
        } catch (Exception e) {
            logError("Error opening file '" + meta.getFilename() + "'", e);
        }
    }
    return false;
}

Example 23 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class ExcelOutput method addFilenameToResult.

private void addFilenameToResult() throws KettleException {
    try {
        if (meta.isAddToResultFiles()) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
            resultFile.setComment("This file was created with an Excel output step by Pentaho Data Integration");
            addResultFile(resultFile);
        }
    } catch (Exception e) {
        throw new KettleException("Unable to add filename to the result", e);
    }
}

Example 24 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class LDIFInput method openNextFile.

private boolean openNextFile() {
    try {
        if (!meta.isFileField()) {
            if (data.filenr >= data.files.nrOfFiles()) {
                if (isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "LDIFInput.Log.FinishedProcessing"));
                }
                return false;
            }
            // Is this the last file?
            data.last_file = (data.filenr == data.files.nrOfFiles() - 1);
            data.file = data.files.getFile(data.filenr);
            // Move file pointer ahead!
            data.filenr++;
        } else {
            // Get row from input rowset & set row busy!
            data.readrow = getRow();
            if (data.readrow == null) {
                if (isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "LDIFInput.Log.FinishedProcessing"));
                }
                return false;
            }
            if (first) {
                first = false;
                data.inputRowMeta = getInputRowMeta();
                data.outputRowMeta = data.inputRowMeta.clone();
                meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
                // Get total previous fields
                data.totalpreviousfields = data.inputRowMeta.size();
                // Create convert meta-data objects that will contain Date & Number formatters
                data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
                // Check is filename field is provided
                if (Utils.isEmpty(meta.getDynamicFilenameField())) {
                    logError(BaseMessages.getString(PKG, "LDIFInput.Log.NoField"));
                    throw new KettleException(BaseMessages.getString(PKG, "LDIFInput.Log.NoField"));
                }
                // cache the position of the field
                if (data.indexOfFilenameField < 0) {
                    data.indexOfFilenameField = getInputRowMeta().indexOfValue(meta.getDynamicFilenameField());
                    if (data.indexOfFilenameField < 0) {
                        // The field is unreachable !
                        logError(BaseMessages.getString(PKG, "LDIFInput.Log.ErrorFindingField") + "[" + meta.getDynamicFilenameField() + "]");
                        throw new KettleException(BaseMessages.getString(PKG, "LDIFInput.Exception.CouldnotFindField", meta.getDynamicFilenameField()));
                    }
                }
            }
            // End if first
            String filename = getInputRowMeta().getString(data.readrow, data.indexOfFilenameField);
            if (isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "LDIFInput.Log.FilenameInStream", meta.getDynamicFilenameField(), filename));
            }
            data.file = KettleVFS.getFileObject(filename, getTransMeta());
        }
        data.filename = KettleVFS.getFilename(data.file);
        // Add additional fields?
        if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
            data.shortFilename = data.file.getName().getBaseName();
        }
        try {
            if (meta.getPathField() != null && meta.getPathField().length() > 0) {
                data.path = KettleVFS.getFilename(data.file.getParent());
            }
            if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
                data.hidden = data.file.isHidden();
            }
            if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
                data.extension = data.file.getName().getExtension();
            }
            if (meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0) {
                data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
            }
            if (meta.getUriField() != null && meta.getUriField().length() > 0) {
                data.uriName = data.file.getName().getURI();
            }
            if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
                data.rootUriName = data.file.getName().getRootURI();
            }
            if (meta.getSizeField() != null && meta.getSizeField().length() > 0) {
                data.size = new Long(data.file.getContent().getSize());
            }
        } catch (Exception e) {
            throw new KettleException(e);
        }
        if (isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "LDIFInput.Log.OpeningFile", data.file.toString()));
        }
        if (meta.AddToResultFilename()) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
            resultFile.setComment(BaseMessages.getString(PKG, "LDIFInput.Log.FileAddedResult"));
            addResultFile(resultFile);
        }
        data.InputLDIF = new LDIF(KettleVFS.getFilename(data.file));
        if (isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "LDIFInput.Log.FileOpened", data.file.toString()));
        }
    } catch (Exception e) {
        logError(BaseMessages.getString(PKG, "LDIFInput.Log.UnableToOpenFile", "" + data.filenr, data.file.toString(), e.toString()));
        stopAll();
        setErrors(1);
        return false;
    }
    return true;
}

Example 25 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class CsvInput method openNextFile.

private boolean openNextFile() throws KettleException {
    try {
        // Close the previous file...
        // 
        data.closeFile();
        if (data.filenr >= data.filenames.length) {
            return false;
        }
        // Open the next one...
        // 
        data.fieldsMapping = createFieldMapping(data.filenames[data.filenr], meta);
        FileObject fileObject = KettleVFS.getFileObject(data.filenames[data.filenr], getTransMeta());
        if (!(fileObject instanceof LocalFile)) {
            // 
            throw new KettleException(BaseMessages.getString(PKG, "CsvInput.Log.OnlyLocalFilesAreSupported"));
        }
        if (meta.isLazyConversionActive()) {
            data.binaryFilename = data.filenames[data.filenr].getBytes();
        }
        data.fis = new FileInputStream(KettleVFS.getFilename(fileObject));
        data.fc = data.fis.getChannel();
        data.bb = ByteBuffer.allocateDirect(data.preferredBufferSize);
        // 
        if (data.parallel) {
            if (data.bytesToSkipInFirstFile > 0) {
                data.fc.position(data.bytesToSkipInFirstFile);
                // evaluate whether there is a need to skip a row
                if (needToSkipRow()) {
                    // on windows systems, it's a sequence of '\r' and '\n'. finally we set the start of the buffer to the end buffer position.
                    while (!data.newLineFound()) {
                        data.moveEndBufferPointer();
                    }
                    data.moveEndBufferPointer();
                    if (data.newLineFound()) {
                        data.moveEndBufferPointer();
                    }
                }
                data.setStartBuffer(data.getEndBuffer());
            }
        }
        // Add filename to result filenames ?
        if (meta.isAddResultFile()) {
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, getTransMeta().getName(), toString());
            resultFile.setComment("File was read by a Csv input step");
            addResultFile(resultFile);
        }
        // Move to the next filename
        // 
        data.filenr++;
        // 
        if (meta.isHeaderPresent()) {
            // Standard flat file : skip header
            if (!data.parallel || data.bytesToSkipInFirstFile <= 0) {
                // skip this row.
                readOneRow(true, false);
                logBasic(BaseMessages.getString(PKG, "CsvInput.Log.HeaderRowSkipped", data.filenames[data.filenr - 1]));
            }
        }
        // Reset the row number pointer...
        // 
        data.rowNumber = 1L;
        // Don't skip again in the next file...
        // 
        data.bytesToSkipInFirstFile = -1L;
        return true;
    } catch (KettleException e) {
        throw e;
    } catch (Exception e) {
        throw new KettleException(e);
    }
}