Examples with ResultFile org.pentaho.di.core.ResultFile used on opensource projects

Example 56 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class BaseFileInputStep method init.

/**
 * Initialize step before execute.
 */
@Override
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (M) smi;
    data = (D) sdi;
    if (!super.init(smi, sdi)) {
        return false;
    }
    // Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS
    if (getTransMeta().getNamedClusterEmbedManager() != null) {
        getTransMeta().getNamedClusterEmbedManager().passEmbeddedMetastoreKey(this, getTransMeta().getEmbeddedMetastoreProviderKey());
    }
    initErrorHandling();
    meta.additionalOutputFields.normalize();
    data.files = meta.getFileInputList(this);
    data.currentFileIndex = 0;
    // If there are missing files,
    // fail if we don't ignore errors
    // 
    Result previousResult = getTrans().getPreviousResult();
    Map<String, ResultFile> resultFiles = (previousResult != null) ? previousResult.getResultFiles() : null;
    if ((previousResult == null || resultFiles == null || resultFiles.size() == 0) && data.files.nrOfMissingFiles() > 0 && !meta.inputFiles.acceptingFilenames && !meta.errorHandling.errorIgnored) {
        logError(BaseMessages.getString(PKG, "TextFileInput.Log.Error.NoFilesSpecified"));
        return false;
    }
    String clusterSize = getVariable(Const.INTERNAL_VARIABLE_CLUSTER_SIZE);
    if (!Utils.isEmpty(clusterSize) && Integer.valueOf(clusterSize) > 1) {
        // TODO: add metadata to configure this.
        String nr = getVariable(Const.INTERNAL_VARIABLE_SLAVE_SERVER_NUMBER);
        if (log.isDetailed()) {
            logDetailed("Running on slave server #" + nr + " : assuming that each slave reads a dedicated part of the same file(s).");
        }
    }
    return init();
}

Example 57 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class BaseFileInputStep method openNextFile.

/**
 * Open next VFS file for processing.
 *
 * This method will support different parallelization methods later.
 */
protected boolean openNextFile() {
    try {
        if (data.currentFileIndex >= data.files.nrOfFiles()) {
            // all files already processed
            return false;
        }
        // Is this the last file?
        data.file = data.files.getFile(data.currentFileIndex);
        data.filename = KettleVFS.getFilename(data.file);
        fillFileAdditionalFields(data, data.file);
        if (meta.inputFiles.passingThruFields) {
            data.currentPassThruFieldsRow = data.passThruFields.get(data.file);
        }
        // 
        if (meta.inputFiles.isaddresult) {
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), toString());
            resultFile.setComment("File was read by an Text File input step");
            addResultFile(resultFile);
        }
        if (log.isBasic()) {
            logBasic("Opening file: " + data.file.getName().getFriendlyURI());
        }
        data.dataErrorLineHandler.handleFile(data.file);
        data.reader = createReader(meta, data, data.file);
    } catch (Exception e) {
        if (!handleOpenFileException(e)) {
            return false;
        }
        data.reader = null;
    }
    // Move file pointer ahead!
    data.currentFileIndex++;
    return true;
}

Example 58 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class FileExists method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (FileExistsMeta) smi;
    data = (FileExistsData) sdi;
    boolean sendToErrorRow = false;
    String errorMessage = null;
    // Get row from input rowset & set row busy!
    Object[] r = getRow();
    if (r == null) {
        // no more input to be expected...
        setOutputDone();
        return false;
    }
    boolean fileexists = false;
    String filetype = null;
    try {
        if (first) {
            first = false;
            // get the RowMeta
            data.previousRowMeta = getInputRowMeta().clone();
            data.NrPrevFields = data.previousRowMeta.size();
            data.outputRowMeta = data.previousRowMeta;
            meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
            // Check is tablename field is provided
            if (Utils.isEmpty(meta.getDynamicFilenameField())) {
                logError(BaseMessages.getString(PKG, "FileExists.Error.FilenameFieldMissing"));
                throw new KettleException(BaseMessages.getString(PKG, "FileExists.Error.FilenameFieldMissing"));
            }
            // cache the position of the field
            if (data.indexOfFileename < 0) {
                data.indexOfFileename = data.previousRowMeta.indexOfValue(meta.getDynamicFilenameField());
                if (data.indexOfFileename < 0) {
                    // The field is unreachable !
                    logError(BaseMessages.getString(PKG, "FileExists.Exception.CouldnotFindField") + "[" + meta.getDynamicFilenameField() + "]");
                    throw new KettleException(BaseMessages.getString(PKG, "FileExists.Exception.CouldnotFindField", meta.getDynamicFilenameField()));
                }
            }
        }
        // End If first
        Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size());
        for (int i = 0; i < data.NrPrevFields; i++) {
            outputRow[i] = r[i];
        }
        // get filename
        String filename = data.previousRowMeta.getString(r, data.indexOfFileename);
        if (!Utils.isEmpty(filename)) {
            data.file = KettleVFS.getFileObject(filename, getTransMeta());
            // Check if file
            fileexists = data.file.exists();
            // include file type?
            if (meta.includeFileType() && fileexists && !Utils.isEmpty(meta.getFileTypeFieldName())) {
                filetype = data.file.getType().toString();
            }
            // add filename to result filenames?
            if (meta.addResultFilenames() && fileexists && data.file.getType() == FileType.FILE) {
                // Add this to the result file names...
                ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), getStepname());
                resultFile.setComment(BaseMessages.getString(PKG, "FileExists.Log.FileAddedResult"));
                addResultFile(resultFile);
                if (log.isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "FileExists.Log.FilenameAddResult", data.file.toString()));
                }
            }
        }
        // Add result field to input stream
        outputRow[data.NrPrevFields] = fileexists;
        int rowIndex = data.NrPrevFields;
        rowIndex++;
        if (meta.includeFileType() && !Utils.isEmpty(meta.getFileTypeFieldName())) {
            outputRow[rowIndex] = filetype;
        }
        // add new values to the row.
        // copy row to output rowset(s);
        putRow(data.outputRowMeta, outputRow);
        if (log.isRowLevel()) {
            logRowlevel(BaseMessages.getString(PKG, "FileExists.LineNumber", getLinesRead() + " : " + getInputRowMeta().getString(r)));
        }
    } catch (Exception e) {
        if (getStepMeta().isDoingErrorHandling()) {
            sendToErrorRow = true;
            errorMessage = e.toString();
        } else {
            logError(BaseMessages.getString(PKG, "FileExists.ErrorInStepRunning") + e.getMessage());
            setErrors(1);
            stopAll();
            // signal end to receiver(s)
            setOutputDone();
            return false;
        }
        if (sendToErrorRow) {
            // Simply add this row to the error row
            putError(getInputRowMeta(), r, 1, errorMessage, meta.getResultFieldName(), "FileExistsO01");
        }
    }
    return true;
}

Example 59 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class FileLocked method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (FileLockedMeta) smi;
    data = (FileLockedData) sdi;
    // Get row from input rowset & set row busy!
    Object[] r = getRow();
    if (r == null) {
        // no more input to be expected...
        setOutputDone();
        return false;
    }
    boolean FileLocked = false;
    if (first) {
        first = false;
        // get the RowMeta
        data.previousRowMeta = getInputRowMeta().clone();
        data.NrPrevFields = data.previousRowMeta.size();
        data.outputRowMeta = data.previousRowMeta;
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
        // Check is filename field is provided
        if (Utils.isEmpty(meta.getDynamicFilenameField())) {
            logError(BaseMessages.getString(PKG, "FileLocked.Error.FilenameFieldMissing"));
            throw new KettleException(BaseMessages.getString(PKG, "FileLocked.Error.FilenameFieldMissing"));
        }
        // cache the position of the field
        if (data.indexOfFileename < 0) {
            data.indexOfFileename = data.previousRowMeta.indexOfValue(meta.getDynamicFilenameField());
            if (data.indexOfFileename < 0) {
                // The field is unreachable !
                logError(BaseMessages.getString(PKG, "FileLocked.Exception.CouldnotFindField") + "[" + meta.getDynamicFilenameField() + "]");
                throw new KettleException(BaseMessages.getString(PKG, "FileLocked.Exception.CouldnotFindField", meta.getDynamicFilenameField()));
            }
        }
    }
    try {
        // get filename
        String filename = data.previousRowMeta.getString(r, data.indexOfFileename);
        if (!Utils.isEmpty(filename)) {
            // Check if file
            LockFile locked = new LockFile(filename);
            FileLocked = locked.isLocked();
            // add filename to result filenames?
            if (meta.addResultFilenames()) {
                // Add this to the result file names...
                ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(filename), getTransMeta().getName(), getStepname());
                resultFile.setComment(BaseMessages.getString(PKG, "FileLocked.Log.FileAddedResult"));
                addResultFile(resultFile);
                if (isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "FileLocked.Log.FilenameAddResult", filename));
                }
            }
        }
        // add file locked
        // copy row to output
        putRow(data.outputRowMeta, RowDataUtil.addValueData(r, data.NrPrevFields, FileLocked));
        if (isRowLevel()) {
            logRowlevel(BaseMessages.getString(PKG, "FileLocked.LineNumber", getLinesRead() + " : " + getInputRowMeta().getString(r)));
        }
    } catch (Exception e) {
        boolean sendToErrorRow = false;
        String errorMessage = null;
        if (getStepMeta().isDoingErrorHandling()) {
            sendToErrorRow = true;
            errorMessage = e.toString();
        } else {
            logError(BaseMessages.getString(PKG, "FileLocked.ErrorInStepRunning") + e.getMessage());
            setErrors(1);
            stopAll();
            // signal end to receiver(s)
            setOutputDone();
            return false;
        }
        if (sendToErrorRow) {
            // Simply add this row to the error row
            putError(getInputRowMeta(), r, 1, errorMessage, meta.getResultFieldName(), "FileLocked001");
        }
    }
    return true;
}

Example 60 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class FilesToResult method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (FilesToResultMeta) smi;
    data = (FilesToResultData) sdi;
    // get row, set busy!
    Object[] r = getRow();
    if (r == null) {
        for (ResultFile resultFile : data.filenames) {
            addResultFile(resultFile);
        }
        logBasic(BaseMessages.getString(PKG, "FilesToResult.Log.AddedNrOfFiles", String.valueOf(data.filenames.size())));
        setOutputDone();
        return false;
    }
    if (first) {
        first = false;
        data.filenameIndex = getInputRowMeta().indexOfValue(meta.getFilenameField());
        if (data.filenameIndex < 0) {
            logError(BaseMessages.getString(PKG, "FilesToResult.Log.CouldNotFindField", meta.getFilenameField()));
            setErrors(1);
            stopAll();
            return false;
        }
    }
    // OK, get the filename field from the row
    String filename = getInputRowMeta().getString(r, data.filenameIndex);
    try {
        ResultFile resultFile = new ResultFile(meta.getFileType(), KettleVFS.getFileObject(filename, getTransMeta()), getTrans().getName(), getStepname());
        // Add all rows to rows buffer...
        data.filenames.add(resultFile);
    } catch (Exception e) {
        throw new KettleException(e);
    }
    // Copy to any possible next steps...
    data.outputRowMeta = getInputRowMeta().clone();
    meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
    // copy row to possible alternate
    putRow(data.outputRowMeta, r);
    if (checkFeedback(getLinesRead())) {
        logBasic(BaseMessages.getString(PKG, "FilesToResult.Log.LineNumber") + getLinesRead());
    }
    return true;
}