Examples with ResultFile org.pentaho.di.core.ResultFile used on opensource projects

Example 31 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class ParGzipCsvInput method openNextFile.

private boolean openNextFile() throws KettleException {
    try {
        // Close the previous file...
        // 
        closeFile();
        if (data.filenr >= data.filenames.length) {
            return false;
        }
        // Open the next one...
        // 
        logBasic("Opening file #" + data.filenr + " : " + data.filenames[data.filenr]);
        FileObject fileObject = KettleVFS.getFileObject(data.filenames[data.filenr], getTransMeta());
        data.fis = KettleVFS.getInputStream(fileObject);
        if (meta.isLazyConversionActive()) {
            data.binaryFilename = data.filenames[data.filenr].getBytes();
        }
        data.gzis = new GZIPInputStream(data.fis, data.bufferSize);
        clearBuffer();
        data.fileReadPosition = 0L;
        data.blockNr = 0;
        data.eofReached = false;
        // Skip to the next file...
        // 
        data.filenr++;
        // 
        if (data.parallel) {
            // Calculate the first block of data to read from the file
            // If the buffer size is 500, we read 0-499 for the first file,
            // 500-999 for the second, 1000-1499 for the third, etc.
            // 
            // After that we need to get 1500-1999 for the first step again,
            // 2000-2499 for the second, 2500-2999 for the third, etc.
            // 
            // This is equivalent :
            // 
            // FROM : stepNumber * bufferSize + blockNr*bufferSize*nrOfSteps
            // TO : FROM + bufferSize - 1
            // 
            // Example : step 0, block 0, size 500:
            // From: 0*500+0*500*3=0 To: 0+500-1=499
            // 
            // Example : step 0, block 1, size 500:
            // From: 0*500+1*500*3=1500 To: 1500+500-1=1999
            // 
            // So our first act is to skip to the correct position in the compressed stream...
            // 
            // for now.
            data.blockSize = 2 * data.bufferSize;
            long bytesToSkip = data.stepNumber * data.blockSize;
            if (bytesToSkip > 0) {
                // Get into position for block 0
                // 
                logBasic("Skipping " + bytesToSkip + " bytes to go to position " + bytesToSkip + " for step copy " + data.stepNumber);
                long bytesSkipped = 0L;
                while (bytesSkipped < bytesToSkip) {
                    long n = data.gzis.skip(bytesToSkip - bytesSkipped);
                    if (n <= 0) {
                        // EOF in this file, can't read a block in this step copy
                        data.eofReached = true;
                        return false;
                    }
                    bytesSkipped += n;
                }
                // Keep track of the file pointer!
                // 
                data.fileReadPosition += bytesSkipped;
                // Reset the bytes read in the current block of data
                // 
                data.totalBytesRead = 0L;
                // Skip the first row until the next CR
                // 
                readOneRow(false);
            } else {
                // Reset the bytes read in the current block of data
                // 
                data.totalBytesRead = 0L;
                // 
                if (meta.isHeaderPresent()) {
                    readOneRow(false);
                }
            }
        } else {
            // Just one block: read it all until we hit an EOF.
            // 
            // 9,223,372,036 GB
            data.blockSize = Long.MAX_VALUE;
            // 
            if (meta.isHeaderPresent()) {
                readOneRow(false);
            }
        }
        // Add filename to result filenames ?
        if (meta.isAddResultFile()) {
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, getTransMeta().getName(), toString());
            resultFile.setComment("File was read by a Csv input step");
            addResultFile(resultFile);
        }
        // Reset the row number pointer...
        // 
        data.rowNumber = 1L;
        return true;
    } catch (Exception e) {
        throw new KettleException(e);
    }
}

Example 32 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class PentahoReportingOutput method processReport.

private void processReport(Object[] r, String sourceFilename, String targetFilename, ProcessorType outputProcessorType, Boolean createParentFolder) throws KettleException {
    try {
        // Load the master report from the PRPT
        // 
        MasterReport report = loadMasterReport(sourceFilename);
        // Set the parameters values that are present in the various fields...
        // 
        ReportParameterValues values = report.getParameterValues();
        ReportParameterDefinition definition = report.getParameterDefinition();
        for (String parameterName : meta.getParameterFieldMap().keySet()) {
            String fieldName = meta.getParameterFieldMap().get(parameterName);
            if (fieldName != null) {
                int index = getInputRowMeta().indexOfValue(fieldName);
                if (index < 0) {
                    throw new KettleException(BaseMessages.getString(PKG, "PentahoReportingOutput.Exception.CanNotFindField", fieldName));
                }
                Class<?> clazz = findParameterClass(definition, parameterName);
                Object value = null;
                if (clazz != null) {
                    if (clazz.equals(String.class)) {
                        value = getInputRowMeta().getString(r, index);
                    } else if (clazz.equals((new String[0]).getClass())) {
                        value = getInputRowMeta().getString(r, index).split("\t");
                    } else if (clazz.equals(Date.class)) {
                        value = getInputRowMeta().getDate(r, index);
                    } else if (clazz.equals(byte.class) || clazz.equals(Byte.class)) {
                        value = getInputRowMeta().getInteger(r, index).byteValue();
                    } else if (clazz.equals(Short.class) || clazz.equals(short.class)) {
                        value = getInputRowMeta().getInteger(r, index).shortValue();
                    } else if (clazz.equals(Integer.class) || clazz.equals(int.class)) {
                        value = getInputRowMeta().getInteger(r, index).intValue();
                    } else if (clazz.equals(Long.class) || clazz.equals(long.class)) {
                        value = getInputRowMeta().getInteger(r, index);
                    } else if (clazz.equals(Double.class) || clazz.equals(double.class)) {
                        value = getInputRowMeta().getNumber(r, index);
                    } else if (clazz.equals(Float.class) || clazz.equals(float.class)) {
                        value = getInputRowMeta().getNumber(r, index).floatValue();
                    } else if (clazz.equals(Number.class)) {
                        value = getInputRowMeta().getBigNumber(r, index).floatValue();
                    } else if (clazz.equals(Boolean.class) || clazz.equals(boolean.class)) {
                        value = getInputRowMeta().getBoolean(r, index);
                    } else if (clazz.equals(BigDecimal.class)) {
                        value = getInputRowMeta().getBigNumber(r, index);
                    } else if (clazz.equals((new byte[0]).getClass())) {
                        value = getInputRowMeta().getBinary(r, index);
                    } else {
                        value = getInputRowMeta().getValueMeta(index).convertToNormalStorageType(r[index]);
                    }
                    values.put(parameterName, value);
                } else {
                    // This parameter was not found, log this as a warning...
                    // 
                    logBasic(BaseMessages.getString(PKG, "PentahoReportingOutput.Log.ParameterNotFoundInReport", parameterName, sourceFilename));
                }
            }
        }
        Runnable exportTask;
        PentahoReportingSwingGuiContext context = new PentahoReportingSwingGuiContext();
        switch(outputProcessorType) {
            case PDF:
                exportTask = new ReportExportTask(report, context, targetFilename, createParentFolder) {

                    protected ReportProcessor createReportProcessor(OutputStream fout) throws Exception {
                        PdfOutputProcessor outputProcessor = new PdfOutputProcessor(report.getConfiguration(), fout, report.getResourceManager());
                        return new PageableReportProcessor(report, outputProcessor);
                    }
                };
                break;
            case CSV:
                exportTask = new ReportExportTask(report, context, targetFilename, createParentFolder) {

                    protected ReportProcessor createReportProcessor(OutputStream fout) throws Exception {
                        ReportStructureValidator validator = new ReportStructureValidator();
                        if (validator.isValidForFastProcessing(report) == false) {
                            StreamCSVOutputProcessor target = new StreamCSVOutputProcessor(fout);
                            return new StreamReportProcessor(report, target);
                        } else {
                            return new FastCsvExportProcessor(report, fout);
                        }
                    }
                };
                break;
            case Excel:
                exportTask = new ReportExportTask(report, context, targetFilename, createParentFolder) {

                    protected ReportProcessor createReportProcessor(OutputStream fout) throws Exception {
                        ReportStructureValidator validator = new ReportStructureValidator();
                        if (validator.isValidForFastProcessing(report) == false) {
                            final FlowExcelOutputProcessor target = new FlowExcelOutputProcessor(report.getConfiguration(), fout, report.getResourceManager());
                            target.setUseXlsxFormat(false);
                            return new FlowReportProcessor(report, target);
                        } else {
                            return new FastExcelExportProcessor(report, fout, false);
                        }
                    }
                };
                break;
            case Excel_2007:
                exportTask = new ReportExportTask(report, context, targetFilename, createParentFolder) {

                    protected ReportProcessor createReportProcessor(OutputStream fout) throws Exception {
                        ReportStructureValidator validator = new ReportStructureValidator();
                        if (validator.isValidForFastProcessing(report) == false) {
                            final FlowExcelOutputProcessor target = new FlowExcelOutputProcessor(report.getConfiguration(), fout, report.getResourceManager());
                            target.setUseXlsxFormat(true);
                            return new FlowReportProcessor(report, target);
                        } else {
                            return new FastExcelExportProcessor(report, fout, true);
                        }
                    }
                };
                break;
            case StreamingHTML:
                exportTask = new ReportExportTask(report, context, targetFilename, createParentFolder) {

                    protected String filename, suffix;

                    protected ContentLocation targetRoot;

                    @Override
                    protected void execute() throws Exception {
                        FileObject targetDirectory = targetFile.getParent();
                        FileObjectRepository targetRepository = new FileObjectRepository(targetDirectory);
                        targetRoot = targetRepository.getRoot();
                        suffix = getSuffix(targetPath);
                        filename = IOUtils.getInstance().stripFileExtension(targetFile.getName().toString());
                        ReportProcessor reportProcessor = createReportProcessor(null);
                        try {
                            reportProcessor.processReport();
                        } finally {
                            reportProcessor.close();
                        }
                    }

                    protected ReportProcessor createReportProcessor(OutputStream fout) throws Exception {
                        ReportStructureValidator validator = new ReportStructureValidator();
                        if (validator.isValidForFastProcessing(report) == false) {
                            final HtmlOutputProcessor outputProcessor = new StreamHtmlOutputProcessor(report.getConfiguration());
                            final HtmlPrinter printer = new AllItemsHtmlPrinter(report.getResourceManager());
                            printer.setContentWriter(targetRoot, new DefaultNameGenerator(targetRoot, filename, suffix));
                            // $NON-NLS-1$
                            printer.setDataWriter(null, null);
                            printer.setUrlRewriter(new FileSystemURLRewriter());
                            outputProcessor.setPrinter(printer);
                            return new StreamReportProcessor(report, outputProcessor);
                        } else {
                            FastHtmlContentItems printer = new FastHtmlContentItems();
                            printer.setContentWriter(targetRoot, new DefaultNameGenerator(targetRoot, filename, suffix));
                            // $NON-NLS-1$
                            printer.setDataWriter(null, null);
                            printer.setUrlRewriter(new FileSystemURLRewriter());
                            return new FastHtmlExportProcessor(report, printer);
                        }
                    }
                };
                break;
            case PagedHTML:
                exportTask = new ReportExportTask(report, context, targetFilename, createParentFolder) {

                    protected String filename, suffix;

                    protected ContentLocation targetRoot;

                    @Override
                    protected void execute() throws Exception {
                        FileObject targetDirectory = targetFile.getParent();
                        FileObjectRepository targetRepository = new FileObjectRepository(targetDirectory);
                        targetRoot = targetRepository.getRoot();
                        suffix = getSuffix(targetPath);
                        filename = IOUtils.getInstance().stripFileExtension(targetFile.getName().toString());
                        ReportProcessor reportProcessor = createReportProcessor(null);
                        try {
                            reportProcessor.processReport();
                        } finally {
                            reportProcessor.close();
                        }
                    }

                    protected ReportProcessor createReportProcessor(OutputStream fout) throws Exception {
                        final FlowHtmlOutputProcessor outputProcessor = new FlowHtmlOutputProcessor();
                        final HtmlPrinter printer = new AllItemsHtmlPrinter(report.getResourceManager());
                        printer.setContentWriter(targetRoot, new DefaultNameGenerator(targetRoot, filename, suffix));
                        printer.setDataWriter(targetRoot, new DefaultNameGenerator(targetRoot, "content"));
                        printer.setUrlRewriter(new FileSystemURLRewriter());
                        outputProcessor.setPrinter(printer);
                        return new FlowReportProcessor(report, outputProcessor);
                    }
                };
                break;
            case RTF:
                exportTask = new ReportExportTask(report, context, targetFilename, createParentFolder) {

                    protected ReportProcessor createReportProcessor(OutputStream fout) throws Exception {
                        StreamRTFOutputProcessor target = new StreamRTFOutputProcessor(report.getConfiguration(), fout, report.getResourceManager());
                        return new StreamReportProcessor(report, target);
                    }
                };
                break;
            default:
                exportTask = null;
                break;
        }
        if (exportTask != null) {
            exportTask.run();
        }
        if (context.getStatusType() == StatusType.ERROR) {
            KettleVFS.getFileObject(targetFilename, getTransMeta()).delete();
            if (context.getCause() != null) {
                throw context.getCause();
            }
            throw new KettleStepException(context.getMessage());
        }
        ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(targetFilename, getTransMeta()), getTransMeta().getName(), getStepname());
        resultFile.setComment("This file was created with a Pentaho Reporting Output step");
        addResultFile(resultFile);
    } catch (Throwable e) {
        throw new KettleException(BaseMessages.getString(PKG, "PentahoReportingOutput.Exception.UnexpectedErrorRenderingReport", sourceFilename, targetFilename, outputProcessorType.getDescription()), e);
    }
}

Example 33 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class ProcessFiles method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (ProcessFilesMeta) smi;
    data = (ProcessFilesData) sdi;
    // Get row from input rowset & set row busy!
    Object[] r = getRow();
    if (r == null) {
        // no more input to be expected...
        setOutputDone();
        return false;
    }
    if (first) {
        first = false;
        // Check is source filename field is provided
        if (Utils.isEmpty(meta.getDynamicSourceFileNameField())) {
            throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFilenameFieldMissing"));
        }
        // Check is target filename field is provided
        if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && Utils.isEmpty(meta.getDynamicTargetFileNameField())) {
            throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFilenameFieldMissing"));
        }
        // cache the position of the source filename field
        if (data.indexOfSourceFilename < 0) {
            data.indexOfSourceFilename = getInputRowMeta().indexOfValue(meta.getDynamicSourceFileNameField());
            if (data.indexOfSourceFilename < 0) {
                // The field is unreachable !
                throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Exception.CouldnotFindField", meta.getDynamicSourceFileNameField()));
            }
        }
        // cache the position of the source filename field
        if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && data.indexOfTargetFilename < 0) {
            data.indexOfTargetFilename = getInputRowMeta().indexOfValue(meta.getDynamicTargetFileNameField());
            if (data.indexOfTargetFilename < 0) {
                // The field is unreachable !
                throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Exception.CouldnotFindField", meta.getDynamicTargetFileNameField()));
            }
        }
        if (meta.simulate) {
            if (log.isBasic()) {
                logBasic(BaseMessages.getString(PKG, "ProcessFiles.Log.SimulationModeON"));
            }
        }
    }
    // End If first
    try {
        // get source filename
        String sourceFilename = getInputRowMeta().getString(r, data.indexOfSourceFilename);
        if (Utils.isEmpty(sourceFilename)) {
            logError(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileEmpty"));
            throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileEmpty"));
        }
        data.sourceFile = KettleVFS.getFileObject(sourceFilename, getTransMeta());
        if (!data.sourceFile.exists()) {
            logError(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotExist", sourceFilename));
            throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotExist", sourceFilename));
        }
        if (data.sourceFile.getType() != FileType.FILE) {
            logError(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotFile", sourceFilename));
            throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotFile", sourceFilename));
        }
        String targetFilename = null;
        if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE) {
            // get value for target filename
            targetFilename = getInputRowMeta().getString(r, data.indexOfTargetFilename);
            if (Utils.isEmpty(targetFilename)) {
                logError(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileEmpty"));
                throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileEmpty"));
            }
            data.targetFile = KettleVFS.getFileObject(targetFilename, getTransMeta());
            if (data.targetFile.exists()) {
                if (log.isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.TargetFileExists", targetFilename));
                }
                // check if target is really a file otherwise it could overwrite a complete folder by copy or move operations
                if (data.targetFile.getType() != FileType.FILE) {
                    logError(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileNotFile", targetFilename));
                    throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileNotFile", targetFilename));
                }
            } else {
                // let's check parent folder
                FileObject parentFolder = data.targetFile.getParent();
                if (!parentFolder.exists()) {
                    if (!meta.isCreateParentFolder()) {
                        throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetParentFolderNotExists", parentFolder.toString()));
                    } else {
                        parentFolder.createFolder();
                    }
                }
                if (parentFolder != null) {
                    parentFolder.close();
                }
            }
        }
        switch(meta.getOperationType()) {
            case ProcessFilesMeta.OPERATION_TYPE_COPY:
                if (((meta.isOverwriteTargetFile() && data.targetFile.exists()) || !data.targetFile.exists()) && !meta.simulate) {
                    data.targetFile.copyFrom(data.sourceFile, new TextOneToOneFileSelector());
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.SourceFileCopied", sourceFilename, targetFilename));
                    }
                } else {
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.TargetNotOverwritten", sourceFilename, targetFilename));
                    }
                }
                break;
            case ProcessFilesMeta.OPERATION_TYPE_MOVE:
                if (((meta.isOverwriteTargetFile() && data.targetFile.exists()) || !data.targetFile.exists()) && !meta.simulate) {
                    data.sourceFile.moveTo(KettleVFS.getFileObject(targetFilename, getTransMeta()));
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.SourceFileMoved", sourceFilename, targetFilename));
                    }
                } else {
                    if (log.isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.TargetNotOverwritten", sourceFilename, targetFilename));
                    }
                }
                break;
            case ProcessFilesMeta.OPERATION_TYPE_DELETE:
                if (!meta.simulate) {
                    if (!data.sourceFile.delete()) {
                        throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.CanNotDeleteFile", data.sourceFile.toString()));
                    }
                }
                if (log.isDetailed()) {
                    logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.SourceFileDeleted", sourceFilename));
                }
                break;
            default:
                break;
        }
        // add filename to result filenames?
        if (meta.isaddTargetFileNametoResult() && meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && data.sourceFile.getType() == FileType.FILE) {
            // Add this to the result file names...
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.targetFile, getTransMeta().getName(), getStepname());
            resultFile.setComment(BaseMessages.getString(PKG, "ProcessFiles.Log.FileAddedResult"));
            addResultFile(resultFile);
            if (log.isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.FilenameAddResult", data.sourceFile.toString()));
            }
        }
        // copy row to possible alternate rowset(s).
        putRow(getInputRowMeta(), r);
        if (checkFeedback(getLinesRead())) {
            if (log.isBasic()) {
                logBasic(BaseMessages.getString(PKG, "ProcessFiles.LineNumber") + getLinesRead());
            }
        }
    } catch (Exception e) {
        boolean sendToErrorRow = false;
        String errorMessage = null;
        if (getStepMeta().isDoingErrorHandling()) {
            sendToErrorRow = true;
            errorMessage = e.toString();
        } else {
            logError(BaseMessages.getString(PKG, "ProcessFiles.ErrorInStepRunning") + e.getMessage());
            setErrors(1);
            stopAll();
            // signal end to receiver(s)
            setOutputDone();
            return false;
        }
        if (sendToErrorRow) {
            // Simply add this row to the error row
            putError(getInputRowMeta(), r, 1, errorMessage, null, "ProcessFiles001");
        }
    }
    return true;
}

Example 34 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class SFTPPut method finishTheJob.

protected void finishTheJob(FileObject file, String sourceData, FileObject destinationFolder) throws KettleException {
    try {
        switch(meta.getAfterFTPS()) {
            case JobEntrySFTPPUT.AFTER_FTPSPUT_DELETE:
                // Delete source file
                if (!file.exists()) {
                    file.delete();
                    if (isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "SFTPPut.Log.DeletedFile", sourceData));
                    }
                }
                break;
            case JobEntrySFTPPUT.AFTER_FTPSPUT_MOVE:
                // Move source file
                FileObject destination = null;
                try {
                    destination = KettleVFS.getFileObject(destinationFolder.getName().getBaseName() + Const.FILE_SEPARATOR + file.getName().getBaseName(), this);
                    file.moveTo(destination);
                    if (isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "SFTPPut.Log.FileMoved", file, destination));
                    }
                } finally {
                    if (destination != null) {
                        destination.close();
                    }
                }
                break;
            default:
                if (meta.isAddFilenameResut()) {
                    // Add this to the result file names...
                    ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, file, getTransMeta().getName(), getStepname());
                    resultFile.setComment(BaseMessages.getString(PKG, "SFTPPut.Log.FilenameAddedToResultFilenames"));
                    addResultFile(resultFile);
                    if (isDetailed()) {
                        logDetailed(BaseMessages.getString(PKG, "SFTPPut.Log.FilenameAddedToResultFilenames", sourceData));
                    }
                }
                break;
        }
    } catch (Exception e) {
        throw new KettleException(e);
    }
}

Example 35 with ResultFile

use of org.pentaho.di.core.ResultFile in project pentaho-kettle by pentaho.

the class TextFileInput method openNextFile.

private boolean openNextFile() {
    try {
        lineNumberInFile = 0;
        if (!closeLastFile() && failAfterBadFile(null)) {
            // (!meta.isSkipBadFiles() || data.isLastFile) ) return false;
            return false;
        }
        if (data.getFiles().nrOfFiles() == 0) {
            return false;
        }
        // Is this the last file?
        data.isLastFile = (data.filenr == data.getFiles().nrOfFiles() - 1);
        data.file = data.getFiles().getFile(data.filenr);
        data.filename = KettleVFS.getFilename(data.file);
        // Move file pointer ahead!
        data.filenr++;
        // Add additional fields?
        if (data.addShortFilename) {
            data.shortFilename = data.file.getName().getBaseName();
        }
        if (data.addPath) {
            data.path = KettleVFS.getFilename(data.file.getParent());
        }
        if (data.addIsHidden) {
            data.hidden = data.file.isHidden();
        }
        if (data.addExtension) {
            data.extension = data.file.getName().getExtension();
        }
        if (data.addLastModificationDate) {
            data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
        }
        if (data.addUri) {
            data.uriName = data.file.getName().getURI();
        }
        if (data.addRootUri) {
            data.rootUriName = data.file.getName().getRootURI();
        }
        if (data.addSize) {
            data.size = new Long(data.file.getContent().getSize());
        }
        data.lineInFile = 0;
        if (meta.isPassingThruFields()) {
            data.currentPassThruFieldsRow = data.passThruFields.get(data.file);
        }
        // 
        if (meta.isAddResultFile()) {
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), toString());
            resultFile.setComment("File was read by an Text File input step");
            addResultFile(resultFile);
        }
        if (log.isBasic()) {
            logBasic("Opening file: " + data.file.getName().getFriendlyURI());
        }
        CompressionProvider provider = CompressionProviderFactory.getInstance().getCompressionProviderByName(meta.getFileCompression());
        data.in = provider.createInputStream(KettleVFS.getInputStream(data.file));
        data.dataErrorLineHandler.handleFile(data.file);
        data.in.nextEntry();
        if (log.isDetailed()) {
            logDetailed("This is a compressed file being handled by the " + provider.getName() + " provider");
        }
        if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
            data.isr = new InputStreamReader(new BufferedInputStream(data.in, BUFFER_SIZE_INPUT_STREAM), meta.getEncoding());
        } else {
            data.isr = new InputStreamReader(new BufferedInputStream(data.in, BUFFER_SIZE_INPUT_STREAM));
        }
        String encoding = data.isr.getEncoding();
        data.encodingType = EncodingType.guessEncodingType(encoding);
        // /////////////////////////////////////////////////////////////////////////////
        // Read the first lines...
        /*
       * Keep track of the status of the file: are there any lines left to read?
       */
        data.doneReading = false;
        /*
       * OK, read a number of lines in the buffer: The header rows The nr rows in the page : optional The footer rows
       */
        int bufferSize = 1;
        bufferSize += meta.hasHeader() ? meta.getNrHeaderLines() : 0;
        bufferSize += meta.isLayoutPaged() ? meta.getNrLinesPerPage() * (Math.max(0, meta.getNrWraps()) + 1) : // it helps when we have wrapped input w/o header
        Math.max(0, meta.getNrWraps());
        bufferSize += meta.hasFooter() ? meta.getNrFooterLines() : 0;
        // See if we need to skip the document header lines...
        if (meta.isLayoutPaged()) {
            for (int i = 0; i < meta.getNrLinesDocHeader(); i++) {
                // Just skip these...
                // header and
                getLine(log, data.isr, data.encodingType, data.fileFormatType, data.lineStringBuilder);
                // footer: not
                // wrapped
                lineNumberInFile++;
            }
        }
        for (int i = 0; i < bufferSize && !data.doneReading; i++) {
            boolean wasNotFiltered = tryToReadLine(!meta.hasHeader() || i >= meta.getNrHeaderLines());
            if (!wasNotFiltered) {
                // grab another line, this one got filtered
                bufferSize++;
            }
        }
        // Reset counters etc.
        data.headerLinesRead = 0;
        data.footerLinesRead = 0;
        data.pageLinesRead = 0;
        // Set a flags
        data.doneWithHeader = !meta.hasHeader();
    } catch (Exception e) {
        String errorMsg = "Couldn't open file #" + data.filenr + " : " + data.file.getName().getFriendlyURI() + " --> " + e.toString();
        logError(errorMsg);
        if (failAfterBadFile(errorMsg)) {
            // !meta.isSkipBadFiles()) stopAll();
            stopAll();
        }
        setErrors(getErrors() + 1);
        return false;
    }
    return true;
}