Examples with CompressionProvider org.pentaho.di.core.compress.CompressionProvider used on opensource projects

Example 16 with CompressionProvider

use of org.pentaho.di.core.compress.CompressionProvider in project pentaho-kettle by pentaho.

the class TextFileInput method openNextFile.

private boolean openNextFile() {
    try {
        lineNumberInFile = 0;
        if (!closeLastFile() && failAfterBadFile(null)) {
            // (!meta.isSkipBadFiles() || data.isLastFile) ) return false;
            return false;
        }
        if (data.getFiles().nrOfFiles() == 0) {
            return false;
        }
        // Is this the last file?
        data.isLastFile = (data.filenr == data.getFiles().nrOfFiles() - 1);
        data.file = data.getFiles().getFile(data.filenr);
        data.filename = KettleVFS.getFilename(data.file);
        // Move file pointer ahead!
        data.filenr++;
        // Add additional fields?
        if (data.addShortFilename) {
            data.shortFilename = data.file.getName().getBaseName();
        }
        if (data.addPath) {
            data.path = KettleVFS.getFilename(data.file.getParent());
        }
        if (data.addIsHidden) {
            data.hidden = data.file.isHidden();
        }
        if (data.addExtension) {
            data.extension = data.file.getName().getExtension();
        }
        if (data.addLastModificationDate) {
            data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
        }
        if (data.addUri) {
            data.uriName = data.file.getName().getURI();
        }
        if (data.addRootUri) {
            data.rootUriName = data.file.getName().getRootURI();
        }
        if (data.addSize) {
            data.size = new Long(data.file.getContent().getSize());
        }
        data.lineInFile = 0;
        if (meta.isPassingThruFields()) {
            data.currentPassThruFieldsRow = data.passThruFields.get(data.file);
        }
        // 
        if (meta.isAddResultFile()) {
            ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), toString());
            resultFile.setComment("File was read by an Text File input step");
            addResultFile(resultFile);
        }
        if (log.isBasic()) {
            logBasic("Opening file: " + data.file.getName().getFriendlyURI());
        }
        CompressionProvider provider = CompressionProviderFactory.getInstance().getCompressionProviderByName(meta.getFileCompression());
        data.in = provider.createInputStream(KettleVFS.getInputStream(data.file));
        data.dataErrorLineHandler.handleFile(data.file);
        data.in.nextEntry();
        if (log.isDetailed()) {
            logDetailed("This is a compressed file being handled by the " + provider.getName() + " provider");
        }
        if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
            data.isr = new InputStreamReader(new BufferedInputStream(data.in, BUFFER_SIZE_INPUT_STREAM), meta.getEncoding());
        } else {
            data.isr = new InputStreamReader(new BufferedInputStream(data.in, BUFFER_SIZE_INPUT_STREAM));
        }
        String encoding = data.isr.getEncoding();
        data.encodingType = EncodingType.guessEncodingType(encoding);
        // /////////////////////////////////////////////////////////////////////////////
        // Read the first lines...
        /*
       * Keep track of the status of the file: are there any lines left to read?
       */
        data.doneReading = false;
        /*
       * OK, read a number of lines in the buffer: The header rows The nr rows in the page : optional The footer rows
       */
        int bufferSize = 1;
        bufferSize += meta.hasHeader() ? meta.getNrHeaderLines() : 0;
        bufferSize += meta.isLayoutPaged() ? meta.getNrLinesPerPage() * (Math.max(0, meta.getNrWraps()) + 1) : // it helps when we have wrapped input w/o header
        Math.max(0, meta.getNrWraps());
        bufferSize += meta.hasFooter() ? meta.getNrFooterLines() : 0;
        // See if we need to skip the document header lines...
        if (meta.isLayoutPaged()) {
            for (int i = 0; i < meta.getNrLinesDocHeader(); i++) {
                // Just skip these...
                // header and
                getLine(log, data.isr, data.encodingType, data.fileFormatType, data.lineStringBuilder);
                // footer: not
                // wrapped
                lineNumberInFile++;
            }
        }
        for (int i = 0; i < bufferSize && !data.doneReading; i++) {
            boolean wasNotFiltered = tryToReadLine(!meta.hasHeader() || i >= meta.getNrHeaderLines());
            if (!wasNotFiltered) {
                // grab another line, this one got filtered
                bufferSize++;
            }
        }
        // Reset counters etc.
        data.headerLinesRead = 0;
        data.footerLinesRead = 0;
        data.pageLinesRead = 0;
        // Set a flags
        data.doneWithHeader = !meta.hasHeader();
    } catch (Exception e) {
        String errorMsg = "Couldn't open file #" + data.filenr + " : " + data.file.getName().getFriendlyURI() + " --> " + e.toString();
        logError(errorMsg);
        if (failAfterBadFile(errorMsg)) {
            // !meta.isSkipBadFiles()) stopAll();
            stopAll();
        }
        setErrors(getErrors() + 1);
        return false;
    }
    return true;
}

Example 17 with CompressionProvider

use of org.pentaho.di.core.compress.CompressionProvider in project pentaho-kettle by pentaho.

the class TextFileOutput method getCompressionProvider.

private CompressionProvider getCompressionProvider() throws KettleException {
    String compressionType = meta.getFileCompression();
    if (Utils.isEmpty(compressionType)) {
        compressionType = FILE_COMPRESSION_TYPE_NONE;
    }
    CompressionProvider compressionProvider = CompressionProviderFactory.getInstance().getCompressionProviderByName(compressionType);
    if (compressionProvider == null) {
        throw new KettleException("No compression provider found with name = " + compressionType);
    }
    if (!compressionProvider.supportsOutput()) {
        throw new KettleException("Compression provider " + compressionType + " does not support output streams!");
    }
    return compressionProvider;
}

Example 18 with CompressionProvider

use of org.pentaho.di.core.compress.CompressionProvider in project pentaho-kettle by pentaho.

the class TextFileOutput method initFileStreamWriter.

public void initFileStreamWriter(String filename) throws KettleException {
    data.writer = null;
    try {
        BufferedOutputStream bufferedOutputStream;
        OutputStream fileOutputStream;
        CompressionOutputStream compressionOutputStream;
        TextFileOutputData.FileStream fileStreams = null;
        try {
            if (meta.getSplitEvery() > 0) {
                if (filename.equals(data.getFileStreamsCollection().getLastFileName())) {
                    fileStreams = data.getFileStreamsCollection().getLastStream();
                }
            } else {
                fileStreams = data.getFileStreamsCollection().getStream(filename);
            }
            boolean writingToFileForFirstTime = fileStreams != null;
            boolean createParentDirIfNotExists = meta.isCreateParentFolder();
            boolean appendToExistingFile = meta.isFileAppended();
            if (fileStreams == null) {
                // Opening file for first time
                CompressionProvider compressionProvider = getCompressionProvider();
                boolean isZipFile = compressionProvider instanceof ZIPCompressionProvider;
                if (appendToExistingFile && isZipFile && isFileExists(filename)) {
                    throw new KettleException("Can not append to an existing zip file : " + filename);
                }
                int maxOpenFiles = getMaxOpenFiles();
                if ((maxOpenFiles > 0) && (data.getFileStreamsCollection().getNumOpenFiles() >= maxOpenFiles)) {
                    // If the file we're going to close is a zip file,  going to remove it from the collection of files
                    // that have been opened. We do this because it is not possible to reopen a
                    // zip file for append. By removing it from the collection, if the same file is referenced later, it will look
                    // like we're opening the file for the first time, and if we're set up to append to existing files it will cause and
                    // exception to be thrown, which is the desired result.
                    data.getFileStreamsCollection().closeOldestOpenFile(isZipFile);
                }
                if (createParentDirIfNotExists && ((data.getFileStreamsCollection().size() == 0) || meta.isFileNameInField())) {
                    createParentFolder(filename);
                }
                if (log.isDetailed()) {
                    logDetailed("Opening output stream using provider: " + compressionProvider.getName());
                }
                fileOutputStream = getOutputStream(filename, getTransMeta(), !isZipFile && appendToExistingFile);
                compressionOutputStream = compressionProvider.createOutputStream(fileOutputStream);
                // The compression output stream may also archive entries. For this we create the filename
                // (with appropriate extension) and add it as an entry to the output stream. For providers
                // that do not archive entries, they should use the default no-op implementation.
                compressionOutputStream.addEntry(filename, environmentSubstitute(meta.getExtension()));
                if (log.isDetailed()) {
                    if (!Utils.isEmpty(meta.getEncoding())) {
                        logDetailed("Opening output stream in encoding: " + meta.getEncoding());
                    } else {
                        logDetailed("Opening output stream in default encoding");
                    }
                }
                bufferedOutputStream = new BufferedOutputStream(compressionOutputStream, 5000);
                fileStreams = data.new FileStream(fileOutputStream, compressionOutputStream, bufferedOutputStream);
                data.getFileStreamsCollection().add(filename, fileStreams);
                if (log.isDetailed()) {
                    logDetailed("Opened new file with name [" + KettleVFS.getFriendlyURI(filename) + "]");
                }
            } else if (fileStreams.getBufferedOutputStream() == null) {
                // File was previously opened and now needs to be reopened.
                int maxOpenFiles = getMaxOpenFiles();
                if ((maxOpenFiles > 0) && (data.getFileStreamsCollection().getNumOpenFiles() >= maxOpenFiles)) {
                    data.getFileStreamsCollection().closeOldestOpenFile(false);
                }
                fileOutputStream = getOutputStream(filename, getTransMeta(), true);
                CompressionProvider compressionProvider = getCompressionProvider();
                compressionOutputStream = compressionProvider.createOutputStream(fileOutputStream);
                compressionOutputStream.addEntry(filename, environmentSubstitute(meta.getExtension()));
                bufferedOutputStream = new BufferedOutputStream(compressionOutputStream, 5000);
                fileStreams.setFileOutputStream(fileOutputStream);
                fileStreams.setCompressedOutputStream(compressionOutputStream);
                fileStreams.setBufferedOutputStream(bufferedOutputStream);
            }
            if (writingToFileForFirstTime) {
                if (meta.isAddToResultFiles()) {
                    // Add this to the result file names...
                    ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, getFileObject(filename, getTransMeta()), getTransMeta().getName(), getStepname());
                    if (resultFile != null) {
                        resultFile.setComment(BaseMessages.getString(PKG, "TextFileOutput.AddResultFile"));
                        addResultFile(resultFile);
                    }
                }
            }
        } catch (Exception e) {
            if (!(e instanceof KettleException)) {
                throw new KettleException("Error opening new file : " + e.toString());
            } else {
                throw (KettleException) e;
            }
        }
        fileStreams.setDirty(true);
        data.fos = fileStreams.getFileOutputStream();
        data.out = fileStreams.getCompressedOutputStream();
        data.writer = fileStreams.getBufferedOutputStream();
    } catch (KettleException ke) {
        throw ke;
    } catch (Exception e) {
        throw new KettleException("Error opening new file : " + e.toString());
    }
}

Example 19 with CompressionProvider

use of org.pentaho.di.core.compress.CompressionProvider in project pentaho-kettle by pentaho.

the class TextFileInputDialog method open.

public String open() {
    Shell parent = getParent();
    Display display = parent.getDisplay();
    shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN);
    props.setLook(shell);
    setShellImage(shell, input);
    lsMod = new ModifyListener() {

        public void modifyText(ModifyEvent e) {
            input.setChanged();
        }
    };
    changed = input.hasChanged();
    FormLayout formLayout = new FormLayout();
    formLayout.marginWidth = Const.FORM_MARGIN;
    formLayout.marginHeight = Const.FORM_MARGIN;
    shell.setLayout(formLayout);
    shell.setText(BaseMessages.getString(PKG, "TextFileInputDialog.DialogTitle"));
    middle = props.getMiddlePct();
    margin = Const.MARGIN;
    // Stepname line
    wlStepname = new Label(shell, SWT.RIGHT);
    wlStepname.setText(BaseMessages.getString(PKG, "System.Label.StepName"));
    props.setLook(wlStepname);
    fdlStepname = new FormData();
    fdlStepname.left = new FormAttachment(0, 0);
    fdlStepname.top = new FormAttachment(0, margin);
    fdlStepname.right = new FormAttachment(middle, -margin);
    wlStepname.setLayoutData(fdlStepname);
    wStepname = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    wStepname.setText(stepname);
    props.setLook(wStepname);
    wStepname.addModifyListener(lsMod);
    fdStepname = new FormData();
    fdStepname.left = new FormAttachment(middle, 0);
    fdStepname.top = new FormAttachment(0, margin);
    fdStepname.right = new FormAttachment(100, 0);
    wStepname.setLayoutData(fdStepname);
    wTabFolder = new CTabFolder(shell, SWT.BORDER);
    props.setLook(wTabFolder, Props.WIDGET_STYLE_TAB);
    wTabFolder.setSimple(false);
    addFilesTab();
    addContentTab();
    addErrorTab();
    addFiltersTabs();
    addFieldsTabs();
    addAdditionalFieldsTab();
    fdTabFolder = new FormData();
    fdTabFolder.left = new FormAttachment(0, 0);
    fdTabFolder.top = new FormAttachment(wStepname, margin);
    fdTabFolder.right = new FormAttachment(100, 0);
    fdTabFolder.bottom = new FormAttachment(100, -50);
    wTabFolder.setLayoutData(fdTabFolder);
    wOK = new Button(shell, SWT.PUSH);
    wOK.setText(BaseMessages.getString(PKG, "System.Button.OK"));
    wPreview = new Button(shell, SWT.PUSH);
    wPreview.setText(BaseMessages.getString(PKG, "TextFileInputDialog.Preview.Button"));
    wCancel = new Button(shell, SWT.PUSH);
    wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel"));
    setButtonPositions(new Button[] { wOK, wPreview, wCancel }, margin, wTabFolder);
    // Add listeners
    lsOK = new Listener() {

        public void handleEvent(Event e) {
            ok();
        }
    };
    lsFirst = new Listener() {

        public void handleEvent(Event e) {
            first(false);
        }
    };
    lsFirstHeader = new Listener() {

        public void handleEvent(Event e) {
            first(true);
        }
    };
    lsGet = new Listener() {

        public void handleEvent(Event e) {
            get();
        }
    };
    lsPreview = new Listener() {

        public void handleEvent(Event e) {
            preview();
        }
    };
    lsMinWidth = new Listener() {

        public void handleEvent(Event e) {
            setMinimalWidth();
        }
    };
    lsCancel = new Listener() {

        public void handleEvent(Event e) {
            cancel();
        }
    };
    wOK.addListener(SWT.Selection, lsOK);
    wFirst.addListener(SWT.Selection, lsFirst);
    wFirstHeader.addListener(SWT.Selection, lsFirstHeader);
    wGet.addListener(SWT.Selection, lsGet);
    wMinWidth.addListener(SWT.Selection, lsMinWidth);
    wPreview.addListener(SWT.Selection, lsPreview);
    wCancel.addListener(SWT.Selection, lsCancel);
    lsDef = new SelectionAdapter() {

        public void widgetDefaultSelected(SelectionEvent e) {
            ok();
        }
    };
    wAccFilenames.addSelectionListener(lsDef);
    wStepname.addSelectionListener(lsDef);
    // wFilename.addSelectionListener( lsDef );
    wSeparator.addSelectionListener(lsDef);
    wLimit.addSelectionListener(lsDef);
    wInclRownumField.addSelectionListener(lsDef);
    wInclFilenameField.addSelectionListener(lsDef);
    wNrHeader.addSelectionListener(lsDef);
    wNrFooter.addSelectionListener(lsDef);
    wNrWraps.addSelectionListener(lsDef);
    wWarnDestDir.addSelectionListener(lsDef);
    wWarnExt.addSelectionListener(lsDef);
    wErrorDestDir.addSelectionListener(lsDef);
    wErrorExt.addSelectionListener(lsDef);
    wLineNrDestDir.addSelectionListener(lsDef);
    wLineNrExt.addSelectionListener(lsDef);
    wAccField.addSelectionListener(lsDef);
    // Add the file to the list of files...
    SelectionAdapter selA = new SelectionAdapter() {

        public void widgetSelected(SelectionEvent arg0) {
            wFilenameList.add(wFilename.getText(), wFilemask.getText(), wExcludeFilemask.getText(), TextFileInputMeta.RequiredFilesCode[0], TextFileInputMeta.RequiredFilesCode[0]);
            wFilename.setText("");
            wFilemask.setText("");
            wExcludeFilemask.setText("");
            wFilenameList.removeEmptyRows();
            wFilenameList.setRowNums();
            wFilenameList.optWidth(true);
        }
    };
    wbaFilename.addSelectionListener(selA);
    wFilename.addSelectionListener(selA);
    // Delete files from the list of files...
    wbdFilename.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent arg0) {
            int[] idx = wFilenameList.getSelectionIndices();
            wFilenameList.remove(idx);
            wFilenameList.removeEmptyRows();
            wFilenameList.setRowNums();
        }
    });
    // Edit the selected file & remove from the list...
    wbeFilename.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent arg0) {
            int idx = wFilenameList.getSelectionIndex();
            if (idx >= 0) {
                String[] string = wFilenameList.getItem(idx);
                wFilename.setText(string[0]);
                wFilemask.setText(string[1]);
                wExcludeFilemask.setText(string[2]);
                wFilenameList.remove(idx);
            }
            wFilenameList.removeEmptyRows();
            wFilenameList.setRowNums();
        }
    });
    // Show the files that are selected at this time...
    wbShowFiles.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            showFiles();
        }
    });
    // Allow the insertion of tabs as separator...
    wbSeparator.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent se) {
            wSeparator.getTextWidget().insert("\t");
        }
    });
    SelectionAdapter lsFlags = new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            setFlags();
        }
    };
    // Enable/disable the right fields...
    wInclFilename.addSelectionListener(lsFlags);
    wInclRownum.addSelectionListener(lsFlags);
    wRownumByFile.addSelectionListener(lsFlags);
    wErrorIgnored.addSelectionListener(lsFlags);
    wSkipBadFiles.addSelectionListener(lsFlags);
    wHeader.addSelectionListener(lsFlags);
    wFooter.addSelectionListener(lsFlags);
    wWraps.addSelectionListener(lsFlags);
    wLayoutPaged.addSelectionListener(lsFlags);
    wAccFilenames.addSelectionListener(lsFlags);
    // Listen to the Browse... button
    wbbFilename.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            if (wFilemask.getText() != null && wFilemask.getText().length() > 0) {
                // A mask: a directory!
                DirectoryDialog dialog = new DirectoryDialog(shell, SWT.OPEN);
                if (wFilename.getText() != null) {
                    String fpath = transMeta.environmentSubstitute(wFilename.getText());
                    dialog.setFilterPath(fpath);
                }
                if (dialog.open() != null) {
                    String str = dialog.getFilterPath();
                    wFilename.setText(str);
                }
            } else {
                FileDialog dialog = new FileDialog(shell, SWT.OPEN);
                CompressionProvider provider = CompressionProviderFactory.getInstance().getCompressionProviderByName(wCompression.getText());
                List<String> filterExtensions = new ArrayList<String>();
                List<String> filterNames = new ArrayList<String>();
                if (!Utils.isEmpty(provider.getDefaultExtension()) && !Utils.isEmpty(provider.getName())) {
                    filterExtensions.add("*." + provider.getDefaultExtension());
                    filterNames.add(provider.getName() + " files");
                }
                filterExtensions.add("*.txt;*.csv");
                filterNames.add(BaseMessages.getString(PKG, "TextFileInputDialog.FileType.TextAndCSVFiles"));
                filterExtensions.add("*.csv");
                filterNames.add(BaseMessages.getString(PKG, "System.FileType.CSVFiles"));
                filterExtensions.add("*.txt");
                filterNames.add(BaseMessages.getString(PKG, "System.FileType.TextFiles"));
                filterExtensions.add("*");
                filterNames.add(BaseMessages.getString(PKG, "System.FileType.AllFiles"));
                dialog.setFilterExtensions(filterExtensions.toArray(new String[filterExtensions.size()]));
                if (wFilename.getText() != null) {
                    String fname = transMeta.environmentSubstitute(wFilename.getText());
                    dialog.setFileName(fname);
                }
                dialog.setFilterNames(filterNames.toArray(new String[filterNames.size()]));
                if (dialog.open() != null) {
                    String str = dialog.getFilterPath() + System.getProperty("file.separator") + dialog.getFileName();
                    wFilename.setText(str);
                }
            }
        }
    });
    // Detect X or ALT-F4 or something that kills this window...
    shell.addShellListener(new ShellAdapter() {

        public void shellClosed(ShellEvent e) {
            cancel();
        }
    });
    wTabFolder.setSelection(0);
    // Set the shell size, based upon previous time...
    getData(input);
    setSize();
    shell.open();
    while (!shell.isDisposed()) {
        if (!display.readAndDispatch()) {
            display.sleep();
        }
    }
    return stepname;
}

Example 20 with CompressionProvider

use of org.pentaho.di.core.compress.CompressionProvider in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    TextFileInputMeta meta = new TextFileInputMeta();
    getInfo(meta);
    TextFileInputMeta previousMeta = (TextFileInputMeta) meta.clone();
    FileInputList textFileList = meta.getTextFileList(transMeta);
    InputStream fileInputStream;
    CompressionInputStream inputStream = null;
    StringBuilder lineStringBuilder = new StringBuilder(256);
    int fileFormatType = meta.getFileFormatTypeNr();
    String delimiter = transMeta.environmentSubstitute(meta.getSeparator());
    String enclosure = transMeta.environmentSubstitute(meta.getEnclosure());
    String escapeCharacter = transMeta.environmentSubstitute(meta.getEscapeCharacter());
    if (textFileList.nrOfFiles() > 0) {
        int clearFields = meta.hasHeader() ? SWT.YES : SWT.NO;
        int nrInputFields = meta.getInputFields().length;
        if (meta.hasHeader() && nrInputFields > 0) {
            MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION);
            mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.ClearFieldList.DialogMessage"));
            mb.setText(BaseMessages.getString(PKG, "TextFileInputDialog.ClearFieldList.DialogTitle"));
            clearFields = mb.open();
            if (clearFields == SWT.CANCEL) {
                return;
            }
        }
        try {
            wFields.table.removeAll();
            FileObject fileObject = textFileList.getFile(0);
            fileInputStream = KettleVFS.getInputStream(fileObject);
            Table table = wFields.table;
            CompressionProvider provider = CompressionProviderFactory.getInstance().createCompressionProviderInstance(meta.getFileCompression());
            inputStream = provider.createInputStream(fileInputStream);
            InputStreamReader reader;
            if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
                reader = new InputStreamReader(inputStream, meta.getEncoding());
            } else {
                reader = new InputStreamReader(inputStream);
            }
            EncodingType encodingType = EncodingType.guessEncodingType(reader.getEncoding());
            if (clearFields == SWT.YES || !meta.hasHeader() || nrInputFields > 0) {
                // Scan the header-line, determine fields...
                String line;
                if (meta.hasHeader() || meta.getInputFields().length == 0) {
                    line = TextFileInput.getLine(log, reader, encodingType, fileFormatType, lineStringBuilder);
                    if (line != null) {
                        // Estimate the number of input fields...
                        // Chop up the line using the delimiter
                        String[] fields = TextFileInput.guessStringsFromLine(transMeta, log, line, meta, delimiter, enclosure, escapeCharacter);
                        for (int i = 0; i < fields.length; i++) {
                            String field = fields[i];
                            if (field == null || field.length() == 0 || (nrInputFields == 0 && !meta.hasHeader())) {
                                field = "Field" + (i + 1);
                            } else {
                                // Trim the field
                                field = Const.trim(field);
                                // Replace all spaces & - with underscore _
                                field = Const.replace(field, " ", "_");
                                field = Const.replace(field, "-", "_");
                            }
                            TableItem item = new TableItem(table, SWT.NONE);
                            item.setText(1, field);
                            // The default type is String...
                            item.setText(2, "String");
                        }
                        wFields.setRowNums();
                        wFields.optWidth(true);
                        // Copy it...
                        getInfo(meta);
                    }
                }
                // Sample a few lines to determine the correct type of the fields...
                String shellText = BaseMessages.getString(PKG, "TextFileInputDialog.LinesToSample.DialogTitle");
                String lineText = BaseMessages.getString(PKG, "TextFileInputDialog.LinesToSample.DialogMessage");
                EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
                int samples = end.open();
                if (samples >= 0) {
                    getInfo(meta);
                    TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, clearFields == SWT.YES);
                    String message = pd.open();
                    if (message != null) {
                        wFields.removeAll();
                        // OK, what's the result of our search?
                        getData(meta);
                        // 
                        if (clearFields == SWT.NO) {
                            getFieldsData(previousMeta, true);
                            wFields.table.setSelection(previousMeta.getInputFields().length, wFields.table.getItemCount() - 1);
                        }
                        wFields.removeEmptyRows();
                        wFields.setRowNums();
                        wFields.optWidth(true);
                        EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "TextFileInputDialog.ScanResults.DialogTitle"), BaseMessages.getString(PKG, "TextFileInputDialog.ScanResults.DialogMessage"), message, true);
                        etd.setReadOnly();
                        etd.open();
                    }
                }
            } else {
                MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
                mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.UnableToReadHeaderLine.DialogMessage"));
                mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
                mb.open();
            }
        } catch (IOException e) {
            new ErrorDialog(shell, BaseMessages.getString(PKG, "TextFileInputDialog.IOError.DialogTitle"), BaseMessages.getString(PKG, "TextFileInputDialog.IOError.DialogMessage"), e);
        } catch (KettleException e) {
            new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "TextFileInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
        } finally {
            try {
                if (inputStream != null) {
                    inputStream.close();
                }
            } catch (Exception e) {
            // Ignore errors
            }
        }
    } else {
        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
        mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.NoValidFileFound.DialogMessage"));
        mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
        mb.open();
    }
}