Examples with TextFileFilter org.pentaho.di.trans.steps.fileinput.text.TextFileFilter used on opensource projects

Example 1 with TextFileFilter

use of org.pentaho.di.trans.steps.fileinput.text.TextFileFilter in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getData.

/**
 * Read the data from the TextFileInputMeta object and show it in this dialog.
 *
 * @param meta
 *          The TextFileInputMeta object to obtain the data from.
 */
public void getData(TextFileInputMeta meta) {
    wAccFilenames.setSelection(meta.inputFiles.acceptingFilenames);
    wPassThruFields.setSelection(meta.inputFiles.passingThruFields);
    if (meta.inputFiles.acceptingField != null) {
        wAccField.setText(meta.inputFiles.acceptingField);
    }
    if (meta.getAcceptingStep() != null) {
        wAccStep.setText(meta.getAcceptingStep().getName());
    }
    if (meta.getFileName() != null) {
        wFilenameList.removeAll();
        for (int i = 0; i < meta.getFileName().length; i++) {
            wFilenameList.add(meta.getFileName()[i], meta.inputFiles.fileMask[i], meta.inputFiles.excludeFileMask[i], meta.getRequiredFilesDesc(meta.inputFiles.fileRequired[i]), meta.getRequiredFilesDesc(meta.inputFiles.includeSubFolders[i]));
        }
        wFilenameList.removeEmptyRows();
        wFilenameList.setRowNums();
        wFilenameList.optWidth(true);
    }
    if (meta.content.fileType != null) {
        wFiletype.setText(meta.content.fileType);
    }
    if (meta.content.separator != null) {
        wSeparator.setText(meta.content.separator);
    }
    if (meta.content.enclosure != null) {
        wEnclosure.setText(meta.content.enclosure);
    }
    if (meta.content.escapeCharacter != null) {
        wEscape.setText(meta.content.escapeCharacter);
    }
    wHeader.setSelection(meta.content.header);
    wNrHeader.setText("" + meta.content.nrHeaderLines);
    wFooter.setSelection(meta.content.footer);
    wNrFooter.setText("" + meta.content.nrFooterLines);
    wWraps.setSelection(meta.content.lineWrapped);
    wNrWraps.setText("" + meta.content.nrWraps);
    wLayoutPaged.setSelection(meta.content.layoutPaged);
    wNrLinesPerPage.setText("" + meta.content.nrLinesPerPage);
    wNrLinesDocHeader.setText("" + meta.content.nrLinesDocHeader);
    if (meta.content.fileCompression != null) {
        wCompression.setText(meta.content.fileCompression);
    }
    wNoempty.setSelection(meta.content.noEmptyLines);
    wInclFilename.setSelection(meta.content.includeFilename);
    wInclRownum.setSelection(meta.content.includeRowNumber);
    wRownumByFile.setSelection(meta.content.rowNumberByFile);
    wDateLenient.setSelection(meta.content.dateFormatLenient);
    wAddResult.setSelection(meta.inputFiles.isaddresult);
    if (meta.content.filenameField != null) {
        wInclFilenameField.setText(meta.content.filenameField);
    }
    if (meta.content.rowNumberField != null) {
        wInclRownumField.setText(meta.content.rowNumberField);
    }
    if (meta.content.fileFormat != null) {
        wFormat.setText(meta.content.fileFormat);
    }
    if (meta.content.length != null) {
        wLength.setText(meta.content.length);
    }
    wLimit.setText("" + meta.content.rowLimit);
    logDebug("getting fields info...");
    getFieldsData(meta, false);
    if (meta.getEncoding() != null) {
        wEncoding.setText(meta.getEncoding());
    }
    // Error handling fields...
    wErrorIgnored.setSelection(meta.errorHandling.errorIgnored);
    wSkipBadFiles.setSelection(meta.errorHandling.skipBadFiles);
    wSkipErrorLines.setSelection(meta.isErrorLineSkipped());
    if (meta.errorHandling.fileErrorField != null) {
        wBadFileField.setText(meta.errorHandling.fileErrorField);
    }
    if (meta.errorHandling.fileErrorMessageField != null) {
        wBadFileMessageField.setText(meta.errorHandling.fileErrorMessageField);
    }
    if (meta.getErrorCountField() != null) {
        wErrorCount.setText(meta.getErrorCountField());
    }
    if (meta.getErrorFieldsField() != null) {
        wErrorFields.setText(meta.getErrorFieldsField());
    }
    if (meta.getErrorTextField() != null) {
        wErrorText.setText(meta.getErrorTextField());
    }
    if (meta.errorHandling.warningFilesDestinationDirectory != null) {
        wWarnDestDir.setText(meta.errorHandling.warningFilesDestinationDirectory);
    }
    if (meta.errorHandling.warningFilesExtension != null) {
        wWarnExt.setText(meta.errorHandling.warningFilesExtension);
    }
    if (meta.errorHandling.errorFilesDestinationDirectory != null) {
        wErrorDestDir.setText(meta.errorHandling.errorFilesDestinationDirectory);
    }
    if (meta.errorHandling.errorFilesExtension != null) {
        wErrorExt.setText(meta.errorHandling.errorFilesExtension);
    }
    if (meta.errorHandling.lineNumberFilesDestinationDirectory != null) {
        wLineNrDestDir.setText(meta.errorHandling.lineNumberFilesDestinationDirectory);
    }
    if (meta.errorHandling.lineNumberFilesExtension != null) {
        wLineNrExt.setText(meta.errorHandling.lineNumberFilesExtension);
    }
    for (int i = 0; i < meta.getFilter().length; i++) {
        TableItem item = wFilter.table.getItem(i);
        TextFileFilter filter = meta.getFilter()[i];
        if (filter.getFilterString() != null) {
            item.setText(1, filter.getFilterString());
        }
        if (filter.getFilterPosition() >= 0) {
            item.setText(2, "" + filter.getFilterPosition());
        }
        item.setText(3, filter.isFilterLastLine() ? BaseMessages.getString(PKG, "System.Combo.Yes") : BaseMessages.getString(PKG, "System.Combo.No"));
        item.setText(4, filter.isFilterPositive() ? BaseMessages.getString(PKG, "System.Combo.Yes") : BaseMessages.getString(PKG, "System.Combo.No"));
    }
    // Date locale
    wDateLocale.setText(meta.content.dateFormatLocale.toString());
    wFields.removeEmptyRows();
    wFields.setRowNums();
    wFields.optWidth(true);
    wFilter.removeEmptyRows();
    wFilter.setRowNums();
    wFilter.optWidth(true);
    if (meta.additionalOutputFields.shortFilenameField != null) {
        wShortFileFieldName.setText(meta.additionalOutputFields.shortFilenameField);
    }
    if (meta.additionalOutputFields.pathField != null) {
        wPathFieldName.setText(meta.additionalOutputFields.pathField);
    }
    if (meta.additionalOutputFields.hiddenField != null) {
        wIsHiddenName.setText(meta.additionalOutputFields.hiddenField);
    }
    if (meta.additionalOutputFields.lastModificationField != null) {
        wLastModificationTimeName.setText(meta.additionalOutputFields.lastModificationField);
    }
    if (meta.additionalOutputFields.uriField != null) {
        wUriName.setText(meta.additionalOutputFields.uriField);
    }
    if (meta.additionalOutputFields.rootUriField != null) {
        wRootUriName.setText(meta.additionalOutputFields.rootUriField);
    }
    if (meta.additionalOutputFields.extensionField != null) {
        wExtensionFieldName.setText(meta.additionalOutputFields.extensionField);
    }
    if (meta.additionalOutputFields.sizeField != null) {
        wSizeFieldName.setText(meta.additionalOutputFields.sizeField);
    }
    setFlags();
    wStepname.selectAll();
    wStepname.setFocus();
}

Example 2 with TextFileFilter

use of org.pentaho.di.trans.steps.fileinput.text.TextFileFilter in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getInfo.

/**
 * Fill meta object from UI options.
 *
 * @param meta
 *          meta object
 * @param preview
 *          flag for preview or real options should be used. Currently, only one option is differ for preview - EOL
 *          chars. It uses as "mixed" for be able to preview any file.
 */
private void getInfo(TextFileInputMeta meta, boolean preview) {
    // return value
    stepname = wStepname.getText();
    // copy info to TextFileInputMeta class (input)
    meta.inputFiles.acceptingFilenames = wAccFilenames.getSelection();
    meta.inputFiles.passingThruFields = wPassThruFields.getSelection();
    meta.inputFiles.acceptingField = wAccField.getText();
    meta.inputFiles.acceptingStepName = wAccStep.getText();
    meta.setAcceptingStep(transMeta.findStep(wAccStep.getText()));
    meta.content.fileType = wFiletype.getText();
    if (preview) {
        // mixed type for preview, for be able to eat any EOL chars
        meta.content.fileFormat = "mixed";
    } else {
        meta.content.fileFormat = wFormat.getText();
    }
    meta.content.separator = wSeparator.getText();
    meta.content.enclosure = wEnclosure.getText();
    meta.content.escapeCharacter = wEscape.getText();
    meta.content.rowLimit = Const.toLong(wLimit.getText(), 0L);
    meta.content.filenameField = wInclFilenameField.getText();
    meta.content.rowNumberField = wInclRownumField.getText();
    meta.inputFiles.isaddresult = wAddResult.getSelection();
    meta.content.includeFilename = wInclFilename.getSelection();
    meta.content.includeRowNumber = wInclRownum.getSelection();
    meta.content.rowNumberByFile = wRownumByFile.getSelection();
    meta.content.header = wHeader.getSelection();
    meta.content.nrHeaderLines = Const.toInt(wNrHeader.getText(), 1);
    meta.content.footer = wFooter.getSelection();
    meta.content.nrFooterLines = Const.toInt(wNrFooter.getText(), 1);
    meta.content.lineWrapped = wWraps.getSelection();
    meta.content.nrWraps = Const.toInt(wNrWraps.getText(), 1);
    meta.content.layoutPaged = wLayoutPaged.getSelection();
    meta.content.nrLinesPerPage = Const.toInt(wNrLinesPerPage.getText(), 80);
    meta.content.nrLinesDocHeader = Const.toInt(wNrLinesDocHeader.getText(), 0);
    meta.content.fileCompression = wCompression.getText();
    meta.content.dateFormatLenient = wDateLenient.getSelection();
    meta.content.noEmptyLines = wNoempty.getSelection();
    meta.content.encoding = wEncoding.getText();
    meta.content.length = wLength.getText();
    int nrfiles = wFilenameList.getItemCount();
    int nrfields = wFields.nrNonEmpty();
    int nrfilters = wFilter.nrNonEmpty();
    meta.allocate(nrfiles, nrfields, nrfilters);
    meta.setFileName(wFilenameList.getItems(0));
    meta.inputFiles.fileMask = wFilenameList.getItems(1);
    meta.inputFiles.excludeFileMask = wFilenameList.getItems(2);
    meta.inputFiles_fileRequired(wFilenameList.getItems(3));
    meta.inputFiles_includeSubFolders(wFilenameList.getItems(4));
    for (int i = 0; i < nrfields; i++) {
        BaseFileField field = new BaseFileField();
        TableItem item = wFields.getNonEmpty(i);
        field.setName(item.getText(1));
        field.setType(ValueMetaFactory.getIdForValueMeta(item.getText(2)));
        field.setFormat(item.getText(3));
        field.setPosition(Const.toInt(item.getText(4), -1));
        field.setLength(Const.toInt(item.getText(5), -1));
        field.setPrecision(Const.toInt(item.getText(6), -1));
        field.setCurrencySymbol(item.getText(7));
        field.setDecimalSymbol(item.getText(8));
        field.setGroupSymbol(item.getText(9));
        field.setNullString(item.getText(10));
        field.setIfNullValue(item.getText(11));
        field.setTrimType(ValueMetaString.getTrimTypeByDesc(item.getText(12)));
        field.setRepeated(BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(13)));
        // CHECKSTYLE:Indentation:OFF
        meta.inputFields[i] = field;
    }
    for (int i = 0; i < nrfilters; i++) {
        TableItem item = wFilter.getNonEmpty(i);
        TextFileFilter filter = new TextFileFilter();
        // CHECKSTYLE:Indentation:OFF
        meta.getFilter()[i] = filter;
        filter.setFilterString(item.getText(1));
        filter.setFilterPosition(Const.toInt(item.getText(2), -1));
        filter.setFilterLastLine(BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(3)));
        filter.setFilterPositive(BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(4)));
    }
    // Error handling fields...
    meta.errorHandling.errorIgnored = wErrorIgnored.getSelection();
    meta.errorHandling.skipBadFiles = wSkipBadFiles.getSelection();
    meta.errorHandling.fileErrorField = wBadFileField.getText();
    meta.errorHandling.fileErrorMessageField = wBadFileMessageField.getText();
    meta.setErrorLineSkipped(wSkipErrorLines.getSelection());
    meta.setErrorCountField(wErrorCount.getText());
    meta.setErrorFieldsField(wErrorFields.getText());
    meta.setErrorTextField(wErrorText.getText());
    meta.errorHandling.warningFilesDestinationDirectory = wWarnDestDir.getText();
    meta.errorHandling.warningFilesExtension = wWarnExt.getText();
    meta.errorHandling.errorFilesDestinationDirectory = wErrorDestDir.getText();
    meta.errorHandling.errorFilesExtension = wErrorExt.getText();
    meta.errorHandling.lineNumberFilesDestinationDirectory = wLineNrDestDir.getText();
    meta.errorHandling.lineNumberFilesExtension = wLineNrExt.getText();
    // Date format Locale
    Locale locale = EnvUtil.createLocale(wDateLocale.getText());
    if (!locale.equals(Locale.getDefault())) {
        meta.content.dateFormatLocale = locale;
    } else {
        meta.content.dateFormatLocale = Locale.getDefault();
    }
    meta.additionalOutputFields.shortFilenameField = wShortFileFieldName.getText();
    meta.additionalOutputFields.pathField = wPathFieldName.getText();
    meta.additionalOutputFields.hiddenField = wIsHiddenName.getText();
    meta.additionalOutputFields.lastModificationField = wLastModificationTimeName.getText();
    meta.additionalOutputFields.uriField = wUriName.getText();
    meta.additionalOutputFields.rootUriField = wRootUriName.getText();
    meta.additionalOutputFields.extensionField = wExtensionFieldName.getText();
    meta.additionalOutputFields.sizeField = wSizeFieldName.getText();
}

Example 3 with TextFileFilter

use of org.pentaho.di.trans.steps.fileinput.text.TextFileFilter in project pentaho-kettle by pentaho.

the class TextFileInputDialogTest method testMinimalWidth_PDI_14253.

@Test
public void testMinimalWidth_PDI_14253() throws Exception {
    final String virtualFile = "ram://pdi-14253.txt";
    KettleVFS.getFileObject(virtualFile).createFile();
    final String content = "r1c1,  r1c2\nr2c1  ,  r2c2  ";
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    bos.write(content.getBytes());
    OutputStream os = KettleVFS.getFileObject(virtualFile).getContent().getOutputStream();
    IOUtils.copy(new ByteArrayInputStream(bos.toByteArray()), os);
    os.close();
    TextFileInputMeta meta = new TextFileInputMeta();
    meta.content.lineWrapped = false;
    meta.inputFields = new BaseFileField[] { new BaseFileField("col1", -1, -1), new BaseFileField("col2", -1, -1) };
    meta.content.fileCompression = "None";
    meta.content.fileType = "CSV";
    meta.content.header = false;
    meta.content.nrHeaderLines = -1;
    meta.content.footer = false;
    meta.content.nrFooterLines = -1;
    TextFileInputData data = new TextFileInputData();
    data.files = new FileInputList();
    data.files.addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.dataErrorLineHandler = mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ",";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0], new Variables() {
    });
    data.filePlayList = new FilePlayListAll();
    TextFileInputDialog dialog = new TextFileInputDialog(mock(Shell.class), meta, mock(TransMeta.class), "TFIMinimalWidthTest");
    TableView tv = mock(TableView.class);
    when(tv.nrNonEmpty()).thenReturn(0);
    // click the Minimal width button
    dialog.setMinimalWidth(tv);
    RowSet output = new BlockingRowSet(5);
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    input.setOutputRowSets(Collections.singletonList(output));
    while (input.processRow(meta, data)) {
    // wait until the step completes executing
    }
    Object[] row1 = output.getRowImmediate();
    assertRow(row1, "r1c1", "r1c2");
    Object[] row2 = output.getRowImmediate();
    assertRow(row2, "r2c1", "r2c2");
    KettleVFS.getFileObject(virtualFile).delete();
}