Examples with BaseFileField org.pentaho.di.trans.steps.file.BaseFileField used on opensource projects

Example 31 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileInputTest method readInputWithNonEmptyNullif.

@Test
public void readInputWithNonEmptyNullif() throws Exception {
    final String virtualFile = createVirtualFile("pdi-14358.txt", "-,-\n");
    BaseFileField col2 = field("col2");
    col2.setNullString("-");
    TextFileInputMeta meta = createMetaObject(field("col1"), col2);
    TextFileInputData data = createDataObject(virtualFile, ",", "col1", "col2");
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    List<Object[]> output = TransTestingUtil.execute(input, meta, data, 1, false);
    TransTestingUtil.assertResult(new Object[] { "-" }, output.get(0));
    deleteVfsFile(virtualFile);
}

Example 32 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileInputTest method readInputWithDefaultValues.

@Test
public void readInputWithDefaultValues() throws Exception {
    final String virtualFile = createVirtualFile("pdi-14832.txt", "1,\n");
    BaseFileField col2 = field("col2");
    col2.setIfNullValue("DEFAULT");
    TextFileInputMeta meta = createMetaObject(field("col1"), col2);
    TextFileInputData data = createDataObject(virtualFile, ",", "col1", "col2");
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    List<Object[]> output = TransTestingUtil.execute(input, meta, data, 1, false);
    TransTestingUtil.assertResult(new Object[] { "1", "DEFAULT" }, output.get(0));
    deleteVfsFile(virtualFile);
}

Example 33 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileInputDialogTest method testMinimalWidth_PDI_14253.

@Test
public void testMinimalWidth_PDI_14253() throws Exception {
    final String virtualFile = "ram://pdi-14253.txt";
    KettleVFS.getFileObject(virtualFile).createFile();
    final String content = "r1c1,  r1c2\nr2c1  ,  r2c2  ";
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    bos.write(content.getBytes());
    OutputStream os = KettleVFS.getFileObject(virtualFile).getContent().getOutputStream();
    IOUtils.copy(new ByteArrayInputStream(bos.toByteArray()), os);
    os.close();
    TextFileInputMeta meta = new TextFileInputMeta();
    meta.content.lineWrapped = false;
    meta.inputFields = new BaseFileField[] { new BaseFileField("col1", -1, -1), new BaseFileField("col2", -1, -1) };
    meta.content.fileCompression = "None";
    meta.content.fileType = "CSV";
    meta.content.header = false;
    meta.content.nrHeaderLines = -1;
    meta.content.footer = false;
    meta.content.nrFooterLines = -1;
    TextFileInputData data = new TextFileInputData();
    data.files = new FileInputList();
    data.files.addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.dataErrorLineHandler = mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ",";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0], new Variables() {
    });
    data.filePlayList = new FilePlayListAll();
    TextFileInputDialog dialog = new TextFileInputDialog(mock(Shell.class), meta, mock(TransMeta.class), "TFIMinimalWidthTest");
    TableView tv = mock(TableView.class);
    when(tv.nrNonEmpty()).thenReturn(0);
    // click the Minimal width button
    dialog.setMinimalWidth(tv);
    RowSet output = new BlockingRowSet(5);
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    input.setOutputRowSets(Collections.singletonList(output));
    while (input.processRow(meta, data)) {
    // wait until the step completes executing
    }
    Object[] row1 = output.getRowImmediate();
    assertRow(row1, "r1c1", "r1c2");
    Object[] row2 = output.getRowImmediate();
    assertRow(row2, "r2c1", "r2c2");
    KettleVFS.getFileObject(virtualFile).delete();
}

Example 34 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getFields.

private Vector<TextFileInputFieldInterface> getFields(TextFileInputMeta info, List<String> rows) {
    Vector<TextFileInputFieldInterface> fields = new Vector<>();
    int maxsize = 0;
    for (String row : rows) {
        int len = row.length();
        if (len > maxsize) {
            maxsize = len;
        }
    }
    int prevEnd = 0;
    int dummynr = 1;
    for (int i = 0; i < info.inputFields.length; i++) {
        BaseFileField f = info.inputFields[i];
        // See if positions are skipped, if this is the case, add dummy fields...
        if (f.getPosition() != prevEnd) {
            // gap
            BaseFileField field = new BaseFileField("Dummy" + dummynr, prevEnd, f.getPosition() - prevEnd);
            // don't include in result by default.
            field.setIgnored(true);
            fields.add(field);
            dummynr++;
        }
        BaseFileField field = new BaseFileField(f.getName(), f.getPosition(), f.getLength());
        field.setType(f.getType());
        field.setIgnored(false);
        field.setFormat(f.getFormat());
        field.setPrecision(f.getPrecision());
        field.setTrimType(f.getTrimType());
        field.setDecimalSymbol(f.getDecimalSymbol());
        field.setGroupSymbol(f.getGroupSymbol());
        field.setCurrencySymbol(f.getCurrencySymbol());
        field.setRepeated(f.isRepeated());
        field.setNullString(f.getNullString());
        fields.add(field);
        prevEnd = field.getPosition() + field.getLength();
    }
    if (info.inputFields.length == 0) {
        BaseFileField field = new BaseFileField("Field1", 0, maxsize);
        fields.add(field);
    } else {
        // Take the last field and see if it reached until the maximum...
        BaseFileField f = info.inputFields[info.inputFields.length - 1];
        int pos = f.getPosition();
        int len = f.getLength();
        if (pos + len < maxsize) {
            // If not, add an extra trailing field!
            BaseFileField field = new BaseFileField("Dummy" + dummynr, pos + len, maxsize - pos - len);
            // don't include in result by default.
            field.setIgnored(true);
            fields.add(field);
        }
    }
    Collections.sort(fields);
    return fields;
}

Example 35 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getFieldsData.

private void getFieldsData(TextFileInputMeta in, boolean insertAtTop) {
    for (int i = 0; i < in.inputFields.length; i++) {
        BaseFileField field = in.inputFields[i];
        TableItem item;
        if (insertAtTop) {
            item = new TableItem(wFields.table, SWT.NONE, i);
        } else {
            if (i >= wFields.table.getItemCount()) {
                item = wFields.table.getItem(i);
            } else {
                item = new TableItem(wFields.table, SWT.NONE);
            }
        }
        item.setText(1, Const.NVL(field.getName(), ""));
        String type = field.getTypeDesc();
        String format = field.getFormat();
        String position = "" + field.getPosition();
        String length = "" + field.getLength();
        String prec = "" + field.getPrecision();
        String curr = field.getCurrencySymbol();
        String group = field.getGroupSymbol();
        String decim = field.getDecimalSymbol();
        String def = field.getNullString();
        String ifNull = field.getIfNullValue();
        String trim = field.getTrimTypeDesc();
        String rep = field.isRepeated() ? BaseMessages.getString(PKG, "System.Combo.Yes") : BaseMessages.getString(PKG, "System.Combo.No");
        if (type != null) {
            item.setText(2, type);
        }
        if (format != null) {
            item.setText(3, format);
        }
        if (position != null && !"-1".equals(position)) {
            item.setText(4, position);
        }
        if (length != null && !"-1".equals(length)) {
            item.setText(5, length);
        }
        if (prec != null && !"-1".equals(prec)) {
            item.setText(6, prec);
        }
        if (curr != null) {
            item.setText(7, curr);
        }
        if (decim != null) {
            item.setText(8, decim);
        }
        if (group != null) {
            item.setText(9, group);
        }
        if (def != null) {
            item.setText(10, def);
        }
        if (ifNull != null) {
            item.setText(11, ifNull);
        }
        if (trim != null) {
            item.setText(12, trim);
        }
        if (rep != null) {
            item.setText(13, rep);
        }
    }
}