Examples with FilePlayListAll org.pentaho.di.core.playlist.FilePlayListAll used on opensource projects

Example 1 with FilePlayListAll

use of org.pentaho.di.core.playlist.FilePlayListAll in project pentaho-kettle by pentaho.

the class TextFileInputTest method readWrappedInputWithoutHeaders.

@Test
public void readWrappedInputWithoutHeaders() throws Exception {
    final String content = new StringBuilder().append("r1c1").append('\n').append(";r1c2\n").append("r2c1").append('\n').append(";r2c2").toString();
    final String virtualFile = createVirtualFile("pdi-2607.txt", content);
    TextFileInputMeta meta = new TextFileInputMeta();
    meta.setLineWrapped(true);
    meta.setNrWraps(1);
    meta.setInputFields(new TextFileInputField[] { field("col1"), field("col2") });
    meta.setFileCompression("None");
    meta.setFileType("CSV");
    meta.setHeader(false);
    meta.setNrHeaderLines(-1);
    meta.setFooter(false);
    meta.setNrFooterLines(-1);
    TextFileInputData data = new TextFileInputData();
    data.setFiles(new FileInputList());
    data.getFiles().addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.dataErrorLineHandler = Mockito.mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ";";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0]);
    data.filePlayList = new FilePlayListAll();
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    List<Object[]> output = TransTestingUtil.execute(input, meta, data, 2, false);
    TransTestingUtil.assertResult(new Object[] { "r1c1", "r1c2" }, output.get(0));
    TransTestingUtil.assertResult(new Object[] { "r2c1", "r2c2" }, output.get(1));
    deleteVfsFile(virtualFile);
}

Example 2 with FilePlayListAll

use of org.pentaho.di.core.playlist.FilePlayListAll in project pentaho-kettle by pentaho.

the class TextFileInputTest method readInputWithDefaultValues.

@Test
public void readInputWithDefaultValues() throws Exception {
    final String virtualFile = createVirtualFile("pdi-14832.txt", "1,\n");
    TextFileInputMeta meta = new TextFileInputMeta();
    TextFileInputField field2 = field("col2");
    field2.setIfNullValue("DEFAULT");
    meta.setInputFields(new TextFileInputField[] { field("col1"), field2 });
    meta.setFileCompression("None");
    meta.setFileType("CSV");
    meta.setHeader(false);
    meta.setNrHeaderLines(-1);
    meta.setFooter(false);
    meta.setNrFooterLines(-1);
    TextFileInputData data = new TextFileInputData();
    data.setFiles(new FileInputList());
    data.getFiles().addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.dataErrorLineHandler = Mockito.mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ",";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0]);
    data.filePlayList = new FilePlayListAll();
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    List<Object[]> output = TransTestingUtil.execute(input, meta, data, 1, false);
    TransTestingUtil.assertResult(new Object[] { "1", "DEFAULT" }, output.get(0));
    deleteVfsFile(virtualFile);
}

Example 3 with FilePlayListAll

use of org.pentaho.di.core.playlist.FilePlayListAll in project pentaho-kettle by pentaho.

the class TextFileInputTest method readInputWithMissedValues.

@Test
public void readInputWithMissedValues() throws Exception {
    final String virtualFile = createVirtualFile("pdi-14172.txt", "1,1,1\n", "2,,2\n");
    TextFileInputMeta meta = new TextFileInputMeta();
    TextFileInputField field2 = field("col2");
    field2.setRepeated(true);
    meta.setInputFields(new TextFileInputField[] { field("col1"), field2, field("col3") });
    meta.setFileCompression("None");
    meta.setFileType("CSV");
    meta.setHeader(false);
    meta.setNrHeaderLines(-1);
    meta.setFooter(false);
    meta.setNrFooterLines(-1);
    TextFileInputData data = new TextFileInputData();
    data.setFiles(new FileInputList());
    data.getFiles().addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col3"));
    data.dataErrorLineHandler = Mockito.mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ",";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0]);
    data.filePlayList = new FilePlayListAll();
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    List<Object[]> output = TransTestingUtil.execute(input, meta, data, 2, false);
    TransTestingUtil.assertResult(new Object[] { "1", "1", "1" }, output.get(0));
    TransTestingUtil.assertResult(new Object[] { "2", "1", "2" }, output.get(1));
    deleteVfsFile(virtualFile);
}

Example 4 with FilePlayListAll

use of org.pentaho.di.core.playlist.FilePlayListAll in project pentaho-kettle by pentaho.

the class TextFileInputTest method createDataObject.

private TextFileInputData createDataObject(String file, String separator, String... outputFields) throws Exception {
    TextFileInputData data = new TextFileInputData();
    data.files = new FileInputList();
    data.files.addFile(KettleVFS.getFileObject(file));
    data.separator = separator;
    data.outputRowMeta = new RowMeta();
    if (outputFields != null) {
        for (String field : outputFields) {
            data.outputRowMeta.addValueMeta(new ValueMetaString(field));
        }
    }
    data.dataErrorLineHandler = mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0], new Variables());
    data.filePlayList = new FilePlayListAll();
    return data;
}

Example 5 with FilePlayListAll

use of org.pentaho.di.core.playlist.FilePlayListAll in project pentaho-kettle by pentaho.

the class TextFileInputDialogTest method testMinimalWidth_PDI_14253.

@Test
public void testMinimalWidth_PDI_14253() throws Exception {
    final String virtualFile = "ram://pdi-14253.txt";
    KettleVFS.getFileObject(virtualFile).createFile();
    final String content = "r1c1,  r1c2\nr2c1  ,  r2c2  ";
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    bos.write(content.getBytes());
    OutputStream os = KettleVFS.getFileObject(virtualFile).getContent().getOutputStream();
    IOUtils.copy(new ByteArrayInputStream(bos.toByteArray()), os);
    os.close();
    TextFileInputMeta meta = new TextFileInputMeta();
    meta.content.lineWrapped = false;
    meta.inputFields = new BaseFileField[] { new BaseFileField("col1", -1, -1), new BaseFileField("col2", -1, -1) };
    meta.content.fileCompression = "None";
    meta.content.fileType = "CSV";
    meta.content.header = false;
    meta.content.nrHeaderLines = -1;
    meta.content.footer = false;
    meta.content.nrFooterLines = -1;
    TextFileInputData data = new TextFileInputData();
    data.files = new FileInputList();
    data.files.addFile(KettleVFS.getFileObject(virtualFile));
    data.outputRowMeta = new RowMeta();
    data.outputRowMeta.addValueMeta(new ValueMetaString("col1"));
    data.outputRowMeta.addValueMeta(new ValueMetaString("col2"));
    data.dataErrorLineHandler = mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.separator = ",";
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0], new Variables() {
    });
    data.filePlayList = new FilePlayListAll();
    TextFileInputDialog dialog = new TextFileInputDialog(mock(Shell.class), meta, mock(TransMeta.class), "TFIMinimalWidthTest");
    TableView tv = mock(TableView.class);
    when(tv.nrNonEmpty()).thenReturn(0);
    // click the Minimal width button
    dialog.setMinimalWidth(tv);
    RowSet output = new BlockingRowSet(5);
    TextFileInput input = StepMockUtil.getStep(TextFileInput.class, TextFileInputMeta.class, "test");
    input.setOutputRowSets(Collections.singletonList(output));
    while (input.processRow(meta, data)) {
    // wait until the step completes executing
    }
    Object[] row1 = output.getRowImmediate();
    assertRow(row1, "r1c1", "r1c2");
    Object[] row2 = output.getRowImmediate();
    assertRow(row2, "r2c1", "r2c2");
    KettleVFS.getFileObject(virtualFile).delete();
}