Examples with FileInputList org.pentaho.di.core.fileinput.FileInputList used on opensource projects

Example 36 with FileInputList

use of org.pentaho.di.core.fileinput.FileInputList in project pentaho-kettle by pentaho.

the class GetFilesRowsCountMeta method check.

public void check(List<CheckResultInterface> remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, Repository repository, IMetaStore metaStore) {
    CheckResult cr;
    // See if we get input...
    if (input.length > 0) {
        cr = new CheckResult(CheckResult.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "GetFilesRowsCountMeta.CheckResult.NoInputExpected"), stepMeta);
        remarks.add(cr);
    } else {
        cr = new CheckResult(CheckResult.TYPE_RESULT_OK, BaseMessages.getString(PKG, "GetFilesRowsCountMeta.CheckResult.NoInput"), stepMeta);
        remarks.add(cr);
    }
    FileInputList fileInputList = getFiles(transMeta);
    if (fileInputList == null || fileInputList.getFiles().size() == 0) {
        cr = new CheckResult(CheckResult.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "GetFilesRowsCountMeta.CheckResult.NoFiles"), stepMeta);
        remarks.add(cr);
    } else {
        cr = new CheckResult(CheckResult.TYPE_RESULT_OK, BaseMessages.getString(PKG, "GetFilesRowsCountMeta.CheckResult.FilesOk", "" + fileInputList.getFiles().size()), stepMeta);
        remarks.add(cr);
    }
    if ((RowSeparator_format.equals("CUSTOM")) && (RowSeparator == null)) {
        cr = new CheckResult(CheckResult.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "GetFilesRowsCountMeta.CheckResult.NoSeparator"), stepMeta);
        remarks.add(cr);
    } else {
        cr = new CheckResult(CheckResult.TYPE_RESULT_OK, BaseMessages.getString(PKG, "GetFilesRowsCountMeta.CheckResult.SeparatorOk"), stepMeta);
        remarks.add(cr);
    }
}

Example 37 with FileInputList

use of org.pentaho.di.core.fileinput.FileInputList in project pentaho-kettle by pentaho.

the class TextFileInputMeta method check.

@Override
public void check(List<CheckResultInterface> remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, Repository repository, IMetaStore metaStore) {
    CheckResult cr;
    // See if we get input...
    if (input.length > 0) {
        if (!inputFiles.acceptingFilenames) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TextFileInputMeta.CheckResult.NoInputError"), stepMeta);
            remarks.add(cr);
        } else {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "TextFileInputMeta.CheckResult.AcceptFilenamesOk"), stepMeta);
            remarks.add(cr);
        }
    } else {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "TextFileInputMeta.CheckResult.NoInputOk"), stepMeta);
        remarks.add(cr);
    }
    FileInputList textFileList = getFileInputList(transMeta);
    if (textFileList.nrOfFiles() == 0) {
        if (!inputFiles.acceptingFilenames) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "TextFileInputMeta.CheckResult.ExpectedFilesError"), stepMeta);
            remarks.add(cr);
        }
    } else {
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "TextFileInputMeta.CheckResult.ExpectedFilesOk", "" + textFileList.nrOfFiles()), stepMeta);
        remarks.add(cr);
    }
}

Example 38 with FileInputList

use of org.pentaho.di.core.fileinput.FileInputList in project pentaho-kettle by pentaho.

the class TextFileInputTest method createDataObject.

private TextFileInputData createDataObject(String file, String separator, String... outputFields) throws Exception {
    TextFileInputData data = new TextFileInputData();
    data.files = new FileInputList();
    data.files.addFile(KettleVFS.getFileObject(file));
    data.separator = separator;
    data.outputRowMeta = new RowMeta();
    if (outputFields != null) {
        for (String field : outputFields) {
            data.outputRowMeta.addValueMeta(new ValueMetaString(field));
        }
    }
    data.dataErrorLineHandler = mock(FileErrorHandler.class);
    data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX;
    data.filterProcessor = new TextFileFilterProcessor(new TextFileFilter[0], new Variables());
    data.filePlayList = new FilePlayListAll();
    return data;
}

Example 39 with FileInputList

use of org.pentaho.di.core.fileinput.FileInputList in project pentaho-kettle by pentaho.

the class PDI_6976_Test method testVerifyNoPreviousStep.

@Test
public void testVerifyNoPreviousStep() {
    LoadFileInputMeta spy = spy(new LoadFileInputMeta());
    FileInputList fileInputList = mock(FileInputList.class);
    List<FileObject> files = when(mock(List.class).size()).thenReturn(1).getMock();
    doReturn(files).when(fileInputList).getFiles();
    doReturn(fileInputList).when(spy).getFiles(any(VariableSpace.class));
    @SuppressWarnings("unchecked") List<CheckResultInterface> validationResults = mock(List.class);
    // Check we do not get validation errors
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            if (((CheckResultInterface) invocation.getArguments()[0]).getType() != CheckResultInterface.TYPE_RESULT_OK) {
                TestCase.fail("We've got validation error");
            }
            return null;
        }
    }).when(validationResults).add(any(CheckResultInterface.class));
    spy.check(validationResults, mock(TransMeta.class), mock(StepMeta.class), mock(RowMetaInterface.class), new String[] {}, new String[] { "File content", "File size" }, mock(RowMetaInterface.class), mock(VariableSpace.class), mock(Repository.class), mock(IMetaStore.class));
}

Example 40 with FileInputList

use of org.pentaho.di.core.fileinput.FileInputList in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    TextFileInputMeta meta = new TextFileInputMeta();
    getInfo(meta);
    TextFileInputMeta previousMeta = (TextFileInputMeta) meta.clone();
    FileInputList textFileList = meta.getTextFileList(transMeta);
    InputStream fileInputStream;
    CompressionInputStream inputStream = null;
    StringBuilder lineStringBuilder = new StringBuilder(256);
    int fileFormatType = meta.getFileFormatTypeNr();
    String delimiter = transMeta.environmentSubstitute(meta.getSeparator());
    String enclosure = transMeta.environmentSubstitute(meta.getEnclosure());
    String escapeCharacter = transMeta.environmentSubstitute(meta.getEscapeCharacter());
    if (textFileList.nrOfFiles() > 0) {
        int clearFields = meta.hasHeader() ? SWT.YES : SWT.NO;
        int nrInputFields = meta.getInputFields().length;
        if (meta.hasHeader() && nrInputFields > 0) {
            MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION);
            mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.ClearFieldList.DialogMessage"));
            mb.setText(BaseMessages.getString(PKG, "TextFileInputDialog.ClearFieldList.DialogTitle"));
            clearFields = mb.open();
            if (clearFields == SWT.CANCEL) {
                return;
            }
        }
        try {
            wFields.table.removeAll();
            FileObject fileObject = textFileList.getFile(0);
            fileInputStream = KettleVFS.getInputStream(fileObject);
            Table table = wFields.table;
            CompressionProvider provider = CompressionProviderFactory.getInstance().createCompressionProviderInstance(meta.getFileCompression());
            inputStream = provider.createInputStream(fileInputStream);
            InputStreamReader reader;
            if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
                reader = new InputStreamReader(inputStream, meta.getEncoding());
            } else {
                reader = new InputStreamReader(inputStream);
            }
            EncodingType encodingType = EncodingType.guessEncodingType(reader.getEncoding());
            if (clearFields == SWT.YES || !meta.hasHeader() || nrInputFields > 0) {
                // Scan the header-line, determine fields...
                String line;
                if (meta.hasHeader() || meta.getInputFields().length == 0) {
                    line = TextFileInput.getLine(log, reader, encodingType, fileFormatType, lineStringBuilder);
                    if (line != null) {
                        // Estimate the number of input fields...
                        // Chop up the line using the delimiter
                        String[] fields = TextFileInput.guessStringsFromLine(transMeta, log, line, meta, delimiter, enclosure, escapeCharacter);
                        for (int i = 0; i < fields.length; i++) {
                            String field = fields[i];
                            if (field == null || field.length() == 0 || (nrInputFields == 0 && !meta.hasHeader())) {
                                field = "Field" + (i + 1);
                            } else {
                                // Trim the field
                                field = Const.trim(field);
                                // Replace all spaces & - with underscore _
                                field = Const.replace(field, " ", "_");
                                field = Const.replace(field, "-", "_");
                            }
                            TableItem item = new TableItem(table, SWT.NONE);
                            item.setText(1, field);
                            // The default type is String...
                            item.setText(2, "String");
                        }
                        wFields.setRowNums();
                        wFields.optWidth(true);
                        // Copy it...
                        getInfo(meta);
                    }
                }
                // Sample a few lines to determine the correct type of the fields...
                String shellText = BaseMessages.getString(PKG, "TextFileInputDialog.LinesToSample.DialogTitle");
                String lineText = BaseMessages.getString(PKG, "TextFileInputDialog.LinesToSample.DialogMessage");
                EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
                int samples = end.open();
                if (samples >= 0) {
                    getInfo(meta);
                    TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, clearFields == SWT.YES);
                    String message = pd.open();
                    if (message != null) {
                        wFields.removeAll();
                        // OK, what's the result of our search?
                        getData(meta);
                        // 
                        if (clearFields == SWT.NO) {
                            getFieldsData(previousMeta, true);
                            wFields.table.setSelection(previousMeta.getInputFields().length, wFields.table.getItemCount() - 1);
                        }
                        wFields.removeEmptyRows();
                        wFields.setRowNums();
                        wFields.optWidth(true);
                        EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "TextFileInputDialog.ScanResults.DialogTitle"), BaseMessages.getString(PKG, "TextFileInputDialog.ScanResults.DialogMessage"), message, true);
                        etd.setReadOnly();
                        etd.open();
                    }
                }
            } else {
                MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
                mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.UnableToReadHeaderLine.DialogMessage"));
                mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
                mb.open();
            }
        } catch (IOException e) {
            new ErrorDialog(shell, BaseMessages.getString(PKG, "TextFileInputDialog.IOError.DialogTitle"), BaseMessages.getString(PKG, "TextFileInputDialog.IOError.DialogMessage"), e);
        } catch (KettleException e) {
            new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "TextFileInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
        } finally {
            try {
                if (inputStream != null) {
                    inputStream.close();
                }
            } catch (Exception e) {
            // Ignore errors
            }
        }
    } else {
        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
        mb.setMessage(BaseMessages.getString(PKG, "TextFileInputDialog.NoValidFileFound.DialogMessage"));
        mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
        mb.open();
    }
}