Examples with TextFileCSVImportProgressDialog org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog used on opensource projects

Example 1 with TextFileCSVImportProgressDialog

use of org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog in project pentaho-kettle by pentaho.

the class S3CsvInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    InputStream inputStream = null;
    try {
        S3CsvInputMeta meta = new S3CsvInputMeta();
        getInfo(meta);
        String filename = transMeta.environmentSubstitute(meta.getFilename());
        String bucketname = transMeta.environmentSubstitute(meta.getBucket());
        int maxLineSize = Const.toInt(transMeta.environmentSubstitute(meta.getMaxLineSize()), 2000);
        wFields.table.removeAll();
        S3ObjectsProvider s3ObjProvider = new S3ObjectsProvider(meta.getS3Service(transMeta));
        S3Bucket s3bucket = s3ObjProvider.getBucket(bucketname);
        if (s3bucket == null) {
            throw new Exception(Messages.getString("S3DefaultService.Exception.UnableToFindBucket.Message", bucketname));
        }
        // Now we can continue reading the rows of data and we can guess the
        // Sample a few lines to determine the correct type of the fields...
        // 
        String shellText = Messages.getString("S3CsvInputDialog.LinesToSample.DialogTitle");
        String lineText = Messages.getString("S3CsvInputDialog.LinesToSample.DialogMessage");
        EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
        int samples = end.open();
        if (samples < 0) {
            return;
        }
        // Only get the first lines, not the complete file
        // And grab an input stream to the data...
        inputStream = s3ObjProvider.getS3Object(s3bucket, filename, 0L, (long) samples * (long) maxLineSize).getDataInputStream();
        InputStreamReader reader = new InputStreamReader(inputStream);
        // Read a line of data to determine the number of rows...
        // 
        String line = TextFileInput.getLine(log, reader, TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder(1000));
        // Split the string, header or data into parts...
        // 
        String[] fieldNames = Const.splitString(line, meta.getDelimiter());
        if (!meta.isHeaderPresent()) {
            // Don't use field names from the header...
            // Generate field names F1 ... F10
            // 
            // $NON-NLS-1$
            DecimalFormat df = new DecimalFormat("000");
            for (int i = 0; i < fieldNames.length; i++) {
                // $NON-NLS-1$
                fieldNames[i] = "Field_" + df.format(i);
            }
        } else {
            if (!Utils.isEmpty(meta.getEnclosure())) {
                for (int i = 0; i < fieldNames.length; i++) {
                    if (fieldNames[i].startsWith(meta.getEnclosure()) && fieldNames[i].endsWith(meta.getEnclosure()) && fieldNames[i].length() > 1) {
                        fieldNames[i] = fieldNames[i].substring(1, fieldNames[i].length() - 1);
                    }
                }
            }
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            fieldNames[i] = Const.trim(fieldNames[i]);
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            TableItem item = new TableItem(wFields.table, SWT.NONE);
            item.setText(1, fieldNames[i]);
            item.setText(2, ValueMeta.getTypeDesc(ValueMetaInterface.TYPE_STRING));
        }
        wFields.removeEmptyRows();
        wFields.setRowNums();
        wFields.optWidth(true);
        getInfo(meta);
        TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, true);
        String message = pd.open();
        if (message != null) {
            wFields.removeAll();
            // OK, what's the result of our search?
            getData(meta);
            wFields.removeEmptyRows();
            wFields.setRowNums();
            wFields.optWidth(true);
            EnterTextDialog etd = new EnterTextDialog(shell, Messages.getString("S3CsvInputDialog.ScanResults.DialogTitle"), Messages.getString("S3CsvInputDialog.ScanResults.DialogMessage"), message, true);
            etd.setReadOnly();
            etd.open();
        }
    } catch (IOException e) {
        new ErrorDialog(shell, Messages.getString("S3CsvInputDialog.IOError.DialogTitle"), Messages.getString("S3CsvInputDialog.IOError.DialogMessage"), e);
    } catch (Exception e) {
        new ErrorDialog(shell, Messages.getString("System.Dialog.Error.Title"), Messages.getString("S3CsvInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
    } finally {
        try {
            if (inputStream != null) {
                inputStream.close();
            }
        } catch (Exception e) {
            log.logError(stepname, "Error closing s3 data input stream", e);
        }
    }
}

Example 2 with TextFileCSVImportProgressDialog

use of org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog in project pentaho-kettle by pentaho.

the class CsvInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    InputStream inputStream = null;
    try {
        CsvInputMeta meta = new CsvInputMeta();
        getInfo(meta);
        String filename = transMeta.environmentSubstitute(meta.getFilename());
        String delimiter = transMeta.environmentSubstitute(meta.getDelimiter());
        String enclosure = transMeta.environmentSubstitute(meta.getEnclosure());
        FileObject fileObject = KettleVFS.getFileObject(filename);
        if (!(fileObject instanceof LocalFile)) {
            // 
            throw new KettleException(BaseMessages.getString(PKG, "CsvInput.Log.OnlyLocalFilesAreSupported"));
        }
        wFields.table.removeAll();
        inputStream = KettleVFS.getInputStream(fileObject);
        String realEncoding = transMeta.environmentSubstitute(meta.getEncoding());
        InputStreamReader reader;
        if (Utils.isEmpty(realEncoding)) {
            reader = new InputStreamReader(inputStream);
        } else {
            reader = new InputStreamReader(inputStream, realEncoding);
        }
        EncodingType encodingType = EncodingType.guessEncodingType(reader.getEncoding());
        // Read a line of data to determine the number of rows...
        // 
        String line = TextFileInput.getLine(log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder(1000));
        // Split the string, header or data into parts...
        // 
        String[] fieldNames = CsvInput.guessStringsFromLine(log, line, delimiter, enclosure, meta.getEscapeCharacter());
        if (!meta.isHeaderPresent()) {
            // Don't use field names from the header...
            // Generate field names F1 ... F10
            // 
            DecimalFormat df = new DecimalFormat("000");
            for (int i = 0; i < fieldNames.length; i++) {
                fieldNames[i] = "Field_" + df.format(i);
            }
        } else {
            if (!Utils.isEmpty(meta.getEnclosure())) {
                for (int i = 0; i < fieldNames.length; i++) {
                    if (fieldNames[i].startsWith(meta.getEnclosure()) && fieldNames[i].endsWith(meta.getEnclosure()) && fieldNames[i].length() > 1) {
                        fieldNames[i] = fieldNames[i].substring(1, fieldNames[i].length() - 1);
                    }
                }
            }
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            fieldNames[i] = Const.trim(fieldNames[i]);
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            TableItem item = new TableItem(wFields.table, SWT.NONE);
            item.setText(1, fieldNames[i]);
            item.setText(2, ValueMetaFactory.getValueMetaName(ValueMetaInterface.TYPE_STRING));
        }
        wFields.removeEmptyRows();
        wFields.setRowNums();
        wFields.optWidth(true);
        // Now we can continue reading the rows of data and we can guess the
        // Sample a few lines to determine the correct type of the fields...
        // 
        String shellText = BaseMessages.getString(PKG, "CsvInputDialog.LinesToSample.DialogTitle");
        String lineText = BaseMessages.getString(PKG, "CsvInputDialog.LinesToSample.DialogMessage");
        EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
        int samples = end.open();
        if (samples >= 0) {
            getInfo(meta);
            TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, true);
            String message = pd.open();
            if (message != null) {
                wFields.removeAll();
                // OK, what's the result of our search?
                getData(meta, false);
                wFields.removeEmptyRows();
                wFields.setRowNums();
                wFields.optWidth(true);
                EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "CsvInputDialog.ScanResults.DialogTitle"), BaseMessages.getString(PKG, "CsvInputDialog.ScanResults.DialogMessage"), message, true);
                etd.setReadOnly();
                etd.open();
            // asyncUpdatePreview();
            }
        }
    } catch (IOException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "CsvInputDialog.IOError.DialogTitle"), BaseMessages.getString(PKG, "CsvInputDialog.IOError.DialogMessage"), e);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "CsvInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
    } finally {
        try {
            inputStream.close();
        } catch (Exception e) {
        // Ignore close errors
        }
    }
}

Example 3 with TextFileCSVImportProgressDialog

use of org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog in project pentaho-kettle by pentaho.

the class ParGzipCsvInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    InputStream inputStream = null;
    try {
        ParGzipCsvInputMeta meta = new ParGzipCsvInputMeta();
        getInfo(meta);
        String filename = transMeta.environmentSubstitute(meta.getFilename());
        FileObject fileObject = KettleVFS.getFileObject(filename);
        if (!(fileObject instanceof LocalFile)) {
            // 
            throw new KettleException(BaseMessages.getString(PKG, "ParGzipCsvInput.Log.OnlyLocalFilesAreSupported"));
        }
        wFields.table.removeAll();
        inputStream = new GZIPInputStream(KettleVFS.getInputStream(fileObject));
        InputStreamReader reader = new InputStreamReader(inputStream);
        EncodingType encodingType = EncodingType.guessEncodingType(reader.getEncoding());
        // Read a line of data to determine the number of rows...
        // 
        String line = TextFileInput.getLine(log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder(1000));
        // Split the string, header or data into parts...
        // 
        String[] fieldNames = Const.splitString(line, meta.getDelimiter());
        if (!meta.isHeaderPresent()) {
            // Don't use field names from the header...
            // Generate field names F1 ... F10
            // 
            DecimalFormat df = new DecimalFormat("000");
            for (int i = 0; i < fieldNames.length; i++) {
                fieldNames[i] = "Field_" + df.format(i);
            }
        } else {
            if (!Utils.isEmpty(meta.getEnclosure())) {
                for (int i = 0; i < fieldNames.length; i++) {
                    if (fieldNames[i].startsWith(meta.getEnclosure()) && fieldNames[i].endsWith(meta.getEnclosure()) && fieldNames[i].length() > 1) {
                        fieldNames[i] = fieldNames[i].substring(1, fieldNames[i].length() - 1);
                    }
                }
            }
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            fieldNames[i] = Const.trim(fieldNames[i]);
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            TableItem item = new TableItem(wFields.table, SWT.NONE);
            item.setText(1, fieldNames[i]);
            item.setText(2, ValueMetaFactory.getValueMetaName(ValueMetaInterface.TYPE_STRING));
        }
        wFields.removeEmptyRows();
        wFields.setRowNums();
        wFields.optWidth(true);
        // Now we can continue reading the rows of data and we can guess the
        // Sample a few lines to determine the correct type of the fields...
        // 
        String shellText = BaseMessages.getString(PKG, "ParGzipCsvInputDialog.LinesToSample.DialogTitle");
        String lineText = BaseMessages.getString(PKG, "ParGzipCsvInputDialog.LinesToSample.DialogMessage");
        EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
        int samples = end.open();
        if (samples >= 0) {
            getInfo(meta);
            TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, true);
            String message = pd.open();
            if (message != null) {
                wFields.removeAll();
                // OK, what's the result of our search?
                getData(meta);
                wFields.removeEmptyRows();
                wFields.setRowNums();
                wFields.optWidth(true);
                EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "ParGzipCsvInputDialog.ScanResults.DialogTitle"), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.ScanResults.DialogMessage"), message, true);
                etd.setReadOnly();
                etd.open();
            }
        }
    } catch (IOException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "ParGzipCsvInputDialog.IOError.DialogTitle"), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.IOError.DialogMessage"), e);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
    } finally {
        try {
            inputStream.close();
        } catch (Exception e) {
        // Ignore errors
        }
    }
}