Examples with ParGzipCsvInputMeta org.pentaho.di.trans.steps.parallelgzipcsv.ParGzipCsvInputMeta used on opensource projects

Example 1 with ParGzipCsvInputMeta

use of org.pentaho.di.trans.steps.parallelgzipcsv.ParGzipCsvInputMeta in project pentaho-kettle by pentaho.

the class ParGzipCsvInputDialog method preview.

// Preview the data
private void preview() {
    // Create the XML input step
    ParGzipCsvInputMeta oneMeta = new ParGzipCsvInputMeta();
    getInfo(oneMeta);
    TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation(transMeta, oneMeta, wStepname.getText());
    EnterNumberDialog numberDialog = new EnterNumberDialog(shell, props.getDefaultPreviewSize(), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.PreviewSize.DialogTitle"), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.PreviewSize.DialogMessage"));
    int previewSize = numberDialog.open();
    if (previewSize > 0) {
        TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog(shell, previewMeta, new String[] { wStepname.getText() }, new int[] { previewSize });
        progressDialog.open();
        Trans trans = progressDialog.getTrans();
        String loggingText = progressDialog.getLoggingText();
        if (!progressDialog.isCancelled()) {
            if (trans.getResult() != null && trans.getResult().getNrErrors() > 0) {
                EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "System.Dialog.PreviewError.Title"), BaseMessages.getString(PKG, "System.Dialog.PreviewError.Message"), loggingText, true);
                etd.setReadOnly();
                etd.open();
            }
        }
        PreviewRowsDialog prd = new PreviewRowsDialog(shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog.getPreviewRowsMeta(wStepname.getText()), progressDialog.getPreviewRows(wStepname.getText()), loggingText);
        prd.open();
    }
}

Example 2 with ParGzipCsvInputMeta

use of org.pentaho.di.trans.steps.parallelgzipcsv.ParGzipCsvInputMeta in project pentaho-kettle by pentaho.

the class ParGzipCsvInputDialog method getCSV.

// Get the data layout
private void getCSV() {
    InputStream inputStream = null;
    try {
        ParGzipCsvInputMeta meta = new ParGzipCsvInputMeta();
        getInfo(meta);
        String filename = transMeta.environmentSubstitute(meta.getFilename());
        FileObject fileObject = KettleVFS.getFileObject(filename);
        if (!(fileObject instanceof LocalFile)) {
            // 
            throw new KettleException(BaseMessages.getString(PKG, "ParGzipCsvInput.Log.OnlyLocalFilesAreSupported"));
        }
        wFields.table.removeAll();
        inputStream = new GZIPInputStream(KettleVFS.getInputStream(fileObject));
        InputStreamReader reader = new InputStreamReader(inputStream);
        EncodingType encodingType = EncodingType.guessEncodingType(reader.getEncoding());
        // Read a line of data to determine the number of rows...
        // 
        String line = TextFileInput.getLine(log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder(1000));
        // Split the string, header or data into parts...
        // 
        String[] fieldNames = Const.splitString(line, meta.getDelimiter());
        if (!meta.isHeaderPresent()) {
            // Don't use field names from the header...
            // Generate field names F1 ... F10
            // 
            DecimalFormat df = new DecimalFormat("000");
            for (int i = 0; i < fieldNames.length; i++) {
                fieldNames[i] = "Field_" + df.format(i);
            }
        } else {
            if (!Utils.isEmpty(meta.getEnclosure())) {
                for (int i = 0; i < fieldNames.length; i++) {
                    if (fieldNames[i].startsWith(meta.getEnclosure()) && fieldNames[i].endsWith(meta.getEnclosure()) && fieldNames[i].length() > 1) {
                        fieldNames[i] = fieldNames[i].substring(1, fieldNames[i].length() - 1);
                    }
                }
            }
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            fieldNames[i] = Const.trim(fieldNames[i]);
        }
        // 
        for (int i = 0; i < fieldNames.length; i++) {
            TableItem item = new TableItem(wFields.table, SWT.NONE);
            item.setText(1, fieldNames[i]);
            item.setText(2, ValueMetaFactory.getValueMetaName(ValueMetaInterface.TYPE_STRING));
        }
        wFields.removeEmptyRows();
        wFields.setRowNums();
        wFields.optWidth(true);
        // Now we can continue reading the rows of data and we can guess the
        // Sample a few lines to determine the correct type of the fields...
        // 
        String shellText = BaseMessages.getString(PKG, "ParGzipCsvInputDialog.LinesToSample.DialogTitle");
        String lineText = BaseMessages.getString(PKG, "ParGzipCsvInputDialog.LinesToSample.DialogMessage");
        EnterNumberDialog end = new EnterNumberDialog(shell, 100, shellText, lineText);
        int samples = end.open();
        if (samples >= 0) {
            getInfo(meta);
            TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, true);
            String message = pd.open();
            if (message != null) {
                wFields.removeAll();
                // OK, what's the result of our search?
                getData(meta);
                wFields.removeEmptyRows();
                wFields.setRowNums();
                wFields.optWidth(true);
                EnterTextDialog etd = new EnterTextDialog(shell, BaseMessages.getString(PKG, "ParGzipCsvInputDialog.ScanResults.DialogTitle"), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.ScanResults.DialogMessage"), message, true);
                etd.setReadOnly();
                etd.open();
            }
        }
    } catch (IOException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "ParGzipCsvInputDialog.IOError.DialogTitle"), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.IOError.DialogMessage"), e);
    } catch (KettleException e) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "System.Dialog.Error.Title"), BaseMessages.getString(PKG, "ParGzipCsvInputDialog.ErrorGettingFileDesc.DialogMessage"), e);
    } finally {
        try {
            inputStream.close();
        } catch (Exception e) {
        // Ignore errors
        }
    }
}