Examples with BaseFileField org.pentaho.di.trans.steps.file.BaseFileField used on opensource projects

Example 1 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileCSVImportProgressDialog method doScan.

private String doScan(IProgressMonitor monitor) throws KettleException {
    if (samples > 0) {
        monitor.beginTask(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.ScanningFile"), samples + 1);
    } else {
        monitor.beginTask(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.ScanningFile"), 2);
    }
    String line = "";
    long fileLineNumber = 0;
    DecimalFormatSymbols dfs = new DecimalFormatSymbols();
    int nrfields = meta.inputFields.length;
    RowMetaInterface outputRowMeta = new RowMeta();
    meta.getFields(outputRowMeta, null, null, null, transMeta, null, null);
    // Remove the storage meta-data (don't go for lazy conversion during scan)
    for (ValueMetaInterface valueMeta : outputRowMeta.getValueMetaList()) {
        valueMeta.setStorageMetadata(null);
        valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
    }
    RowMetaInterface convertRowMeta = outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
    // How many null values?
    // How many times null value?
    int[] nrnull = new int[nrfields];
    // String info
    // min string
    String[] minstr = new String[nrfields];
    // max string
    String[] maxstr = new String[nrfields];
    // first occ. of string?
    boolean[] firststr = new boolean[nrfields];
    // Date info
    // is the field perhaps a Date?
    boolean[] isDate = new boolean[nrfields];
    // How many date formats work?
    int[] dateFormatCount = new int[nrfields];
    // What are the date formats that
    boolean[][] dateFormat = new boolean[nrfields][Const.getDateFormats().length];
    // work?
    // min date value
    Date[][] minDate = new Date[nrfields][Const.getDateFormats().length];
    // max date value
    Date[][] maxDate = new Date[nrfields][Const.getDateFormats().length];
    // Number info
    // is the field perhaps a Number?
    boolean[] isNumber = new boolean[nrfields];
    // How many number formats work?
    int[] numberFormatCount = new int[nrfields];
    // What are the number format
    boolean[][] numberFormat = new boolean[nrfields][Const.getNumberFormats().length];
    // that work?
    // min number value
    double[][] minValue = new double[nrfields][Const.getDateFormats().length];
    // max number value
    double[][] maxValue = new double[nrfields][Const.getDateFormats().length];
    // remember the precision?
    int[][] numberPrecision = new int[nrfields][Const.getNumberFormats().length];
    // remember the length?
    int[][] numberLength = new int[nrfields][Const.getNumberFormats().length];
    for (int i = 0; i < nrfields; i++) {
        BaseFileField field = meta.inputFields[i];
        if (log.isDebug()) {
            debug = "init field #" + i;
        }
        if (replaceMeta) {
            // Clear previous info...
            field.setName(meta.inputFields[i].getName());
            field.setType(meta.inputFields[i].getType());
            field.setFormat("");
            field.setLength(-1);
            field.setPrecision(-1);
            field.setCurrencySymbol(dfs.getCurrencySymbol());
            field.setDecimalSymbol("" + dfs.getDecimalSeparator());
            field.setGroupSymbol("" + dfs.getGroupingSeparator());
            field.setNullString("-");
            field.setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
        }
        nrnull[i] = 0;
        minstr[i] = "";
        maxstr[i] = "";
        firststr[i] = true;
        // Init data guess
        isDate[i] = true;
        for (int j = 0; j < Const.getDateFormats().length; j++) {
            dateFormat[i][j] = true;
            minDate[i][j] = Const.MAX_DATE;
            maxDate[i][j] = Const.MIN_DATE;
        }
        dateFormatCount[i] = Const.getDateFormats().length;
        // Init number guess
        isNumber[i] = true;
        for (int j = 0; j < Const.getNumberFormats().length; j++) {
            numberFormat[i][j] = true;
            minValue[i][j] = Double.MAX_VALUE;
            maxValue[i][j] = -Double.MAX_VALUE;
            numberPrecision[i][j] = -1;
            numberLength[i][j] = -1;
        }
        numberFormatCount[i] = Const.getNumberFormats().length;
    }
    TextFileInputMeta strinfo = (TextFileInputMeta) meta.clone();
    for (int i = 0; i < nrfields; i++) {
        strinfo.inputFields[i].setType(ValueMetaInterface.TYPE_STRING);
    }
    // Sample <samples> rows...
    debug = "get first line";
    StringBuilder lineBuffer = new StringBuilder(256);
    int fileFormatType = meta.getFileFormatTypeNr();
    // If the file has a header we overwrite the first line
    // However, if it doesn't have a header, take a new line
    // 
    line = TextFileInputUtils.getLine(log, reader, encodingType, fileFormatType, lineBuffer);
    fileLineNumber++;
    int skipped = 1;
    if (meta.content.header) {
        while (line != null && skipped < meta.content.nrHeaderLines) {
            line = TextFileInputUtils.getLine(log, reader, encodingType, fileFormatType, lineBuffer);
            skipped++;
            fileLineNumber++;
        }
    }
    int linenr = 1;
    List<StringEvaluator> evaluators = new ArrayList<StringEvaluator>();
    // Allocate number and date parsers
    DecimalFormat df2 = (DecimalFormat) NumberFormat.getInstance();
    DecimalFormatSymbols dfs2 = new DecimalFormatSymbols();
    SimpleDateFormat daf2 = new SimpleDateFormat();
    boolean errorFound = false;
    while (!errorFound && line != null && (linenr <= samples || samples == 0) && !monitor.isCanceled()) {
        monitor.subTask(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.ScanningLine", "" + linenr));
        if (samples > 0) {
            monitor.worked(1);
        }
        if (log.isDebug()) {
            debug = "convert line #" + linenr + " to row";
        }
        RowMetaInterface rowMeta = new RowMeta();
        meta.getFields(rowMeta, "stepname", null, null, transMeta, null, null);
        // Remove the storage meta-data (don't go for lazy conversion during scan)
        for (ValueMetaInterface valueMeta : rowMeta.getValueMetaList()) {
            valueMeta.setStorageMetadata(null);
            valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
        }
        String delimiter = transMeta.environmentSubstitute(meta.content.separator);
        String enclosure = transMeta.environmentSubstitute(meta.content.enclosure);
        String escapeCharacter = transMeta.environmentSubstitute(meta.content.escapeCharacter);
        Object[] r = TextFileInputUtils.convertLineToRow(log, new TextFileLine(line, fileLineNumber, null), strinfo, null, 0, outputRowMeta, convertRowMeta, FileInputList.createFilePathList(transMeta, meta.inputFiles.fileName, meta.inputFiles.fileMask, meta.inputFiles.excludeFileMask, meta.inputFiles.fileRequired, meta.inputFiles.includeSubFolderBoolean())[0], rownumber, delimiter, enclosure, escapeCharacter, null, new BaseFileInputAdditionalField(), null, null, false, null, null, null, null, null);
        if (r == null) {
            errorFound = true;
            continue;
        }
        rownumber++;
        for (int i = 0; i < nrfields && i < r.length; i++) {
            StringEvaluator evaluator;
            if (i >= evaluators.size()) {
                evaluator = new StringEvaluator(true);
                evaluators.add(evaluator);
            } else {
                evaluator = evaluators.get(i);
            }
            String string = rowMeta.getString(r, i);
            if (i == 0) {
                System.out.println();
            }
            evaluator.evaluateString(string);
        }
        fileLineNumber++;
        if (r != null) {
            linenr++;
        }
        // Grab another line...
        // 
        line = TextFileInputUtils.getLine(log, reader, encodingType, fileFormatType, lineBuffer);
    }
    monitor.worked(1);
    monitor.setTaskName(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Task.AnalyzingResults"));
    // Show information on items using a dialog box
    // 
    StringBuilder message = new StringBuilder();
    message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.ResultAfterScanning", "" + (linenr - 1)));
    message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.HorizontalLine"));
    for (int i = 0; i < nrfields; i++) {
        BaseFileField field = meta.inputFields[i];
        StringEvaluator evaluator = evaluators.get(i);
        List<StringEvaluationResult> evaluationResults = evaluator.getStringEvaluationResults();
        // 
        if (evaluationResults.isEmpty()) {
            field.setType(ValueMetaInterface.TYPE_STRING);
            field.setLength(evaluator.getMaxLength());
        } else {
            StringEvaluationResult result = evaluator.getAdvicedResult();
            if (result != null) {
                // Take the first option we find, list the others below...
                // 
                ValueMetaInterface conversionMeta = result.getConversionMeta();
                field.setType(conversionMeta.getType());
                field.setTrimType(conversionMeta.getTrimType());
                field.setFormat(conversionMeta.getConversionMask());
                field.setDecimalSymbol(conversionMeta.getDecimalSymbol());
                field.setGroupSymbol(conversionMeta.getGroupingSymbol());
                field.setLength(conversionMeta.getLength());
                field.setPrecision(conversionMeta.getPrecision());
                nrnull[i] = result.getNrNull();
                minstr[i] = result.getMin() == null ? "" : result.getMin().toString();
                maxstr[i] = result.getMax() == null ? "" : result.getMax().toString();
            }
        }
        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.FieldNumber", "" + (i + 1)));
        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.FieldName", field.getName()));
        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.FieldType", field.getTypeDesc()));
        switch(field.getType()) {
            case ValueMetaInterface.TYPE_NUMBER:
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.EstimatedLength", (field.getLength() < 0 ? "-" : "" + field.getLength())));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.EstimatedPrecision", field.getPrecision() < 0 ? "-" : "" + field.getPrecision()));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat", field.getFormat()));
                if (!evaluationResults.isEmpty()) {
                    if (evaluationResults.size() > 1) {
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.WarnNumberFormat"));
                    }
                    for (StringEvaluationResult seResult : evaluationResults) {
                        String mask = seResult.getConversionMeta().getConversionMask();
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat2", mask));
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.TrimType", seResult.getConversionMeta().getTrimType()));
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberMinValue", seResult.getMin()));
                        message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberMaxValue", seResult.getMax()));
                        try {
                            df2.applyPattern(mask);
                            df2.setDecimalFormatSymbols(dfs2);
                            double mn = df2.parse(seResult.getMin().toString()).doubleValue();
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberExample", mask, seResult.getMin(), Double.toString(mn)));
                        } catch (Exception e) {
                            if (log.isDetailed()) {
                                log.logDetailed("This is unexpected: parsing [" + seResult.getMin() + "] with format [" + mask + "] did not work.");
                            }
                        }
                    }
                }
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.NumberNrNullValues", "" + nrnull[i]));
                break;
            case ValueMetaInterface.TYPE_STRING:
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringMaxLength", "" + field.getLength()));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringMinValue", minstr[i]));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringMaxValue", maxstr[i]));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.StringNrNullValues", "" + nrnull[i]));
                break;
            case ValueMetaInterface.TYPE_DATE:
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateMaxLength", field.getLength() < 0 ? "-" : "" + field.getLength()));
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateFormat", field.getFormat()));
                if (dateFormatCount[i] > 1) {
                    message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.WarnDateFormat"));
                }
                if (!Utils.isEmpty(minstr[i])) {
                    for (int x = 0; x < Const.getDateFormats().length; x++) {
                        if (dateFormat[i][x]) {
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateFormat2", Const.getDateFormats()[x]));
                            Date mindate = minDate[i][x];
                            Date maxdate = maxDate[i][x];
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateMinValue", mindate.toString()));
                            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateMaxValue", maxdate.toString()));
                            daf2.applyPattern(Const.getDateFormats()[x]);
                            try {
                                Date md = daf2.parse(minstr[i]);
                                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateExample", Const.getDateFormats()[x], minstr[i], md.toString()));
                            } catch (Exception e) {
                                if (log.isDetailed()) {
                                    log.logDetailed("This is unexpected: parsing [" + minstr[i] + "] with format [" + Const.getDateFormats()[x] + "] did not work.");
                                }
                            }
                        }
                    }
                }
                message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.DateNrNullValues", "" + nrnull[i]));
                break;
            default:
                break;
        }
        if (nrnull[i] == linenr - 1) {
            message.append(BaseMessages.getString(PKG, "TextFileCSVImportProgressDialog.Info.AllNullValues"));
        }
        message.append(Const.CR);
    }
    monitor.worked(1);
    monitor.done();
    return message.toString();
}

Example 2 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileImportWizardPage2 method createControl.

public void createControl(Composite parent) {
    shell = parent.getShell();
    int margin = Const.MARGIN;
    int left = props.getMiddlePct() / 2;
    int middle = props.getMiddlePct();
    int right = middle + left;
    // create the composite to hold the widgets
    Composite composite = new Composite(parent, SWT.NONE);
    props.setLook(composite);
    FormLayout compLayout = new FormLayout();
    compLayout.marginHeight = Const.FORM_MARGIN;
    compLayout.marginWidth = Const.FORM_MARGIN;
    composite.setLayout(compLayout);
    wFields = new List(composite, SWT.SINGLE | SWT.BORDER | SWT.V_SCROLL | SWT.H_SCROLL);
    props.setLook(wFields);
    fdFields = new FormData();
    fdFields.top = new FormAttachment(0, 0);
    fdFields.left = new FormAttachment(0, 0);
    fdFields.right = new FormAttachment(left, 0);
    fdFields.bottom = new FormAttachment(100, 0);
    wFields.setLayoutData(fdFields);
    refreshFields();
    // Fieldname line
    wlFieldname = new Label(composite, SWT.RIGHT);
    wlFieldname.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Fieldname.Label"));
    props.setLook(wlFieldname);
    fdlFieldname = new FormData();
    fdlFieldname.left = new FormAttachment(wFields, 0);
    fdlFieldname.top = new FormAttachment(0, 0);
    fdlFieldname.right = new FormAttachment(middle, 0);
    wlFieldname.setLayoutData(fdlFieldname);
    wFieldname = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wFieldname);
    fdFieldname = new FormData();
    fdFieldname.left = new FormAttachment(middle, margin);
    fdFieldname.right = new FormAttachment(right, 0);
    fdFieldname.top = new FormAttachment(0, 0);
    wFieldname.setLayoutData(fdFieldname);
    // Position line
    wlPosition = new Label(composite, SWT.RIGHT);
    wlPosition.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.FieldPosition.Label"));
    props.setLook(wlPosition);
    fdlPosition = new FormData();
    fdlPosition.left = new FormAttachment(wFields, 0);
    fdlPosition.top = new FormAttachment(wFieldname, margin);
    fdlPosition.right = new FormAttachment(middle, 0);
    wlPosition.setLayoutData(fdlPosition);
    wPosition = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wPosition);
    fdPosition = new FormData();
    fdPosition.left = new FormAttachment(middle, margin);
    fdPosition.top = new FormAttachment(wFieldname, margin);
    fdPosition.right = new FormAttachment(right, 0);
    wPosition.setLayoutData(fdPosition);
    wPosition.setEnabled(false);
    // Topos line
    wlLength = new Label(composite, SWT.RIGHT);
    wlLength.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.FieldLength.Label"));
    props.setLook(wlLength);
    fdlLength = new FormData();
    fdlLength.left = new FormAttachment(wFields, 0);
    fdlLength.top = new FormAttachment(wPosition, margin);
    fdlLength.right = new FormAttachment(middle, 0);
    wlLength.setLayoutData(fdlLength);
    wLength = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wLength);
    fdLength = new FormData();
    fdLength.left = new FormAttachment(middle, margin);
    fdLength.top = new FormAttachment(wPosition, margin);
    fdLength.right = new FormAttachment(right, 0);
    wLength.setLayoutData(fdLength);
    wLength.setEnabled(false);
    // Fieldtype line
    wlFieldtype = new Label(composite, SWT.RIGHT);
    wlFieldtype.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.FieldType.Label"));
    props.setLook(wlFieldtype);
    fdlFieldtype = new FormData();
    fdlFieldtype.left = new FormAttachment(wFields, 0);
    fdlFieldtype.top = new FormAttachment(wLength, margin);
    fdlFieldtype.right = new FormAttachment(middle, 0);
    wlFieldtype.setLayoutData(fdlFieldtype);
    wFieldtype = new CCombo(composite, SWT.BORDER | SWT.READ_ONLY);
    props.setLook(wFieldtype);
    for (int i = 0; i < ValueMetaFactory.getValueMetaNames().length; i++) {
        wFieldtype.add(ValueMetaFactory.getValueMetaNames()[i]);
    }
    fdFieldtype = new FormData();
    fdFieldtype.left = new FormAttachment(middle, margin);
    fdFieldtype.top = new FormAttachment(wLength, margin);
    fdFieldtype.right = new FormAttachment(right, 0);
    wFieldtype.setLayoutData(fdFieldtype);
    // Format line
    wlFormat = new Label(composite, SWT.RIGHT);
    wlFormat.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.FieldFormatter.Label"));
    props.setLook(wlFormat);
    fdlFormat = new FormData();
    fdlFormat.left = new FormAttachment(wFields, 0);
    fdlFormat.top = new FormAttachment(wFieldtype, margin);
    fdlFormat.right = new FormAttachment(middle, 0);
    wlFormat.setLayoutData(fdlFormat);
    wFormat = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wFormat);
    fdFormat = new FormData();
    fdFormat.left = new FormAttachment(middle, margin);
    fdFormat.top = new FormAttachment(wFieldtype, margin);
    fdFormat.right = new FormAttachment(right, 0);
    wFormat.setLayoutData(fdFormat);
    // Ignore checkbox
    wlIgnore = new Label(composite, SWT.RIGHT);
    wlIgnore.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Ignore.Label"));
    props.setLook(wlIgnore);
    fdlIgnore = new FormData();
    fdlIgnore.left = new FormAttachment(wFields, 0);
    fdlIgnore.top = new FormAttachment(wFormat, margin);
    fdlIgnore.right = new FormAttachment(middle, 0);
    wlIgnore.setLayoutData(fdlIgnore);
    wIgnore = new Button(composite, SWT.CHECK);
    props.setLook(wIgnore);
    fdIgnore = new FormData();
    fdIgnore.left = new FormAttachment(middle, margin);
    fdIgnore.top = new FormAttachment(wFormat, margin);
    fdIgnore.right = new FormAttachment(right, 0);
    wIgnore.setLayoutData(fdIgnore);
    // Trimtype line
    wlTrimtype = new Label(composite, SWT.RIGHT);
    wlTrimtype.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.TrimType.Label"));
    props.setLook(wlTrimtype);
    fdlTrimtype = new FormData();
    fdlTrimtype.left = new FormAttachment(wFields, 0);
    fdlTrimtype.top = new FormAttachment(wIgnore, margin);
    fdlTrimtype.right = new FormAttachment(middle, 0);
    wlTrimtype.setLayoutData(fdlTrimtype);
    wTrimtype = new CCombo(composite, SWT.BORDER | SWT.READ_ONLY);
    props.setLook(wTrimtype);
    for (int i = 0; i < ValueMetaString.trimTypeDesc.length; i++) {
        wTrimtype.add(ValueMetaString.trimTypeDesc[i]);
    }
    fdTrimtype = new FormData();
    fdTrimtype.left = new FormAttachment(middle, margin);
    fdTrimtype.top = new FormAttachment(wIgnore, margin);
    fdTrimtype.right = new FormAttachment(right, 0);
    wTrimtype.setLayoutData(fdTrimtype);
    // Precision line
    wlPrecision = new Label(composite, SWT.RIGHT);
    wlPrecision.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Precision.Label"));
    props.setLook(wlPrecision);
    fdlPrecision = new FormData();
    fdlPrecision.left = new FormAttachment(wFields, 0);
    fdlPrecision.top = new FormAttachment(wTrimtype, margin);
    fdlPrecision.right = new FormAttachment(middle, 0);
    wlPrecision.setLayoutData(fdlPrecision);
    wPrecision = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wPrecision);
    fdPrecision = new FormData();
    fdPrecision.left = new FormAttachment(middle, margin);
    fdPrecision.top = new FormAttachment(wTrimtype, margin);
    fdPrecision.right = new FormAttachment(right, 0);
    wPrecision.setLayoutData(fdPrecision);
    // Currency line
    wlCurrency = new Label(composite, SWT.RIGHT);
    wlCurrency.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Currency.Label"));
    props.setLook(wlCurrency);
    fdlCurrency = new FormData();
    fdlCurrency.left = new FormAttachment(wFields, 0);
    fdlCurrency.top = new FormAttachment(wPrecision, margin);
    fdlCurrency.right = new FormAttachment(middle, 0);
    wlCurrency.setLayoutData(fdlCurrency);
    wCurrency = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wCurrency);
    fdCurrency = new FormData();
    fdCurrency.left = new FormAttachment(middle, margin);
    fdCurrency.top = new FormAttachment(wPrecision, margin);
    fdCurrency.right = new FormAttachment(right, 0);
    wCurrency.setLayoutData(fdCurrency);
    // Decimal line
    wlDecimal = new Label(composite, SWT.RIGHT);
    wlDecimal.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Decimal.Label"));
    props.setLook(wlDecimal);
    fdlDecimal = new FormData();
    fdlDecimal.left = new FormAttachment(wFields, 0);
    fdlDecimal.top = new FormAttachment(wCurrency, margin);
    fdlDecimal.right = new FormAttachment(middle, 0);
    wlDecimal.setLayoutData(fdlDecimal);
    wDecimal = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wDecimal);
    fdDecimal = new FormData();
    fdDecimal.left = new FormAttachment(middle, margin);
    fdDecimal.top = new FormAttachment(wCurrency, margin);
    fdDecimal.right = new FormAttachment(right, 0);
    wDecimal.setLayoutData(fdDecimal);
    // Group line
    wlGroup = new Label(composite, SWT.RIGHT);
    wlGroup.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Group.Label"));
    props.setLook(wlGroup);
    fdlGroup = new FormData();
    fdlGroup.left = new FormAttachment(wFields, 0);
    fdlGroup.top = new FormAttachment(wDecimal, margin);
    fdlGroup.right = new FormAttachment(middle, 0);
    wlGroup.setLayoutData(fdlGroup);
    wGroup = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wGroup);
    fdGroup = new FormData();
    fdGroup.left = new FormAttachment(middle, margin);
    fdGroup.top = new FormAttachment(wDecimal, margin);
    fdGroup.right = new FormAttachment(right, 0);
    wGroup.setLayoutData(fdGroup);
    // Ignore checkbox
    wlRepeat = new Label(composite, SWT.RIGHT);
    wlRepeat.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Repeat.Label"));
    props.setLook(wlRepeat);
    fdlRepeat = new FormData();
    fdlRepeat.left = new FormAttachment(wFields, 0);
    fdlRepeat.top = new FormAttachment(wGroup, margin);
    fdlRepeat.right = new FormAttachment(middle, 0);
    wlRepeat.setLayoutData(fdlRepeat);
    wRepeat = new Button(composite, SWT.CHECK);
    props.setLook(wRepeat);
    fdRepeat = new FormData();
    fdRepeat.left = new FormAttachment(middle, margin);
    fdRepeat.top = new FormAttachment(wGroup, margin);
    fdRepeat.right = new FormAttachment(right, 0);
    wRepeat.setLayoutData(fdRepeat);
    // Null line
    wlNull = new Label(composite, SWT.RIGHT);
    wlNull.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Null.Label"));
    props.setLook(wlNull);
    fdlNull = new FormData();
    fdlNull.left = new FormAttachment(wFields, 0);
    fdlNull.top = new FormAttachment(wRepeat, margin);
    fdlNull.right = new FormAttachment(middle, 0);
    wlNull.setLayoutData(fdlNull);
    wNull = new Text(composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
    props.setLook(wNull);
    fdNull = new FormData();
    fdNull.left = new FormAttachment(middle, margin);
    fdNull.top = new FormAttachment(wRepeat, margin);
    fdNull.right = new FormAttachment(right, 0);
    wNull.setLayoutData(fdNull);
    // The buttons
    wPrev = new Button(composite, SWT.PUSH);
    wPrev.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Previous.Button"));
    fdPrev = new FormData();
    fdPrev.left = new FormAttachment(left, margin);
    fdPrev.bottom = new FormAttachment(100, 0);
    wPrev.setLayoutData(fdPrev);
    // Guess button
    wGuess = new Button(composite, SWT.PUSH);
    wGuess.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Guess.Button"));
    fdGuess = new FormData();
    fdGuess.left = new FormAttachment(wPrev, margin);
    fdGuess.bottom = new FormAttachment(100, 0);
    wGuess.setLayoutData(fdGuess);
    // GuessAll button
    wGuessAll = new Button(composite, SWT.PUSH);
    wGuessAll.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.GuessAll.Button"));
    fdGuessAll = new FormData();
    fdGuessAll.left = new FormAttachment(wGuess, 0);
    fdGuessAll.bottom = new FormAttachment(100, 0);
    wGuessAll.setLayoutData(fdGuessAll);
    wNext = new Button(composite, SWT.PUSH);
    wNext.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.Next.Button"));
    fdNext = new FormData();
    fdNext.left = new FormAttachment(wGuessAll, 0);
    fdNext.bottom = new FormAttachment(100, 0);
    wNext.setLayoutData(fdNext);
    // Sample list on the right...
    wSamples = new List(composite, SWT.SINGLE | SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL);
    props.setLook(wSamples, Props.WIDGET_STYLE_FIXED);
    fdSamples = new FormData();
    fdSamples.top = new FormAttachment(0, 0);
    fdSamples.left = new FormAttachment(right, 0);
    fdSamples.right = new FormAttachment(100, 0);
    fdSamples.bottom = new FormAttachment(100, 0);
    wSamples.setLayoutData(fdSamples);
    wFields.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            showInfo();
        }
    });
    if (wFields.getItemCount() > 0) {
        wFields.select(0);
        showInfo();
    }
    wFieldtype.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                int valtype = ValueMetaFactory.getIdForValueMeta(wFieldtype.getText());
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setType(valtype);
            }
        }
    });
    wFieldname.addModifyListener(new ModifyListener() {

        public void modifyText(ModifyEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setName(wFieldname.getText());
                wFields.setItem(idx, wFieldname.getText());
            }
        }
    });
    wFormat.addModifyListener(new ModifyListener() {

        public void modifyText(ModifyEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setFormat(wFormat.getText());
            }
        }
    });
    wNext.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0 && idx < wFields.getItemCount() - 1) {
                wFields.select(idx + 1);
                wFields.showSelection();
                showInfo();
            }
        }
    });
    wPrev.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx > 0) {
                wFields.select(idx - 1);
                wFields.showSelection();
                showInfo();
            }
        }
    });
    wIgnore.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.flipIgnored();
            }
        }
    });
    wGuess.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setSamples(wSamples.getItems());
                field.guess();
                showInfo();
            }
        }
    });
    wGuessAll.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
            mb.setMessage(BaseMessages.getString(PKG, "TextFileImportWizardPage2.OverwriteTypeSettings.DialogMessage"));
            mb.setText(BaseMessages.getString(PKG, "TextFileImportWizardPage2.OverwriteTypeSettings.DialogTitle"));
            int answer = mb.open();
            if (answer == SWT.YES) {
                for (int i = 0; i < fields.size(); i++) {
                    BaseFileField field = (BaseFileField) fields.get(i);
                    field.setSamples(getRowSamples(field.getPosition(), field.getLength()));
                    field.guess();
                    wFields.select(i);
                    wFields.showSelection();
                    showInfo();
                }
            }
            if (wFields.getItemCount() > 0) {
                wFields.select(0);
                wFields.showSelection();
            }
            showInfo();
        }
    });
    wRepeat.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.flipRepeated();
            }
        }
    });
    wCurrency.addModifyListener(new ModifyListener() {

        public void modifyText(ModifyEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setCurrencySymbol(wCurrency.getText());
            }
        }
    });
    wGroup.addModifyListener(new ModifyListener() {

        public void modifyText(ModifyEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setGroupSymbol(wGroup.getText());
            }
        }
    });
    wDecimal.addModifyListener(new ModifyListener() {

        public void modifyText(ModifyEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setDecimalSymbol(wDecimal.getText());
            }
        }
    });
    wNull.addModifyListener(new ModifyListener() {

        public void modifyText(ModifyEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setNullString(wNull.getText());
            }
        }
    });
    wTrimtype.addSelectionListener(new SelectionAdapter() {

        public void widgetSelected(SelectionEvent e) {
            int idx = wFields.getSelectionIndex();
            if (idx >= 0) {
                int trimType = ValueMetaString.getTrimTypeByDesc(wTrimtype.getText());
                BaseFileField field = (BaseFileField) fields.get(idx);
                field.setTrimType(trimType);
            }
        }
    });
    // set the composite as the control for this page
    setControl(composite);
}

Example 3 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileImportWizardPage2 method showInfo.

private void showInfo() {
    int idx = wFields.getSelectionIndex();
    if (idx >= 0) {
        BaseFileField field = (BaseFileField) fields.get(idx);
        String name = field.getName();
        int from = field.getPosition();
        int length = field.getLength();
        String type = field.getTypeDesc();
        boolean ignore = field.isIgnored();
        String format = field.getFormat();
        String trimtype = field.getTrimTypeDesc();
        int precision = field.getPrecision();
        String currency = field.getCurrencySymbol();
        String decimal = field.getDecimalSymbol();
        String group = field.getGroupSymbol();
        boolean repeat = field.isRepeated();
        String nullif = field.getNullString();
        if (name != null) {
            wFieldname.setText(name);
        }
        wPosition.setText("" + from);
        wLength.setText("" + length);
        if (type != null) {
            wFieldtype.setText(type);
        }
        wIgnore.setSelection(ignore);
        if (format != null) {
            wFormat.setText(format);
        }
        if (trimtype != null) {
            wTrimtype.setText(trimtype);
        }
        wPrecision.setText("" + precision);
        if (currency != null) {
            wCurrency.setText(currency);
        }
        if (decimal != null) {
            wDecimal.setText(decimal);
        }
        if (group != null) {
            wGroup.setText(group);
        }
        wRepeat.setSelection(repeat);
        if (nullif != null) {
            wNull.setText(nullif);
        }
        // Clear the sample list...
        wSamples.removeAll();
        String[] samples = getRowSamples(from, length);
        for (int i = 0; i < samples.length; i++) {
            wSamples.add(samples[i]);
        }
    }
}

Example 4 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileInputDialog method getInfo.

/**
 * Fill meta object from UI options.
 *
 * @param meta
 *          meta object
 * @param preview
 *          flag for preview or real options should be used. Currently, only one option is differ for preview - EOL
 *          chars. It uses as "mixed" for be able to preview any file.
 */
private void getInfo(TextFileInputMeta meta, boolean preview) {
    // return value
    stepname = wStepname.getText();
    // copy info to TextFileInputMeta class (input)
    meta.inputFiles.acceptingFilenames = wAccFilenames.getSelection();
    meta.inputFiles.passingThruFields = wPassThruFields.getSelection();
    meta.inputFiles.acceptingField = wAccField.getText();
    meta.inputFiles.acceptingStepName = wAccStep.getText();
    meta.setAcceptingStep(transMeta.findStep(wAccStep.getText()));
    meta.content.fileType = wFiletype.getText();
    if (preview) {
        // mixed type for preview, for be able to eat any EOL chars
        meta.content.fileFormat = "mixed";
    } else {
        meta.content.fileFormat = wFormat.getText();
    }
    meta.content.separator = wSeparator.getText();
    meta.content.enclosure = wEnclosure.getText();
    meta.content.escapeCharacter = wEscape.getText();
    meta.content.rowLimit = Const.toLong(wLimit.getText(), 0L);
    meta.content.filenameField = wInclFilenameField.getText();
    meta.content.rowNumberField = wInclRownumField.getText();
    meta.inputFiles.isaddresult = wAddResult.getSelection();
    meta.content.includeFilename = wInclFilename.getSelection();
    meta.content.includeRowNumber = wInclRownum.getSelection();
    meta.content.rowNumberByFile = wRownumByFile.getSelection();
    meta.content.header = wHeader.getSelection();
    meta.content.nrHeaderLines = Const.toInt(wNrHeader.getText(), 1);
    meta.content.footer = wFooter.getSelection();
    meta.content.nrFooterLines = Const.toInt(wNrFooter.getText(), 1);
    meta.content.lineWrapped = wWraps.getSelection();
    meta.content.nrWraps = Const.toInt(wNrWraps.getText(), 1);
    meta.content.layoutPaged = wLayoutPaged.getSelection();
    meta.content.nrLinesPerPage = Const.toInt(wNrLinesPerPage.getText(), 80);
    meta.content.nrLinesDocHeader = Const.toInt(wNrLinesDocHeader.getText(), 0);
    meta.content.fileCompression = wCompression.getText();
    meta.content.dateFormatLenient = wDateLenient.getSelection();
    meta.content.noEmptyLines = wNoempty.getSelection();
    meta.content.encoding = wEncoding.getText();
    meta.content.length = wLength.getText();
    int nrfiles = wFilenameList.getItemCount();
    int nrfields = wFields.nrNonEmpty();
    int nrfilters = wFilter.nrNonEmpty();
    meta.allocate(nrfiles, nrfields, nrfilters);
    meta.setFileName(wFilenameList.getItems(0));
    meta.inputFiles.fileMask = wFilenameList.getItems(1);
    meta.inputFiles.excludeFileMask = wFilenameList.getItems(2);
    meta.inputFiles_fileRequired(wFilenameList.getItems(3));
    meta.inputFiles_includeSubFolders(wFilenameList.getItems(4));
    for (int i = 0; i < nrfields; i++) {
        BaseFileField field = new BaseFileField();
        TableItem item = wFields.getNonEmpty(i);
        field.setName(item.getText(1));
        field.setType(ValueMetaFactory.getIdForValueMeta(item.getText(2)));
        field.setFormat(item.getText(3));
        field.setPosition(Const.toInt(item.getText(4), -1));
        field.setLength(Const.toInt(item.getText(5), -1));
        field.setPrecision(Const.toInt(item.getText(6), -1));
        field.setCurrencySymbol(item.getText(7));
        field.setDecimalSymbol(item.getText(8));
        field.setGroupSymbol(item.getText(9));
        field.setNullString(item.getText(10));
        field.setIfNullValue(item.getText(11));
        field.setTrimType(ValueMetaString.getTrimTypeByDesc(item.getText(12)));
        field.setRepeated(BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(13)));
        // CHECKSTYLE:Indentation:OFF
        meta.inputFields[i] = field;
    }
    for (int i = 0; i < nrfilters; i++) {
        TableItem item = wFilter.getNonEmpty(i);
        TextFileFilter filter = new TextFileFilter();
        // CHECKSTYLE:Indentation:OFF
        meta.getFilter()[i] = filter;
        filter.setFilterString(item.getText(1));
        filter.setFilterPosition(Const.toInt(item.getText(2), -1));
        filter.setFilterLastLine(BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(3)));
        filter.setFilterPositive(BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(item.getText(4)));
    }
    // Error handling fields...
    meta.errorHandling.errorIgnored = wErrorIgnored.getSelection();
    meta.errorHandling.skipBadFiles = wSkipBadFiles.getSelection();
    meta.errorHandling.fileErrorField = wBadFileField.getText();
    meta.errorHandling.fileErrorMessageField = wBadFileMessageField.getText();
    meta.setErrorLineSkipped(wSkipErrorLines.getSelection());
    meta.setErrorCountField(wErrorCount.getText());
    meta.setErrorFieldsField(wErrorFields.getText());
    meta.setErrorTextField(wErrorText.getText());
    meta.errorHandling.warningFilesDestinationDirectory = wWarnDestDir.getText();
    meta.errorHandling.warningFilesExtension = wWarnExt.getText();
    meta.errorHandling.errorFilesDestinationDirectory = wErrorDestDir.getText();
    meta.errorHandling.errorFilesExtension = wErrorExt.getText();
    meta.errorHandling.lineNumberFilesDestinationDirectory = wLineNrDestDir.getText();
    meta.errorHandling.lineNumberFilesExtension = wLineNrExt.getText();
    // Date format Locale
    Locale locale = EnvUtil.createLocale(wDateLocale.getText());
    if (!locale.equals(Locale.getDefault())) {
        meta.content.dateFormatLocale = locale;
    } else {
        meta.content.dateFormatLocale = Locale.getDefault();
    }
    meta.additionalOutputFields.shortFilenameField = wShortFileFieldName.getText();
    meta.additionalOutputFields.pathField = wPathFieldName.getText();
    meta.additionalOutputFields.hiddenField = wIsHiddenName.getText();
    meta.additionalOutputFields.lastModificationField = wLastModificationTimeName.getText();
    meta.additionalOutputFields.uriField = wUriName.getText();
    meta.additionalOutputFields.rootUriField = wRootUriName.getText();
    meta.additionalOutputFields.extensionField = wExtensionFieldName.getText();
    meta.additionalOutputFields.sizeField = wSizeFieldName.getText();
}

Example 5 with BaseFileField

use of org.pentaho.di.trans.steps.file.BaseFileField in project pentaho-kettle by pentaho.

the class TextFileInputMeta method setDefault.

@Override
public void setDefault() {
    additionalOutputFields.shortFilenameField = null;
    additionalOutputFields.pathField = null;
    additionalOutputFields.hiddenField = null;
    additionalOutputFields.lastModificationField = null;
    additionalOutputFields.uriField = null;
    additionalOutputFields.rootUriField = null;
    additionalOutputFields.extensionField = null;
    additionalOutputFields.sizeField = null;
    inputFiles.isaddresult = true;
    content.separator = ";";
    content.enclosure = "\"";
    content.breakInEnclosureAllowed = false;
    content.header = true;
    content.nrHeaderLines = 1;
    content.footer = false;
    content.nrFooterLines = 1;
    content.lineWrapped = false;
    content.nrWraps = 1;
    content.layoutPaged = false;
    content.nrLinesPerPage = 80;
    content.nrLinesDocHeader = 0;
    content.fileCompression = "None";
    content.noEmptyLines = true;
    content.fileFormat = "DOS";
    content.fileType = "CSV";
    content.includeFilename = false;
    content.filenameField = "";
    content.includeRowNumber = false;
    content.rowNumberField = "";
    content.dateFormatLenient = true;
    content.rowNumberByFile = false;
    errorHandling.errorIgnored = false;
    errorHandling.skipBadFiles = false;
    errorLineSkipped = false;
    errorHandling.warningFilesDestinationDirectory = null;
    errorHandling.warningFilesExtension = "warning";
    errorHandling.errorFilesDestinationDirectory = null;
    errorHandling.errorFilesExtension = "error";
    errorHandling.lineNumberFilesDestinationDirectory = null;
    errorHandling.lineNumberFilesExtension = "line";
    int nrfiles = 0;
    int nrfields = 0;
    int nrfilters = 0;
    allocate(nrfiles, nrfields, nrfilters);
    for (int i = 0; i < nrfiles; i++) {
        inputFiles.fileName[i] = "filename" + (i + 1);
        inputFiles.fileMask[i] = "";
        inputFiles.excludeFileMask[i] = "";
        inputFiles.fileRequired[i] = NO;
        inputFiles.includeSubFolders[i] = NO;
    }
    for (int i = 0; i < nrfields; i++) {
        inputFields[i] = new BaseFileField("field" + (i + 1), 1, -1);
    }
    content.dateFormatLocale = Locale.getDefault();
    content.rowLimit = 0L;
}