Examples with TextFileInputField org.pentaho.di.trans.steps.textfileinput.TextFileInputField used on opensource projects

Example 1 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project data-access by pentaho.

the class CsvTransformGenerator method createInputStep.

protected StepMeta createInputStep(TransMeta transMeta) {
    CsvInputMeta csvInputMeta = new CsvInputMeta();
    CsvFileInfo fileInfo = getModelInfo().getFileInfo();
    String fileName = fileInfo.getTmpFilename();
    String path;
    if (fileName.endsWith(".tmp")) {
        // $NON-NLS-1$
        path = PentahoSystem.getApplicationContext().getSolutionPath(TMP_FILE_PATH);
    } else {
        String relativePath = PentahoSystem.getSystemSetting("file-upload-defaults/relative-path", // $NON-NLS-1$
        String.valueOf(DEFAULT_RELATIVE_UPLOAD_FILE_PATH));
        path = PentahoSystem.getApplicationContext().getSolutionPath(relativePath);
    }
    File file = new File(path + fileInfo.getTmpFilename());
    String filename = file.getAbsolutePath();
    ColumnInfo[] columns = getModelInfo().getColumns();
    TextFileInputField[] inputFields = new TextFileInputField[columns.length];
    int idx = 0;
    for (ColumnInfo column : columns) {
        TextFileInputField field = new TextFileInputField();
        field.setCurrencySymbol(fileInfo.getCurrencySymbol());
        field.setDecimalSymbol(fileInfo.getCurrencySymbol());
        field.setFormat(column.getFormat());
        field.setGroupSymbol(fileInfo.getGroupSymbol());
        field.setIfNullValue(fileInfo.getIfNull());
        field.setIgnored(column.isIgnore());
        field.setLength(column.getLength());
        field.setName(column.getId());
        field.setNullString(fileInfo.getNullStr());
        // field.setPosition(position);
        field.setPrecision(column.getPrecision());
        field.setRepeated(false);
        field.setSamples(null);
        field.setTrimType(ValueMeta.TRIM_TYPE_BOTH);
        field.setType(convertDataType(column));
        inputFields[idx] = field;
        idx++;
    }
    csvInputMeta.setAddResultFile(false);
    // $NON-NLS-1$
    csvInputMeta.setBufferSize("5000");
    csvInputMeta.setDelimiter(fileInfo.getDelimiter());
    csvInputMeta.setEnclosure(fileInfo.getEnclosure());
    csvInputMeta.setEncoding(fileInfo.getEncoding());
    csvInputMeta.setFilename(filename);
    csvInputMeta.setFilenameField(null);
    // TODO strip off more than one row if present...
    csvInputMeta.setHeaderPresent(fileInfo.getHeaderRows() > 0);
    // inputMeta.get.setID(1);
    csvInputMeta.setIncludingFilename(false);
    csvInputMeta.setInputFields(inputFields);
    csvInputMeta.setLazyConversionActive(true);
    // $NON-NLS-1$
    csvInputMeta.setRowNumField("");
    csvInputMeta.setRunningInParallel(false);
    // inputMeta.setTargetSteps(null);
    StepMeta csvInputStepMeta = new StepMeta(CSV_INPUT, CSV_INPUT, csvInputMeta);
    csvInputStepMeta.setStepErrorMeta(new StepErrorMeta(transMeta, csvInputStepMeta));
    transMeta.addStep(csvInputStepMeta);
    csvErrorRowCount = 0;
    final FileTransformStats stats = getTransformStats();
    StepErrorMeta csvInputErrorMeta = new StepErrorMeta(transMeta, csvInputStepMeta) {

        public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
            if (csvErrorRowCount < maxErrorRows) {
                StringBuffer sb = new StringBuffer();
                sb.append("Rejected Row: ");
                for (Object rowData : row) {
                    sb.append(rowData);
                    sb.append(", ");
                }
                sb.append("\r\n");
                stats.getErrors().add(sb.toString() + errorDescriptions);
            }
            csvErrorRowCount++;
            stats.setErrorCount(csvErrorRowCount);
            super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
        }
    };
    StepMeta outputDummyStepMeta = addDummyStep(transMeta, "CSVInputErrorDummy");
    csvInputErrorMeta.setTargetStep(outputDummyStepMeta);
    csvInputErrorMeta.setEnabled(true);
    csvInputStepMeta.setStepErrorMeta(csvInputErrorMeta);
    return csvInputStepMeta;
}

Example 2 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class ParGzipCsvInputMeta method getFields.

@Override
public void getFields(RowMetaInterface rowMeta, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    try {
        // Start with a clean slate, eats the input
        rowMeta.clear();
        for (int i = 0; i < inputFields.length; i++) {
            TextFileInputField field = inputFields[i];
            ValueMetaInterface valueMeta = ValueMetaFactory.createValueMeta(field.getName(), field.getType());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setLength(field.getLength());
            valueMeta.setPrecision(field.getPrecision());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setDecimalSymbol(field.getDecimalSymbol());
            valueMeta.setGroupingSymbol(field.getGroupSymbol());
            valueMeta.setCurrencySymbol(field.getCurrencySymbol());
            valueMeta.setTrimType(field.getTrimType());
            if (lazyConversionActive) {
                valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
            }
            valueMeta.setStringEncoding(space.environmentSubstitute(encoding));
            // In case we want to convert Strings...
            // Using a copy of the valueMeta object means that the inner and outer representation format is the same.
            // Preview will show the data the same way as we read it.
            // This layout is then taken further down the road by the metadata through the transformation.
            // 
            ValueMetaInterface storageMetadata = ValueMetaFactory.cloneValueMeta(valueMeta, ValueMetaInterface.TYPE_STRING);
            storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            // we don't really know the lengths of the strings read in advance.
            storageMetadata.setLength(-1, -1);
            valueMeta.setStorageMetadata(storageMetadata);
            valueMeta.setOrigin(origin);
            rowMeta.addValueMeta(valueMeta);
        }
        if (!Utils.isEmpty(filenameField) && includingFilename) {
            ValueMetaInterface filenameMeta = new ValueMetaString(filenameField);
            filenameMeta.setOrigin(origin);
            if (lazyConversionActive) {
                filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
                filenameMeta.setStorageMetadata(new ValueMetaString(filenameField));
            }
            rowMeta.addValueMeta(filenameMeta);
        }
        if (!Utils.isEmpty(rowNumField)) {
            ValueMetaInterface rowNumMeta = new ValueMetaInteger(rowNumField);
            rowNumMeta.setLength(10);
            rowNumMeta.setOrigin(origin);
            rowMeta.addValueMeta(rowNumMeta);
        }
    } catch (Exception e) {
        throw new KettleStepException(e);
    }
}

Example 3 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class ParGzipCsvInputMeta method readData.

private void readData(Node stepnode) throws KettleXMLException {
    try {
        filename = XMLHandler.getTagValue(stepnode, "filename");
        filenameField = XMLHandler.getTagValue(stepnode, "filename_field");
        rowNumField = XMLHandler.getTagValue(stepnode, "rownum_field");
        includingFilename = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "include_filename"));
        delimiter = XMLHandler.getTagValue(stepnode, "separator");
        enclosure = XMLHandler.getTagValue(stepnode, "enclosure");
        bufferSize = XMLHandler.getTagValue(stepnode, "buffer_size");
        headerPresent = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "header"));
        lazyConversionActive = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "lazy_conversion"));
        isaddresult = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "add_filename_result"));
        runningInParallel = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "parallel"));
        encoding = XMLHandler.getTagValue(stepnode, "encoding");
        Node fields = XMLHandler.getSubNode(stepnode, "fields");
        int nrfields = XMLHandler.countNodes(fields, "field");
        allocate(nrfields);
        for (int i = 0; i < nrfields; i++) {
            inputFields[i] = new TextFileInputField();
            Node fnode = XMLHandler.getSubNodeByNr(fields, "field", i);
            inputFields[i].setName(XMLHandler.getTagValue(fnode, "name"));
            inputFields[i].setType(ValueMetaFactory.getIdForValueMeta(XMLHandler.getTagValue(fnode, "type")));
            inputFields[i].setFormat(XMLHandler.getTagValue(fnode, "format"));
            inputFields[i].setCurrencySymbol(XMLHandler.getTagValue(fnode, "currency"));
            inputFields[i].setDecimalSymbol(XMLHandler.getTagValue(fnode, "decimal"));
            inputFields[i].setGroupSymbol(XMLHandler.getTagValue(fnode, "group"));
            inputFields[i].setLength(Const.toInt(XMLHandler.getTagValue(fnode, "length"), -1));
            inputFields[i].setPrecision(Const.toInt(XMLHandler.getTagValue(fnode, "precision"), -1));
            inputFields[i].setTrimType(ValueMetaString.getTrimTypeByCode(XMLHandler.getTagValue(fnode, "trim_type")));
        }
    } catch (Exception e) {
        throw new KettleXMLException("Unable to load step info from XML", e);
    }
}

Example 4 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class CsvInputMeta method getXML.

@Override
public String getXML() {
    StringBuilder retval = new StringBuilder(500);
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("FILENAME"), filename));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("FILENAME_FIELD"), filenameField));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("ROW_NUM_FIELD"), rowNumField));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("INCLUDE_FILENAME"), includingFilename));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("DELIMITER"), delimiter));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("ENCLOSURE"), enclosure));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("HEADER_PRESENT"), headerPresent));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("BUFFERSIZE"), bufferSize));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("LAZY_CONVERSION"), lazyConversionActive));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("ADD_FILENAME_RESULT"), isaddresult));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("PARALLEL"), runningInParallel));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("NEWLINE_POSSIBLE"), newlinePossibleInFields));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("FORMAT"), fileFormat));
    retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("ENCODING"), encoding));
    retval.append("    ").append(XMLHandler.openTag(getXmlCode("FIELDS"))).append(Const.CR);
    for (int i = 0; i < inputFields.length; i++) {
        TextFileInputField field = inputFields[i];
        retval.append("      ").append(XMLHandler.openTag(getXmlCode("FIELD"))).append(Const.CR);
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_NAME"), field.getName()));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_TYPE"), ValueMetaFactory.getValueMetaName(field.getType())));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_FORMAT"), field.getFormat()));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_CURRENCY"), field.getCurrencySymbol()));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_DECIMAL"), field.getDecimalSymbol()));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_GROUP"), field.getGroupSymbol()));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_LENGTH"), field.getLength()));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_PRECISION"), field.getPrecision()));
        retval.append("        ").append(XMLHandler.addTagValue(getXmlCode("FIELD_TRIM_TYPE"), ValueMetaString.getTrimTypeCode(field.getTrimType())));
        retval.append("      ").append(XMLHandler.closeTag(getXmlCode("FIELD"))).append(Const.CR);
    }
    retval.append("    ").append(XMLHandler.closeTag(getXmlCode("FIELDS"))).append(Const.CR);
    return retval.toString();
}

Example 5 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class CsvInputMeta method getFields.

@Override
public void getFields(RowMetaInterface rowMeta, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    try {
        // Start with a clean slate, eats the input
        rowMeta.clear();
        for (int i = 0; i < inputFields.length; i++) {
            TextFileInputField field = inputFields[i];
            ValueMetaInterface valueMeta = ValueMetaFactory.createValueMeta(field.getName(), field.getType());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setLength(field.getLength());
            valueMeta.setPrecision(field.getPrecision());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setDecimalSymbol(field.getDecimalSymbol());
            valueMeta.setGroupingSymbol(field.getGroupSymbol());
            valueMeta.setCurrencySymbol(field.getCurrencySymbol());
            valueMeta.setTrimType(field.getTrimType());
            if (lazyConversionActive) {
                valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
            }
            valueMeta.setStringEncoding(space.environmentSubstitute(encoding));
            // In case we want to convert Strings...
            // Using a copy of the valueMeta object means that the inner and outer representation format is the same.
            // Preview will show the data the same way as we read it.
            // This layout is then taken further down the road by the metadata through the transformation.
            // 
            ValueMetaInterface storageMetadata = ValueMetaFactory.cloneValueMeta(valueMeta, ValueMetaInterface.TYPE_STRING);
            storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            // we don't really know the lengths of the strings read in advance.
            storageMetadata.setLength(-1, -1);
            valueMeta.setStorageMetadata(storageMetadata);
            valueMeta.setOrigin(origin);
            rowMeta.addValueMeta(valueMeta);
        }
        if (!Utils.isEmpty(filenameField) && includingFilename) {
            ValueMetaInterface filenameMeta = new ValueMetaString(filenameField);
            filenameMeta.setOrigin(origin);
            if (lazyConversionActive) {
                filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
                filenameMeta.setStorageMetadata(new ValueMetaString(filenameField));
            }
            rowMeta.addValueMeta(filenameMeta);
        }
        if (!Utils.isEmpty(rowNumField)) {
            ValueMetaInterface rowNumMeta = new ValueMetaInteger(rowNumField);
            rowNumMeta.setLength(10);
            rowNumMeta.setOrigin(origin);
            rowMeta.addValueMeta(rowNumMeta);
        }
    } catch (Exception e) {
        throw new KettleStepException(e);
    }
}