Examples with TextFileInputField org.pentaho.di.trans.steps.textfileinput.TextFileInputField used on opensource projects

Example 21 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class ParGzipCsvInputMeta method getFields.

@Override
public void getFields(RowMetaInterface rowMeta, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    try {
        // Start with a clean slate, eats the input
        rowMeta.clear();
        for (int i = 0; i < inputFields.length; i++) {
            TextFileInputField field = inputFields[i];
            ValueMetaInterface valueMeta = ValueMetaFactory.createValueMeta(field.getName(), field.getType());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setLength(field.getLength());
            valueMeta.setPrecision(field.getPrecision());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setDecimalSymbol(field.getDecimalSymbol());
            valueMeta.setGroupingSymbol(field.getGroupSymbol());
            valueMeta.setCurrencySymbol(field.getCurrencySymbol());
            valueMeta.setTrimType(field.getTrimType());
            if (lazyConversionActive) {
                valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
            }
            valueMeta.setStringEncoding(space.environmentSubstitute(encoding));
            // In case we want to convert Strings...
            // Using a copy of the valueMeta object means that the inner and outer representation format is the same.
            // Preview will show the data the same way as we read it.
            // This layout is then taken further down the road by the metadata through the transformation.
            // 
            ValueMetaInterface storageMetadata = ValueMetaFactory.cloneValueMeta(valueMeta, ValueMetaInterface.TYPE_STRING);
            storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
            // we don't really know the lengths of the strings read in advance.
            storageMetadata.setLength(-1, -1);
            valueMeta.setStorageMetadata(storageMetadata);
            valueMeta.setOrigin(origin);
            rowMeta.addValueMeta(valueMeta);
        }
        if (!Utils.isEmpty(filenameField) && includingFilename) {
            ValueMetaInterface filenameMeta = new ValueMetaString(filenameField);
            filenameMeta.setOrigin(origin);
            if (lazyConversionActive) {
                filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
                filenameMeta.setStorageMetadata(new ValueMetaString(filenameField));
            }
            rowMeta.addValueMeta(filenameMeta);
        }
        if (!Utils.isEmpty(rowNumField)) {
            ValueMetaInterface rowNumMeta = new ValueMetaInteger(rowNumField);
            rowNumMeta.setLength(10);
            rowNumMeta.setOrigin(origin);
            rowMeta.addValueMeta(rowNumMeta);
        }
    } catch (Exception e) {
        throw new KettleStepException(e);
    }
}

Example 22 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project data-access by pentaho.

the class CsvTransformGenerator method createInputStep.

protected StepMeta createInputStep(TransMeta transMeta) {
    CsvInputMeta csvInputMeta = new CsvInputMeta();
    CsvFileInfo fileInfo = getModelInfo().getFileInfo();
    String fileName = fileInfo.getTmpFilename();
    String path;
    if (fileName.endsWith(".tmp")) {
        // $NON-NLS-1$
        path = PentahoSystem.getApplicationContext().getSolutionPath(TMP_FILE_PATH);
    } else {
        String relativePath = PentahoSystem.getSystemSetting("file-upload-defaults/relative-path", // $NON-NLS-1$
        String.valueOf(DEFAULT_RELATIVE_UPLOAD_FILE_PATH));
        path = PentahoSystem.getApplicationContext().getSolutionPath(relativePath);
    }
    File file = new File(path + fileInfo.getTmpFilename());
    String filename = file.getAbsolutePath();
    ColumnInfo[] columns = getModelInfo().getColumns();
    TextFileInputField[] inputFields = new TextFileInputField[columns.length];
    int idx = 0;
    for (ColumnInfo column : columns) {
        TextFileInputField field = new TextFileInputField();
        field.setCurrencySymbol(fileInfo.getCurrencySymbol());
        field.setDecimalSymbol(fileInfo.getCurrencySymbol());
        field.setFormat(column.getFormat());
        field.setGroupSymbol(fileInfo.getGroupSymbol());
        field.setIfNullValue(fileInfo.getIfNull());
        field.setIgnored(column.isIgnore());
        field.setLength(column.getLength());
        field.setName(column.getId());
        field.setNullString(fileInfo.getNullStr());
        // field.setPosition(position);
        field.setPrecision(column.getPrecision());
        field.setRepeated(false);
        field.setSamples(null);
        field.setTrimType(ValueMeta.TRIM_TYPE_BOTH);
        field.setType(convertDataType(column));
        inputFields[idx] = field;
        idx++;
    }
    csvInputMeta.setAddResultFile(false);
    // $NON-NLS-1$
    csvInputMeta.setBufferSize("5000");
    csvInputMeta.setDelimiter(fileInfo.getDelimiter());
    csvInputMeta.setEnclosure(fileInfo.getEnclosure());
    csvInputMeta.setEncoding(fileInfo.getEncoding());
    csvInputMeta.setFilename(filename);
    csvInputMeta.setFilenameField(null);
    // TODO strip off more than one row if present...
    csvInputMeta.setHeaderPresent(fileInfo.getHeaderRows() > 0);
    // inputMeta.get.setID(1);
    csvInputMeta.setIncludingFilename(false);
    csvInputMeta.setInputFields(inputFields);
    csvInputMeta.setLazyConversionActive(true);
    // $NON-NLS-1$
    csvInputMeta.setRowNumField("");
    csvInputMeta.setRunningInParallel(false);
    // inputMeta.setTargetSteps(null);
    StepMeta csvInputStepMeta = new StepMeta(CSV_INPUT, CSV_INPUT, csvInputMeta);
    csvInputStepMeta.setStepErrorMeta(new StepErrorMeta(transMeta, csvInputStepMeta));
    transMeta.addStep(csvInputStepMeta);
    csvErrorRowCount = 0;
    final FileTransformStats stats = getTransformStats();
    StepErrorMeta csvInputErrorMeta = new StepErrorMeta(transMeta, csvInputStepMeta) {

        public void addErrorRowData(Object[] row, int startIndex, long nrErrors, String errorDescriptions, String fieldNames, String errorCodes) {
            if (csvErrorRowCount < maxErrorRows) {
                StringBuffer sb = new StringBuffer();
                sb.append("Rejected Row: ");
                for (Object rowData : row) {
                    sb.append(rowData);
                    sb.append(", ");
                }
                sb.append("\r\n");
                stats.getErrors().add(sb.toString() + errorDescriptions);
            }
            csvErrorRowCount++;
            stats.setErrorCount(csvErrorRowCount);
            super.addErrorRowData(row, startIndex, nrErrors, errorDescriptions, fieldNames, errorCodes);
        }
    };
    StepMeta outputDummyStepMeta = addDummyStep(transMeta, "CSVInputErrorDummy");
    csvInputErrorMeta.setTargetStep(outputDummyStepMeta);
    csvInputErrorMeta.setEnabled(true);
    csvInputStepMeta.setStepErrorMeta(csvInputErrorMeta);
    return csvInputStepMeta;
}

Example 23 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class ParGzipCsvInputMeta method saveRep.

@Override
public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step) throws KettleException {
    try {
        rep.saveStepAttribute(id_transformation, id_step, "filename", filename);
        rep.saveStepAttribute(id_transformation, id_step, "filename_field", filenameField);
        rep.saveStepAttribute(id_transformation, id_step, "rownum_field", rowNumField);
        rep.saveStepAttribute(id_transformation, id_step, "include_filename", includingFilename);
        rep.saveStepAttribute(id_transformation, id_step, "separator", delimiter);
        rep.saveStepAttribute(id_transformation, id_step, "enclosure", enclosure);
        rep.saveStepAttribute(id_transformation, id_step, "buffer_size", bufferSize);
        rep.saveStepAttribute(id_transformation, id_step, "header", headerPresent);
        rep.saveStepAttribute(id_transformation, id_step, "lazy_conversion", lazyConversionActive);
        rep.saveStepAttribute(id_transformation, id_step, "add_filename_result", isaddresult);
        rep.saveStepAttribute(id_transformation, id_step, "parallel", runningInParallel);
        rep.saveStepAttribute(id_transformation, id_step, "encoding", encoding);
        for (int i = 0; i < inputFields.length; i++) {
            TextFileInputField field = inputFields[i];
            rep.saveStepAttribute(id_transformation, id_step, i, "field_name", field.getName());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_type", ValueMetaFactory.getValueMetaName(field.getType()));
            rep.saveStepAttribute(id_transformation, id_step, i, "field_format", field.getFormat());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_currency", field.getCurrencySymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_decimal", field.getDecimalSymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_group", field.getGroupSymbol());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_length", field.getLength());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_precision", field.getPrecision());
            rep.saveStepAttribute(id_transformation, id_step, i, "field_trim_type", ValueMetaString.getTrimTypeCode(field.getTrimType()));
        }
    } catch (Exception e) {
        throw new KettleException("Unable to save step information to the repository for id_step=" + id_step, e);
    }
}

Example 24 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class ParGzipCsvInputMeta method readRep.

@Override
public void readRep(Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases) throws KettleException {
    try {
        filename = rep.getStepAttributeString(id_step, "filename");
        filenameField = rep.getStepAttributeString(id_step, "filename_field");
        rowNumField = rep.getStepAttributeString(id_step, "rownum_field");
        includingFilename = rep.getStepAttributeBoolean(id_step, "include_filename");
        delimiter = rep.getStepAttributeString(id_step, "separator");
        enclosure = rep.getStepAttributeString(id_step, "enclosure");
        headerPresent = rep.getStepAttributeBoolean(id_step, "header");
        bufferSize = rep.getStepAttributeString(id_step, "buffer_size");
        lazyConversionActive = rep.getStepAttributeBoolean(id_step, "lazy_conversion");
        isaddresult = rep.getStepAttributeBoolean(id_step, "add_filename_result");
        runningInParallel = rep.getStepAttributeBoolean(id_step, "parallel");
        encoding = rep.getStepAttributeString(id_step, "encoding");
        int nrfields = rep.countNrStepAttributes(id_step, "field_name");
        allocate(nrfields);
        for (int i = 0; i < nrfields; i++) {
            inputFields[i] = new TextFileInputField();
            inputFields[i].setName(rep.getStepAttributeString(id_step, i, "field_name"));
            inputFields[i].setType(ValueMetaFactory.getIdForValueMeta(rep.getStepAttributeString(id_step, i, "field_type")));
            inputFields[i].setFormat(rep.getStepAttributeString(id_step, i, "field_format"));
            inputFields[i].setCurrencySymbol(rep.getStepAttributeString(id_step, i, "field_currency"));
            inputFields[i].setDecimalSymbol(rep.getStepAttributeString(id_step, i, "field_decimal"));
            inputFields[i].setGroupSymbol(rep.getStepAttributeString(id_step, i, "field_group"));
            inputFields[i].setLength((int) rep.getStepAttributeInteger(id_step, i, "field_length"));
            inputFields[i].setPrecision((int) rep.getStepAttributeInteger(id_step, i, "field_precision"));
            inputFields[i].setTrimType(ValueMetaString.getTrimTypeByCode(rep.getStepAttributeString(id_step, i, "field_trim_type")));
        }
    } catch (Exception e) {
        throw new KettleException("Unexpected error reading step information from the repository", e);
    }
}

Example 25 with TextFileInputField

use of org.pentaho.di.trans.steps.textfileinput.TextFileInputField in project pentaho-kettle by pentaho.

the class ParGzipCsvInputMeta method getXML.

@Override
public String getXML() {
    StringBuilder retval = new StringBuilder(500);
    retval.append("    ").append(XMLHandler.addTagValue("filename", filename));
    retval.append("    ").append(XMLHandler.addTagValue("filename_field", filenameField));
    retval.append("    ").append(XMLHandler.addTagValue("rownum_field", rowNumField));
    retval.append("    ").append(XMLHandler.addTagValue("include_filename", includingFilename));
    retval.append("    ").append(XMLHandler.addTagValue("separator", delimiter));
    retval.append("    ").append(XMLHandler.addTagValue("enclosure", enclosure));
    retval.append("    ").append(XMLHandler.addTagValue("header", headerPresent));
    retval.append("    ").append(XMLHandler.addTagValue("buffer_size", bufferSize));
    retval.append("    ").append(XMLHandler.addTagValue("lazy_conversion", lazyConversionActive));
    retval.append("    ").append(XMLHandler.addTagValue("add_filename_result", isaddresult));
    retval.append("    ").append(XMLHandler.addTagValue("parallel", runningInParallel));
    retval.append("    ").append(XMLHandler.addTagValue("encoding", encoding));
    retval.append("    <fields>").append(Const.CR);
    for (int i = 0; i < inputFields.length; i++) {
        TextFileInputField field = inputFields[i];
        retval.append("      <field>").append(Const.CR);
        retval.append("        ").append(XMLHandler.addTagValue("name", field.getName()));
        retval.append("        ").append(XMLHandler.addTagValue("type", ValueMetaFactory.getValueMetaName(field.getType())));
        retval.append("        ").append(XMLHandler.addTagValue("format", field.getFormat()));
        retval.append("        ").append(XMLHandler.addTagValue("currency", field.getCurrencySymbol()));
        retval.append("        ").append(XMLHandler.addTagValue("decimal", field.getDecimalSymbol()));
        retval.append("        ").append(XMLHandler.addTagValue("group", field.getGroupSymbol()));
        retval.append("        ").append(XMLHandler.addTagValue("length", field.getLength()));
        retval.append("        ").append(XMLHandler.addTagValue("precision", field.getPrecision()));
        retval.append("        ").append(XMLHandler.addTagValue("trim_type", ValueMetaString.getTrimTypeCode(field.getTrimType())));
        retval.append("      </field>").append(Const.CR);
    }
    retval.append("    </fields>").append(Const.CR);
    return retval.toString();
}