Examples with TextFileInputField org.apache.hop.core.file.TextFileInputField used on opensource projects

Example 1 with TextFileInputField

use of org.apache.hop.core.file.TextFileInputField in project hop by apache.

the class TextFileInputDialog method getFields.

private Vector<ITextFileInputField> getFields(TextFileInputMeta info, List<String> rows) {
    Vector<ITextFileInputField> fields = new Vector<>();
    int maxsize = 0;
    for (String row : rows) {
        int len = row.length();
        if (len > maxsize) {
            maxsize = len;
        }
    }
    int prevEnd = 0;
    int dummynr = 1;
    for (int i = 0; i < info.getInputFields().length; i++) {
        TextFileInputField f = info.getInputFields()[i];
        // See if positions are skipped, if this is the case, add dummy fields...
        if (f.getPosition() != prevEnd) {
            // gap
            TextFileInputField field = new TextFileInputField("Dummy" + dummynr, prevEnd, f.getPosition() - prevEnd);
            // don't include in result by default.
            field.setIgnored(true);
            fields.add(field);
            dummynr++;
        }
        TextFileInputField field = new TextFileInputField(f.getName(), f.getPosition(), f.getLength());
        field.setType(f.getType());
        field.setIgnored(false);
        field.setFormat(f.getFormat());
        field.setPrecision(f.getPrecision());
        field.setTrimType(f.getTrimType());
        field.setDecimalSymbol(f.getDecimalSymbol());
        field.setGroupSymbol(f.getGroupSymbol());
        field.setCurrencySymbol(f.getCurrencySymbol());
        field.setRepeated(f.isRepeated());
        field.setNullString(f.getNullString());
        fields.add(field);
        prevEnd = field.getPosition() + field.getLength();
    }
    if (info.getInputFields().length == 0) {
        TextFileInputField field = new TextFileInputField("Field1", 0, maxsize);
        fields.add(field);
    } else {
        // Take the last field and see if it reached until the maximum...
        TextFileInputField f = info.getInputFields()[info.getInputFields().length - 1];
        int pos = f.getPosition();
        int len = f.getLength();
        if (pos + len < maxsize) {
            // If not, add an extra trailing field!
            TextFileInputField field = new TextFileInputField("Dummy" + dummynr, pos + len, maxsize - pos - len);
            // don't include in result by default.
            field.setIgnored(true);
            fields.add(field);
        }
    }
    Collections.sort(fields);
    return fields;
}

Example 2 with TextFileInputField

use of org.apache.hop.core.file.TextFileInputField in project hop by apache.

the class TextFileInputMeta method getXml.

@Override
public String getXml() {
    StringBuilder retval = new StringBuilder(1500);
    retval.append("    ").append(XmlHandler.addTagValue("accept_filenames", acceptingFilenames));
    retval.append("    ").append(XmlHandler.addTagValue("passing_through_fields", passingThruFields));
    retval.append("    ").append(XmlHandler.addTagValue("accept_field", acceptingField));
    retval.append("    ").append(XmlHandler.addTagValue("accept_transform_name", (acceptingTransform != null ? acceptingTransform.getName() : "")));
    retval.append("    ").append(XmlHandler.addTagValue("separator", separator));
    retval.append("    ").append(XmlHandler.addTagValue("enclosure", enclosure));
    retval.append("    ").append(XmlHandler.addTagValue("enclosure_breaks", breakInEnclosureAllowed));
    retval.append("    ").append(XmlHandler.addTagValue("escapechar", escapeCharacter));
    retval.append("    ").append(XmlHandler.addTagValue("header", header));
    retval.append("    ").append(XmlHandler.addTagValue("nr_headerlines", nrHeaderLines));
    retval.append("    ").append(XmlHandler.addTagValue("footer", footer));
    retval.append("    ").append(XmlHandler.addTagValue("nr_footerlines", nrFooterLines));
    retval.append("    ").append(XmlHandler.addTagValue("line_wrapped", lineWrapped));
    retval.append("    ").append(XmlHandler.addTagValue("nr_wraps", nrWraps));
    retval.append("    ").append(XmlHandler.addTagValue("layout_paged", layoutPaged));
    retval.append("    ").append(XmlHandler.addTagValue("nr_lines_per_page", nrLinesPerPage));
    retval.append("    ").append(XmlHandler.addTagValue("nr_lines_doc_header", nrLinesDocHeader));
    retval.append("    ").append(XmlHandler.addTagValue("noempty", noEmptyLines));
    retval.append("    ").append(XmlHandler.addTagValue("include", includeFilename));
    retval.append("    ").append(XmlHandler.addTagValue("include_field", filenameField));
    retval.append("    ").append(XmlHandler.addTagValue("rownum", includeRowNumber));
    retval.append("    ").append(XmlHandler.addTagValue("rownumByFile", rowNumberByFile));
    retval.append("    ").append(XmlHandler.addTagValue("rownum_field", rowNumberField));
    retval.append("    ").append(XmlHandler.addTagValue("format", fileFormat));
    retval.append("    ").append(XmlHandler.addTagValue("encoding", encoding));
    retval.append("    " + XmlHandler.addTagValue("add_to_result_filenames", isaddresult));
    retval.append("    <file>").append(Const.CR);
    for (int i = 0; i < fileName.length; i++) {
        saveSource(retval, fileName[i]);
        retval.append("      ").append(XmlHandler.addTagValue("filemask", fileMask[i]));
        retval.append("      ").append(XmlHandler.addTagValue("exclude_filemask", excludeFileMask[i]));
        retval.append("      ").append(XmlHandler.addTagValue("file_required", fileRequired[i]));
        retval.append("      ").append(XmlHandler.addTagValue("include_subfolders", includeSubFolders[i]));
    }
    retval.append("      ").append(XmlHandler.addTagValue("type", fileType));
    retval.append("      ").append(XmlHandler.addTagValue("compression", (fileCompression == null) ? "None" : fileCompression));
    retval.append("    </file>").append(Const.CR);
    retval.append("    <filters>").append(Const.CR);
    for (int i = 0; i < filter.length; i++) {
        String filterString = filter[i].getFilterString();
        byte[] filterBytes = new byte[] {};
        String filterPrefix = "";
        if (filterString != null) {
            filterBytes = filterString.getBytes();
            filterPrefix = STRING_BASE64_PREFIX;
        }
        String filterEncoded = filterPrefix + new String(Base64.encodeBase64(filterBytes));
        retval.append("      <filter>").append(Const.CR);
        retval.append("        ").append(XmlHandler.addTagValue("filter_string", filterEncoded, false));
        retval.append("        ").append(XmlHandler.addTagValue("filter_position", filter[i].getFilterPosition(), false));
        retval.append("        ").append(XmlHandler.addTagValue("filter_is_last_line", filter[i].isFilterLastLine(), false));
        retval.append("        ").append(XmlHandler.addTagValue("filter_is_positive", filter[i].isFilterPositive(), false));
        retval.append("      </filter>").append(Const.CR);
    }
    retval.append("    </filters>").append(Const.CR);
    retval.append("    <fields>").append(Const.CR);
    for (int i = 0; i < inputFields.length; i++) {
        TextFileInputField field = inputFields[i];
        retval.append("      <field>").append(Const.CR);
        retval.append("        ").append(XmlHandler.addTagValue("name", field.getName()));
        retval.append("        ").append(XmlHandler.addTagValue("type", field.getTypeDesc()));
        retval.append("        ").append(XmlHandler.addTagValue("format", field.getFormat()));
        retval.append("        ").append(XmlHandler.addTagValue("currency", field.getCurrencySymbol()));
        retval.append("        ").append(XmlHandler.addTagValue("decimal", field.getDecimalSymbol()));
        retval.append("        ").append(XmlHandler.addTagValue("group", field.getGroupSymbol()));
        retval.append("        ").append(XmlHandler.addTagValue("nullif", field.getNullString()));
        retval.append("        ").append(XmlHandler.addTagValue("ifnull", field.getIfNullValue()));
        retval.append("        ").append(XmlHandler.addTagValue("position", field.getPosition()));
        retval.append("        ").append(XmlHandler.addTagValue("length", field.getLength()));
        retval.append("        ").append(XmlHandler.addTagValue("precision", field.getPrecision()));
        retval.append("        ").append(XmlHandler.addTagValue("trim_type", field.getTrimTypeCode()));
        retval.append("        ").append(XmlHandler.addTagValue("repeat", field.isRepeated()));
        retval.append("      </field>").append(Const.CR);
    }
    retval.append("    </fields>").append(Const.CR);
    retval.append("    ").append(XmlHandler.addTagValue("limit", rowLimit));
    // ERROR HANDLING
    retval.append("    ").append(XmlHandler.addTagValue("error_ignored", errorIgnored));
    retval.append("    ").append(XmlHandler.addTagValue("skip_bad_files", skipBadFiles));
    retval.append("    ").append(XmlHandler.addTagValue("file_error_field", fileErrorField));
    retval.append("    ").append(XmlHandler.addTagValue("file_error_message_field", fileErrorMessageField));
    retval.append("    ").append(XmlHandler.addTagValue("error_line_skipped", errorLineSkipped));
    retval.append("    ").append(XmlHandler.addTagValue("error_count_field", errorCountField));
    retval.append("    ").append(XmlHandler.addTagValue("error_fields_field", errorFieldsField));
    retval.append("    ").append(XmlHandler.addTagValue("error_text_field", errorTextField));
    retval.append("    ").append(XmlHandler.addTagValue("bad_line_files_destination_directory", warningFilesDestinationDirectory));
    retval.append("    ").append(XmlHandler.addTagValue("bad_line_files_extension", warningFilesExtension));
    retval.append("    ").append(XmlHandler.addTagValue("error_line_files_destination_directory", errorFilesDestinationDirectory));
    retval.append("    ").append(XmlHandler.addTagValue("error_line_files_extension", errorFilesExtension));
    retval.append("    ").append(XmlHandler.addTagValue("line_number_files_destination_directory", lineNumberFilesDestinationDirectory));
    retval.append("    ").append(XmlHandler.addTagValue("line_number_files_extension", lineNumberFilesExtension));
    retval.append("    ").append(XmlHandler.addTagValue("date_format_lenient", dateFormatLenient));
    retval.append("    ").append(XmlHandler.addTagValue("date_format_locale", dateFormatLocale != null ? dateFormatLocale.toString() : Locale.getDefault().toString()));
    retval.append("    ").append(XmlHandler.addTagValue("shortFileFieldName", shortFileFieldName));
    retval.append("    ").append(XmlHandler.addTagValue("pathFieldName", pathFieldName));
    retval.append("    ").append(XmlHandler.addTagValue("hiddenFieldName", hiddenFieldName));
    retval.append("    ").append(XmlHandler.addTagValue("lastModificationTimeFieldName", lastModificationTimeFieldName));
    retval.append("    ").append(XmlHandler.addTagValue("uriNameFieldName", uriNameFieldName));
    retval.append("    ").append(XmlHandler.addTagValue("rootUriNameFieldName", rootUriNameFieldName));
    retval.append("    ").append(XmlHandler.addTagValue("extensionFieldName", extensionFieldName));
    retval.append("    ").append(XmlHandler.addTagValue("sizeFieldName", sizeFieldName));
    return retval.toString();
}

Example 3 with TextFileInputField

use of org.apache.hop.core.file.TextFileInputField in project hop by apache.

the class TextFileInputMeta method getFields.

@Override
public void getFields(IRowMeta row, String name, IRowMeta[] info, TransformMeta nextTransform, IVariables variables, IHopMetadataProvider metadataProvider) throws HopTransformException {
    if (!isPassingThruFields()) {
        // all incoming fields are not transmitted !
        row.clear();
    } else {
        if (info != null) {
            boolean found = false;
            for (int i = 0; i < info.length && !found; i++) {
                if (info[i] != null) {
                    row.mergeRowMeta(info[i]);
                    found = true;
                }
            }
        }
    }
    for (int i = 0; i < inputFields.length; i++) {
        TextFileInputField field = inputFields[i];
        int type = field.getType();
        if (type == IValueMeta.TYPE_NONE) {
            type = IValueMeta.TYPE_STRING;
        }
        try {
            IValueMeta v = ValueMetaFactory.createValueMeta(field.getName(), type);
            v.setLength(field.getLength());
            v.setPrecision(field.getPrecision());
            v.setOrigin(name);
            v.setConversionMask(field.getFormat());
            v.setDecimalSymbol(field.getDecimalSymbol());
            v.setGroupingSymbol(field.getGroupSymbol());
            v.setCurrencySymbol(field.getCurrencySymbol());
            v.setDateFormatLenient(dateFormatLenient);
            v.setDateFormatLocale(dateFormatLocale);
            v.setTrimType(field.getTrimType());
            row.addValueMeta(v);
        } catch (Exception e) {
            throw new HopTransformException(e);
        }
    }
    if (errorIgnored) {
        if (errorCountField != null && errorCountField.length() > 0) {
            IValueMeta v = new ValueMetaInteger(errorCountField);
            v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0);
            v.setOrigin(name);
            row.addValueMeta(v);
        }
        if (errorFieldsField != null && errorFieldsField.length() > 0) {
            IValueMeta v = new ValueMetaString(errorFieldsField);
            v.setOrigin(name);
            row.addValueMeta(v);
        }
        if (errorTextField != null && errorTextField.length() > 0) {
            IValueMeta v = new ValueMetaString(errorTextField);
            v.setOrigin(name);
            row.addValueMeta(v);
        }
    }
    if (includeFilename) {
        IValueMeta v = new ValueMetaString(filenameField);
        v.setLength(100);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (includeRowNumber) {
        IValueMeta v = new ValueMetaInteger(rowNumberField);
        v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (getShortFileNameField() != null && getShortFileNameField().length() > 0) {
        IValueMeta v = new ValueMetaString(variables.resolve(getShortFileNameField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (getExtensionField() != null && getExtensionField().length() > 0) {
        IValueMeta v = new ValueMetaString(variables.resolve(getExtensionField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (getPathField() != null && getPathField().length() > 0) {
        IValueMeta v = new ValueMetaString(variables.resolve(getPathField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (getSizeField() != null && getSizeField().length() > 0) {
        IValueMeta v = new ValueMetaInteger(variables.resolve(getSizeField()));
        v.setOrigin(name);
        v.setLength(9);
        row.addValueMeta(v);
    }
    if (isHiddenField() != null && isHiddenField().length() > 0) {
        IValueMeta v = new ValueMetaBoolean(variables.resolve(isHiddenField()));
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (getLastModificationDateField() != null && getLastModificationDateField().length() > 0) {
        IValueMeta v = new ValueMetaDate(variables.resolve(getLastModificationDateField()));
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (getUriField() != null && getUriField().length() > 0) {
        IValueMeta v = new ValueMetaString(variables.resolve(getUriField()));
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
    if (getRootUriField() != null && getRootUriField().length() > 0) {
        IValueMeta v = new ValueMetaString(getRootUriField());
        v.setLength(100, -1);
        v.setOrigin(name);
        row.addValueMeta(v);
    }
}

Example 4 with TextFileInputField

use of org.apache.hop.core.file.TextFileInputField in project hop by apache.

the class XsltTest method runTestWithParams.

public void runTestWithParams(String xmlFieldname, String resultFieldname, boolean xslInField, boolean xslFileInField, String xslFileField, String xslFilename, String xslFactory) throws Exception {
    HopEnvironment.init();
    // 
    // Create a new pipeline...
    // 
    PipelineMeta pipelineMeta = new PipelineMeta();
    pipelineMeta.setName("xslt");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector transform...
    // 
    String injectorTransformName = "injector transform";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(TransformPluginType.class, im);
    TransformMeta injectorTransform = new TransformMeta(injectorPid, injectorTransformName, im);
    pipelineMeta.addTransform(injectorTransform);
    // 
    // Create a XSLT transform
    // 
    String xsltName = "xslt transform";
    XsltMeta xm = new XsltMeta();
    String xsltPid = registry.getPluginId(TransformPluginType.class, xm);
    TransformMeta xsltTransform = new TransformMeta(xsltPid, xsltName, xm);
    pipelineMeta.addTransform(xsltTransform);
    TextFileInputField[] fields = new TextFileInputField[3];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new TextFileInputField();
    }
    fields[0].setName("XML");
    fields[0].setType(IValueMeta.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(IValueMeta.TRIM_TYPE_NONE);
    fields[1].setName("XSL");
    fields[1].setType(IValueMeta.TYPE_STRING);
    fields[1].setFormat("");
    fields[1].setLength(-1);
    fields[1].setPrecision(-1);
    fields[1].setCurrencySymbol("");
    fields[1].setDecimalSymbol("");
    fields[1].setGroupSymbol("");
    fields[1].setTrimType(IValueMeta.TRIM_TYPE_NONE);
    fields[2].setName("filename");
    fields[2].setType(IValueMeta.TYPE_STRING);
    fields[2].setFormat("");
    fields[2].setLength(-1);
    fields[2].setPrecision(-1);
    fields[2].setCurrencySymbol("");
    fields[2].setDecimalSymbol("");
    fields[2].setGroupSymbol("");
    fields[2].setTrimType(IValueMeta.TRIM_TYPE_NONE);
    xm.setFieldname(xmlFieldname);
    xm.setResultfieldname(resultFieldname);
    xm.setXSLField(xslInField);
    xm.setXSLFileField(xslFileField);
    xm.setXSLFieldIsAFile(xslFileInField);
    xm.setXslFilename(xslFilename);
    xm.setXSLFactory(xslFactory);
    PipelineHopMeta hi = new PipelineHopMeta(injectorTransform, xsltTransform);
    pipelineMeta.addPipelineHop(hi);
    // 
    // Create a dummy transform 1
    // 
    String dummyTransformName1 = "dummy transform 1";
    DummyMeta dm1 = new DummyMeta();
    String dummyPid1 = registry.getPluginId(TransformPluginType.class, dm1);
    TransformMeta dummyTransform1 = new TransformMeta(dummyPid1, dummyTransformName1, dm1);
    pipelineMeta.addTransform(dummyTransform1);
    PipelineHopMeta hi1 = new PipelineHopMeta(xsltTransform, dummyTransform1);
    pipelineMeta.addPipelineHop(hi1);
    // Now execute the pipeline...
    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
    pipeline.prepareExecution();
    ITransform si = pipeline.getTransform(dummyTransformName1, 0);
    RowTransformCollector dummyRc1 = new RowTransformCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = pipeline.addRowProducer(injectorTransformName, 0);
    pipeline.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData(xslFilename);
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    pipeline.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows, 2);
}

Example 5 with TextFileInputField

use of org.apache.hop.core.file.TextFileInputField in project hop by apache.

the class CsvInputDialog method getData.

@Override
public void getData(final CsvInputMeta inputMeta, final boolean copyTransformName, final boolean reloadAllFields, final Set<String> newFieldNames) {
    if (copyTransformName) {
        wTransformName.setText(transformName);
    }
    if (isReceivingInput) {
        wFilenameField.setText(Const.NVL(inputMeta.getFilenameField(), ""));
        wIncludeFilename.setSelection(inputMeta.isIncludingFilename());
    } else {
        wFilename.setText(Const.NVL(inputMeta.getFilename(), ""));
    }
    wDelimiter.setText(Const.NVL(inputMeta.getDelimiter(), ""));
    wEnclosure.setText(Const.NVL(inputMeta.getEnclosure(), ""));
    wBufferSize.setText(Const.NVL(inputMeta.getBufferSize(), ""));
    wLazyConversion.setSelection(inputMeta.isLazyConversionActive());
    wHeaderPresent.setSelection(inputMeta.isHeaderPresent());
    wRunningInParallel.setSelection(inputMeta.isRunningInParallel());
    wNewlinePossible.setSelection(inputMeta.isNewlinePossibleInFields());
    wRowNumField.setText(Const.NVL(inputMeta.getRowNumField(), ""));
    wAddResult.setSelection(inputMeta.isAddResultFile());
    wEncoding.setText(Const.NVL(inputMeta.getEncoding(), ""));
    final List<String> fieldName = newFieldNames == null ? new ArrayList() : newFieldNames.stream().map(String::toString).collect(Collectors.toList());
    for (int i = 0; i < inputMeta.getInputFields().length; i++) {
        TextFileInputField field = inputMeta.getInputFields()[i];
        final TableItem item = getTableItem(field.getName());
        // update the item only if we are reloading all fields, or the field is new
        if (!reloadAllFields && !fieldName.contains(field.getName())) {
            continue;
        }
        int colnr = 1;
        item.setText(colnr++, Const.NVL(field.getName(), ""));
        item.setText(colnr++, ValueMetaFactory.getValueMetaName(field.getType()));
        item.setText(colnr++, Const.NVL(field.getFormat(), ""));
        item.setText(colnr++, field.getLength() >= 0 ? Integer.toString(field.getLength()) : "");
        item.setText(colnr++, field.getPrecision() >= 0 ? Integer.toString(field.getPrecision()) : "");
        item.setText(colnr++, Const.NVL(field.getCurrencySymbol(), ""));
        item.setText(colnr++, Const.NVL(field.getDecimalSymbol(), ""));
        item.setText(colnr++, Const.NVL(field.getGroupSymbol(), ""));
        item.setText(colnr++, Const.NVL(field.getTrimTypeDesc(), ""));
    }
    wFields.removeEmptyRows();
    wFields.setRowNums();
    wFields.optWidth(true);
    setFlags();
    wTransformName.selectAll();
    wTransformName.setFocus();
}