Examples with ValueMetaDate org.apache.hop.core.row.value.ValueMetaDate used on opensource projects

Example 6 with ValueMetaDate

use of org.apache.hop.core.row.value.ValueMetaDate in project hop by apache.

the class RowGenerator method buildRow.

public static final RowMetaAndData buildRow(RowGeneratorMeta meta, List<ICheckResult> remarks, String origin) throws HopPluginException {
    IRowMeta rowMeta = new RowMeta();
    Object[] rowData = RowDataUtil.allocateRowData(meta.getFields().size() + 2);
    int index = 0;
    if (meta.isNeverEnding()) {
        if (!Utils.isEmpty(meta.getRowTimeField())) {
            rowMeta.addValueMeta(new ValueMetaDate(meta.getRowTimeField()));
            rowData[index++] = null;
        }
        if (!Utils.isEmpty(meta.getLastTimeField())) {
            rowMeta.addValueMeta(new ValueMetaDate(meta.getLastTimeField()));
            rowData[index++] = null;
        }
    }
    for (GeneratorField field : meta.getFields()) {
        int typeString = ValueMetaFactory.getIdForValueMeta(field.getType());
        if (StringUtils.isNotEmpty(field.getType())) {
            IValueMeta valueMeta = // build a
            ValueMetaFactory.createValueMeta(field.getName(), typeString);
            // value!
            valueMeta.setLength(field.getLength());
            valueMeta.setPrecision(field.getPrecision());
            valueMeta.setConversionMask(field.getFormat());
            valueMeta.setCurrencySymbol(field.getCurrency());
            valueMeta.setGroupingSymbol(field.getGroup());
            valueMeta.setDecimalSymbol(field.getDecimal());
            valueMeta.setOrigin(origin);
            IValueMeta stringMeta = ValueMetaFactory.cloneValueMeta(valueMeta, IValueMeta.TYPE_STRING);
            if (field.isSetEmptyString()) {
                // Set empty string
                rowData[index] = StringUtil.EMPTY_STRING;
            } else {
                String stringValue = field.getValue();
                // If the value is empty: consider it to be NULL.
                if (Utils.isEmpty(stringValue)) {
                    rowData[index] = null;
                    if (valueMeta.getType() == IValueMeta.TYPE_NONE) {
                        String message = BaseMessages.getString(PKG, "RowGenerator.CheckResult.SpecifyTypeError", valueMeta.getName(), stringValue);
                        remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message, null));
                    }
                } else {
                    // 
                    try {
                        rowData[index] = valueMeta.convertData(stringMeta, stringValue);
                    } catch (HopValueException e) {
                        switch(valueMeta.getType()) {
                            case IValueMeta.TYPE_NUMBER:
                                String message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Number", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message, null));
                                break;
                            case IValueMeta.TYPE_DATE:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Date", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message, null));
                                break;
                            case IValueMeta.TYPE_INTEGER:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Integer", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message, null));
                                break;
                            case IValueMeta.TYPE_BIGNUMBER:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.BigNumber", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message, null));
                                break;
                            case IValueMeta.TYPE_TIMESTAMP:
                                message = BaseMessages.getString(PKG, "RowGenerator.BuildRow.Error.Parsing.Timestamp", valueMeta.getName(), stringValue, e.toString());
                                remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message, null));
                                break;
                            default:
                                // Boolean and binary don't throw errors normally, so it's probably an unspecified
                                // error problem...
                                message = BaseMessages.getString(PKG, "RowGenerator.CheckResult.SpecifyTypeError", valueMeta.getName(), stringValue);
                                remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, message, null));
                                break;
                        }
                    }
                }
            }
            // Now add value to the row!
            // This is in fact a copy from the fields row, but now with data.
            rowMeta.addValueMeta(valueMeta);
            index++;
        }
    }
    return new RowMetaAndData(rowMeta, rowData);
}

Example 7 with ValueMetaDate

use of org.apache.hop.core.row.value.ValueMetaDate in project hop by apache.

the class MemoryGroupByAggregationTest method testDefault.

@Test
@Ignore
public void testDefault() throws Exception {
    addColumn(new ValueMetaInteger("intg"), 0L, 1L, 1L, 10L);
    addColumn(new ValueMetaInteger("nul"));
    addColumn(new ValueMetaInteger("mix1"), -1L, 2L);
    addColumn(new ValueMetaInteger("mix2"), null, 7L);
    addColumn(new ValueMetaNumber("mix3"), -1.0, 2.5);
    addColumn(new ValueMetaDate("date1"), new Date(1L), new Date(2L));
    RowMetaAndData output = runTransform();
    assertThat(output.getInteger("intg_min"), is(0L));
    assertThat(output.getInteger("intg_max"), is(10L));
    assertThat(output.getInteger("intg_sum"), is(12L));
    assertThat(output.getInteger("intg_ave"), is(3L));
    assertThat(output.getInteger("intg_count"), is(4L));
    assertThat(output.getInteger("intg_count_any"), is(4L));
    assertThat(output.getInteger("intg_count_distinct"), is(3L));
    assertThat(output.getInteger("nul_min"), nullValue());
    assertThat(output.getInteger("nul_max"), nullValue());
    assertThat(output.getInteger("nul_sum"), nullValue());
    assertThat(output.getInteger("nul_ave"), nullValue());
    assertThat(output.getInteger("nul_count"), is(0L));
    assertThat(output.getInteger("nul_count_any"), is(4L));
    assertThat(output.getInteger("nul_count_distinct"), is(0L));
    assertThat(output.getInteger("mix1_max"), is(2L));
    assertThat(output.getInteger("mix1_min"), is(-1L));
    assertThat(output.getInteger("mix1_sum"), is(1L));
    assertThat(output.getInteger("mix1_ave"), is(0L));
    assertThat(output.getInteger("mix1_count"), is(2L));
    assertThat(output.getInteger("mix1_count_any"), is(4L));
    assertThat(output.getInteger("mix1_count_distinct"), is(2L));
    assertThat(output.getInteger("mix2_max"), is(7L));
    assertThat(output.getInteger("mix2_min"), is(7L));
    assertThat(output.getInteger("mix2_sum"), is(7L));
    assertThat(output.getNumber("mix2_ave", Double.NaN), is(7.0));
    assertThat(output.getInteger("mix2_count"), is(1L));
    assertThat(output.getInteger("mix2_count_any"), is(4L));
    assertThat(output.getInteger("mix2_count_distinct"), is(1L));
    assertThat(output.getNumber("mix3_max", Double.NaN), is(2.5));
    assertThat(output.getNumber("mix3_min", Double.NaN), is(-1.0));
    assertThat(output.getNumber("mix3_sum", Double.NaN), is(1.5));
    assertThat(output.getNumber("mix3_ave", Double.NaN), is(0.75));
    assertThat(output.getInteger("mix3_count"), is(2L));
    assertThat(output.getInteger("mix3_count_any"), is(4L));
    assertThat(output.getInteger("mix3_count_distinct"), is(2L));
    assertThat(output.getNumber("date1_min", Double.NaN), is(1.0));
    assertThat(output.getNumber("date1_max", Double.NaN), is(2.0));
    assertThat(output.getNumber("date1_sum", Double.NaN), is(3.0));
    assertThat(output.getNumber("date1_ave", Double.NaN), is(1.5));
    assertThat(output.getInteger("date1_count"), is(2L));
    assertThat(output.getInteger("date1_count_any"), is(4L));
    assertThat(output.getInteger("date1_count_distinct"), is(2L));
}

Example 8 with ValueMetaDate

use of org.apache.hop.core.row.value.ValueMetaDate in project hop by apache.

the class MailInputMeta method getFields.

@Override
public void getFields(IRowMeta r, String name, IRowMeta[] info, TransformMeta nextTransform, IVariables variables, IHopMetadataProvider metadataProvider) throws HopTransformException {
    int i;
    for (i = 0; i < inputFields.length; i++) {
        MailInputField field = inputFields[i];
        IValueMeta v = new ValueMetaString(variables.resolve(field.getName()));
        switch(field.getColumn()) {
            case MailInputField.COLUMN_MESSAGE_NR:
            case MailInputField.COLUMN_SIZE:
            case MailInputField.COLUMN_ATTACHED_FILES_COUNT:
                v = new ValueMetaInteger(variables.resolve(field.getName()));
                v.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0);
                break;
            case MailInputField.COLUMN_RECEIVED_DATE:
            case MailInputField.COLUMN_SENT_DATE:
                v = new ValueMetaDate(variables.resolve(field.getName()));
                break;
            case MailInputField.COLUMN_FLAG_DELETED:
            case MailInputField.COLUMN_FLAG_DRAFT:
            case MailInputField.COLUMN_FLAG_FLAGGED:
            case MailInputField.COLUMN_FLAG_NEW:
            case MailInputField.COLUMN_FLAG_READ:
                v = new ValueMetaBoolean(variables.resolve(field.getName()));
                break;
            default:
                // STRING
                v.setLength(250);
                v.setPrecision(-1);
                break;
        }
        v.setOrigin(name);
        r.addValueMeta(v);
    }
}

Example 9 with ValueMetaDate

use of org.apache.hop.core.row.value.ValueMetaDate in project hop by apache.

the class SelectValues_LocaleHandling_Test method executeAndCheck.

private void executeAndCheck(String locale, String expectedWeekNumber) throws Exception {
    SelectValuesMeta transformMeta = new SelectValuesMeta();
    transformMeta.allocate(1, 0, 1);
    transformMeta.getSelectFields()[0] = new SelectField();
    transformMeta.getSelectFields()[0].setName("field");
    transformMeta.getMeta()[0] = new SelectMetadataChange("field", null, IValueMeta.TYPE_STRING, -2, -2, IValueMeta.STORAGE_TYPE_NORMAL, "ww", false, locale, null, false, null, null, null);
    SelectValuesData transformData = new SelectValuesData();
    transformData.select = true;
    transformData.metadata = true;
    transformData.firstselect = true;
    transformData.firstmetadata = true;
    configureTransform(transformMeta, transformData);
    RowMeta inputRowMeta = new RowMeta();
    inputRowMeta.addValueMeta(new ValueMetaDate("field"));
    transform.setInputRowMeta(inputRowMeta);
    List<Object[]> execute = PipelineTestingUtil.execute(transform, 1, true);
    PipelineTestingUtil.assertResult(execute, Collections.singletonList(new Object[] { expectedWeekNumber }));
}

Example 10 with ValueMetaDate

use of org.apache.hop.core.row.value.ValueMetaDate in project hop by apache.

the class SasInput method processRow.

@Override
public boolean processRow() throws HopException {
    final Object[] fileRowData = getRow();
    if (fileRowData == null) {
        // No more work to do...
        // 
        setOutputDone();
        return false;
    }
    // 
    if (first) {
        // The output row meta data, what does it look like?
        // 
        data.outputRowMeta = new RowMeta();
        // See if the input row contains the filename field...
        // 
        int idx = getInputRowMeta().indexOfValue(meta.getAcceptingField());
        if (idx < 0) {
            throw new HopException(BaseMessages.getString(PKG, "SASInput.Log.Error.UnableToFindFilenameField", meta.getAcceptingField()));
        }
        // Determine the output row layout
        // 
        data.outputRowMeta = getInputRowMeta().clone();
        meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider);
    }
    String rawFilename = getInputRowMeta().getString(fileRowData, meta.getAcceptingField(), null);
    final String filename = resolve(rawFilename);
    // Add this to the result file names...
    // 
    ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, HopVfs.getFileObject(filename), getPipelineMeta().getName(), getTransformName());
    resultFile.setComment(BaseMessages.getString(PKG, "SASInput.ResultFile.Comment"));
    addResultFile(resultFile);
    // 
    try (InputStream inputStream = HopVfs.getInputStream(filename)) {
        SasFileReaderImpl sasFileReader = new SasFileReaderImpl(inputStream);
        SasFileProperties sasFileProperties = sasFileReader.getSasFileProperties();
        logBasic(BaseMessages.getString(PKG, "SASInput.Log.OpenedSASFile") + " : [" + filename + "]");
        // What are the columns in the file?
        // 
        List<Column> columns = sasFileReader.getColumns();
        // Map this to the columns we want...
        // 
        List<Integer> indexes = new ArrayList<>();
        for (SasInputField field : meta.getOutputFields()) {
            int index = -1;
            for (int c = 0; c < columns.size(); c++) {
                if (columns.get(c).getName().equalsIgnoreCase(field.getName())) {
                    index = c;
                    break;
                }
            }
            if (index < 0) {
                throw new HopException("Field '" + field.getName() + " could not be found in input file '" + filename);
            }
            indexes.add(index);
        }
        // Now we have the indexes of the output fields to grab.
        // Let's grab them...
        // 
        Object[] sasRow;
        while ((sasRow = sasFileReader.readNext()) != null) {
            Object[] outputRow = RowDataUtil.createResizedCopy(fileRowData, data.outputRowMeta.size());
            for (int i = 0; i < meta.getOutputFields().size(); i++) {
                SasInputField field = meta.getOutputFields().get(i);
                int index = indexes.get(i);
                Column column = columns.get(index);
                ColumnFormat columnFormat = column.getFormat();
                Object sasValue = sasRow[index];
                Object value = null;
                IValueMeta inputValueMeta = null;
                String fieldName = Const.NVL(field.getRename(), field.getName());
                int outputIndex = getInputRowMeta().size() + i;
                if (sasValue instanceof byte[]) {
                    inputValueMeta = new ValueMetaString(fieldName);
                    if (sasFileProperties.getEncoding() != null) {
                        value = new String((byte[]) sasValue, sasFileProperties.getEncoding());
                    } else {
                        // TODO: user defined encoding.
                        value = new String((byte[]) sasValue);
                    }
                }
                if (sasValue instanceof String) {
                    inputValueMeta = new ValueMetaString(fieldName);
                    value = sasValue;
                }
                if (sasValue instanceof Double) {
                    inputValueMeta = new ValueMetaNumber(fieldName);
                    value = sasValue;
                }
                if (sasValue instanceof Float) {
                    inputValueMeta = new ValueMetaNumber(fieldName);
                    value = Double.valueOf((double) sasValue);
                }
                if (sasValue instanceof Long) {
                    inputValueMeta = new ValueMetaInteger(fieldName);
                    value = sasValue;
                }
                if (sasValue instanceof Integer) {
                    inputValueMeta = new ValueMetaInteger(fieldName);
                    value = Long.valueOf((int) sasValue);
                }
                if (sasValue instanceof Date) {
                    inputValueMeta = new ValueMetaDate(fieldName);
                    value = sasValue;
                }
                if (inputValueMeta != null) {
                    inputValueMeta.setLength(field.getLength());
                    inputValueMeta.setPrecision(field.getPrecision());
                    inputValueMeta.setConversionMask(field.getConversionMask());
                    IValueMeta outputValueMeta = data.outputRowMeta.getValueMeta(outputIndex);
                    outputRow[outputIndex] = outputValueMeta.convertData(inputValueMeta, value);
                }
            }
            // Send the row on its way...
            // 
            putRow(data.outputRowMeta, outputRow);
        }
    } catch (Exception e) {
        throw new HopException("Error reading from file " + filename, e);
    }
    return true;
}