use of org.knime.core.data.DataValue in project knime-core by knime.
the class RuleEngineFilter2PortsNodeModel method createStreamableOperator.
/**
* {@inheritDoc}
*/
@Override
public StreamableOperator createStreamableOperator(final PartitionInfo partitionInfo, final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
return new StreamableOperator() {
private SimpleStreamableOperatorInternals m_internals;
/**
* {@inheritDoc}
*/
@Override
public void loadInternals(final StreamableOperatorInternals internals) {
m_internals = (SimpleStreamableOperatorInternals) internals;
}
/**
* {@inheritDoc}
*/
@Override
public void runIntermediate(final PortInput[] inputs, final ExecutionContext exec) throws Exception {
// count number of rows
long count = 0;
RowInput rowInput = (RowInput) inputs[DATA_PORT];
while (rowInput.poll() != null) {
count++;
}
m_internals.getConfig().addLong(CFG_ROW_COUNT, count);
}
/**
* {@inheritDoc}
*/
@Override
public StreamableOperatorInternals saveInternals() {
return m_internals;
}
@Override
public void runFinal(final PortInput[] inputs, final PortOutput[] outputs, final ExecutionContext exec) throws Exception {
long rowCount = -1L;
if (m_internals.getConfig().containsKey(CFG_ROW_COUNT)) {
rowCount = m_internals.getConfig().getLong(CFG_ROW_COUNT);
}
m_rulesList.clear();
final PortInput rulePort = inputs[RULE_PORT];
if (rulePort instanceof PortObjectInput) {
PortObjectInput poRule = (PortObjectInput) rulePort;
m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules((BufferedDataTable) poRule.getPortObject(), m_settings, RuleNodeSettings.RuleFilter));
} else if (rulePort instanceof RowInput) {
RowInput riRule = (RowInput) rulePort;
m_rulesList.addAll(RuleEngineVariable2PortsNodeModel.rules(riRule, m_settings, RuleNodeSettings.RuleFilter));
}
final DataTableSpec spec = (DataTableSpec) inSpecs[DATA_PORT];
try {
parseRules(spec, RuleNodeSettings.RuleSplitter);
} catch (final ParseException e) {
throw new InvalidSettingsException(e);
}
final RowInput inputPartitions = (RowInput) inputs[DATA_PORT];
final List<Rule> rules = parseRules(inputPartitions.getDataTableSpec(), RuleNodeSettings.RuleFilter);
final RowOutput first = (RowOutput) outputs[0];
final int nrOutPorts = getNrOutPorts();
final RowOutput second = nrOutPorts > 1 ? (RowOutput) outputs[1] : new RowOutput() {
@Override
public void push(final DataRow row) throws InterruptedException {
// do nothing
}
@Override
public void close() throws InterruptedException {
// do nothing
}
};
final RowOutput[] containers = new RowOutput[] { first, second };
final int matchIndex = m_includeOnMatch ? 0 : 1;
final int otherIndex = 1 - matchIndex;
try {
final MutableLong rowIdx = new MutableLong(0L);
final long rows = rowCount;
final VariableProvider provider = new VariableProvider() {
@Override
public Object readVariable(final String name, final Class<?> type) {
return RuleEngineFilter2PortsNodeModel.this.readVariable(name, type);
}
@Override
@Deprecated
public int getRowCount() {
throw new UnsupportedOperationException();
}
@Override
public long getRowCountLong() {
return rows;
}
@Override
@Deprecated
public int getRowIndex() {
throw new UnsupportedOperationException();
}
@Override
public long getRowIndexLong() {
return rowIdx.longValue();
}
};
DataRow row;
while ((row = inputPartitions.poll()) != null) {
rowIdx.increment();
if (rows > 0) {
exec.setProgress(rowIdx.longValue() / (double) rows, () -> "Adding row " + rowIdx.longValue() + " of " + rows);
} else {
exec.setMessage(() -> "Adding row " + rowIdx.longValue() + " of " + rows);
}
exec.checkCanceled();
boolean wasMatch = false;
for (Rule r : rules) {
if (r.getCondition().matches(row, provider).getOutcome() == MatchState.matchedAndStop) {
// r.getSideEffect().perform(row, provider);
DataValue value = r.getOutcome().getComputedResult(row, provider);
if (value instanceof BooleanValue) {
final BooleanValue bv = (BooleanValue) value;
containers[bv.getBooleanValue() ? matchIndex : otherIndex].push(row);
} else {
containers[matchIndex].push(row);
}
wasMatch = true;
break;
}
}
if (!wasMatch) {
containers[otherIndex].push(row);
}
}
} finally {
try {
second.close();
} finally {
first.close();
}
}
}
};
}
use of org.knime.core.data.DataValue in project knime-core by knime.
the class HistogramColumn method loadNominalHistogramsPrivate.
private static Map<Integer, HistogramNominalModel> loadNominalHistogramsPrivate(final File histogramsGz, final int[] nominalKeysSize) throws IOException, InvalidSettingsException {
final FileInputStream is = new FileInputStream(histogramsGz);
final GZIPInputStream inData = new GZIPInputStream(is);
final ConfigRO config = NodeSettings.loadFromXML(inData);
Map<Integer, HistogramNominalModel> histograms = new HashMap<Integer, HistogramNominalModel>();
// .getConfig(HISTOGRAMS);
ConfigRO hs = config;
int[] nomColumnIndices = config.getIntArray(NOMINAL_COLUMNS);
for (int colIdx : nomColumnIndices) {
Config h = hs.getConfig(HISTOGRAM + colIdx);
int maxCount = h.getInt(MAX_COUNT);
int rowCount = h.getInt(ROW_COUNT);
String colName = h.getString(COL_NAME);
String[] values = h.getStringArray(BIN_VALUES);
int[] binCounts = h.getIntArray(BIN_COUNTS);
Map<DataValue, Integer> bins = new HashMap<DataValue, Integer>();
for (int i = binCounts.length; i-- > 0; ) {
if (values[i] == "?") {
bins.put(new MissingCell(null), binCounts[i]);
} else {
bins.put(new StringCell(values[i]), binCounts[i]);
}
}
HistogramNominalModel histogramData = new HistogramNominalModel(bins, colIdx, colName, rowCount);
histogramData.setMaxCount(maxCount);
histogramData.setRowCount(rowCount);
// assert Math.abs(histogramData.m_width - width) < 1e-9: "histogram data width: " + histogramData.m_width + " width: " + width;
assert nominalKeysSize[colIdx] == bins.size() : "Saved size of nominal bins: " + nominalKeysSize[colIdx] + ", restored from the file: " + bins.size();
histograms.put(colIdx, histogramData);
}
return histograms;
}
use of org.knime.core.data.DataValue in project knime-core by knime.
the class HistogramColumn method constructFromDataArray.
/**
* Constructs the helper data structures from the numeric hostigran models and the data as {@link DataArray}.
*
* @param histograms The numeric histograms.
* @param data The input data.
* @param nominalColumnNames The nominal column names.
* @return The helper data structures.
* @see #construct(Map, DataTable, Set)
*/
protected static Pair<Map<Integer, Map<Integer, Set<RowKey>>>, Map<Integer, Map<DataValue, Set<RowKey>>>> constructFromDataArray(final Map<Integer, HistogramNumericModel> histograms, final DataTable data, final Set<String> nominalColumnNames) {
Map<Integer, Map<Integer, Set<RowKey>>> numericMapping = new HashMap<Integer, Map<Integer, Set<RowKey>>>();
Map<Integer, Map<DataValue, Set<RowKey>>> nominalMapping = new HashMap<Integer, Map<DataValue, Set<RowKey>>>();
DataTableSpec tableSpec = data.getDataTableSpec();
for (DataColumnSpec colSpec : tableSpec) {
int colIndex = tableSpec.findColumnIndex(colSpec.getName());
if (colSpec.getType().isCompatible(DoubleValue.class)) {
// + colIndex;
if (histograms.containsKey(Integer.valueOf(colIndex)) && histograms.get(colIndex) != null) {
numericMapping.put(colIndex, new HashMap<Integer, Set<RowKey>>());
}
}
if (colSpec.getDomain().hasValues() || nominalColumnNames.contains(colSpec.getName())) {
nominalMapping.put(colIndex, new HashMap<DataValue, Set<RowKey>>());
}
}
for (DataRow dataRow : data) {
for (Entry<Integer, Map<Integer, Set<RowKey>>> outer : numericMapping.entrySet()) {
Integer key = outer.getKey();
DataCell cell = dataRow.getCell(key);
if (cell instanceof DoubleValue) {
DoubleValue dv = (DoubleValue) cell;
Integer bin = Integer.valueOf(histograms.get(key).findBin(dv));
Map<Integer, Set<RowKey>> inner = outer.getValue();
if (!inner.containsKey(bin)) {
inner.put(bin, new HashSet<RowKey>());
}
inner.get(bin).add(dataRow.getKey());
}
}
for (Entry<Integer, Map<DataValue, Set<RowKey>>> outer : nominalMapping.entrySet()) {
int key = outer.getKey().intValue();
DataCell cell = dataRow.getCell(key);
if (!cell.isMissing()) /* && cell instanceof NominalValue*/
{
Map<DataValue, Set<RowKey>> inner = outer.getValue();
if (!inner.containsKey(cell)) {
inner.put(cell, new HashSet<RowKey>());
}
inner.get(cell).add(dataRow.getKey());
}
}
}
return Pair.create(numericMapping, nominalMapping);
}
use of org.knime.core.data.DataValue in project knime-core by knime.
the class HistogramColumn method loadHistograms.
/**
* Loads the histograms from the saved internal files.
*
* @param histogramsGz The file for the histograms.
* @param dataArrayGz The data array file for the row keys.
* @param nominalColumns The nominal columns.
* @param strategy The strategy used to compute the bins.
* @param means The mean values for the numeric columns.
* @return A triple (Pair(Pair(,),)) of histograms, numeric and nominal row keys.
* @throws IOException Failed to read the files.
* @throws InvalidSettingsException Something went wrong.
*/
public static Pair<Pair<Map<Integer, ? extends HistogramModel<?>>, Map<Integer, Map<Integer, Set<RowKey>>>>, Map<Integer, Map<DataValue, Set<RowKey>>>> loadHistograms(final File histogramsGz, final File dataArrayGz, final Set<String> nominalColumns, final BinNumberSelectionStrategy strategy, final double[] means) throws IOException, InvalidSettingsException {
Map<Integer, Map<Integer, Set<RowKey>>> numericKeys = new HashMap<Integer, Map<Integer, Set<RowKey>>>();
Map<Integer, HistogramNumericModel> histograms = loadHistogramsPrivate(histogramsGz, numericKeys, strategy, means);
Map<Integer, Map<DataValue, Set<RowKey>>> nominalKeys = new HashMap<Integer, Map<DataValue, Set<RowKey>>>();
ContainerTable table = DataContainer.readFromZip(dataArrayGz);
Set<Integer> numericColIndices = numericKeys.keySet();
for (String colName : nominalColumns) {
int colIndex = table.getDataTableSpec().findColumnIndex(colName);
if (colIndex < 0) {
continue;
}
nominalKeys.put(Integer.valueOf(colIndex), new HashMap<DataValue, Set<RowKey>>());
}
for (DataRow dataRow : table) {
for (Integer col : numericColIndices) {
// Integer col = Integer.valueOf(colIdx);
HistogramNumericModel hd = histograms.get(col);
Map<Integer, Set<RowKey>> map = numericKeys.get(col);
DataCell cell = dataRow.getCell(col.intValue());
if (!cell.isMissing() && cell instanceof DoubleValue) {
DoubleValue dv = (DoubleValue) cell;
Integer bin = Integer.valueOf(hd.findBin(dv));
if (!map.containsKey(bin)) {
map.put(bin, new HashSet<RowKey>());
}
map.get(bin).add(dataRow.getKey());
}
}
for (Entry<Integer, Map<DataValue, Set<RowKey>>> entry : nominalKeys.entrySet()) {
DataCell value = dataRow.getCell(entry.getKey().intValue());
Map<DataValue, Set<RowKey>> map = entry.getValue();
if (!map.containsKey(value)) {
map.put(value, new HashSet<RowKey>());
}
map.get(value).add(dataRow.getKey());
}
}
return Pair.create(new Pair<Map<Integer, ? extends HistogramModel<?>>, Map<Integer, Map<Integer, Set<RowKey>>>>(histograms, numericKeys), nominalKeys);
}
use of org.knime.core.data.DataValue in project knime-core by knime.
the class DateTimeDifferenceNodeDialog method updateDateTimeComponents.
@SuppressWarnings("unchecked")
private void updateDateTimeComponents(final SettingsModelDateTime dateTimeModel) {
if (m_dialogComp1stColSelect.getSelectedAsSpec() != null) {
final DataType type = m_dialogComp1stColSelect.getSelectedAsSpec().getType();
// adapt column filter of column selection for 2nd column
try {
final List<Class<? extends DataValue>> valueClasses = type.getValueClasses();
if (valueClasses.contains(ZonedDateTimeValue.class)) {
m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(ZonedDateTimeValue.class));
} else if (valueClasses.contains(LocalDateTimeValue.class)) {
m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalDateTimeValue.class));
} else if (valueClasses.contains(LocalDateValue.class)) {
m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalDateValue.class));
} else if (valueClasses.contains(LocalTimeValue.class)) {
m_dialogComp2ndColSelect.setColumnFilter(new DataValueColumnFilter(LocalTimeValue.class));
}
} catch (NotConfigurableException ex) {
// will never happen, because there is always at least the selected 1st column available
}
// adapt date&time component
dateTimeModel.setUseDate(!type.isCompatible(LocalTimeValue.class));
dateTimeModel.setUseTime(!type.isCompatible(LocalDateValue.class));
dateTimeModel.setUseZone(type.isCompatible(ZonedDateTimeValue.class));
}
updateWarningLabel();
}
Aggregations