use of org.knime.core.data.container.CellFactory in project knime-core by knime.
the class ConstantValueColumnNodeModel method createColumnRearranger.
/**
* {@inheritDoc}
*/
@Override
protected ColumnRearranger createColumnRearranger(final DataTableSpec in) throws InvalidSettingsException {
final String value = m_config.getValue();
checkSetting(value != null, "Configuration missing.");
checkSetting(!(m_config.getReplacedColumn() == null && m_config.getNewColumnName() == null), "Either a replacing column or a new column name must be specified");
String colName = m_config.getReplacedColumn();
final int replacedColumn = in.findColumnIndex(m_config.getReplacedColumn());
checkSetting(!(colName != null && replacedColumn < 0), "Column to replace: '%s' does not exist in input table", colName);
String newName = replacedColumn >= 0 ? colName : DataTableSpec.getUniqueColumnName(in, m_config.getNewColumnName());
DataColumnSpec outColumnSpec = new DataColumnSpecCreator(newName, m_config.getCellFactory().getDataType()).createSpec();
final DataCell constantCell = m_config.getCellFactory().createCell(value, m_config.getDateFormat());
ColumnRearranger rearranger = new ColumnRearranger(in);
CellFactory fac = new SingleCellFactory(outColumnSpec) {
@Override
public DataCell getCell(final DataRow row) {
return constantCell;
}
};
if (replacedColumn >= 0) {
rearranger.replace(fac, replacedColumn);
} else {
rearranger.append(fac);
}
return rearranger;
}
use of org.knime.core.data.container.CellFactory in project knime-core by knime.
the class HiLiteCollectorNodeModel method execute.
/**
* {@inheritDoc}
*/
@Override
protected PortObject[] execute(final PortObject[] inData, final ExecutionContext exec) throws Exception {
if (m_annotationMap.isEmpty()) {
return inData;
}
DataTableSpec inSpec = (DataTableSpec) inData[0].getSpec();
final DataColumnSpec[] cspecs = createSpecs(inSpec);
ColumnRearranger cr = new ColumnRearranger(inSpec);
cr.append(new CellFactory() {
/**
* {@inheritDoc}
*/
@Override
public DataCell[] getCells(final DataRow row) {
if (m_annotationMap.isEmpty()) {
return new DataCell[0];
}
DataCell[] cells = new DataCell[m_lastIndex + 1];
for (int i = 0; i < cells.length; i++) {
Map<Integer, String> map = m_annotationMap.get(row.getKey());
if (map == null) {
cells[i] = DataType.getMissingCell();
} else {
String str = map.get(i);
if (str == null) {
cells[i] = DataType.getMissingCell();
} else {
cells[i] = new StringCell(str);
}
}
}
return cells;
}
@Override
public DataColumnSpec[] getColumnSpecs() {
return cspecs;
}
/**
* {@inheritDoc}
*/
@Override
public void setProgress(final int curRowNr, final int rowCount, final RowKey lastKey, final ExecutionMonitor em) {
em.setProgress((double) curRowNr / rowCount);
}
});
return new BufferedDataTable[] { exec.createColumnRearrangeTable((BufferedDataTable) inData[0], cr, exec) };
}
use of org.knime.core.data.container.CellFactory in project knime-core by knime.
the class CollectionCreateNodeModel method createColumnRearranger.
private ColumnRearranger createColumnRearranger(final DataTableSpec in) throws InvalidSettingsException {
List<String> includes = m_includeModel.getIncludeList();
if (includes == null || includes.isEmpty()) {
throw new InvalidSettingsException("Select columns to aggregate");
}
String[] names = includes.toArray(new String[includes.size()]);
final int[] colIndices = new int[names.length];
for (int i = 0; i < names.length; i++) {
int index = in.findColumnIndex(names[i]);
if (index < 0) {
throw new InvalidSettingsException("No column \"" + names[i] + "\" in input table");
}
colIndices[i] = index;
}
DataType comType = CollectionCellFactory.getElementType(in, colIndices);
String newColName = m_newColName.getStringValue();
DataType type;
if (m_createSet.getBooleanValue()) {
type = SetCell.getCollectionType(comType);
} else {
type = ListCell.getCollectionType(comType);
}
DataColumnSpecCreator newColSpecC = new DataColumnSpecCreator(newColName, type);
newColSpecC.setElementNames(names);
DataColumnSpec newColSpec = newColSpecC.createSpec();
CellFactory appendFactory = new SingleCellFactory(newColSpec) {
/**
* {@inheritDoc}
*/
@Override
public DataCell getCell(final DataRow row) {
if (m_createSet.getBooleanValue()) {
return CollectionCellFactory.createSetCell(row, colIndices);
} else {
return CollectionCellFactory.createListCell(row, colIndices);
}
}
};
ColumnRearranger rearranger = new ColumnRearranger(in);
if (m_removeCols.getBooleanValue()) {
rearranger.remove(colIndices);
}
rearranger.append(appendFactory);
return rearranger;
}
use of org.knime.core.data.container.CellFactory in project knime-core by knime.
the class One2ManyCol2PMMLNodeModel method configure.
/**
* {@inheritDoc}
*/
@Override
protected PortObjectSpec[] configure(final PortObjectSpec[] inSpecs) throws InvalidSettingsException {
DataTableSpec inDataSpec = (DataTableSpec) inSpecs[0];
m_appendOrgColName = false;
String[] includes = m_includedColumns.applyTo(inDataSpec).getIncludes();
if (includes.length <= 0) {
setWarningMessage("No columns to transfrom selected. Will have no effect!");
}
// check if the values are present in the current spec
if (includes.length > 0) {
checkColumnsSpecs(inDataSpec);
}
CellFactory cellFactory = new One2ManyCellFactory(inDataSpec, Arrays.asList(includes), m_appendOrgColName);
ColumnRearranger rearranger = createRearranger(inDataSpec, cellFactory);
if (m_pmmlOutEnabled) {
PMMLPortObjectSpec pmmlSpec = m_pmmlInEnabled ? (PMMLPortObjectSpec) inSpecs[1] : null;
PMMLPortObjectSpecCreator pmmlSpecCreator = new PMMLPortObjectSpecCreator(pmmlSpec, inDataSpec);
return new PortObjectSpec[] { rearranger.createSpec(), pmmlSpecCreator.createSpec() };
} else {
return new PortObjectSpec[] { rearranger.createSpec() };
}
}
use of org.knime.core.data.container.CellFactory in project knime-core by knime.
the class Pivot2NodeModel method execute.
/**
* {@inheritDoc}
*/
@Override
protected PortObject[] execute(final PortObject[] inData, final ExecutionContext exec) throws Exception {
final BufferedDataTable table = (BufferedDataTable) inData[0];
final List<String> groupAndPivotCols = createAllColumns();
final BufferedDataTable groupTable;
final String orderPivotColumnName;
ExecutionContext groupAndPivotExec = exec.createSubExecutionContext(0.5);
ExecutionContext groupExec = exec.createSubExecutionContext(0.25);
ExecutionContext pivotExec = exec.createSubExecutionContext(0.25);
double progMainTotal = 0.0;
double progMainTableAppendIndexForSort = isProcessInMemory() || isRetainOrder() ? 1.0 : 0.0;
progMainTotal += progMainTableAppendIndexForSort;
double progMainTableGroup = 5.0;
progMainTotal += progMainTableGroup;
double progMainTableInMemSort = isProcessInMemory() ? 3.0 : 0.0;
progMainTotal += progMainTableInMemSort;
double progMainTableGetPivots = 1.0;
progMainTotal += progMainTableGetPivots;
double progMainTableFillPivots = 1.0;
progMainTotal += progMainTableFillPivots;
double progMainTableRestoreSort = isProcessInMemory() || isRetainOrder() ? 1.0 : 0.0;
progMainTotal += progMainTableRestoreSort;
double progMainTableReplaceRowKey = isProcessInMemory() ? 1.0 : 0.0;
progMainTotal += progMainTableReplaceRowKey;
if (isProcessInMemory() || isRetainOrder()) {
exec.setMessage("Keeping row order");
final String retainOrderCol = DataTableSpec.getUniqueColumnName(table.getDataTableSpec(), "#pivot_order#");
// append temp. id column with minimum-aggregation method
final ColumnAggregator[] colAggregators = getColumnAggregators().toArray(new ColumnAggregator[0]);
final Set<String> workingCols = new LinkedHashSet<String>();
workingCols.addAll(groupAndPivotCols);
for (final ColumnAggregator ca : colAggregators) {
workingCols.add(ca.getOriginalColName());
}
workingCols.add(retainOrderCol);
final BufferedDataTable appTable = GroupByTable.appendOrderColumn(groupAndPivotExec.createSubExecutionContext(progMainTableAppendIndexForSort / progMainTotal), table, workingCols, retainOrderCol);
final DataColumnSpec retainOrderColSpec = appTable.getSpec().getColumnSpec(retainOrderCol);
final ColumnAggregator[] aggrs = new ColumnAggregator[colAggregators.length + 1];
System.arraycopy(colAggregators, 0, aggrs, 0, colAggregators.length);
aggrs[colAggregators.length] = new ColumnAggregator(retainOrderColSpec, AggregationMethods.getRowOrderMethod(), true);
orderPivotColumnName = getColumnNamePolicy().createColumName(aggrs[colAggregators.length]);
exec.setMessage("Grouping main table");
final GroupByTable groupByTable = createGroupByTable(groupAndPivotExec.createSubExecutionContext(progMainTableGroup / progMainTotal), appTable, groupAndPivotCols, isProcessInMemory(), false, /* retain order always false; handled by pivoting */
Arrays.asList(aggrs));
// true then sort table by group&pivot columns
if (isProcessInMemory()) {
exec.setMessage("Sorting group table");
final boolean[] sortDirection = new boolean[groupAndPivotCols.size()];
// ensure that missing values are at the end by sorting in ascending order
Arrays.fill(sortDirection, true);
final SortedTable sortedGroupByTable = new SortedTable(groupByTable.getBufferedTable(), groupAndPivotCols, sortDirection, groupAndPivotExec.createSubExecutionContext(progMainTableInMemSort / progMainTotal));
groupTable = sortedGroupByTable.getBufferedDataTable();
} else {
groupTable = groupByTable.getBufferedTable();
}
} else {
exec.setMessage("Grouping main table");
final GroupByTable groupByTable = createGroupByTable(groupAndPivotExec.createSubExecutionContext(progMainTableGroup / progMainTotal), table, groupAndPivotCols, isProcessInMemory(), false, getColumnAggregators());
groupTable = groupByTable.getBufferedTable();
orderPivotColumnName = null;
}
final List<String> pivotCols = m_pivotCols.getIncludeList();
final int[] pivotIdx = new int[pivotCols.size()];
final DataTableSpec groupSpec = groupTable.getSpec();
final Set<String>[] combPivots = createCombinedPivots(groupSpec, pivotCols);
for (int i = 0; i < pivotIdx.length; i++) {
pivotIdx[i] = groupSpec.findColumnIndex(pivotCols.get(i));
}
exec.setProgress("Determining pivots...");
ExecutionContext fillExec = groupAndPivotExec.createSubExecutionContext(progMainTableGetPivots / progMainTotal);
final long groupTableSize = groupTable.size();
long groupIndex = 0;
for (final DataRow row : groupTable) {
for (int i = 0; i < pivotIdx.length; i++) {
if (combPivots[i] == null) {
combPivots[i] = new LinkedHashSet<String>();
}
final DataCell cell = row.getCell(pivotIdx[i]);
if (cell.isMissing()) {
if (!m_ignoreMissValues.getBooleanValue()) {
combPivots[i].add(cell.toString());
}
} else {
combPivots[i].add(cell.toString());
}
}
fillExec.setProgress(groupIndex++ / (double) groupTableSize, String.format("Group \"%s\" (%d/%d)", row.getKey(), groupIndex, groupTableSize));
fillExec.checkCanceled();
}
final Map<String, Integer> pivotStarts = new LinkedHashMap<String, Integer>();
final DataTableSpec outSpec = createOutSpec(groupSpec, combPivots, pivotStarts, orderPivotColumnName);
exec.setProgress("Filling pivot table");
BufferedDataTable pivotTable = fillPivotTable(groupTable, outSpec, pivotStarts, groupAndPivotExec.createSubExecutionContext(progMainTableFillPivots / progMainTotal), orderPivotColumnName);
if (orderPivotColumnName != null) {
exec.setMessage("Restoring row order");
final SortedTable sortedPivotTable = new SortedTable(pivotTable, Arrays.asList(new String[] { orderPivotColumnName }), new boolean[] { true }, groupAndPivotExec.createSubExecutionContext(progMainTableRestoreSort / progMainTotal));
pivotTable = sortedPivotTable.getBufferedDataTable();
final ColumnRearranger colre = new ColumnRearranger(pivotTable.getSpec());
colre.remove(orderPivotColumnName);
pivotTable = exec.createColumnRearrangeTable(pivotTable, colre, exec.createSilentSubProgress(0.0));
}
// temp fix for bug 3286
if (isProcessInMemory()) {
// if process in memory is true, RowKey's needs to be re-computed
final BufferedDataContainer rowkeyBuf = groupAndPivotExec.createSubExecutionContext(progMainTableReplaceRowKey / progMainTotal).createDataContainer(pivotTable.getSpec());
long rowIndex = 0;
for (DataRow row : pivotTable) {
rowkeyBuf.addRowToTable(new DefaultRow(RowKey.createRowKey(rowIndex++), row));
}
rowkeyBuf.close();
pivotTable = rowkeyBuf.getTable();
}
groupAndPivotExec.setProgress(1.0);
/* Fill the 3rd port */
exec.setMessage("Determining pivot totals");
double progPivotTotal = 0.0;
double progPivotGroup = 5.0;
progPivotTotal += progPivotGroup;
double progPivotFillMissing = 1.0;
progPivotTotal += progPivotFillMissing;
double progPivotFillPivots = 1.0;
progPivotTotal += progPivotFillPivots;
double progPivotOverallTotals = m_totalAggregation.getBooleanValue() ? 5.0 : 0.0;
progPivotTotal += progPivotOverallTotals;
// create pivot table only on pivot columns (for grouping)
// perform pivoting: result in single line
final GroupByTable rowGroup = createGroupByTable(pivotExec.createSubExecutionContext(progPivotGroup / progPivotTotal), table, m_pivotCols.getIncludeList(), isProcessInMemory(), isRetainOrder(), getColumnAggregators());
final BufferedDataTable rowGroupTable = rowGroup.getBufferedTable();
// fill group columns with missing cells
final ColumnRearranger colre = new ColumnRearranger(rowGroupTable.getDataTableSpec());
for (int i = 0; i < getGroupByColumns().size(); i++) {
final DataColumnSpec cspec = outSpec.getColumnSpec(i);
final CellFactory factory = new SingleCellFactory(cspec) {
/**
* {@inheritDoc}
*/
@Override
public DataCell getCell(final DataRow row) {
return DataType.getMissingCell();
}
};
colre.insertAt(i, factory);
}
final BufferedDataTable groupedRowTable = exec.createColumnRearrangeTable(rowGroupTable, colre, pivotExec.createSubExecutionContext(progPivotFillMissing / progPivotTotal));
BufferedDataTable pivotRowsTable = fillPivotTable(groupedRowTable, outSpec, pivotStarts, pivotExec.createSubExecutionContext(progPivotFillPivots / progPivotTotal), null);
if (orderPivotColumnName != null) {
final ColumnRearranger colre2 = new ColumnRearranger(pivotRowsTable.getSpec());
colre2.remove(orderPivotColumnName);
pivotRowsTable = exec.createColumnRearrangeTable(pivotRowsTable, colre2, exec.createSilentSubProgress(0.0));
}
// total aggregation without grouping
if (m_totalAggregation.getBooleanValue()) {
@SuppressWarnings("unchecked") final GroupByTable totalGroup = createGroupByTable(pivotExec.createSubExecutionContext(progPivotOverallTotals / progPivotTotal), table, Collections.EMPTY_LIST, isProcessInMemory(), isRetainOrder(), getColumnAggregators());
final BufferedDataTable totalGroupTable = totalGroup.getBufferedTable();
final DataTableSpec pivotsRowsSpec = pivotRowsTable.getSpec();
final DataTableSpec totalGroupSpec = totalGroupTable.getSpec();
final DataTableSpec overallTotalSpec = new DataTableSpec(pivotsRowsSpec, totalGroupSpec);
final BufferedDataContainer buf = exec.createDataContainer(overallTotalSpec);
if (pivotRowsTable.size() > 0) {
final List<DataCell> pivotTotalsCells = new ArrayList<DataCell>();
final DataRow pivotsRow = pivotRowsTable.iterator().next();
for (final DataCell cell : pivotsRow) {
pivotTotalsCells.add(cell);
}
final DataRow totalGroupRow = totalGroupTable.iterator().next();
for (final DataCell cell : totalGroupRow) {
pivotTotalsCells.add(cell);
}
buf.addRowToTable(new DefaultRow(new RowKey("Totals"), pivotTotalsCells));
}
buf.close();
pivotRowsTable = buf.getTable();
}
pivotExec.setProgress(1.0);
/* Fill the 2nd port: important to create this last since it will create
* the final hilite handler (mapping) for port #1 AND #2 (bug 3270) */
exec.setMessage("Creating group totals");
// create group table only on group columns; no pivoting
final BufferedDataTable columnGroupTable = createGroupByTable(groupExec, table, getGroupByColumns()).getBufferedTable();
return new PortObject[] { // pivot table
pivotTable, // group totals
columnGroupTable, // pivot and overall totals
pivotRowsTable };
}
Aggregations