Examples with DataContainer org.knime.core.data.container.DataContainer used on opensource projects

Example 31 with DataContainer

use of org.knime.core.data.container.DataContainer in project knime-core by knime.

the class CrossJoinerNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected BufferedDataTable[] execute(final BufferedDataTable[] inData, final ExecutionContext exec) throws Exception {
    boolean showLeft = m_showLeft.getBooleanValue();
    boolean showRight = m_showRight.getBooleanValue();
    int chunksize = m_cacheSize.getIntValue();
    String sep = m_rkseparator.getStringValue();
    DataContainer dc = exec.createDataContainer(createSpec(inData[0].getDataTableSpec(), inData[1].getDataTableSpec(), showLeft, showRight));
    long numOutRows = inData[0].size() * inData[1].size();
    long rowcounter = 0;
    CloseableRowIterator leftit = inData[0].iterator();
    // iterate over all possible chunks of left table
    for (long chunkcount = 0; chunkcount < Math.ceil(inData[0].size() * 1.0 / chunksize); chunkcount++) {
        // read one chunk of left table
        List<DataRow> rowsleft = new LinkedList<DataRow>();
        for (int i = 0; i < chunksize && leftit.hasNext(); i++) {
            rowsleft.add(leftit.next());
            exec.checkCanceled();
        }
        // iterate over all possible chunks of right table
        CloseableRowIterator rightit = inData[1].iterator();
        for (long chunkcount2 = 0; chunkcount2 < Math.ceil(inData[1].size() * 1.0 / chunksize); chunkcount2++) {
            // read  one chunk of right table
            List<DataRow> rowsright = new LinkedList<DataRow>();
            for (int i = 0; i < chunksize && rightit.hasNext(); i++) {
                rowsright.add(rightit.next());
            }
            for (DataRow left : rowsleft) {
                for (DataRow right : rowsright) {
                    DataRow newRow = joinRow(left, right, showLeft, showRight, sep);
                    dc.addRowToTable(newRow);
                    exec.checkCanceled();
                    exec.setProgress(rowcounter++ / (double) numOutRows, "Generating Row " + newRow.getKey().toString());
                }
            }
        }
        rightit.close();
    }
    leftit.close();
    dc.close();
    return new BufferedDataTable[] { (BufferedDataTable) dc.getTable() };
}

Example 32 with DataContainer

use of org.knime.core.data.container.DataContainer in project knime-core by knime.

the class DataTableSpecExtractor method extract.

/**
 * Creates and returns a data table containing the meta information of the given spec. The meta information is
 * referred to as the table data specification and contains information such as column names, types, domain values
 * (list of possible values for categorical columns) and lower and upper bounds. It also contains the information
 * which of the columns have a view handler associated, as well the possible values, if specified. Each column in
 * the given table spec is represented as a row in the returned table.
 *
 * @param spec The spec to extract the meta information from.
 * @return The data table containing the meta information of the given spec.
 */
public DataTable extract(final DataTableSpec spec) {
    List<DataColumnSpec> colSpecs = new ArrayList<DataColumnSpec>();
    if (m_extractColumnNameAsColumn) {
        colSpecs.add(new DataColumnSpecCreator("Column Name", StringCell.TYPE).createSpec());
    }
    colSpecs.add(new DataColumnSpecCreator("Column Type", StringCell.TYPE).createSpec());
    colSpecs.add(new DataColumnSpecCreator("Column Index", IntCell.TYPE).createSpec());
    switch(m_propertyHandlerOutputFormat) {
        case Hide:
            break;
        case Boolean:
            colSpecs.add(new DataColumnSpecCreator("Color Handler", BooleanCell.TYPE).createSpec());
            colSpecs.add(new DataColumnSpecCreator("Size Handler", BooleanCell.TYPE).createSpec());
            colSpecs.add(new DataColumnSpecCreator("Shape Handler", BooleanCell.TYPE).createSpec());
            colSpecs.add(new DataColumnSpecCreator("Filter Handler", BooleanCell.TYPE).createSpec());
            break;
        default:
            colSpecs.add(new DataColumnSpecCreator("Color Handler", StringCell.TYPE).createSpec());
            colSpecs.add(new DataColumnSpecCreator("Size Handler", StringCell.TYPE).createSpec());
            colSpecs.add(new DataColumnSpecCreator("Shape Handler", StringCell.TYPE).createSpec());
            colSpecs.add(new DataColumnSpecCreator("Filter Handler", StringCell.TYPE).createSpec());
    }
    // likely number (important for sorting)
    DataType lowerBoundColType = null;
    DataType upperBoundColType = null;
    for (DataColumnSpec c : spec) {
        DataColumnDomain domain = c.getDomain();
        if (domain.hasLowerBound()) {
            DataType curLowerType = domain.getLowerBound().getType();
            if (lowerBoundColType == null) {
                lowerBoundColType = curLowerType;
            } else {
                lowerBoundColType = DataType.getCommonSuperType(lowerBoundColType, curLowerType);
            }
        }
        if (domain.hasUpperBound()) {
            DataType curUpperType = domain.getUpperBound().getType();
            if (upperBoundColType == null) {
                upperBoundColType = curUpperType;
            } else {
                upperBoundColType = DataType.getCommonSuperType(upperBoundColType, curUpperType);
            }
        }
    }
    lowerBoundColType = lowerBoundColType == null ? GENERIC_DATA_TYPE : lowerBoundColType;
    upperBoundColType = upperBoundColType == null ? GENERIC_DATA_TYPE : upperBoundColType;
    colSpecs.add(new DataColumnSpecCreator("Lower Bound", lowerBoundColType).createSpec());
    colSpecs.add(new DataColumnSpecCreator("Upper Bound", upperBoundColType).createSpec());
    int maxPossValues = 0;
    switch(m_possibleValueOutputFormat) {
        case Hide:
            break;
        case Collection:
            colSpecs.add(new DataColumnSpecCreator("Possible Values", SetCell.getCollectionType(GENERIC_DATA_TYPE)).createSpec());
            break;
        default:
            for (DataColumnSpec c : spec) {
                DataColumnDomain domain = c.getDomain();
                if (domain.hasValues()) {
                    maxPossValues = Math.max(domain.getValues().size(), maxPossValues);
                }
            }
            for (int i = 0; i < maxPossValues; i++) {
                colSpecs.add(new DataColumnSpecCreator("Value " + i, GENERIC_DATA_TYPE).createSpec());
            }
    }
    /* fill it */
    DataContainer dc = new DataContainer(new DataTableSpec(colSpecs.toArray(new DataColumnSpec[colSpecs.size()])));
    for (int i = 0; i < spec.getNumColumns(); i++) {
        DataColumnSpec colSpec = spec.getColumnSpec(i);
        List<DataCell> cells = new ArrayList<DataCell>();
        if (m_extractColumnNameAsColumn) {
            cells.add(new StringCell(colSpec.getName()));
        }
        cells.add(new StringCell(colSpec.getType().toString()));
        cells.add(new IntCell(i));
        ColorHandler colorHandler = colSpec.getColorHandler();
        SizeHandler sizeHandler = colSpec.getSizeHandler();
        ShapeHandler shapeHandler = colSpec.getShapeHandler();
        Optional<FilterHandler> filterHandler = colSpec.getFilterHandler();
        switch(m_propertyHandlerOutputFormat) {
            case Hide:
                break;
            case Boolean:
                cells.add(BooleanCellFactory.create(colorHandler != null));
                cells.add(BooleanCellFactory.create(sizeHandler != null));
                cells.add(BooleanCellFactory.create(shapeHandler != null));
                cells.add(BooleanCellFactory.create(filterHandler.isPresent()));
                break;
            default:
                cells.add(new StringCell(colorHandler == null ? "" : colorHandler.toString()));
                cells.add(new StringCell(sizeHandler == null ? "" : sizeHandler.toString()));
                cells.add(new StringCell(shapeHandler == null ? "" : shapeHandler.toString()));
                cells.add(new StringCell(filterHandler.map(f -> f.toString()).orElse("")));
        }
        DataColumnDomain domain = colSpec.getDomain();
        DataCell lb = domain.getLowerBound();
        if (lb != null) {
            cells.add(lb);
        } else {
            cells.add(DataType.getMissingCell());
        }
        DataCell ub = domain.getUpperBound();
        if (ub != null) {
            cells.add(ub);
        } else {
            cells.add(DataType.getMissingCell());
        }
        switch(m_possibleValueOutputFormat) {
            case Hide:
                break;
            case Collection:
                if (domain.hasValues()) {
                    cells.add(CollectionCellFactory.createSetCell(domain.getValues()));
                } else {
                    cells.add(DataType.getMissingCell());
                }
                break;
            default:
                Set<DataCell> set = domain.hasValues() ? domain.getValues() : Collections.EMPTY_SET;
                int nrColsToWrite = maxPossValues;
                for (DataCell c : set) {
                    cells.add(c);
                    nrColsToWrite -= 1;
                }
                while (nrColsToWrite > 0) {
                    cells.add(DataType.getMissingCell());
                    nrColsToWrite -= 1;
                }
        }
        dc.addRowToTable(new DefaultRow(new RowKey(colSpec.getName()), cells));
    }
    dc.close();
    return dc.getTable();
}

Example 33 with DataContainer

use of org.knime.core.data.container.DataContainer in project knime-core by knime.

the class DoubleVectorCellTest method testSerialization.

@Test
public void testSerialization() throws Exception {
    double[] d = IntStream.range(0, 10000).mapToDouble(i -> i).toArray();
    DataCell cell = DoubleVectorCellFactory.createCell(d);
    DataContainer c = new DataContainer(new DataTableSpec(new DataColumnSpecCreator("foo", DoubleVectorCellFactory.TYPE).createSpec()));
    c.addRowToTable(new DefaultRow("row", cell));
    c.close();
    DataTable table = c.getTable();
    byte[] bytes;
    try (ByteArrayOutputStream output = new ByteArrayOutputStream()) {
        DataContainer.writeToStream(table, output, new ExecutionMonitor());
        output.close();
        bytes = output.toByteArray();
    }
    ContainerTable containerTable;
    try (ByteArrayInputStream input = new ByteArrayInputStream(bytes)) {
        containerTable = DataContainer.readFromStream(input);
    }
    DataCell cell2 = containerTable.iterator().next().getCell(0);
    Assert.assertNotSame(c, cell2);
    Assert.assertEquals(cell, cell2);
}

Example 34 with DataContainer

use of org.knime.core.data.container.DataContainer in project knime-core by knime.

the class HiLiteCollectorNodeModel method getHiLiteAnnotationsTable.

/**
 * @return table with hilit rows first and then all rows with annotations
 */
DataTable getHiLiteAnnotationsTable() {
    DataContainer buf;
    if (m_annotationMap.isEmpty()) {
        buf = new DataContainer(new DataTableSpec());
    } else {
        buf = new DataContainer(new DataTableSpec(createSpecs(null)));
    }
    HiLiteHandler hdl = getInHiLiteHandler(0);
    if (hdl != null) {
        for (RowKey key : hdl.getHiLitKeys()) {
            DataCell[] cells = new DataCell[buf.getTableSpec().getNumColumns()];
            for (int i = 0; i < cells.length; i++) {
                Map<Integer, String> map = m_annotationMap.get(key);
                if (map == null) {
                    cells[i] = DataType.getMissingCell();
                } else {
                    String str = m_annotationMap.get(key).get(i);
                    if (str == null) {
                        cells[i] = DataType.getMissingCell();
                    } else {
                        cells[i] = new StringCell(str);
                    }
                }
            }
            buf.addRowToTable(new DefaultRow(key, cells));
        }
        for (RowKey key : m_annotationMap.keySet()) {
            if (!hdl.isHiLit(key)) {
                DataCell[] cells = new DataCell[buf.getTableSpec().getNumColumns()];
                for (int i = 0; i < cells.length; i++) {
                    String str = m_annotationMap.get(key).get(i);
                    if (str == null) {
                        cells[i] = DataType.getMissingCell();
                    } else {
                        cells[i] = new StringCell(str);
                    }
                }
                buf.addRowToTable(new DefaultRow(key, cells));
            }
        }
    }
    buf.close();
    return buf.getTable();
}

Example 35 with DataContainer

use of org.knime.core.data.container.DataContainer in project knime-core by knime.

the class DataTableSpecView method createTableSpecTable.

private DataTable createTableSpecTable(final DataTableSpec spec) {
    if (spec != null) {
        DataTableSpecExtractor e = new DataTableSpecExtractor();
        e.setExtractColumnNameAsColumn(false);
        e.setPossibleValueOutputFormat(PossibleValueOutputFormat.Columns);
        e.setPropertyHandlerOutputFormat(PropertyHandlerOutputFormat.ToString);
        return e.extract(spec);
    } else {
        String[] names = new String[] { "No outgoing table spec" };
        DataType[] types = new DataType[] { StringCell.TYPE };
        DataContainer result = new DataContainer(new DataTableSpec(names, types));
        result.close();
        return result.getTable();
    }
}