Examples with BufferedDataTable org.knime.core.node.BufferedDataTable used on opensource projects

Example 81 with BufferedDataTable

use of org.knime.core.node.BufferedDataTable in project knime-core by knime.

the class TreeEnsembleModelExtractorNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected PortObject[] execute(final PortObject[] inObjects, final ExecutionContext exec) throws Exception {
    TreeEnsembleModelPortObject treeEnsembleModel = (TreeEnsembleModelPortObject) inObjects[0];
    DataTableSpec outSpec = createOutSpec();
    BufferedDataContainer container = exec.createDataContainer(outSpec, false, 0);
    int nrModels = treeEnsembleModel.getEnsembleModel().getNrModels();
    for (int i = 0; i < nrModels; i++) {
        PMMLPortObject pmmlObject = treeEnsembleModel.createDecisionTreePMMLPortObject(i);
        DataCell cell = PMMLCellFactory.create(pmmlObject.getPMMLValue().toString());
        RowKey key = RowKey.createRowKey(i);
        container.addRowToTable(new DefaultRow(key, cell));
        exec.checkCanceled();
        exec.setProgress(i / (double) nrModels, "Exported model " + (i + 1) + "/" + nrModels);
    }
    container.close();
    return new BufferedDataTable[] { container.getTable() };
}

Example 82 with BufferedDataTable

use of org.knime.core.node.BufferedDataTable in project knime-core by knime.

the class RandomForestRegressionLearnerNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected PortObject[] execute(final PortObject[] inObjects, final ExecutionContext exec) throws Exception {
    BufferedDataTable t = (BufferedDataTable) inObjects[0];
    DataTableSpec spec = t.getDataTableSpec();
    final FilterLearnColumnRearranger learnRearranger = m_configuration.filterLearnColumns(spec);
    String warn = learnRearranger.getWarning();
    BufferedDataTable learnTable = exec.createColumnRearrangeTable(t, learnRearranger, exec.createSubProgress(0.0));
    DataTableSpec learnSpec = learnTable.getDataTableSpec();
    TreeEnsembleModelPortObjectSpec ensembleSpec = m_configuration.createPortObjectSpec(learnSpec);
    ExecutionMonitor readInExec = exec.createSubProgress(0.1);
    ExecutionMonitor learnExec = exec.createSubProgress(0.8);
    ExecutionMonitor outOfBagExec = exec.createSubProgress(0.1);
    TreeDataCreator dataCreator = new TreeDataCreator(m_configuration, learnSpec, learnTable.getRowCount());
    exec.setProgress("Reading data into memory");
    TreeData data = dataCreator.readData(learnTable, m_configuration, readInExec);
    m_hiliteRowSample = dataCreator.getDataRowsForHilite();
    m_viewMessage = dataCreator.getViewMessage();
    String dataCreationWarning = dataCreator.getAndClearWarningMessage();
    if (dataCreationWarning != null) {
        if (warn == null) {
            warn = dataCreationWarning;
        } else {
            warn = warn + "\n" + dataCreationWarning;
        }
    }
    readInExec.setProgress(1.0);
    exec.setMessage("Learning trees");
    TreeEnsembleLearner learner = new TreeEnsembleLearner(m_configuration, data);
    TreeEnsembleModel model;
    try {
        model = learner.learnEnsemble(learnExec);
    } catch (ExecutionException e) {
        Throwable cause = e.getCause();
        if (cause instanceof Exception) {
            throw (Exception) cause;
        }
        throw e;
    }
    TreeEnsembleModelPortObject modelPortObject = new TreeEnsembleModelPortObject(ensembleSpec, model);
    learnExec.setProgress(1.0);
    exec.setMessage("Out of bag prediction");
    TreeEnsemblePredictor outOfBagPredictor = createOutOfBagPredictor(ensembleSpec, modelPortObject, spec);
    outOfBagPredictor.setOutofBagFilter(learner.getRowSamples(), data.getTargetColumn());
    ColumnRearranger outOfBagRearranger = outOfBagPredictor.getPredictionRearranger();
    BufferedDataTable outOfBagTable = exec.createColumnRearrangeTable(t, outOfBagRearranger, outOfBagExec);
    BufferedDataTable colStatsTable = learner.createColumnStatisticTable(exec.createSubExecutionContext(0.0));
    m_ensembleModelPortObject = modelPortObject;
    if (warn != null) {
        setWarningMessage(warn);
    }
    return new PortObject[] { outOfBagTable, colStatsTable, modelPortObject };
}

Example 83 with BufferedDataTable

use of org.knime.core.node.BufferedDataTable in project knime-core by knime.

the class TreeEnsembleClassificationPredictorNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected PortObject[] execute(final PortObject[] inObjects, final ExecutionContext exec) throws Exception {
    TreeEnsembleModelPortObject model = (TreeEnsembleModelPortObject) inObjects[0];
    TreeEnsembleModelPortObjectSpec modelSpec = model.getSpec();
    BufferedDataTable data = (BufferedDataTable) inObjects[1];
    DataTableSpec dataSpec = data.getDataTableSpec();
    final TreeEnsemblePredictor pred = new TreeEnsemblePredictor(modelSpec, model, dataSpec, m_configuration);
    ColumnRearranger rearranger = pred.getPredictionRearranger();
    BufferedDataTable outTable = exec.createColumnRearrangeTable(data, rearranger, exec);
    return new BufferedDataTable[] { outTable };
}

Example 84 with BufferedDataTable

use of org.knime.core.node.BufferedDataTable in project knime-core by knime.

the class TreeEnsembleShrinkerNodeModel method execute.

@Override
protected PortObject[] execute(final PortObject[] inObjects, final ExecutionContext exec) throws Exception {
    TreeEnsembleModel treeEnsemble = ((TreeEnsembleModelPortObject) inObjects[0]).getEnsembleModel();
    TreeEnsembleModelPortObject resultEnsemble;
    int resultSize = m_config.getResultSize(treeEnsemble.getNrModels());
    boolean shrink = true;
    if (!m_config.isResultSizeAutomatic()) {
        // Check if result size is valid
        if (resultSize < 1) {
            // Result size is to small, use 1
            setWarningMessage("The configured result size is smaller than 1, defaulting to 1");
            resultSize = 1;
        } else if (resultSize > treeEnsemble.getNrModels()) {
            // Result size is to big, just keep current ensemble
            setWarningMessage("The configured result size is bigger than the size of the input ensemble, defaulting to the input ensembles size");
            shrink = false;
        } else if (resultSize == treeEnsemble.getNrModels()) {
            // Result size is ensemble size -> we don't need to shrink
            shrink = false;
        }
    }
    // If our result size is not smaller than the current ensemble we don't have to do the following and therefore can save time
    if (shrink) {
        BufferedDataTable inData = (BufferedDataTable) inObjects[1];
        // Create shrinker
        TreeEnsembleShrinker shrinker = new TreeEnsembleShrinker(treeEnsemble, inData, m_config.getTargetColumn(), exec);
        // Shrink ensemble
        if (m_config.isResultSizeAutomatic()) {
            shrinker.autoShrink();
        } else {
            shrinker.shrinkTo(resultSize);
        }
        // Get shrunk ensemble
        TreeEnsembleModel newEnsemble = shrinker.getModel();
        // Push flow variable with archived accuracy
        pushFlowVariableDouble("Tree Ensemble Shrinker Prediction Accuracy", shrinker.getAccuracy());
        // Create port object for tree ensemble
        resultEnsemble = new TreeEnsembleModelPortObject(((TreeEnsembleModelPortObject) inObjects[0]).getSpec(), newEnsemble);
    } else {
        // We did not need to shrink just use input tree ensemble port object
        resultEnsemble = (TreeEnsembleModelPortObject) inObjects[0];
    }
    // Convert tree ensemble port object to PMML
    PMMLPortObject pmmlEnsemble = convertToPmmlEnsemble(resultEnsemble, exec);
    return new PortObject[] { pmmlEnsemble };
}

Example 85 with BufferedDataTable

use of org.knime.core.node.BufferedDataTable in project knime-core by knime.

the class StatisticCalculatorTest method testGeneralStats.

/**
 * Tests common statistics.
 *
 * @throws InvalidSettingsException
 * @throws CanceledExecutionException
 */
@Test
public void testGeneralStats() throws InvalidSettingsException, CanceledExecutionException {
    BufferedDataTable createRandomTable = createRandomTableWithMissingValues(20, 8);
    Skewness skewness = new Skewness();
    MinMax minMax = new MinMax();
    DoubleMinMax doubleMinMax = new DoubleMinMax(true);
    Kurtosis kurtosis = new Kurtosis();
    Mean mean = new Mean();
    Variance variance = new Variance();
    StandardDeviation sDev = new StandardDeviation();
    MissingValue missingValue = new MissingValue();
    SpecialDoubleCells sdc = new SpecialDoubleCells();
    StatisticCalculator statisticCalculator = new StatisticCalculator(createRandomTable.getDataTableSpec(), createRandomTable.getSpec().getColumnNames(), skewness, minMax, kurtosis, mean, variance, missingValue, sDev, doubleMinMax, sdc);
    statisticCalculator.evaluate(createRandomTable, EXEC_CONTEXT);
    Statistics3Table statistics3Table = new Statistics3Table(createRandomTable, false, 0, Collections.<String>emptyList(), EXEC_CONTEXT);
    for (int i = 0; i < createRandomTable.getDataTableSpec().getNumColumns(); i++) {
        String colName = "" + i;
        assertEquals(statistics3Table.getMean(i), mean.getResult(colName), 0.0001);
        assertEquals(statistics3Table.getKurtosis(i), kurtosis.getResult(colName), 0.0001);
        checkValueOrMissingIfNaN(statistics3Table.getMin()[i], minMax.getMin(colName));
        assertEquals(statistics3Table.getMinCells()[i], minMax.getMin(colName));
        checkValueOrMissingIfNaN(doubleMinMax.getMin(colName), statistics3Table.getNonInfMin(i));
        checkValueOrMissingIfNaN(statistics3Table.getMax()[i], minMax.getMax(colName));
        assertEquals(statistics3Table.getMaxCells()[i], minMax.getMax(colName));
        checkValueOrMissingIfNaN(doubleMinMax.getMax(colName), statistics3Table.getNonInfMax(i));
        assertEquals(statistics3Table.getVariance(i), variance.getResult(colName), 0.0001);
        assertEquals(statistics3Table.getStandardDeviation(i), sDev.getResult(colName), 0.0001);
        assertEquals(statistics3Table.getNumberMissingValues(i), missingValue.getNumberMissingValues(colName), 0.0001);
        assertEquals(statistics3Table.getNumberNegativeInfiniteValues(i), sdc.getNumberNegativeInfiniteValues(colName), 0.0001);
        assertEquals(statistics3Table.getNumberPositiveInfiniteValues(i), sdc.getNumberPositiveInfiniteValues(colName), 0.0001);
        assertEquals(statistics3Table.getNumberNaNValues(i), sdc.getNumberNaNValues(colName), 0.0001);
    }
}