Examples with ThreadPool org.knime.core.util.ThreadPool used on opensource projects

Example 1 with ThreadPool

use of org.knime.core.util.ThreadPool in project knime-core by knime.

the class PathProximity method calculatePathProximities.

public ProximityMatrix calculatePathProximities(final ExecutionContext exec) throws InterruptedException, CanceledExecutionException {
    final ThreadPool tp = KNIMEConstants.GLOBAL_THREAD_POOL;
    final int procCount = 3 * Runtime.getRuntime().availableProcessors() / 2;
    final Semaphore semaphore = new Semaphore(procCount);
    final AtomicReference<Throwable> proxThrowableRef = new AtomicReference<Throwable>();
    // The path proximity matrix is not symmetric if applied for a single table
    // therefore we have to use the two table approach even it is only a single table
    ProximityMatrix proximityMatrix = new TwoTablesProximityMatrix(m_tables[0], m_tables[1]);
    final int nrTrees = m_modelPO.getEnsembleModel().getNrModels();
    final Future<?>[] calcFutures = new Future<?>[nrTrees];
    exec.setProgress(0, "Starting proximity calculation per tree.");
    for (int i = 0; i < nrTrees; i++) {
        semaphore.acquire();
        finishedTree(i, exec, nrTrees);
        checkThrowable(proxThrowableRef);
        ExecutionMonitor subExec = exec.createSubProgress(0.0);
        calcFutures[i] = tp.enqueue(new PathProximityCalcRunnable(i, proximityMatrix, semaphore, proxThrowableRef, subExec));
    }
    for (int i = 0; i < procCount; i++) {
        semaphore.acquire();
        finishedTree(nrTrees - procCount + i, exec, nrTrees);
    }
    for (Future<?> future : calcFutures) {
        try {
            future.get();
        } catch (Exception e) {
            proxThrowableRef.compareAndSet(null, e);
        }
    }
    checkThrowable(proxThrowableRef);
    proximityMatrix.normalize(1.0 / nrTrees);
    return proximityMatrix;
}

Example 2 with ThreadPool

use of org.knime.core.util.ThreadPool in project knime-core by knime.

the class NodeExecutionJobManagerPool method collectJobManagerFactories.

private static void collectJobManagerFactories() {
    managerFactories = new LinkedHashMap<String, NodeExecutionJobManagerFactory>();
    IExtensionRegistry registry = Platform.getExtensionRegistry();
    IExtensionPoint point = registry.getExtensionPoint(EXT_POINT_ID);
    if (point == null) {
        // let's throw in the default manager - otherwise things fail badly
        managerFactories.put(getDefaultJobManagerFactory().getID(), getDefaultJobManagerFactory());
        LOGGER.error("Invalid extension point: " + EXT_POINT_ID);
        throw new IllegalStateException("ACTIVATION ERROR: " + " --> Invalid extension point: " + EXT_POINT_ID);
    }
    for (IConfigurationElement elem : point.getConfigurationElements()) {
        String jobMgr = elem.getAttribute(EXT_POINT_ATTR_JOBMGR);
        String decl = elem.getDeclaringExtension().getUniqueIdentifier();
        if (jobMgr == null || jobMgr.isEmpty()) {
            LOGGER.error("The extension '" + decl + "' doesn't provide the required attribute '" + EXT_POINT_ATTR_JOBMGR + "'");
            LOGGER.error("Extension " + decl + " ignored.");
            continue;
        }
        // try instantiating the job manager.
        NodeExecutionJobManagerFactory instance = null;
        try {
            // TODO: THE THREADED MANAGER NEEDS TO BE RE-WRITTEN!
            if (jobMgr.equals(getDefaultJobManagerFactory().getID())) {
                instance = getDefaultJobManagerFactory();
            } else {
                instance = (NodeExecutionJobManagerFactory) elem.createExecutableExtension(EXT_POINT_ATTR_JOBMGR);
            }
        } catch (UnsatisfiedLinkError ule) {
            // in case an implementation tries to load an external lib
            // when the factory class gets loaded
            LOGGER.error("Unable to load a library required for '" + jobMgr + "'");
            LOGGER.error("Either specify it in the -Djava.library.path " + "option at the program's command line, or");
            LOGGER.error("include it in the LD_LIBRARY_PATH variable.");
            LOGGER.error("Extension " + jobMgr + " ('" + decl + "') ignored.", ule);
        } catch (CoreException ex) {
            Throwable cause = ex.getStatus().getException();
            if (cause != null) {
                LOGGER.error("Problems during initialization of job manager (with id '" + jobMgr + "'): " + cause.getMessage(), ex);
                if (decl != null) {
                    LOGGER.error("Extension " + decl + " ignored.");
                }
            } else {
                LOGGER.error("Problems during initialization of job manager (with id '" + jobMgr + "')", ex);
                if (decl != null) {
                    LOGGER.error("Extension " + decl + " ignored.");
                }
            }
        } catch (Throwable t) {
            LOGGER.error("Problems during initialization of job manager (with id '" + jobMgr + "')", t);
            if (decl != null) {
                LOGGER.error("Extension " + decl + " ignored.");
            }
        }
        if (instance != null) {
            /*
                 * make sure the ThreadedJobManagerFactory is always the first
                 * in the list
                 */
            if ((instance instanceof ThreadPool) && managerFactories.size() > 0) {
                Map<String, NodeExecutionJobManagerFactory> old = managerFactories;
                managerFactories = new LinkedHashMap<String, NodeExecutionJobManagerFactory>();
                managerFactories.put(instance.getID(), instance);
                for (Map.Entry<String, NodeExecutionJobManagerFactory> e : old.entrySet()) {
                    managerFactories.put(e.getKey(), e.getValue());
                }
            } else {
                managerFactories.put(instance.getID(), instance);
            }
        }
    }
}

Example 3 with ThreadPool

use of org.knime.core.util.ThreadPool in project knime-core by knime.

the class LKGradientBoostedTreesLearner method learn.

/**
 * {@inheritDoc}
 *
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Override
public MultiClassGradientBoostedTreesModel learn(final ExecutionMonitor exec) throws CanceledExecutionException, InterruptedException, ExecutionException {
    final TreeData data = getData();
    final TreeTargetNominalColumnData target = (TreeTargetNominalColumnData) data.getTargetColumn();
    final NominalValueRepresentation[] classNomVals = target.getMetaData().getValues();
    final int numClasses = classNomVals.length;
    final String[] classLabels = new String[numClasses];
    final int nrModels = getConfig().getNrModels();
    final int nrRows = target.getNrRows();
    final TreeModelRegression[][] models = new TreeModelRegression[nrModels][numClasses];
    final ArrayList<ArrayList<Map<TreeNodeSignature, Double>>> coefficientMaps = new ArrayList<ArrayList<Map<TreeNodeSignature, Double>>>(nrModels);
    // variables for parallelization
    final ThreadPool tp = KNIMEConstants.GLOBAL_THREAD_POOL;
    final AtomicReference<Throwable> learnThrowableRef = new AtomicReference<Throwable>();
    final int procCount = 3 * Runtime.getRuntime().availableProcessors() / 2;
    exec.setMessage("Transforming problem");
    // transform the original k class classification problem into k regression problems
    final TreeData[] actual = new TreeData[numClasses];
    for (int i = 0; i < numClasses; i++) {
        final double[] newTarget = calculateNewTarget(target, i);
        actual[i] = createNumericDataFromArray(newTarget);
        classLabels[i] = classNomVals[i].getNominalValue();
    }
    final RandomData rd = getConfig().createRandomData();
    final double[][] previousFunctions = new double[numClasses][nrRows];
    TreeNodeSignatureFactory signatureFactory = null;
    final int maxLevels = getConfig().getMaxLevels();
    if (maxLevels < TreeEnsembleLearnerConfiguration.MAX_LEVEL_INFINITE) {
        int capacity = IntMath.pow(2, maxLevels - 1);
        signatureFactory = new TreeNodeSignatureFactory(capacity);
    } else {
        signatureFactory = new TreeNodeSignatureFactory();
    }
    exec.setMessage("Learn trees");
    for (int i = 0; i < nrModels; i++) {
        final Semaphore semaphore = new Semaphore(procCount);
        final ArrayList<Map<TreeNodeSignature, Double>> classCoefficientMaps = new ArrayList<Map<TreeNodeSignature, Double>>(numClasses);
        // prepare calculation of pseudoResiduals
        final double[][] probs = new double[numClasses][nrRows];
        for (int r = 0; r < nrRows; r++) {
            double sumExpF = 0;
            for (int j = 0; j < numClasses; j++) {
                sumExpF += Math.exp(previousFunctions[j][r]);
            }
            for (int j = 0; j < numClasses; j++) {
                probs[j][r] = Math.exp(previousFunctions[j][r]) / sumExpF;
            }
        }
        final Future<?>[] treeCoefficientMapPairs = new Future<?>[numClasses];
        for (int j = 0; j < numClasses; j++) {
            checkThrowable(learnThrowableRef);
            final RandomData rdSingle = TreeEnsembleLearnerConfiguration.createRandomData(rd.nextLong(Long.MIN_VALUE, Long.MAX_VALUE));
            final ExecutionMonitor subExec = exec.createSubProgress(0.0);
            semaphore.acquire();
            treeCoefficientMapPairs[j] = tp.enqueue(new TreeLearnerCallable(rdSingle, probs[j], actual[j], subExec, numClasses, previousFunctions[j], semaphore, learnThrowableRef, signatureFactory));
        }
        for (int j = 0; j < numClasses; j++) {
            checkThrowable(learnThrowableRef);
            semaphore.acquire();
            final Pair<TreeModelRegression, Map<TreeNodeSignature, Double>> pair = (Pair<TreeModelRegression, Map<TreeNodeSignature, Double>>) treeCoefficientMapPairs[j].get();
            models[i][j] = pair.getFirst();
            classCoefficientMaps.add(pair.getSecond());
            semaphore.release();
        }
        checkThrowable(learnThrowableRef);
        coefficientMaps.add(classCoefficientMaps);
        exec.setProgress((double) i / nrModels, "Finished level " + i + "/" + nrModels);
    }
    return MultiClassGradientBoostedTreesModel.createMultiClassGradientBoostedTreesModel(getConfig(), data.getMetaData(), models, data.getTreeType(), 0, numClasses, coefficientMaps, classLabels);
}

Example 4 with ThreadPool

use of org.knime.core.util.ThreadPool in project knime-core by knime.

the class Combinations method enumerate.

/**
 * Enumerates all combinations and calls the callback for each combination.
 * The enumeration process is carried out in parallel by several threads.
 *
 * @param threads the number of threads to use
 * @param callback the callback class
 *
 * @throws InterruptedException if this thread was interrupted while waiting
 *             for the enumeration to finish
 */
public void enumerate(final int threads, final Callback callback) throws InterruptedException {
    final long pSize = m_nrOfCombinations / threads;
    ThreadPool pool = new ThreadPool(threads - 1);
    for (int i = 0; i < threads; i++) {
        final int k = i;
        Runnable r = new Runnable() {

            @Override
            public void run() {
                enumerate(k * pSize, (k + 1) * pSize - 1, callback);
            }
        };
        pool.submit(r);
    }
    enumerate((threads - 1) * pSize, m_nrOfCombinations, callback);
    pool.waitForTermination();
}

Example 5 with ThreadPool

use of org.knime.core.util.ThreadPool in project knime-core by knime.

the class SVMLearnerNodeModel method execute.

/**
 * {@inheritDoc}
 */
@Override
protected PortObject[] execute(final PortObject[] inData, final ExecutionContext exec) throws Exception {
    BufferedDataTable inTable = (BufferedDataTable) inData[0];
    DataTableSpec inSpec = inTable.getDataTableSpec();
    LearnColumnsAndColumnRearrangerTuple tuple = createTrainTableColumnRearranger(inSpec);
    // no progress needed as constant operation (column removal only)
    BufferedDataTable trainTable = exec.createColumnRearrangeTable(inTable, tuple.getTrainingRearranger(), exec.createSubProgress(0.0));
    DataTableSpec trainSpec = trainTable.getDataTableSpec();
    int classpos = trainSpec.findColumnIndex(m_classcol.getStringValue());
    CheckUtils.checkArgument(classpos >= 0, "Selected class column not found: " + m_classcol.getStringValue());
    // convert input data
    ArrayList<DoubleVector> inputData = new ArrayList<DoubleVector>();
    List<String> categories = new ArrayList<String>();
    StringValue classvalue = null;
    for (DataRow row : trainTable) {
        exec.checkCanceled();
        ArrayList<Double> values = new ArrayList<Double>();
        boolean add = true;
        for (int i = 0; i < row.getNumCells(); i++) {
            if (row.getCell(i).isMissing()) {
                add = false;
                break;
            }
            if (i != classpos) {
                DoubleValue cell = (DoubleValue) row.getCell(i);
                values.add(cell.getDoubleValue());
            } else {
                classvalue = (StringValue) row.getCell(classpos);
                if (!categories.contains(classvalue.getStringValue())) {
                    categories.add(classvalue.getStringValue());
                }
            }
        }
        if (add) {
            @SuppressWarnings("null") final String nonNullClassValue = classvalue.getStringValue();
            inputData.add(new DoubleVector(row.getKey(), values, nonNullClassValue));
        }
    }
    if (categories.isEmpty()) {
        throw new Exception("No categories found to train SVM. " + "Possibly an empty input table was provided.");
    }
    DoubleVector[] inputDataArr = new DoubleVector[inputData.size()];
    inputDataArr = inputData.toArray(inputDataArr);
    Kernel kernel = KernelFactory.getKernel(m_kernelType);
    Vector<SettingsModelDouble> kernelparams = m_kernelParameters.get(m_kernelType);
    for (int i = 0; i < kernel.getNumberParameters(); ++i) {
        kernel.setParameter(i, kernelparams.get(i).getDoubleValue());
    }
    final Svm[] svms = new Svm[categories.size()];
    exec.setMessage("Training SVM");
    final BinarySvmRunnable[] bst = new BinarySvmRunnable[categories.size()];
    for (int i = 0; i < categories.size(); i++) {
        bst[i] = new BinarySvmRunnable(inputDataArr, categories.get(i), kernel, m_paramC.getDoubleValue(), exec.createSubProgress((1.0 / categories.size())));
    }
    ThreadPool pool = KNIMEConstants.GLOBAL_THREAD_POOL;
    final Future<?>[] fut = new Future<?>[bst.length];
    KNIMETimer timer = KNIMETimer.getInstance();
    TimerTask timerTask = new TimerTask() {

        @Override
        public void run() {
            try {
                exec.checkCanceled();
            } catch (final CanceledExecutionException ce) {
                for (int i = 0; i < fut.length; i++) {
                    if (fut[i] != null) {
                        fut[i].cancel(true);
                    }
                }
                super.cancel();
            }
        }
    };
    timer.scheduleAtFixedRate(timerTask, 0, 3000);
    for (int i = 0; i < bst.length; i++) {
        fut[i] = pool.enqueue(bst[i]);
    }
    try {
        pool.runInvisible(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                for (int i = 0; i < fut.length; ++i) {
                    fut[i].get();
                    bst[i].ok();
                    if (bst[i].getWarning() != null) {
                        setWarningMessage(bst[i].getWarning());
                    }
                    svms[i] = bst[i].getSvm();
                }
                return null;
            }
        });
    } catch (Exception ex) {
        exec.checkCanceled();
        Throwable t = ex;
        if (ex instanceof ExecutionException) {
            t = ex.getCause();
        }
        if (t instanceof Exception) {
            throw (Exception) t;
        } else {
            throw new Exception(t);
        }
    } finally {
        for (int i = 0; i < fut.length; i++) {
            fut[i].cancel(true);
        }
        timerTask.cancel();
    }
    // the optional PMML input (can be null)
    PMMLPortObject inPMMLPort = m_pmmlInEnabled ? (PMMLPortObject) inData[1] : null;
    // create the outgoing PMML spec
    PMMLPortObjectSpecCreator specCreator = new PMMLPortObjectSpecCreator(inPMMLPort, inSpec);
    specCreator.setLearningCols(trainSpec);
    specCreator.setTargetCol(trainSpec.getColumnSpec(m_classcol.getStringValue()));
    // create the outgoing PMML port object
    PMMLPortObject outPMMLPort = new PMMLPortObject(specCreator.createSpec(), inPMMLPort, inSpec);
    outPMMLPort.addModelTranslater(new PMMLSVMTranslator(categories, Arrays.asList(svms), kernel));
    m_svms = svms;
    return new PortObject[] { outPMMLPort };
}