Examples with BiIndex org.apache.ignite.ml.trees.trainers.columnbased.BiIndex used on opensource projects

Example 1 with BiIndex

use of org.apache.ignite.ml.trees.trainers.columnbased.BiIndex in project ignite by apache.

the class DecisionTreesExample method main.

/**
 * Launches example.
 *
 * @param args Program arguments.
 */
public static void main(String[] args) throws IOException {
    System.out.println(">>> Decision trees example started.");
    String igniteCfgPath;
    CommandLineParser parser = new BasicParser();
    String trainingImagesPath;
    String trainingLabelsPath;
    String testImagesPath;
    String testLabelsPath;
    Map<String, String> mnistPaths = new HashMap<>();
    mnistPaths.put(MNIST_TRAIN_IMAGES, "train-images-idx3-ubyte");
    mnistPaths.put(MNIST_TRAIN_LABELS, "train-labels-idx1-ubyte");
    mnistPaths.put(MNIST_TEST_IMAGES, "t10k-images-idx3-ubyte");
    mnistPaths.put(MNIST_TEST_LABELS, "t10k-labels-idx1-ubyte");
    try {
        // Parse the command line arguments.
        CommandLine line = parser.parse(buildOptions(), args);
        if (line.hasOption(MLExamplesCommonArgs.UNATTENDED)) {
            System.out.println(">>> Skipped example execution because 'unattended' mode is used.");
            System.out.println(">>> Decision trees example finished.");
            return;
        }
        igniteCfgPath = line.getOptionValue(CONFIG, DEFAULT_CONFIG);
    } catch (ParseException e) {
        e.printStackTrace();
        return;
    }
    if (!getMNIST(mnistPaths.values())) {
        System.out.println(">>> You should have MNIST dataset in " + MNIST_DIR + " to run this example.");
        return;
    }
    trainingImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + mnistPaths.get(MNIST_TRAIN_IMAGES))).getPath();
    trainingLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + mnistPaths.get(MNIST_TRAIN_LABELS))).getPath();
    testImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + mnistPaths.get(MNIST_TEST_IMAGES))).getPath();
    testLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + mnistPaths.get(MNIST_TEST_LABELS))).getPath();
    try (Ignite ignite = Ignition.start(igniteCfgPath)) {
        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
        int ptsCnt = 60000;
        int featCnt = 28 * 28;
        Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnist(trainingImagesPath, trainingLabelsPath, new Random(123L), ptsCnt);
        Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnist(testImagesPath, testLabelsPath, new Random(123L), 10_000);
        IgniteCache<BiIndex, Double> cache = createBiIndexedCache(ignite);
        loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1, ignite);
        ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer = new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
        System.out.println(">>> Training started");
        long before = System.currentTimeMillis();
        DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
        System.out.println(">>> Training finished in " + (System.currentTimeMillis() - before));
        IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
        Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
        System.out.println(">>> Errs percentage: " + accuracy);
    } catch (IOException e) {
        e.printStackTrace();
    }
    System.out.println(">>> Decision trees example finished.");
}

Example 2 with BiIndex

use of org.apache.ignite.ml.trees.trainers.columnbased.BiIndex in project ignite by apache.

the class DecisionTreesExample method loadVectorsIntoBiIndexedCache.

/**
 * Loads vectors into cache.
 *
 * @param cacheName Name of cache.
 * @param vectorsIter Iterator over vectors to load.
 * @param vectorSize Size of vector.
 * @param ignite Ignite instance.
 */
private static void loadVectorsIntoBiIndexedCache(String cacheName, Iterator<? extends Vector> vectorsIter, int vectorSize, Ignite ignite) {
    try (IgniteDataStreamer<BiIndex, Double> streamer = ignite.dataStreamer(cacheName)) {
        int sampleIdx = 0;
        streamer.perNodeBufferSize(10000);
        while (vectorsIter.hasNext()) {
            org.apache.ignite.ml.math.Vector next = vectorsIter.next();
            for (int i = 0; i < vectorSize; i++) streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
            sampleIdx++;
            if (sampleIdx % 1000 == 0)
                System.out.println(">>> Loaded " + sampleIdx + " vectors.");
        }
    }
}

Example 3 with BiIndex

use of org.apache.ignite.ml.trees.trainers.columnbased.BiIndex in project ignite by apache.

the class ColumnDecisionTreeTrainerBenchmark method tstMNISTBiIndexedCache.

/**
 * Run decision tree classifier on MNIST using bi-indexed cache as a storage for dataset.
 * To run this test rename this method so it starts from 'test'.
 *
 * @throws IOException In case of loading MNIST dataset errors.
 */
public void tstMNISTBiIndexedCache() throws IOException {
    IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
    int ptsCnt = 40_000;
    int featCnt = 28 * 28;
    Properties props = loadMNISTProperties();
    Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnist(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), ptsCnt);
    Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnist(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), 10_000);
    IgniteCache<BiIndex, Double> cache = createBiIndexedCache();
    loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1);
    ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer = new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
    X.println("Training started.");
    long before = System.currentTimeMillis();
    DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
    X.println("Training finished in " + (System.currentTimeMillis() - before));
    IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
    Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
    X.println("Errors percentage: " + accuracy);
    Assert.assertEquals(0, SplitCache.getOrCreate(ignite).size());
    Assert.assertEquals(0, FeaturesCache.getOrCreate(ignite).size());
    Assert.assertEquals(0, ContextCache.getOrCreate(ignite).size());
    Assert.assertEquals(0, ProjectionsCache.getOrCreate(ignite).size());
}

Example 4 with BiIndex

use of org.apache.ignite.ml.trees.trainers.columnbased.BiIndex in project ignite by apache.

the class ColumnDecisionTreeTrainerBenchmark method loadVectorsIntoBiIndexedCache.

/**
 * Load vectors into bi-indexed cache.
 *
 * @param cacheName Name of cache.
 * @param iter Iterator over vectors.
 * @param vectorSize size of vectors.
 */
private void loadVectorsIntoBiIndexedCache(String cacheName, Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
    try (IgniteDataStreamer<BiIndex, Double> streamer = Ignition.localIgnite().dataStreamer(cacheName)) {
        int sampleIdx = 0;
        streamer.perNodeBufferSize(10000);
        while (iter.hasNext()) {
            org.apache.ignite.ml.math.Vector next = iter.next();
            for (int i = 0; i < vectorSize; i++) streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
            sampleIdx++;
            if (sampleIdx % 1000 == 0)
                System.out.println("Loaded: " + sampleIdx + " vectors.");
        }
    }
}