Examples with StreamingSchema water.api.StreamingSchema used on opensource projects

Example 1 with StreamingSchema

use of water.api.StreamingSchema in project h2o-3 by h2oai.

the class Model method testJavaScoring.

public boolean testJavaScoring(Frame data, Frame model_predictions, double rel_epsilon, double abs_epsilon, double fraction) {
    ModelBuilder mb = ModelBuilder.make(_parms.algoName().toLowerCase(), null, null);
    boolean havePojo = mb.havePojo();
    boolean haveMojo = mb.haveMojo();
    Random rnd = RandomUtils.getRNG(data.byteSize());
    assert data.numRows() == model_predictions.numRows();
    Frame fr = new Frame(data);
    boolean computeMetrics = data.vec(_output.responseName()) != null && !data.vec(_output.responseName()).isBad();
    try {
        String[] warns = adaptTestForTrain(fr, true, computeMetrics);
        if (warns.length > 0)
            System.err.println(Arrays.toString(warns));
        // Output is in the model's domain, but needs to be mapped to the scored
        // dataset's domain.
        int[] omap = null;
        if (_output.isClassifier()) {
            Vec actual = fr.vec(_output.responseName());
            // Scored/test domain; can be null
            String[] sdomain = actual == null ? null : actual.domain();
            // Domain of predictions (union of test and train)
            String[] mdomain = model_predictions.vec(0).domain();
            if (sdomain != null && !Arrays.equals(mdomain, sdomain)) {
                // Map from model-domain to scoring-domain
                omap = CategoricalWrappedVec.computeMap(mdomain, sdomain);
            }
        }
        String modelName = JCodeGen.toJavaId(_key.toString());
        boolean preview = false;
        GenModel genmodel = null;
        Vec[] dvecs = fr.vecs();
        Vec[] pvecs = model_predictions.vecs();
        double[] features = null;
        int num_errors = 0;
        int num_total = 0;
        // First try internal POJO via fast double[] API
        if (havePojo) {
            try {
                String java_text = toJava(preview, true);
                Class clz = JCodeGen.compile(modelName, java_text);
                genmodel = (GenModel) clz.newInstance();
            } catch (Exception e) {
                e.printStackTrace();
                throw H2O.fail("Internal POJO compilation failed", e);
            }
            features = MemoryManager.malloc8d(genmodel._names.length);
            double[] predictions = MemoryManager.malloc8d(genmodel.nclasses() + 1);
            // Compare predictions, counting mis-predicts
            for (int row = 0; row < fr.numRows(); row++) {
                // For all rows, single-threaded
                if (rnd.nextDouble() >= fraction)
                    continue;
                num_total++;
                // Native Java API
                for (// Build feature set
                int col = 0; // Build feature set
                col < features.length; // Build feature set
                col++) features[col] = dvecs[col].at(row);
                // POJO predictions
                genmodel.score0(features, predictions);
                for (int col = _output.isClassifier() ? 1 : 0; col < pvecs.length; col++) {
                    // Compare predictions
                    // Load internal scoring predictions
                    double d = pvecs[col].at(row);
                    // map categorical response to scoring domain
                    if (col == 0 && omap != null)
                        d = omap[(int) d];
                    if (!MathUtils.compare(predictions[col], d, abs_epsilon, rel_epsilon)) {
                        if (num_errors++ < 10)
                            System.err.println("Predictions mismatch, row " + row + ", col " + model_predictions._names[col] + ", internal prediction=" + d + ", POJO prediction=" + predictions[col]);
                        break;
                    }
                }
            }
        }
        // EasyPredict API with POJO and/or MOJO
        for (int i = 0; i < 2; ++i) {
            if (i == 0 && !havePojo)
                continue;
            if (i == 1 && !haveMojo)
                continue;
            if (i == 1) {
                // MOJO
                final String filename = modelName + ".zip";
                StreamingSchema ss = new StreamingSchema(getMojo(), filename);
                try {
                    FileOutputStream os = new FileOutputStream(ss.getFilename());
                    ss.getStreamWriter().writeTo(os);
                    os.close();
                    genmodel = MojoModel.load(filename);
                    features = MemoryManager.malloc8d(genmodel._names.length);
                } catch (IOException e1) {
                    e1.printStackTrace();
                    throw H2O.fail("Internal MOJO loading failed", e1);
                } finally {
                    boolean deleted = new File(filename).delete();
                    if (!deleted)
                        Log.warn("Failed to delete the file");
                }
            }
            EasyPredictModelWrapper epmw = new EasyPredictModelWrapper(new EasyPredictModelWrapper.Config().setModel(genmodel).setConvertUnknownCategoricalLevelsToNa(true));
            RowData rowData = new RowData();
            BufferedString bStr = new BufferedString();
            for (int row = 0; row < fr.numRows(); row++) {
                // For all rows, single-threaded
                if (rnd.nextDouble() >= fraction)
                    continue;
                if (genmodel.getModelCategory() == ModelCategory.AutoEncoder)
                    continue;
                // Generate input row
                for (int col = 0; col < features.length; col++) {
                    if (dvecs[col].isString()) {
                        rowData.put(genmodel._names[col], dvecs[col].atStr(bStr, row).toString());
                    } else {
                        double val = dvecs[col].at(row);
                        rowData.put(genmodel._names[col], genmodel._domains[col] == null ? (Double) val : // missing categorical values are kept as NaN, the score0 logic passes it on to bitSetContains()
                        Double.isNaN(val) ? // missing categorical values are kept as NaN, the score0 logic passes it on to bitSetContains()
                        val : //unseen levels are treated as such
                        (int) val < genmodel._domains[col].length ? genmodel._domains[col][(int) val] : "UnknownLevel");
                    }
                }
                // Make a prediction
                AbstractPrediction p;
                try {
                    p = epmw.predict(rowData);
                } catch (PredictException e) {
                    num_errors++;
                    if (num_errors < 20) {
                        System.err.println("EasyPredict threw an exception when predicting row " + rowData);
                        e.printStackTrace();
                    }
                    continue;
                }
                // Convert model predictions and "internal" predictions into the same shape
                double[] expected_preds = new double[pvecs.length];
                double[] actual_preds = new double[pvecs.length];
                for (int col = 0; col < pvecs.length; col++) {
                    // Compare predictions
                    // Load internal scoring predictions
                    double d = pvecs[col].at(row);
                    // map categorical response to scoring domain
                    if (col == 0 && omap != null)
                        d = omap[(int) d];
                    double d2 = Double.NaN;
                    switch(genmodel.getModelCategory()) {
                        case Clustering:
                            d2 = ((ClusteringModelPrediction) p).cluster;
                            break;
                        case Regression:
                            d2 = ((RegressionModelPrediction) p).value;
                            break;
                        case Binomial:
                            BinomialModelPrediction bmp = (BinomialModelPrediction) p;
                            d2 = (col == 0) ? bmp.labelIndex : bmp.classProbabilities[col - 1];
                            break;
                        case Multinomial:
                            MultinomialModelPrediction mmp = (MultinomialModelPrediction) p;
                            d2 = (col == 0) ? mmp.labelIndex : mmp.classProbabilities[col - 1];
                            break;
                        case DimReduction:
                            d2 = ((DimReductionModelPrediction) p).dimensions[col];
                            break;
                    }
                    expected_preds[col] = d;
                    actual_preds[col] = d2;
                }
                // Verify the correctness of the prediction
                num_total++;
                for (int col = genmodel.isClassifier() ? 1 : 0; col < pvecs.length; col++) {
                    if (!MathUtils.compare(actual_preds[col], expected_preds[col], abs_epsilon, rel_epsilon)) {
                        num_errors++;
                        if (num_errors < 20) {
                            System.err.println((i == 0 ? "POJO" : "MOJO") + " EasyPredict Predictions mismatch for row " + rowData);
                            System.err.println("  Expected predictions: " + Arrays.toString(expected_preds));
                            System.err.println("  Actual predictions:   " + Arrays.toString(actual_preds));
                        }
                        break;
                    }
                }
            }
        }
        if (num_errors != 0)
            System.err.println("Number of errors: " + num_errors + (num_errors > 20 ? " (only first 20 are shown)" : "") + " out of " + num_total + " rows tested.");
        return num_errors == 0;
    } finally {
        // Remove temp keys.
        cleanup_adapt(fr, data);
    }
}

Example 2 with StreamingSchema

use of water.api.StreamingSchema in project h2o-3 by h2oai.

the class GBMTest method lowCardinality.

@Test
public void lowCardinality() throws IOException {
    int[] vals = new int[] { 2, 10, 20, 25, 26, 27, 100 };
    double[] maes = new double[vals.length];
    int i = 0;
    for (int nbins_cats : vals) {
        GBMModel model = null;
        GBMModel.GBMParameters parms = new GBMModel.GBMParameters();
        Frame train, train_preds = null;
        Scope.enter();
        train = parse_test_file("smalldata/gbm_test/alphabet_cattest.csv");
        try {
            parms._train = train._key;
            // Train on the outcome
            parms._response_column = "y";
            parms._max_depth = 2;
            parms._min_rows = 1;
            parms._ntrees = 1;
            parms._learn_rate = 1;
            parms._nbins_cats = nbins_cats;
            GBM job = new GBM(parms);
            model = job.trainModel().get();
            StreamingSchema ss = new StreamingSchema(model.getMojo(), "model.zip");
            FileOutputStream fos = new FileOutputStream("model.zip");
            ss.getStreamWriter().writeTo(fos);
            train_preds = model.score(train);
            Assert.assertTrue(model.testJavaScoring(train, train_preds, 1e-15));
            double mae = ModelMetricsRegression.make(train_preds.vec(0), train.vec("y"), gaussian).mae();
            Log.info("Train MAE: " + mae);
            maes[i++] = mae;
            if (//even 25 can do a perfect job
            nbins_cats >= 25)
                Assert.assertEquals(mae, 0, 1e-8);
            else
                Assert.assertNotEquals(mae, 0, 1e-8);
        } finally {
            if (model != null)
                model.delete();
            if (train != null)
                train.remove();
            if (train_preds != null)
                train_preds.remove();
            new File("model.zip").delete();
            Scope.exit();
        }
    }
    Log.info(Arrays.toString(vals));
    Log.info(Arrays.toString(maes));
}