Examples with Chunk water.fvec.Chunk used on opensource projects

Example 26 with Chunk

use of water.fvec.Chunk in project h2o-3 by h2oai.

the class DataInfoTestAdapt method checkFrame.

private void checkFrame(final Frame checkMe, final Frame gold) {
    Vec[] vecs = new Vec[checkMe.numCols() + gold.numCols()];
    new MRTask() {

        @Override
        public void map(Chunk[] cs) {
            int off = checkMe.numCols();
            for (int i = 0; i < off; ++i) {
                for (int r = 0; r < cs[0]._len; ++r) {
                    double check = cs[i].atd(r);
                    double gold = cs[i + off].atd(r);
                    if (Math.abs(check - gold) > 1e-12)
                        throw new RuntimeException("bonk");
                }
            }
        }
    }.doAll(vecs);
}

Example 27 with Chunk

use of water.fvec.Chunk in project h2o-3 by h2oai.

the class XValPredictionsCheck method checkModel.

void checkModel(Model m, Vec foldId, int nclass) {
    if (// DRF does out of back instead of true training, nobs might be different
    !(m instanceof DRFModel))
        assertEquals(m._output._training_metrics._nobs, m._output._cross_validation_metrics._nobs);
    m.delete();
    m.deleteCrossValidationModels();
    Key[] xvalKeys = m._output._cross_validation_predictions;
    Key xvalKey = m._output._cross_validation_holdout_predictions_frame_id;
    final int[] id = new int[1];
    for (Key k : xvalKeys) {
        Frame preds = DKV.getGet(k);
        assert preds.numRows() == foldId.length();
        Vec[] vecs = new Vec[nclass + 1];
        vecs[0] = foldId;
        if (nclass == 1)
            vecs[1] = preds.anyVec();
        else
            System.arraycopy(preds.vecs(ArrayUtils.range(1, nclass)), 0, vecs, 1, nclass);
        new MRTask() {

            @Override
            public void map(Chunk[] cs) {
                Chunk foldId = cs[0];
                for (int r = 0; r < cs[0]._len; ++r) if (foldId.at8(r) != id[0])
                    for (int i = 1; i < cs.length; ++i) // no prediction for this row!
                    assert cs[i].atd(r) == 0;
            }
        }.doAll(vecs);
        id[0]++;
        preds.delete();
    }
    xvalKey.remove();
}

Example 28 with Chunk

use of water.fvec.Chunk in project h2o-3 by h2oai.

the class DeepLearningGradientCheck method gradientCheck.

@Test
public void gradientCheck() {
    Frame tfr = null;
    DeepLearningModel dl = null;
    try {
        tfr = parse_test_file("smalldata/glm_test/cancar_logIn.csv");
        for (String s : new String[] { "Merit", "Class" }) {
            Vec f = tfr.vec(s).toCategoricalVec();
            tfr.remove(s).remove();
            tfr.add(s, f);
        }
        DKV.put(tfr);
        tfr.add("Binary", tfr.anyVec().makeZero());
        new MRTask() {

            public void map(Chunk[] c) {
                for (int i = 0; i < c[0]._len; ++i) if (c[0].at8(i) == 1)
                    c[1].set(i, 1);
            }
        }.doAll(tfr.vecs(new String[] { "Class", "Binary" }));
        Vec cv = tfr.vec("Binary").toCategoricalVec();
        tfr.remove("Binary").remove();
        tfr.add("Binary", cv);
        DKV.put(tfr);
        Random rng = new Random(0xDECAF);
        int count = 0;
        int failedcount = 0;
        double maxRelErr = 0;
        double meanRelErr = 0;
        for (DistributionFamily dist : new DistributionFamily[] { DistributionFamily.gaussian, DistributionFamily.laplace, DistributionFamily.quantile, DistributionFamily.huber, // DistributionFamily.modified_huber,
        DistributionFamily.gamma, DistributionFamily.poisson, DistributionFamily.AUTO, DistributionFamily.tweedie, DistributionFamily.multinomial, DistributionFamily.bernoulli }) {
            for (DeepLearningParameters.Activation act : new DeepLearningParameters.Activation[] { //            DeepLearningParameters.Activation.ExpRectifier,
            DeepLearningParameters.Activation.Tanh, DeepLearningParameters.Activation.Rectifier }) {
                for (String response : new String[] { //binary classification
                "Binary", //multi-class
                "Class", //regression
                "Cost" }) {
                    for (boolean adaptive : new boolean[] { true, false }) {
                        for (int miniBatchSize : new int[] { 1 }) {
                            if (response.equals("Class")) {
                                if (dist != DistributionFamily.multinomial && dist != DistributionFamily.AUTO)
                                    continue;
                            } else if (response.equals("Binary")) {
                                if (dist != DistributionFamily.modified_huber && dist != DistributionFamily.bernoulli && dist != DistributionFamily.AUTO)
                                    continue;
                            } else {
                                if (dist == DistributionFamily.multinomial || dist == DistributionFamily.modified_huber || dist == DistributionFamily.bernoulli)
                                    continue;
                            }
                            DeepLearningParameters parms = new DeepLearningParameters();
                            parms._huber_alpha = rng.nextDouble() + 0.1;
                            parms._tweedie_power = 1.01 + rng.nextDouble() * 0.9;
                            parms._quantile_alpha = 0.05 + rng.nextDouble() * 0.9;
                            parms._train = tfr._key;
                            //converge to a reasonable model to avoid too large gradients
                            parms._epochs = 100;
                            parms._l1 = 1e-3;
                            parms._l2 = 1e-3;
                            parms._force_load_balance = false;
                            parms._hidden = new int[] { 10, 10, 10 };
                            //otherwise we introduce small bprop errors
                            parms._fast_mode = false;
                            parms._response_column = response;
                            parms._distribution = dist;
                            parms._max_w2 = 10;
                            parms._seed = 0xaaabbb;
                            parms._activation = act;
                            parms._adaptive_rate = adaptive;
                            parms._rate = 1e-4;
                            parms._momentum_start = 0.9;
                            parms._momentum_stable = 0.99;
                            parms._mini_batch_size = miniBatchSize;
                            //                DeepLearningModelInfo.gradientCheck = null;
                            //tell it what gradient to collect
                            DeepLearningModelInfo.gradientCheck = new DeepLearningModelInfo.GradientCheck(0, 0, 0);
                            // Build a first model; all remaining models should be equal
                            DeepLearning job = new DeepLearning(parms);
                            try {
                                dl = job.trainModel().get();
                                boolean classification = response.equals("Class") || response.equals("Binary");
                                if (!classification) {
                                    Frame p = dl.score(tfr);
                                    hex.ModelMetrics mm = hex.ModelMetrics.getFromDKV(dl, tfr);
                                    double resdev = ((ModelMetricsRegression) mm)._mean_residual_deviance;
                                    Log.info("Mean residual deviance: " + resdev);
                                    p.delete();
                                }
                                //golden version
                                DeepLearningModelInfo modelInfo = IcedUtils.deepCopy(dl.model_info());
                                //                Log.info(modelInfo.toStringAll());
                                long before = dl.model_info().checksum_impl();
                                float meanLoss = 0;
                                // loop over every row in the dataset and check that the predictions
                                for (int rId = 0; rId < tfr.numRows(); rId += 1) /*miniBatchSize*/
                                {
                                    // start from scratch - with a clean model
                                    dl.set_model_info(IcedUtils.deepCopy(modelInfo));
                                    final DataInfo di = dl.model_info().data_info();
                                    // populate miniBatch (consecutive rows)
                                    final DataInfo.Row[] rowsMiniBatch = new DataInfo.Row[miniBatchSize];
                                    for (int i = 0; i < rowsMiniBatch.length; ++i) {
                                        if (0 <= rId + i && rId + i < tfr.numRows()) {
                                            rowsMiniBatch[i] = new FrameTask.ExtractDenseRow(di, rId + i).doAll(di._adaptedFrame)._row;
                                        }
                                    }
                                    // loss at weight
                                    long cs = dl.model_info().checksum_impl();
                                    double loss = dl.meanLoss(rowsMiniBatch);
                                    assert (cs == before);
                                    assert (before == dl.model_info().checksum_impl());
                                    meanLoss += loss;
                                    for (int layer = 0; layer <= parms._hidden.length; ++layer) {
                                        int rows = dl.model_info().get_weights(layer).rows();
                                        assert (dl.model_info().get_biases(layer).size() == rows);
                                        for (int row = 0; row < rows; ++row) {
                                            //check bias
                                            if (true) {
                                                // start from scratch - with a clean model
                                                dl.set_model_info(IcedUtils.deepCopy(modelInfo));
                                                // do one forward propagation pass (and fill the mini-batch gradients -> set training=true)
                                                Neurons[] neurons = DeepLearningTask.makeNeuronsForTraining(dl.model_info());
                                                double[] responses = new double[miniBatchSize];
                                                double[] offsets = new double[miniBatchSize];
                                                int n = 0;
                                                for (DataInfo.Row myRow : rowsMiniBatch) {
                                                    if (myRow == null)
                                                        continue;
                                                    ((Neurons.Input) neurons[0]).setInput(-1, myRow.numIds, myRow.numVals, myRow.nBins, myRow.binIds, n);
                                                    responses[n] = myRow.response(0);
                                                    offsets[n] = myRow.offset;
                                                    n++;
                                                }
                                                DeepLearningTask.fpropMiniBatch(-1, /*seed doesn't matter*/
                                                neurons, dl.model_info(), null, true, /*training*/
                                                responses, offsets, n);
                                                // check that we didn't change the model's weights/biases
                                                long after = dl.model_info().checksum_impl();
                                                assert (after == before);
                                                // record the gradient since gradientChecking is enabled
                                                //tell it what gradient to collect
                                                DeepLearningModelInfo.gradientCheck = new DeepLearningModelInfo.GradientCheck(layer, row, -1);
                                                //update the weights and biases
                                                DeepLearningTask.bpropMiniBatch(neurons, n);
                                                assert (before != dl.model_info().checksum_impl());
                                                // reset the model back to the trained model
                                                dl.set_model_info(IcedUtils.deepCopy(modelInfo));
                                                assert (before == dl.model_info().checksum_impl());
                                                double bpropGradient = DeepLearningModelInfo.gradientCheck.gradient;
                                                // FIXME: re-enable this once the loss is computed from the de-standardized prediction/response
                                                //                    double actualResponse=myRow.response[0];
                                                //                    double predResponseLinkSpace = neurons[neurons.length-1]._a.get(0);
                                                //                    if (di._normRespMul != null) {
                                                //                      bpropGradient /= di._normRespMul[0]; //no shift for gradient
                                                //                      actualResponse = (actualResponse / di._normRespMul[0] + di._normRespSub[0]);
                                                //                      predResponseLinkSpace = (predResponseLinkSpace / di._normRespMul[0] + di._normRespSub[0]);
                                                //                    }
                                                //                    bpropGradient *= new Distribution(parms._distribution).gradient(actualResponse, predResponseLinkSpace);
                                                final double bias = dl.model_info().get_biases(layer).get(row);
                                                //don't make the weight deltas too small, or the float weights "won't notice"
                                                double eps = 1e-4 * Math.abs(bias);
                                                if (eps == 0)
                                                    eps = 1e-6;
                                                // loss at bias + eps
                                                dl.model_info().get_biases(layer).set(row, bias + eps);
                                                double up = dl.meanLoss(rowsMiniBatch);
                                                // loss at bias - eps
                                                dl.model_info().get_biases(layer).set(row, bias - eps);
                                                double down = dl.meanLoss(rowsMiniBatch);
                                                if (Math.abs(up - down) / Math.abs(up + down) < 1e-8) {
                                                    //relative change in loss function is too small -> skip
                                                    continue;
                                                }
                                                double gradient = ((up - down) / (2. * eps));
                                                double relError = 2 * Math.abs(bpropGradient - gradient) / (Math.abs(gradient) + Math.abs(bpropGradient));
                                                count++;
                                                // if either gradient is tiny, check if both are tiny
                                                if (Math.abs(gradient) < 1e-7 || Math.abs(bpropGradient) < 1e-7) {
                                                    //all good
                                                    if (Math.abs(bpropGradient - gradient) < 1e-7)
                                                        continue;
                                                }
                                                meanRelErr += relError;
                                                if (relError > MAX_TOLERANCE) {
                                                    Log.info("\nDistribution: " + dl._parms._distribution);
                                                    Log.info("\nRow: " + rId);
                                                    Log.info("bias (layer " + layer + ", row " + row + "): " + bias + " +/- " + eps);
                                                    Log.info("loss: " + loss);
                                                    Log.info("losses up/down: " + up + " / " + down);
                                                    Log.info("=> Finite differences gradient: " + gradient);
                                                    Log.info("=> Back-propagation gradient  : " + bpropGradient);
                                                    Log.info("=> Relative error             : " + PrettyPrint.formatPct(relError));
                                                    failedcount++;
                                                }
                                            }
                                            int cols = dl.model_info().get_weights(layer).cols();
                                            for (int col = 0; col < cols; ++col) {
                                                if (rng.nextFloat() >= SAMPLE_RATE)
                                                    continue;
                                                // start from scratch - with a clean model
                                                dl.set_model_info(IcedUtils.deepCopy(modelInfo));
                                                // do one forward propagation pass (and fill the mini-batch gradients -> set training=true)
                                                Neurons[] neurons = DeepLearningTask.makeNeuronsForTraining(dl.model_info());
                                                double[] responses = new double[miniBatchSize];
                                                double[] offsets = new double[miniBatchSize];
                                                int n = 0;
                                                for (DataInfo.Row myRow : rowsMiniBatch) {
                                                    if (myRow == null)
                                                        continue;
                                                    ((Neurons.Input) neurons[0]).setInput(-1, myRow.numIds, myRow.numVals, myRow.nBins, myRow.binIds, n);
                                                    responses[n] = myRow.response(0);
                                                    offsets[n] = myRow.offset;
                                                    n++;
                                                }
                                                DeepLearningTask.fpropMiniBatch(-1, /*seed doesn't matter*/
                                                neurons, dl.model_info(), null, true, /*training*/
                                                responses, offsets, n);
                                                // check that we didn't change the model's weights/biases
                                                long after = dl.model_info().checksum_impl();
                                                assert (after == before);
                                                // record the gradient since gradientChecking is enabled
                                                //tell it what gradient to collect
                                                DeepLearningModelInfo.gradientCheck = new DeepLearningModelInfo.GradientCheck(layer, row, col);
                                                //update the weights
                                                DeepLearningTask.bpropMiniBatch(neurons, n);
                                                assert (before != dl.model_info().checksum_impl());
                                                // reset the model back to the trained model
                                                dl.set_model_info(IcedUtils.deepCopy(modelInfo));
                                                assert (before == dl.model_info().checksum_impl());
                                                double bpropGradient = DeepLearningModelInfo.gradientCheck.gradient;
                                                // FIXME: re-enable this once the loss is computed from the de-standardized prediction/response
                                                //                    double actualResponse=myRow.response[0];
                                                //                    double predResponseLinkSpace = neurons[neurons.length-1]._a.get(0);
                                                //                    if (di._normRespMul != null) {
                                                //                      bpropGradient /= di._normRespMul[0]; //no shift for gradient
                                                //                      actualResponse = (actualResponse / di._normRespMul[0] + di._normRespSub[0]);
                                                //                      predResponseLinkSpace = (predResponseLinkSpace / di._normRespMul[0] + di._normRespSub[0]);
                                                //                    }
                                                //                    bpropGradient *= new Distribution(parms._distribution).gradient(actualResponse, predResponseLinkSpace);
                                                final float weight = dl.model_info().get_weights(layer).get(row, col);
                                                //don't make the weight deltas too small, or the float weights "won't notice"
                                                double eps = 1e-4 * Math.abs(weight);
                                                if (eps == 0)
                                                    eps = 1e-6;
                                                // loss at weight + eps
                                                dl.model_info().get_weights(layer).set(row, col, (float) (weight + eps));
                                                double up = dl.meanLoss(rowsMiniBatch);
                                                // loss at weight - eps
                                                dl.model_info().get_weights(layer).set(row, col, (float) (weight - eps));
                                                double down = dl.meanLoss(rowsMiniBatch);
                                                if (Math.abs(up - down) / Math.abs(up + down) < 1e-8) {
                                                    //relative change in loss function is too small -> skip
                                                    continue;
                                                }
                                                double gradient = ((up - down) / (2. * eps));
                                                double relError = 2 * Math.abs(bpropGradient - gradient) / (Math.abs(gradient) + Math.abs(bpropGradient));
                                                count++;
                                                // if either gradient is tiny, check if both are tiny
                                                if (Math.abs(gradient) < 1e-7 || Math.abs(bpropGradient) < 1e-7) {
                                                    //all good
                                                    if (Math.abs(bpropGradient - gradient) < 1e-7)
                                                        continue;
                                                }
                                                meanRelErr += relError;
                                                if (relError > MAX_TOLERANCE) {
                                                    Log.info("\nDistribution: " + dl._parms._distribution);
                                                    Log.info("\nRow: " + rId);
                                                    Log.info("weight (layer " + layer + ", row " + row + ", col " + col + "): " + weight + " +/- " + eps);
                                                    Log.info("loss: " + loss);
                                                    Log.info("losses up/down: " + up + " / " + down);
                                                    Log.info("=> Finite differences gradient: " + gradient);
                                                    Log.info("=> Back-propagation gradient  : " + bpropGradient);
                                                    Log.info("=> Relative error             : " + PrettyPrint.formatPct(relError));
                                                    failedcount++;
                                                }
                                                //                          Assert.assertTrue(failedcount==0);
                                                maxRelErr = Math.max(maxRelErr, relError);
                                                assert (!Double.isNaN(maxRelErr));
                                            }
                                        }
                                    }
                                }
                                meanLoss /= tfr.numRows();
                                Log.info("Mean loss: " + meanLoss);
                            //                  // FIXME: re-enable this
                            //                  if (parms._l1 == 0 && parms._l2 == 0) {
                            //                    assert(Math.abs(meanLoss-resdev)/Math.abs(resdev) < 1e-5);
                            //                  }
                            } catch (RuntimeException ex) {
                                dl = DKV.getGet(job.dest());
                                if (dl != null)
                                    Assert.assertTrue(dl.model_info().isUnstable());
                                else
                                    Assert.assertTrue(job.isStopped());
                            } finally {
                                if (dl != null)
                                    dl.delete();
                            }
                        }
                    }
                }
            }
        }
        Log.info("Number of tests: " + count);
        Log.info("Number of failed tests: " + failedcount);
        Log.info("Mean. relative error: " + meanRelErr / count);
        Log.info("Max. relative error: " + PrettyPrint.formatPct(maxRelErr));
        Assert.assertTrue("Error too large: " + maxRelErr + " >= " + MAX_TOLERANCE, maxRelErr < MAX_TOLERANCE);
        Assert.assertTrue("Failed count too large: " + failedcount + " > " + MAX_FAILED_COUNT, failedcount <= MAX_FAILED_COUNT);
    } finally {
        if (tfr != null)
            tfr.remove();
    }
}

Example 29 with Chunk

use of water.fvec.Chunk in project h2o-3 by h2oai.

the class TreeMeasuresCollector method map.

@Override
public void map(Chunk[] chks) {
    double[] data = new double[_ncols];
    double[] preds = new double[_nclasses + 1];
    Chunk cresp = _st.chk_resp(chks);
    Chunk weights = _st.hasWeightCol() ? _st.chk_weight(chks) : new C0DChunk(1, chks[0]._len);
    int nrows = cresp._len;
    // preallocate
    int[] oob = new int[2 + Math.round((1f - _rate) * nrows * 1.2f + 0.5f)];
    int[] soob = null;
    // Prepare output data
    _nrows = new double[_ntrees];
    _votes = _classification ? new double[_ntrees] : null;
    _sse = _classification ? null : new float[_ntrees];
    // seed for shuffling oob samples
    long seedForOob = ShuffleTask.seed(cresp.cidx());
    // Start iteration
    for (int tidx = 0; tidx < _ntrees; tidx++) {
        // tree
        // OOB RNG for this tree
        Random rng = rngForTree(_trees[tidx], cresp.cidx());
        // Collect oob rows and permutate them
        // reuse use the same array for sampling
        oob = ModelUtils.sampleOOBRows(nrows, _rate, rng, oob);
        // Get number of sample rows
        int oobcnt = oob[0];
        if (_var >= 0) {
            if (soob == null || soob.length < oobcnt)
                soob = new int[oobcnt];
            // Shuffle array and copy results into <code>soob</code>
            ArrayUtils.shuffleArray(oob, oobcnt, soob, seedForOob, 1);
        }
        for (int j = 1; j < 1 + oobcnt; j++) {
            int row = oob[j];
            double w = weights.atd(row);
            // we cannot deal with this row anyhow
            if (cresp.isNA(row))
                continue;
            if (w == 0)
                continue;
            // 1+i - one free is expected by prediction
            for (int i = 0; i < _ncols; i++) data[i] = chks[i].atd(row);
            // - permute variable
            if (_var >= 0)
                data[_var] = chks[_var].atd(soob[j - 1]);
            else
                assert soob == null;
            // - score data
            Arrays.fill(preds, 0);
            // - score only the tree
            score0(data, preds, _trees[tidx]);
            // - derive a prediction
            if (_classification) {
                int pred = getPrediction(preds, null, /*FIXME: should use model's _priorClassDistribution*/
                data, _threshold);
                int actu = (int) cresp.at8(row);
                // - collect only correct votes
                if (pred == actu)
                    _votes[tidx] += w;
            } else {
                /* regression */
                // Important!
                double pred = preds[0];
                double actu = cresp.atd(row);
                _sse[tidx] += (actu - pred) * (actu - pred);
            }
            // - collect rows which were used for voting
            _nrows[tidx] += w;
        //if (_var<0) System.err.println("VARIMP OOB row: " + (cresp._start+row) + " : " + Arrays.toString(data) + " tree/actu: " + pred + "/" + actu);
        }
    }
    // Clean-up
    _trees = null;
}

Example 30 with Chunk

use of water.fvec.Chunk in project h2o-3 by h2oai.

the class Cleaner method run.

// Cleaner thread runs in a forever loop.  (This call cannot be synchronized,
// lest we hold the lock during a (very long) clean process - and various
// async callbacks attempt to "kick" the Cleaner awake - which will require
// taking the lock... blocking the kicking thread for the duration.
@Override
public /*synchronized*/
void run() {
    boolean diskFull = false;
    while (true) {
        // Sweep the K/V store, writing out Values (cleaning) and free'ing
        // - Clean all "old" values (lazily, optimistically)
        // - Clean and free old values if above the desired cache level
        // Do not let optimistic cleaning get in the way of emergency cleaning.
        // Get a recent histogram, computing one as needed
        Histo h = Histo.current(false);
        long now = System.currentTimeMillis();
        // When things first got dirtied
        long dirty = _dirty;
        // is below desired levels & nothing has been dirty awhile.
        if (// Cache is low and
        h._cached < DESIRED && (now - dirty < 5000)) {
            // not dirty a long time
            // Block asleep, waking every 5 secs to check for stuff, or when poked
            block_store_cleaner();
            // Awoke; loop back and re-check histogram.
            continue;
        }
        now = System.currentTimeMillis();
        // Reset, since we are going write stuff out
        _dirty = Long.MAX_VALUE;
        MemoryManager.set_goals("preclean", false);
        // The age beyond which we need to toss out things to hit the desired
        // caching levels. If forced, be exact (toss out the minimal amount).
        // If lazy, store-to-disk things down to 1/2 the desired cache level
        // and anything older than 5 secs.
        // Forced to clean
        boolean force = (h._cached >= DESIRED || !MemoryManager.CAN_ALLOC);
        if (// Try to clean the diskFull flag
        force && diskFull)
            diskFull = isDiskFull();
        long clean_to_age = h.clean_to(force ? DESIRED : (DESIRED >> 1));
        // more than 5sec old
        if (!force)
            clean_to_age = Math.max(clean_to_age, now - 5000);
        // Test mode: clean all
        if (DESIRED == -1)
            clean_to_age = now;
        // No logging if under memory pressure: can deadlock the cleaner thread
        String s = h + " DESIRED=" + (DESIRED >> 20) + "M dirtysince=" + (now - dirty) + " force=" + force + " clean2age=" + (now - clean_to_age);
        if (MemoryManager.canAlloc())
            Log.debug(s);
        else
            System.err.println(s);
        // Disk i/o bytes
        long cleaned = 0;
        // memory freed bytes
        long freed = 0;
        // i/o ns writing
        long io_ns = 0;
        // For faster K/V store walking get the NBHM raw backing array,
        // and walk it directly.
        Object[] kvs = H2O.STORE.raw_array();
        // Start the walk at slot 2, because slots 0,1 hold meta-data
        for (int i = 2; i < kvs.length; i += 2) {
            // In the raw backing array, Keys and Values alternate in slots
            Object ok = kvs[i], ov = kvs[i + 1];
            // Ignore tombstones and Primes and null's
            if (!(ok instanceof Key))
                continue;
            // Ignore tombstones and Primes and null's
            if (!(ov instanceof Value))
                continue;
            Value val = (Value) ov;
            byte[] m = val.rawMem();
            Object p = val.rawPOJO();
            // Nothing to throw out
            if (m == null && p == null)
                continue;
            // we do not want to throw out Lockables.
            if (val.isLockable())
                continue;
            boolean isChunk = p instanceof Chunk && !((Chunk) p).isVolatile();
            // Ignore things younger than the required age.  In particular, do
            // not spill-to-disk all dirty things we find.
            long touched = val._lastAccessedTime;
            if (touched > clean_to_age) {
                // (no need for both forms).  Note no savings for Chunks, for which m==p._mem
                if (val.isPersisted() && m != null && p != null && !isChunk) {
                    // Toss serialized form, since can rebuild from POJO
                    val.freeMem();
                    freed += val._max;
                }
                // But may write it out later
                dirty_store(touched);
                // Too young
                continue;
            }
            // Spiller turned off?
            if (!H2O.ARGS.cleaner)
                continue;
            // Should I further force it from memory?
            if (isChunk && !val.isPersisted() && !diskFull && ((Key) ok).home()) {
                // && (force || (lazyPersist() && lazy_clean(key)))) {
                long now_ns = System.nanoTime();
                try // Write to disk
                {
                    val.storePersist();
                }// Can happen due to racing key delete/remove
                 catch (FileNotFoundException fnfe) {
                    continue;
                } catch (IOException e) {
                    Log.warn(isDiskFull() ? "Disk full! Disabling swapping to disk." + (force ? " Memory low! Please free some space in " + H2O.ICE_ROOT + "!" : "") : "Disk swapping failed! " + e.getMessage());
                    // Something is wrong so mark disk as full anyways so we do not
                    // attempt to write again.  (will retry next run when memory is low)
                    diskFull = true;
                }
                if (m == null)
                    m = val.rawMem();
                // Accumulate i/o bytes
                if (m != null)
                    cleaned += m.length;
                // Accumulate i/o time
                io_ns += System.nanoTime() - now_ns;
            }
            // And, under pressure, free all
            if (isChunk && force && (val.isPersisted() || !((Key) ok).home())) {
                val.freeMem();
                if (m != null)
                    freed += val._max;
                m = null;
                val.freePOJO();
                if (p != null)
                    freed += val._max;
                p = null;
                // Double-counted freed mem for Chunks since val._pojo._mem & val._mem are the same.
                if (isChunk)
                    freed -= val._max;
            }
            // serializing again.
            if (m != null && p != null && !isChunk) {
                val.freeMem();
                freed += val._max;
            }
            // If a GC cycle happened and we can no longer alloc, start forcing
            // from RAM as we go
            // Forced to clean
            force = (h._cached >= DESIRED || !MemoryManager.CAN_ALLOC);
        }
        String s1 = "Cleaner pass took: " + PrettyPrint.msecs(System.currentTimeMillis() - now, true) + ", spilled " + PrettyPrint.bytes(cleaned) + " in " + PrettyPrint.usecs(io_ns >> 10);
        // Force a new histogram
        h = Histo.current(true);
        MemoryManager.set_goals("postclean", false);
        // No logging if under memory pressure: can deadlock the cleaner thread
        String s2 = h + " diski_o=" + PrettyPrint.bytes(cleaned) + ", freed=" + (freed >> 20) + "M, DESIRED=" + (DESIRED >> 20) + "M";
        if (MemoryManager.canAlloc())
            Log.debug(s1, s2);
        else
            System.err.println(s1 + "\n" + s2);
        // For testing thread
        synchronized (this) {
            _did_sweep = true;
            // Turn off test-mode after 1 sweep
            if (DESIRED == -1)
                DESIRED = 0;
            // Wake up testing thread
            notifyAll();
        }
    }
}