Examples with StatsReport org.deeplearning4j.ui.stats.api.StatsReport used on opensource projects

Example 1 with StatsReport

use of org.deeplearning4j.ui.stats.api.StatsReport in project deeplearning4j by deeplearning4j.

the class TestStatsStorage method getReport.

private static StatsReport getReport(int sid, int tid, int wid, long time, boolean useJ7Storage) {
    StatsReport rep;
    if (useJ7Storage) {
        rep = new JavaStatsReport();
    } else {
        rep = new SbeStatsReport();
    }
    rep.reportIDs("sid" + sid, "tid" + tid, "wid" + wid, time);
    rep.reportScore(100.0);
    rep.reportPerformance(1000, 1001, 1002, 1003.0, 1004.0);
    return rep;
}

Example 2 with StatsReport

use of org.deeplearning4j.ui.stats.api.StatsReport in project deeplearning4j by deeplearning4j.

the class TrainModule method getLayerMeanMagnitudes.

//TODO float precision for smaller transfers?
//First: iteration. Second: ratios, by parameter
private MeanMagnitudes getLayerMeanMagnitudes(int layerIdx, TrainModuleUtils.GraphInfo gi, List<Persistable> updates, List<Integer> iterationCounts, ModelType modelType) {
    if (gi == null) {
        return new MeanMagnitudes(Collections.emptyList(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    }
    String layerName = gi.getVertexNames().get(layerIdx);
    if (modelType != ModelType.CG) {
        //Get the original name, for the index...
        layerName = gi.getOriginalVertexName().get(layerIdx);
    }
    String layerType = gi.getVertexTypes().get(layerIdx);
    if ("input".equalsIgnoreCase(layerType)) {
        //TODO better checking - other vertices, etc
        return new MeanMagnitudes(Collections.emptyList(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    }
    List<Integer> iterCounts = new ArrayList<>();
    Map<String, List<Double>> ratioValues = new HashMap<>();
    Map<String, List<Double>> outParamMM = new HashMap<>();
    Map<String, List<Double>> outUpdateMM = new HashMap<>();
    if (updates != null) {
        int pCount = -1;
        for (Persistable u : updates) {
            pCount++;
            if (!(u instanceof StatsReport))
                continue;
            StatsReport sp = (StatsReport) u;
            if (iterationCounts != null) {
                iterCounts.add(iterationCounts.get(pCount));
            } else {
                int iterCount = sp.getIterationCount();
                iterCounts.add(iterCount);
            }
            //Info we want, for each parameter in this layer: mean magnitudes for parameters, updates AND the ratio of these
            Map<String, Double> paramMM = sp.getMeanMagnitudes(StatsType.Parameters);
            Map<String, Double> updateMM = sp.getMeanMagnitudes(StatsType.Updates);
            for (String s : paramMM.keySet()) {
                String prefix;
                if (modelType == ModelType.Layer) {
                    prefix = layerName;
                } else {
                    prefix = layerName + "_";
                }
                if (s.startsWith(prefix)) {
                    //Relevant parameter for this layer...
                    String layerParam = s.substring(prefix.length());
                    double pmm = paramMM.getOrDefault(s, 0.0);
                    double umm = updateMM.getOrDefault(s, 0.0);
                    if (!Double.isFinite(pmm)) {
                        pmm = NAN_REPLACEMENT_VALUE;
                    }
                    if (!Double.isFinite(umm)) {
                        umm = NAN_REPLACEMENT_VALUE;
                    }
                    double ratio;
                    if (umm == 0.0 && pmm == 0.0) {
                        //To avoid NaN from 0/0
                        ratio = 0.0;
                    } else {
                        ratio = umm / pmm;
                    }
                    List<Double> list = ratioValues.get(layerParam);
                    if (list == null) {
                        list = new ArrayList<>();
                        ratioValues.put(layerParam, list);
                    }
                    list.add(ratio);
                    List<Double> pmmList = outParamMM.get(layerParam);
                    if (pmmList == null) {
                        pmmList = new ArrayList<>();
                        outParamMM.put(layerParam, pmmList);
                    }
                    pmmList.add(pmm);
                    List<Double> ummList = outUpdateMM.get(layerParam);
                    if (ummList == null) {
                        ummList = new ArrayList<>();
                        outUpdateMM.put(layerParam, ummList);
                    }
                    ummList.add(umm);
                }
            }
        }
    }
    return new MeanMagnitudes(iterCounts, ratioValues, outParamMM, outUpdateMM);
}

Example 3 with StatsReport

use of org.deeplearning4j.ui.stats.api.StatsReport in project deeplearning4j by deeplearning4j.

the class TrainModule method getLayerActivations.

private Triple<int[], float[], float[]> getLayerActivations(int index, TrainModuleUtils.GraphInfo gi, List<Persistable> updates, List<Integer> iterationCounts) {
    if (gi == null) {
        return EMPTY_TRIPLE;
    }
    //Index may be for an input, for example
    String type = gi.getVertexTypes().get(index);
    if ("input".equalsIgnoreCase(type)) {
        return EMPTY_TRIPLE;
    }
    List<String> origNames = gi.getOriginalVertexName();
    if (index < 0 || index >= origNames.size()) {
        return EMPTY_TRIPLE;
    }
    String layerName = origNames.get(index);
    int size = (updates == null ? 0 : updates.size());
    int[] iterCounts = new int[size];
    float[] mean = new float[size];
    float[] stdev = new float[size];
    int used = 0;
    if (updates != null) {
        int uCount = -1;
        for (Persistable u : updates) {
            uCount++;
            if (!(u instanceof StatsReport))
                continue;
            StatsReport sp = (StatsReport) u;
            if (iterationCounts == null) {
                iterCounts[used] = sp.getIterationCount();
            } else {
                iterCounts[used] = iterationCounts.get(uCount);
            }
            Map<String, Double> means = sp.getMean(StatsType.Activations);
            Map<String, Double> stdevs = sp.getStdev(StatsType.Activations);
            //TODO PROPER VALIDATION ETC, ERROR HANDLING
            if (means != null && means.containsKey(layerName)) {
                mean[used] = means.get(layerName).floatValue();
                stdev[used] = stdevs.get(layerName).floatValue();
                if (!Float.isFinite(mean[used])) {
                    mean[used] = (float) NAN_REPLACEMENT_VALUE;
                }
                if (!Float.isFinite(stdev[used])) {
                    stdev[used] = (float) NAN_REPLACEMENT_VALUE;
                }
                used++;
            }
        }
    }
    if (used != iterCounts.length) {
        iterCounts = Arrays.copyOf(iterCounts, used);
        mean = Arrays.copyOf(mean, used);
        stdev = Arrays.copyOf(stdev, used);
    }
    return new Triple<>(iterCounts, mean, stdev);
}

Example 4 with StatsReport

use of org.deeplearning4j.ui.stats.api.StatsReport in project deeplearning4j by deeplearning4j.

the class TrainModule method getHistograms.

private static Map<String, Object> getHistograms(int layerIdx, TrainModuleUtils.GraphInfo gi, StatsType statsType, Persistable p) {
    if (p == null)
        return null;
    if (!(p instanceof StatsReport))
        return null;
    StatsReport sr = (StatsReport) p;
    String layerName = gi.getOriginalVertexName().get(layerIdx);
    Map<String, Histogram> map = sr.getHistograms(statsType);
    List<String> paramNames = new ArrayList<>();
    Map<String, Object> ret = new HashMap<>();
    if (layerName != null) {
        for (String s : map.keySet()) {
            if (s.startsWith(layerName)) {
                String paramName;
                if (s.charAt(layerName.length()) == '_') {
                    //MLN or CG parameter naming convention
                    paramName = s.substring(layerName.length() + 1);
                } else {
                    //Pretrain layer (VAE, RBM) naming convention
                    paramName = s.substring(layerName.length());
                }
                paramNames.add(paramName);
                Histogram h = map.get(s);
                Map<String, Object> thisHist = new HashMap<>();
                double min = h.getMin();
                double max = h.getMax();
                if (Double.isNaN(min)) {
                    //If either is NaN, both will be
                    min = NAN_REPLACEMENT_VALUE;
                    max = NAN_REPLACEMENT_VALUE;
                }
                thisHist.put("min", min);
                thisHist.put("max", max);
                thisHist.put("bins", h.getNBins());
                thisHist.put("counts", h.getBinCounts());
                ret.put(paramName, thisHist);
            }
        }
    }
    ret.put("paramNames", paramNames);
    return ret;
}

Example 5 with StatsReport

use of org.deeplearning4j.ui.stats.api.StatsReport in project deeplearning4j by deeplearning4j.

the class TrainModule method getOverviewData.

private Result getOverviewData() {
    Long lastUpdate = lastUpdateForSession.get(currentSessionID);
    if (lastUpdate == null)
        lastUpdate = -1L;
    I18N i18N = I18NProvider.getInstance();
    boolean noData = currentSessionID == null;
    //First pass (optimize later): query all data...
    StatsStorage ss = (noData ? null : knownSessionIDs.get(currentSessionID));
    String wid = getWorkerIdForIndex(currentWorkerIdx);
    if (wid == null) {
        noData = true;
    }
    List<Integer> scoresIterCount = new ArrayList<>();
    List<Double> scores = new ArrayList<>();
    Map<String, Object> result = new HashMap<>();
    result.put("updateTimestamp", lastUpdate);
    result.put("scores", scores);
    result.put("scoresIter", scoresIterCount);
    //Get scores info
    List<Persistable> updates = (noData ? null : ss.getAllUpdatesAfter(currentSessionID, StatsListener.TYPE_ID, wid, 0));
    if (updates == null || updates.size() == 0) {
        noData = true;
    }
    //Collect update ratios for weights
    //Collect standard deviations: activations, gradients, updates
    //Mean magnitude (updates) / mean magnitude (parameters)
    Map<String, List<Double>> updateRatios = new HashMap<>();
    result.put("updateRatios", updateRatios);
    Map<String, List<Double>> stdevActivations = new HashMap<>();
    Map<String, List<Double>> stdevGradients = new HashMap<>();
    Map<String, List<Double>> stdevUpdates = new HashMap<>();
    result.put("stdevActivations", stdevActivations);
    result.put("stdevGradients", stdevGradients);
    result.put("stdevUpdates", stdevUpdates);
    if (!noData) {
        Persistable u = updates.get(0);
        if (u instanceof StatsReport) {
            StatsReport sp = (StatsReport) u;
            Map<String, Double> map = sp.getMeanMagnitudes(StatsType.Parameters);
            if (map != null) {
                for (String s : map.keySet()) {
                    if (!s.toLowerCase().endsWith("w"))
                        //TODO: more robust "weights only" approach...
                        continue;
                    updateRatios.put(s, new ArrayList<>());
                }
            }
            Map<String, Double> stdGrad = sp.getStdev(StatsType.Gradients);
            if (stdGrad != null) {
                for (String s : stdGrad.keySet()) {
                    if (!s.toLowerCase().endsWith("w"))
                        //TODO: more robust "weights only" approach...
                        continue;
                    stdevGradients.put(s, new ArrayList<>());
                }
            }
            Map<String, Double> stdUpdate = sp.getStdev(StatsType.Updates);
            if (stdUpdate != null) {
                for (String s : stdUpdate.keySet()) {
                    if (!s.toLowerCase().endsWith("w"))
                        //TODO: more robust "weights only" approach...
                        continue;
                    stdevUpdates.put(s, new ArrayList<>());
                }
            }
            Map<String, Double> stdAct = sp.getStdev(StatsType.Activations);
            if (stdAct != null) {
                for (String s : stdAct.keySet()) {
                    stdevActivations.put(s, new ArrayList<>());
                }
            }
        }
    }
    StatsReport last = null;
    int lastIterCount = -1;
    //Legacy issue - Spark training - iteration counts are used to be reset... which means: could go 0,1,2,0,1,2, etc...
    //Or, it could equally go 4,8,4,8,... or 5,5,5,5 - depending on the collection and averaging frequencies
    //Now, it should use the proper iteration counts
    boolean needToHandleLegacyIterCounts = false;
    if (!noData) {
        double lastScore;
        int totalUpdates = updates.size();
        int subsamplingFrequency = 1;
        if (totalUpdates > maxChartPoints) {
            subsamplingFrequency = totalUpdates / maxChartPoints;
        }
        int pCount = -1;
        int lastUpdateIdx = updates.size() - 1;
        for (Persistable u : updates) {
            pCount++;
            if (!(u instanceof StatsReport))
                continue;
            last = (StatsReport) u;
            int iterCount = last.getIterationCount();
            if (iterCount <= lastIterCount) {
                needToHandleLegacyIterCounts = true;
            }
            lastIterCount = iterCount;
            if (pCount > 0 && subsamplingFrequency > 1 && pCount % subsamplingFrequency != 0) {
                //Skip this - subsample the data
                if (pCount != lastUpdateIdx)
                    //Always keep the most recent value
                    continue;
            }
            scoresIterCount.add(iterCount);
            lastScore = last.getScore();
            if (Double.isFinite(lastScore)) {
                scores.add(lastScore);
            } else {
                scores.add(NAN_REPLACEMENT_VALUE);
            }
            //Update ratios: mean magnitudes(updates) / mean magnitudes (parameters)
            Map<String, Double> updateMM = last.getMeanMagnitudes(StatsType.Updates);
            Map<String, Double> paramMM = last.getMeanMagnitudes(StatsType.Parameters);
            if (updateMM != null && paramMM != null && updateMM.size() > 0 && paramMM.size() > 0) {
                for (String s : updateRatios.keySet()) {
                    List<Double> ratioHistory = updateRatios.get(s);
                    double currUpdate = updateMM.getOrDefault(s, 0.0);
                    double currParam = paramMM.getOrDefault(s, 0.0);
                    double ratio = currUpdate / currParam;
                    if (Double.isFinite(ratio)) {
                        ratioHistory.add(ratio);
                    } else {
                        ratioHistory.add(NAN_REPLACEMENT_VALUE);
                    }
                }
            }
            //Standard deviations: gradients, updates, activations
            Map<String, Double> stdGrad = last.getStdev(StatsType.Gradients);
            Map<String, Double> stdUpd = last.getStdev(StatsType.Updates);
            Map<String, Double> stdAct = last.getStdev(StatsType.Activations);
            if (stdGrad != null) {
                for (String s : stdevGradients.keySet()) {
                    double d = stdGrad.getOrDefault(s, 0.0);
                    stdevGradients.get(s).add(fixNaN(d));
                }
            }
            if (stdUpd != null) {
                for (String s : stdevUpdates.keySet()) {
                    double d = stdUpd.getOrDefault(s, 0.0);
                    stdevUpdates.get(s).add(fixNaN(d));
                }
            }
            if (stdAct != null) {
                for (String s : stdevActivations.keySet()) {
                    double d = stdAct.getOrDefault(s, 0.0);
                    stdevActivations.get(s).add(fixNaN(d));
                }
            }
        }
    }
    if (needToHandleLegacyIterCounts) {
        cleanLegacyIterationCounts(scoresIterCount);
    }
    //----- Performance Info -----
    String[][] perfInfo = new String[][] { { i18N.getMessage("train.overview.perftable.startTime"), "" }, { i18N.getMessage("train.overview.perftable.totalRuntime"), "" }, { i18N.getMessage("train.overview.perftable.lastUpdate"), "" }, { i18N.getMessage("train.overview.perftable.totalParamUpdates"), "" }, { i18N.getMessage("train.overview.perftable.updatesPerSec"), "" }, { i18N.getMessage("train.overview.perftable.examplesPerSec"), "" } };
    if (last != null) {
        perfInfo[2][1] = String.valueOf(dateFormat.format(new Date(last.getTimeStamp())));
        perfInfo[3][1] = String.valueOf(last.getTotalMinibatches());
        perfInfo[4][1] = String.valueOf(df2.format(last.getMinibatchesPerSecond()));
        perfInfo[5][1] = String.valueOf(df2.format(last.getExamplesPerSecond()));
    }
    result.put("perf", perfInfo);
    // ----- Model Info -----
    String[][] modelInfo = new String[][] { { i18N.getMessage("train.overview.modeltable.modeltype"), "" }, { i18N.getMessage("train.overview.modeltable.nLayers"), "" }, { i18N.getMessage("train.overview.modeltable.nParams"), "" } };
    if (!noData) {
        Persistable p = ss.getStaticInfo(currentSessionID, StatsListener.TYPE_ID, wid);
        if (p != null) {
            StatsInitializationReport initReport = (StatsInitializationReport) p;
            int nLayers = initReport.getModelNumLayers();
            long numParams = initReport.getModelNumParams();
            String className = initReport.getModelClassName();
            String modelType;
            if (className.endsWith("MultiLayerNetwork")) {
                modelType = "MultiLayerNetwork";
            } else if (className.endsWith("ComputationGraph")) {
                modelType = "ComputationGraph";
            } else {
                modelType = className;
                if (modelType.lastIndexOf('.') > 0) {
                    modelType = modelType.substring(modelType.lastIndexOf('.') + 1);
                }
            }
            modelInfo[0][1] = modelType;
            modelInfo[1][1] = String.valueOf(nLayers);
            modelInfo[2][1] = String.valueOf(numParams);
        }
    }
    result.put("model", modelInfo);
    return Results.ok(Json.toJson(result));
}