Examples with Mean org.apache.commons.math3.stat.descriptive.moment.Mean used on opensource projects

Example 86 with Mean

use of org.apache.commons.math3.stat.descriptive.moment.Mean in project vcell by virtualcell.

the class NetCDFWriter method writeHybridInputFile.

/**
 * Write the model to a NetCDF file which serves as an input for stoch hybrid simulator.
 * To write to a NetCDF file is a bit complicated. First, we have to create a NetCDF-3
 * file. And then feed in the data.
 * Creation date: (5/22/2007 5:36:03 PM)
 */
public void writeHybridInputFile(String[] parameterNames) throws Exception, cbit.vcell.parser.ExpressionException, IOException, MathException, InvalidRangeException {
    Simulation simulation = simTask.getSimulation();
    SimulationSymbolTable simSymbolTable = simTask.getSimulationJob().getSimulationSymbolTable();
    if (initialize()) {
        // we need to get model and control information first
        NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(filename, false);
        // Model info. will be extracted from subDomain of mathDescription
        java.util.Enumeration<SubDomain> e = simulation.getMathDescription().getSubDomains();
        // remember we are dealing with compartmental model here. only 1 subdomain.
        SubDomain subDomain = e.nextElement();
        JumpProcess[] reactions = (JumpProcess[]) subDomain.getJumpProcesses().toArray(new JumpProcess[subDomain.getJumpProcesses().size()]);
        // get species variable names
        Variable[] variables = simSymbolTable.getVariables();
        String[] speciesNames = new String[variables.length];
        for (int i = 0; i < variables.length; i++) speciesNames[i] = variables[i].getName();
        // the probabilities for reactions
        Expression[] probs = new Expression[reactions.length];
        for (int i = 0; i < reactions.length; i++) {
            probs[i] = simSymbolTable.substituteFunctions(reactions[i].getProbabilityRate());
            probs[i] = probs[i].flatten();
        }
        VarIniCondition[] varInis = (VarIniCondition[]) subDomain.getVarIniConditions().toArray(new VarIniCondition[subDomain.getVarIniConditions().size()]);
        // the non-constant stoch variables
        Vector<Variable> vars = new Vector<Variable>();
        for (int i = 0; i < varInis.length; i++) {
            if (varInis[i].getVar() instanceof StochVolVariable) {
                vars.addElement(varInis[i].getVar());
            }
        }
        // get reaction rate law types and rate constants
        ReactionRateLaw[] reactionRateLaws = getReactionRateLaws(probs);
        SolverTaskDescription solverTaskDescription = simulation.getSolverTaskDescription();
        TimeBounds timeBounds = solverTaskDescription.getTimeBounds();
        UniformOutputTimeSpec timeSpec = (UniformOutputTimeSpec) solverTaskDescription.getOutputTimeSpec();
        UniformOutputTimeSpec outputTimeSpec = ((UniformOutputTimeSpec) solverTaskDescription.getOutputTimeSpec());
        NonspatialStochSimOptions stochOpt = solverTaskDescription.getStochOpt();
        // create an empty NetCDF-3 file
        // define dimensions
        /* these sizes must match the buffers allocated in corresponding Fortran code -- see globalvariables.f90
			in numerics Hy3S/src directory */
        Dimension numTrial = ncfile.addDimension("NumTrials", (int) stochOpt.getNumOfTrials());
        Dimension numSpecies = ncfile.addDimension("NumSpecies", vars.size());
        Dimension numReactions = ncfile.addDimension("NumReactions", subDomain.getJumpProcesses().size());
        int outPoints = ((int) ((timeBounds.getEndingTime() - timeBounds.getStartingTime()) / outputTimeSpec.getOutputTimeStep())) + 1;
        Dimension numTimePoints = ncfile.addDimension("NumTimePoints", outPoints);
        Dimension numModels = ncfile.addDimension("NumModels", 1);
        Dimension numMaxDepList = ncfile.addDimension("NumMaxDepList", 6);
        Dimension numMaxStoichList = ncfile.addDimension("NumMaxStoichList", 25);
        Dimension stringLen = ncfile.addDimension("StringLen", 72);
        // define variables
        // jms info
        ArrayList<Dimension> dims = new ArrayList<Dimension>();
        dims.add(stringLen);
        if (bMessaging) {
            ncfile.addVariable("JMS_BROKER", DataType.CHAR, dims);
            ncfile.addVariable("JMS_USER", DataType.CHAR, dims);
            ncfile.addVariable("JMS_PASSWORD", DataType.CHAR, dims);
            ncfile.addVariable("JMS_QUEUE", DataType.CHAR, dims);
            ncfile.addVariable("JMS_TOPIC", DataType.CHAR, dims);
            ncfile.addVariable("VCELL_USER", DataType.CHAR, dims);
            ncfile.addVariable("SIMULATION_KEY", DataType.INT, new ArrayList<Dimension>());
            ncfile.addVariable("JOB_INDEX", DataType.INT, new ArrayList<Dimension>());
        }
        // scalars
        ncfile.addVariable("TStart", DataType.DOUBLE, new ArrayList<Dimension>());
        ncfile.addVariable("TEnd", DataType.DOUBLE, new ArrayList<Dimension>());
        ncfile.addVariable("SaveTime", DataType.DOUBLE, new ArrayList<Dimension>());
        ncfile.addVariable("Volume", DataType.DOUBLE, new ArrayList<Dimension>());
        ncfile.addVariable("CellGrowthTime", DataType.DOUBLE, new ArrayList<Dimension>());
        ncfile.addVariable("CellGrowthTimeSD", DataType.DOUBLE, new ArrayList<Dimension>());
        ncfile.addVariable("ExpType", DataType.INT, new ArrayList<Dimension>());
        ncfile.addVariable("LastTrial", DataType.INT, new ArrayList<Dimension>());
        ncfile.addVariable("LastModel", DataType.INT, new ArrayList<Dimension>());
        ncfile.addVariable("MaxNumModels", DataType.INT, new ArrayList<Dimension>());
        ncfile.addVariable("NumModels", DataType.INT, new ArrayList<Dimension>());
        // variables with at least 1 dimension
        ArrayList<Dimension> dimspecies = new ArrayList<Dimension>();
        dimspecies.add(numSpecies);
        ArrayList<Dimension> dimreactions = new ArrayList<Dimension>();
        dimreactions.add(numReactions);
        ncfile.addVariable("SpeciesSplitOnDivision", DataType.INT, dimspecies);
        ncfile.addVariable("SaveSpeciesData", DataType.INT, dimspecies);
        ncfile.addVariable("Reaction_Rate_Laws", DataType.INT, dimreactions);
        ncfile.addVariable("Reaction_DListLen", DataType.INT, dimreactions);
        ncfile.addVariable("Reaction_StoichListLen", DataType.INT, dimreactions);
        ncfile.addVariable("Reaction_OptionalData", DataType.INT, dimreactions);
        dims.clear();
        dims.add(numReactions);
        dims.add(numMaxStoichList);
        ncfile.addVariable("Reaction_StoichCoeff", DataType.INT, dims);
        ncfile.addVariable("Reaction_StoichSpecies", DataType.INT, dims);
        dims.clear();
        dims.add(numReactions);
        dims.add(numMaxDepList);
        ncfile.addVariable("Reaction_DepList", DataType.INT, dims);
        dims.clear();
        dims.add(numReactions);
        dims.add(stringLen);
        ncfile.addVariable("Reaction_names", DataType.CHAR, dims);
        dims.clear();
        dims.add(numSpecies);
        dims.add(stringLen);
        ncfile.addVariable("Species_names", DataType.CHAR, dims);
        ncfile.addVariable("SpeciesIC", DataType.INT, dimspecies);
        dims.clear();
        dims.add(numReactions);
        dims.add(numMaxDepList);
        ncfile.addVariable("Reaction_Rate_Constants", DataType.DOUBLE, dims);
        // create the file
        try {
            ncfile.create();
        } catch (IOException ioe) {
            ioe.printStackTrace(System.err);
            throw new IOException("Error creating hybrid file " + filename + ": " + ioe.getMessage());
        }
        // write data to the NetCDF file
        try {
            // write jms info
            if (bMessaging) {
                ArrayChar.D1 jmsString = new ArrayChar.D1(stringLen.getLength());
                String jmshost = PropertyLoader.getRequiredProperty(PropertyLoader.jmsHostExternal);
                // 
                // Used for new REST HTTP messaging api (USE THIS WHEN Hyrbid Solvers are compiled).
                // 
                // String jmsrestport = PropertyLoader.getRequiredProperty(PropertyLoader.jmsRestPortExternal);
                // String jmsurl = jmshost+":"+jmsrestport;
                // 
                // connect to messaging using legacy AMQP protocol instead of new REST api.  Needed for legacy pre-compiled solvers.
                // 
                String jmsport = PropertyLoader.getRequiredProperty(PropertyLoader.jmsPortExternal);
                String jmsurl = "failover:(tcp://" + jmshost + ":" + jmsport + ")";
                jmsString.setString(jmsurl);
                ncfile.write("JMS_BROKER", jmsString);
                jmsString.setString(PropertyLoader.getRequiredProperty(PropertyLoader.jmsUser));
                ncfile.write("JMS_USER", jmsString);
                String jmsPassword = PropertyLoader.getSecretValue(PropertyLoader.jmsPasswordValue, PropertyLoader.jmsPasswordFile);
                jmsString.setString(jmsPassword);
                ncfile.write("JMS_PASSWORD", jmsString);
                jmsString.setString(VCellQueue.WorkerEventQueue.getName());
                ncfile.write("JMS_QUEUE", jmsString);
                jmsString.setString(VCellTopic.ServiceControlTopic.getName());
                ncfile.write("JMS_TOPIC", jmsString);
                jmsString.setString(simulation.getVersion().getOwner().getName());
                ncfile.write("VCELL_USER", jmsString);
                ArrayInt.D0 scalarJMS = new ArrayInt.D0();
                scalarJMS.set(Integer.parseInt(simulation.getVersion().getVersionKey() + ""));
                ncfile.write("SIMULATION_KEY", scalarJMS);
                scalarJMS.set(simTask.getSimulationJob().getJobIndex());
                ncfile.write("JOB_INDEX", scalarJMS);
            }
            ArrayDouble.D0 scalarDouble = new ArrayDouble.D0();
            // TStart, TEnd, SaveTime
            if ((timeBounds.getEndingTime() > timeBounds.getStartingTime()) && (outputTimeSpec.getOutputTimeStep() > 0)) {
                scalarDouble.set(timeBounds.getStartingTime());
                ncfile.write("TStart", scalarDouble);
                scalarDouble.set(timeBounds.getEndingTime());
                ncfile.write("TEnd", scalarDouble);
                scalarDouble.set(outputTimeSpec.getOutputTimeStep());
                ncfile.write("SaveTime", scalarDouble);
            } else {
                System.err.println("Time setting error. Ending time smaller than starting time or save interval is not a positive value.");
                throw new RuntimeException("Time setting error. Ending time smaller than starting time or save interval is not a positive value.");
            }
            // Volume
            // we set volume to 1. This model file cannot support multi-compartmental sizes.
            // When writting the rate constants, we must take the volume into account according to the reaction type.
            scalarDouble.set(1);
            ncfile.write("Volume", scalarDouble);
            // CellGrowthTime, CellGrowthTimeSD,
            scalarDouble.set(0);
            ncfile.write("CellGrowthTime", scalarDouble);
            ncfile.write("CellGrowthTimeSD", scalarDouble);
            // ExpType, Last Trial, Last Model, MaxNumModels, NumModels
            ArrayInt.D0 scalarInt = new ArrayInt.D0();
            scalarInt.set(0);
            ncfile.write("LastTrial", scalarInt);
            ncfile.write("LastModel", scalarInt);
            scalarInt.set(1);
            ncfile.write("ExpType", scalarInt);
            ncfile.write("MaxNumModels", scalarInt);
            ncfile.write("NumModels", scalarInt);
            // SpeciesSplitOnDivision
            ArrayInt A1 = new ArrayInt.D1(numSpecies.getLength());
            Index idx = A1.getIndex();
            for (int i = 0; i < numSpecies.getLength(); i++) {
                A1.setInt(idx.set(i), 0);
            }
            ncfile.write("SpeciesSplitOnDivision", new int[1], A1);
            // SaveSpeciesData
            ArrayInt A2 = new ArrayInt.D1(numSpecies.getLength());
            idx = A2.getIndex();
            for (int i = 0; i < numSpecies.getLength(); i++) {
                A2.setInt(idx.set(i), 1);
            }
            ncfile.write("SaveSpeciesData", new int[1], A2);
            // Reaction_Rate_Laws
            ArrayInt A3 = new ArrayInt.D1(numReactions.getLength());
            idx = A3.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                A3.setInt(idx.set(i), reactionRateLaws[i].getLawType());
            }
            ncfile.write("Reaction_Rate_Laws", new int[1], A3);
            // Reaction_DListLen
            ArrayInt A4 = new ArrayInt.D1(numReactions.getLength());
            idx = A4.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                if (reactionRateLaws[i].getLawType() == ReactionRateLaw.order_0)
                    A4.setInt(idx.set(i), 0);
                else if ((reactionRateLaws[i].getLawType() == ReactionRateLaw.order_1) || (reactionRateLaws[i].getLawType() == ReactionRateLaw.order_2_1substrate) || (reactionRateLaws[i].getLawType() == ReactionRateLaw.order_3_1substrate))
                    A4.setInt(idx.set(i), 1);
                else if ((reactionRateLaws[i].getLawType() == ReactionRateLaw.order_2_2substrate) || (reactionRateLaws[i].getLawType() == ReactionRateLaw.order_3_2substrate))
                    A4.setInt(idx.set(i), 2);
                else if (reactionRateLaws[i].getLawType() == ReactionRateLaw.order_3_3substrate)
                    A4.setInt(idx.set(i), 3);
            }
            ncfile.write("Reaction_DListLen", new int[1], A4);
            // Reaction_StoichListLen
            ArrayInt A5 = new ArrayInt.D1(numReactions.getLength());
            idx = A5.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                A5.setInt(idx.set(i), reactions[i].getActions().size());
            }
            ncfile.write("Reaction_StoichListLen", new int[1], A5);
            // Reaction_OptionalData
            ArrayInt A6 = new ArrayInt.D1(numReactions.getLength());
            idx = A6.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                A6.setInt(idx.set(i), 0);
            }
            ncfile.write("Reaction_OptionalData", new int[1], A6);
            // Reaction_StoichCoeff
            ArrayInt A7 = new ArrayInt.D2(numReactions.getLength(), numMaxStoichList.getLength());
            idx = A7.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                Action[] actions = (Action[]) reactions[i].getActions().toArray(new Action[reactions[i].getActions().size()]);
                for (int j = 0; j < actions.length; j++) {
                    try {
                        actions[j].getOperand().evaluateConstant();
                        int coeff = (int) Math.round(actions[j].getOperand().evaluateConstant());
                        A7.setInt(idx.set(i, j), coeff);
                    } catch (ExpressionException ex) {
                        ex.printStackTrace(System.err);
                        throw new ExpressionException(ex.getMessage());
                    }
                }
            }
            ncfile.write("Reaction_StoichCoeff", new int[2], A7);
            // Reaction_StoichSpecies
            ArrayInt A8 = new ArrayInt.D2(numReactions.getLength(), numMaxStoichList.getLength());
            idx = A8.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                ArrayList<Action> actions = reactions[i].getActions();
                for (int j = 0; j < actions.size(); j++) {
                    A8.setInt(idx.set(i, j), getVariableIndex(((Action) actions.get(j)).getVar().getName(), vars));
                }
            }
            ncfile.write("Reaction_StoichSpecies", new int[2], A8);
            // Reaction_DepList
            ArrayInt A9 = new ArrayInt.D2(numReactions.getLength(), numMaxDepList.getLength());
            idx = A9.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                ReactionRateLaw rl = reactionRateLaws[i];
                Hashtable<String, Integer> tem = varInProbOrderHash[i];
                Enumeration<String> varnames = tem.keys();
                if (rl.getLawType() == ReactionRateLaw.order_0) {
                // don't do anything here.
                } else if ((rl.getLawType() == ReactionRateLaw.order_1) || (rl.getLawType() == ReactionRateLaw.order_2_1substrate) || (rl.getLawType() == ReactionRateLaw.order_3_1substrate) || (rl.getLawType() == ReactionRateLaw.order_2_2substrate) || (rl.getLawType() == ReactionRateLaw.order_3_3substrate)) {
                    int j = 0;
                    while (varnames.hasMoreElements()) {
                        String name = varnames.nextElement();
                        A9.setInt(idx.set(i, j), getVariableIndex(name, vars));
                        j++;
                    }
                } else if (rl.getLawType() == ReactionRateLaw.order_3_2substrate) {
                    int order = 0;
                    String highOrderName = "";
                    String lowOrderName = "";
                    // we must make sure to put the higher order species first.
                    while (varnames.hasMoreElements()) {
                        lowOrderName = varnames.nextElement();
                        if (tem.get(lowOrderName) > order) {
                            String s = highOrderName;
                            highOrderName = lowOrderName;
                            lowOrderName = s;
                            order = tem.get(highOrderName);
                        }
                    }
                    A9.setInt(idx.set(i, 0), getVariableIndex(highOrderName, vars));
                    A9.setInt(idx.set(i, 1), getVariableIndex(lowOrderName, vars));
                }
            }
            ncfile.write("Reaction_DepList", new int[2], A9);
            // Reaction_names
            ArrayChar A10 = new ArrayChar.D2(numReactions.getLength(), stringLen.getLength());
            for (int i = 0; i < numReactions.getLength(); i++) {
                String name = reactions[i].getName();
                int diff = stringLen.getLength() - name.length();
                if (diff >= 0) {
                    for (int j = 0; j < diff; j++) {
                        name = name + " ";
                    }
                    A10.setString(i, name);
                } else
                    throw new RuntimeException("Name of Reaction:" + name + " is too long. Please shorten to " + stringLen.getLength() + " chars.");
            }
            ncfile.write("Reaction_names", A10);
            // Species_names
            ArrayChar A11 = new ArrayChar.D2(numSpecies.getLength(), stringLen.getLength());
            for (int i = 0; i < numSpecies.getLength(); i++) {
                String name = vars.elementAt(i).getName();
                int diff = stringLen.getLength() - name.length();
                if (diff >= 0) {
                    for (int j = 0; j < diff; j++) {
                        name = name + " ";
                    }
                    A11.setString(i, name);
                } else
                    throw new RuntimeException("Name of Species:" + name + " is too long. Please shorten to " + stringLen.getLength() + " chars.");
            }
            ncfile.write("Species_names", A11);
            // Species Initial Condition (in number of molecules).
            // Species iniCondition are sampled from a poisson distribution(which has a mean of the current iniExp value)
            RandomDataGenerator dist = new RandomDataGenerator();
            if (stochOpt.isUseCustomSeed()) {
                Integer randomSeed = stochOpt.getCustomSeed();
                if (randomSeed != null) {
                    dist.reSeed(randomSeed);
                }
            }
            ArrayLong A12 = new ArrayLong.D1(numSpecies.getLength());
            idx = A12.getIndex();
            for (int i = 0; i < numSpecies.getLength(); i++) {
                try {
                    VarIniCondition varIniCondition = subDomain.getVarIniCondition(vars.elementAt(i));
                    Expression varIniExp = varIniCondition.getIniVal();
                    varIniExp.bindExpression(simSymbolTable);
                    varIniExp = simSymbolTable.substituteFunctions(varIniExp).flatten();
                    double expectedCount = varIniExp.evaluateConstant();
                    long varCount = 0;
                    if (varIniCondition instanceof VarIniCount) {
                        varCount = (long) expectedCount;
                    } else {
                        if (expectedCount > 0) {
                            varCount = dist.nextPoisson(expectedCount);
                        }
                    }
                    A12.setLong(idx.set(i), varCount);
                } catch (ExpressionException ex) {
                    ex.printStackTrace(System.err);
                    throw new ExpressionException(ex.getMessage());
                }
            }
            ncfile.write("SpeciesIC", new int[1], A12);
            // Reaction_Rate_Constants(NumReactions, NumMaxDepList) ;
            ArrayDouble A13 = new ArrayDouble.D2(numReactions.getLength(), numMaxDepList.getLength());
            idx = A13.getIndex();
            for (int i = 0; i < numReactions.getLength(); i++) {
                ReactionRateLaw rl = reactionRateLaws[i];
                A13.setDouble(idx.set(i, 0), rl.getRateConstant());
            }
            ncfile.write("Reaction_Rate_Constants", A13);
        } catch (IOException ioe) {
            ioe.printStackTrace(System.err);
            throw new IOException("Error writing hybrid input file " + filename + ": " + ioe.getMessage());
        } catch (InvalidRangeException ire) {
            ire.printStackTrace(System.err);
            throw new InvalidRangeException("Error writing hybrid input file " + filename + ": " + ire.getMessage());
        }
        try {
            ncfile.close();
        } catch (IOException ioe) {
            throw new IOException("Error closing file " + filename + ". " + ioe.getMessage());
        }
    }
}

Example 87 with Mean

use of org.apache.commons.math3.stat.descriptive.moment.Mean in project vcell by virtualcell.

the class FluorescenceNoiseTest method doit.

private void doit() throws ImageException {
    int[] sizes = new int[] { 1, 2, 4, 8, 16, 32, 64, 128, 256 };
    for (int imageSize : sizes) {
        ISize size = new ISize(imageSize, imageSize, 1);
        Extent extent = new Extent(1, 1, 1);
        Origin origin = new Origin(0, 0, 0);
        // NormalizedSampleFunction sampleFunction = NormalizedSampleFunction.createUniform("uniformROI", origin, extent, size);
        NormalizedSampleFunction sampleFunction = NormalizedSampleFunction.fromGaussian("testGaussian", origin, extent, size, 0.5, 0.2, 0.1);
        SampleStatistics[] samples = new SampleStatistics[NUM_TRIALS];
        for (int i = 0; i < NUM_TRIALS; i++) {
            UShortImage rawImage = getUniformFluorescenceImage(size, extent, origin, MEAN_INTENSITY);
            samples[i] = sampleFunction.sample(rawImage);
        }
        Mean mean = new Mean();
        Variance var = new Variance();
        double[] weightedMeans = getWeightedMeans(samples);
        double[] weightedVariances = getWeightedVariances(samples);
        double weightedMeansVariance = var.evaluate(weightedMeans);
        double weightedMeansMean = mean.evaluate(weightedMeans);
        double weightedVarVariance = var.evaluate(weightedVariances);
        double weightedVarMean = mean.evaluate(weightedVariances);
        double V1 = samples[0].sumOfWeights;
        double V2 = samples[0].sumOfWeightsSquared;
        System.out.println("image is " + imageSize + "x" + imageSize + ", V1=" + V1 + ", V2=" + V2 + ", numTrials=" + NUM_TRIALS + ", sample means (mu=" + weightedMeansMean + ",s=" + weightedMeansVariance + "), sample variances (mu=" + weightedVarMean + ",s=" + weightedVarVariance);
    }
}

Example 88 with Mean

use of org.apache.commons.math3.stat.descriptive.moment.Mean in project vcell by virtualcell.

the class FluorescenceNoiseTest method getUniformFluorescenceImage.

private UShortImage getUniformFluorescenceImage(ISize size, Extent extent, Origin origin, double mean) throws ImageException {
    ExponentialDistribution expDistribution = new ExponentialDistribution(mean);
    short[] shortPixels = new short[size.getXYZ()];
    for (int i = 0; i < shortPixels.length; i++) {
        // consider that we have unsigned short (must use 32 bit integer first)
        double sample = expDistribution.sample();
        if (sample > 65535) {
            sample = 65535;
            System.err.println("sample " + sample + " overflows 16 bit unsigned");
        }
        shortPixels[i] = (short) (0xffff & ((int) Math.round(sample)));
    }
    UShortImage rawImage = new UShortImage(shortPixels, origin, extent, size.getX(), size.getY(), size.getZ());
    return rawImage;
}

Example 89 with Mean

use of org.apache.commons.math3.stat.descriptive.moment.Mean in project metron by apache.

the class StellarStatisticsFunctionsTest method run.

/**
 * Runs a Stellar expression.
 * @param expr The expression to run.
 * @param variables The variables available to the expression.
 */
private static Object run(String expr, Map<String, Object> variables) {
    StellarProcessor processor = new StellarProcessor();
    Object ret = processor.parse(expr, new DefaultVariableResolver(x -> variables.get(x), x -> variables.containsKey(x)), StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
    byte[] raw = SerDeUtils.toBytes(ret);
    Object actual = SerDeUtils.fromBytes(raw, Object.class);
    if (ret instanceof StatisticsProvider) {
        StatisticsProvider left = (StatisticsProvider) ret;
        StatisticsProvider right = (StatisticsProvider) actual;
        // N
        tolerantAssertEquals(prov -> prov.getCount(), left, right);
        // sum
        tolerantAssertEquals(prov -> prov.getSum(), left, right, 1e-3);
        // sum of squares
        tolerantAssertEquals(prov -> prov.getSumSquares(), left, right, 1e-3);
        // sum of squares
        tolerantAssertEquals(prov -> prov.getSumLogs(), left, right, 1e-3);
        // Mean
        tolerantAssertEquals(prov -> prov.getMean(), left, right, 1e-3);
        // Quadratic Mean
        tolerantAssertEquals(prov -> prov.getQuadraticMean(), left, right, 1e-3);
        // SD
        tolerantAssertEquals(prov -> prov.getStandardDeviation(), left, right, 1e-3);
        // Variance
        tolerantAssertEquals(prov -> prov.getVariance(), left, right, 1e-3);
        // Min
        tolerantAssertEquals(prov -> prov.getMin(), left, right, 1e-3);
        // Max
        tolerantAssertEquals(prov -> prov.getMax(), left, right, 1e-3);
        // Kurtosis
        tolerantAssertEquals(prov -> prov.getKurtosis(), left, right, 1e-3);
        // Skewness
        tolerantAssertEquals(prov -> prov.getSkewness(), left, right, 1e-3);
        for (double d = 10.0; d < 100.0; d += 10) {
            final double pctile = d;
            // This is a sketch, so we're a bit more forgiving here in our choice of \epsilon.
            tolerantAssertEquals(prov -> prov.getPercentile(pctile), left, right, 1e-2);
        }
    }
    return ret;
}

Example 90 with Mean

use of org.apache.commons.math3.stat.descriptive.moment.Mean in project metron by apache.

the class MedianAbsoluteDeviationTest method test.

@Test
public void test() {
    GaussianRandomGenerator gaussian = new GaussianRandomGenerator(new MersenneTwister(0L));
    DescriptiveStatistics stats = new DescriptiveStatistics();
    List<MedianAbsoluteDeviationFunctions.State> states = new ArrayList<>();
    MedianAbsoluteDeviationFunctions.State currentState = null;
    // initialize the state
    currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, NULL)", ImmutableMap.of("states", states));
    for (int i = 0, j = 0; i < 10000; ++i, ++j) {
        Double d = gaussian.nextNormalizedDouble();
        stats.addValue(d);
        run("OUTLIER_MAD_ADD(currentState, data)", ImmutableMap.of("currentState", currentState, "data", d));
        if (j >= 1000) {
            j = 0;
            List<MedianAbsoluteDeviationFunctions.State> stateWindow = new ArrayList<>();
            for (int stateIndex = Math.max(0, states.size() - 5); stateIndex < states.size(); ++stateIndex) {
                stateWindow.add(states.get(stateIndex));
            }
            currentState = (MedianAbsoluteDeviationFunctions.State) run("OUTLIER_MAD_STATE_MERGE(states, currentState)", ImmutableMap.of("states", stateWindow, "currentState", currentState));
        }
    }
    {
        Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMin()));
        Assert.assertTrue("Score: " + score + " is not an outlier despite being a minimum.", score > 3.5);
    }
    {
        Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMax()));
        Assert.assertTrue("Score: " + score + " is not an outlier despite being a maximum", score > 3.5);
    }
    {
        Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() + 4 * stats.getStandardDeviation()));
        Assert.assertTrue("Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5);
    }
    {
        Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean() - 4 * stats.getStandardDeviation()));
        Assert.assertTrue("Score: " + score + " is not an outlier despite being 4 std deviations away from the mean", score > 3.5);
    }
    {
        Double score = (Double) run("OUTLIER_MAD_SCORE(currentState, value)", ImmutableMap.of("currentState", currentState, "value", stats.getMean()));
        Assert.assertFalse("Score: " + score + " is an outlier despite being the mean", score > 3.5);
    }
}