Examples with EventRateLimiter cbit.vcell.util.EventRateLimiter used on opensource projects

Example 1 with EventRateLimiter

use of cbit.vcell.util.EventRateLimiter in project vcell by virtualcell.

the class DataSetControllerImpl method getTimeSeriesValues_private.

private TimeSeriesJobResults getTimeSeriesValues_private(OutputContext outputContext, final VCDataIdentifier vcdID, final TimeSeriesJobSpec timeSeriesJobSpec) throws DataAccessException {
    double[] dataTimes = null;
    boolean isPostProcessing = false;
    try {
        if (getVCData(vcdID) instanceof SimulationData && ((SimulationData) getVCData(vcdID)).isPostProcessing(outputContext, timeSeriesJobSpec.getVariableNames()[0])) {
            isPostProcessing = true;
            dataTimes = ((SimulationData) getVCData(vcdID)).getDataTimesPostProcess(outputContext);
        }
    } catch (Exception e) {
        // ignore
        e.printStackTrace();
    }
    if (dataTimes == null) {
        dataTimes = getDataSetTimes(vcdID);
    }
    TimeInfo timeInfo = new TimeInfo(vcdID, timeSeriesJobSpec.getStartTime(), timeSeriesJobSpec.getStep(), timeSeriesJobSpec.getEndTime(), dataTimes);
    if (dataTimes.length <= 0) {
        return null;
    }
    boolean[] wantsTheseTimes = new boolean[dataTimes.length];
    double[] desiredTimeValues = null;
    int desiredNumTimes = 0;
    Arrays.fill(wantsTheseTimes, false);
    double[] tempTimes = new double[dataTimes.length];
    int stepCounter = 0;
    for (int i = 0; i < dataTimes.length; i += 1) {
        if (dataTimes[i] > timeSeriesJobSpec.getEndTime()) {
            break;
        }
        if (dataTimes[i] == timeSeriesJobSpec.getStartTime()) {
            tempTimes[desiredNumTimes] = dataTimes[i];
            desiredNumTimes += 1;
            stepCounter = 0;
            wantsTheseTimes[i] = true;
            if (timeSeriesJobSpec.getStep() == 0) {
                break;
            }
        } else if (desiredNumTimes > 0 && stepCounter % timeSeriesJobSpec.getStep() == 0) {
            tempTimes[desiredNumTimes] = dataTimes[i];
            desiredNumTimes += 1;
            wantsTheseTimes[i] = true;
        }
        stepCounter += 1;
    }
    if (desiredNumTimes == 0) {
        throw new IllegalArgumentException("Couldn't find startTime " + timeSeriesJobSpec.getStartTime());
    }
    desiredTimeValues = new double[desiredNumTimes];
    System.arraycopy(tempTimes, 0, desiredTimeValues, 0, desiredNumTimes);
    // Check timeInfo
    if (desiredTimeValues.length != timeInfo.getDesiredTimeValues().length) {
        throw new DataAccessException("timeInfo check failed");
    }
    for (int i = 0; i < desiredTimeValues.length; i++) {
        if (desiredTimeValues[i] != timeInfo.getDesiredTimeValues()[i]) {
            throw new DataAccessException("timeInfo check failed");
        }
    }
    for (int i = 0; i < wantsTheseTimes.length; i++) {
        if (wantsTheseTimes[i] != timeInfo.getWantsTheseTimes()[i]) {
            throw new DataAccessException("timeInfo check failed");
        }
    }
    try {
        timeSeriesJobSpec.initIndices();
        // See if we need special processing
        TimeSeriesJobResults specialTSJR = getSpecialTimeSeriesValues(outputContext, vcdID, timeSeriesJobSpec, timeInfo);
        if (specialTSJR != null) {
            return specialTSJR;
        }
        // 
        VCData vcData = getVCData(vcdID);
        // 
        // Determine Memory Usage for this job to protect server
        // 
        // No TimeSeries jobs larger than this
        final long MAX_MEM_USAGE = 20000000;
        long memUsage = 0;
        // efficient function stats are not yet implemented so check to adjust calculation
        boolean bHasFunctionVars = false;
        for (int i = 0; i < timeSeriesJobSpec.getVariableNames().length; i += 1) {
            bHasFunctionVars = bHasFunctionVars || (getFunction(outputContext, vcdID, timeSeriesJobSpec.getVariableNames()[i]) != null);
        }
        for (int i = 0; i < timeSeriesJobSpec.getIndices().length; i += 1) {
            memUsage += (timeSeriesJobSpec.isCalcSpaceStats() && !bHasFunctionVars ? NUM_STATS : timeSeriesJobSpec.getIndices()[i].length);
        }
        memUsage *= desiredNumTimes * 8 * 2;
        System.out.println("DataSetControllerImpl.getTimeSeriesValues: job memory=" + memUsage);
        if (memUsage > MAX_MEM_USAGE) {
            throw new DataAccessException("DataSetControllerImpl.getTimeSeriesValues: Job too large" + (bHasFunctionVars ? "(has function vars)" : "") + ", requires approx. " + memUsage + " bytes of memory (only " + MAX_MEM_USAGE + " bytes allowed).  Choose fewer datapoints or times.");
        }
        // 
        Vector<double[][]> valuesV = new Vector<double[][]>();
        SpatialStatsInfo spatialStatsInfo = null;
        if (timeSeriesJobSpec.isCalcSpaceStats()) {
            spatialStatsInfo = calcSpatialStatsInfo(outputContext, timeSeriesJobSpec, vcdID);
        }
        final EventRateLimiter eventRateLimiter = new EventRateLimiter();
        for (int k = 0; k < timeSeriesJobSpec.getVariableNames().length; k += 1) {
            double[][] timeSeries = null;
            String varName = timeSeriesJobSpec.getVariableNames()[k];
            int[] indices = timeSeriesJobSpec.getIndices()[k];
            if (timeSeriesJobSpec.isCalcSpaceStats() && !bHasFunctionVars) {
                timeSeries = new double[NUM_STATS + 1][desiredNumTimes];
            } else {
                timeSeries = new double[indices.length + 1][desiredNumTimes];
            }
            timeSeries[0] = desiredTimeValues;
            ProgressListener progressListener = new ProgressListener() {

                public void updateProgress(double progress) {
                    // System.out.println("Considering firing progress event at "+new Date());
                    if (eventRateLimiter.isOkayToFireEventNow()) {
                        // System.out.println("ACTUALLY firing Progress event at "+new Date());
                        fireDataJobEventIfNecessary(timeSeriesJobSpec.getVcDataJobID(), MessageEvent.DATA_PROGRESS, vcdID, new Double(progress), null, null);
                    }
                }

                public void updateMessage(String message) {
                // ignore
                }
            };
            AnnotatedFunction function = getFunction(outputContext, vcdID, varName);
            if (function != null) {
                if (vcData instanceof SimulationData) {
                    function = ((SimulationData) vcData).simplifyFunction(function);
                } else {
                    throw new Exception("DataSetControllerImpl::getTimeSeriesValues_private(): has to be SimulationData to get time plot.");
                }
                MultiFunctionIndexes mfi = new MultiFunctionIndexes(vcdID, function, indices, wantsTheseTimes, progressListener, outputContext);
                for (int i = 0; i < desiredTimeValues.length; i++) {
                    fireDataJobEventIfNecessary(timeSeriesJobSpec.getVcDataJobID(), MessageEvent.DATA_PROGRESS, vcdID, new Double(NumberUtils.formatNumber(100.0 * (double) (k * desiredTimeValues.length + i) / (double) (timeSeriesJobSpec.getVariableNames().length * desiredTimeValues.length), 3)), null, null);
                    for (int j = 0; j < indices.length; j++) {
                        timeSeries[j + 1][i] = mfi.evaluateTimeFunction(outputContext, i, j);
                    }
                }
            } else {
                double[][][] valuesOverTime = null;
                if (timeSeriesJobSpec.isCalcSpaceStats() && !bHasFunctionVars) {
                    valuesOverTime = vcData.getSimDataTimeSeries(outputContext, new String[] { varName }, new int[][] { indices }, wantsTheseTimes, spatialStatsInfo, progressListener);
                } else {
                    valuesOverTime = vcData.getSimDataTimeSeries(outputContext, new String[] { varName }, new int[][] { indices }, wantsTheseTimes, progressListener);
                }
                for (int i = 0; i < desiredTimeValues.length; i++) {
                    fireDataJobEventIfNecessary(timeSeriesJobSpec.getVcDataJobID(), MessageEvent.DATA_PROGRESS, vcdID, new Double(NumberUtils.formatNumber(100.0 * (double) (k * desiredTimeValues.length + i) / (double) (timeSeriesJobSpec.getVariableNames().length * desiredTimeValues.length), 3)), null, null);
                    if (timeSeriesJobSpec.isCalcSpaceStats() && !bHasFunctionVars) {
                        // min
                        timeSeries[MIN_OFFSET + 1][i] = valuesOverTime[i][0][MIN_OFFSET];
                        // max
                        timeSeries[MAX_OFFSET + 1][i] = valuesOverTime[i][0][MAX_OFFSET];
                        // mean
                        timeSeries[MEAN_OFFSET + 1][i] = valuesOverTime[i][0][MEAN_OFFSET];
                        // wmean
                        timeSeries[WMEAN_OFFSET + 1][i] = valuesOverTime[i][0][WMEAN_OFFSET];
                        // sum
                        timeSeries[SUM_OFFSET + 1][i] = valuesOverTime[i][0][SUM_OFFSET];
                        // wsum
                        timeSeries[WSUM_OFFSET + 1][i] = valuesOverTime[i][0][WSUM_OFFSET];
                    } else {
                        for (int j = 0; j < indices.length; j++) {
                            timeSeries[j + 1][i] = valuesOverTime[i][0][j];
                        }
                    }
                }
            }
            valuesV.add(timeSeries);
        }
        if (timeSeriesJobSpec.isCalcSpaceStats() && !bHasFunctionVars) {
            double[][] min = new double[timeSeriesJobSpec.getVariableNames().length][desiredTimeValues.length];
            double[][] max = new double[timeSeriesJobSpec.getVariableNames().length][desiredTimeValues.length];
            double[][] mean = new double[timeSeriesJobSpec.getVariableNames().length][desiredTimeValues.length];
            double[][] wmean = new double[timeSeriesJobSpec.getVariableNames().length][desiredTimeValues.length];
            double[][] sum = new double[timeSeriesJobSpec.getVariableNames().length][desiredTimeValues.length];
            double[][] wsum = new double[timeSeriesJobSpec.getVariableNames().length][desiredTimeValues.length];
            for (int i = 0; i < valuesV.size(); i += 1) {
                double[][] timeStat = (double[][]) valuesV.elementAt(i);
                for (int j = 0; j < desiredTimeValues.length; j += 1) {
                    min[i][j] = timeStat[MIN_OFFSET + 1][j];
                    max[i][j] = timeStat[MAX_OFFSET + 1][j];
                    mean[i][j] = timeStat[MEAN_OFFSET + 1][j];
                    wmean[i][j] = timeStat[WMEAN_OFFSET + 1][j];
                    sum[i][j] = timeStat[SUM_OFFSET + 1][j];
                    wsum[i][j] = timeStat[WSUM_OFFSET + 1][j];
                }
            }
            return new TSJobResultsSpaceStats(timeSeriesJobSpec.getVariableNames(), timeSeriesJobSpec.getIndices(), desiredTimeValues, min, max, mean, (spatialStatsInfo.bWeightsValid ? wmean : null), sum, (spatialStatsInfo.bWeightsValid ? wsum : null), (spatialStatsInfo.bWeightsValid ? spatialStatsInfo.totalSpace : null));
        } else if (timeSeriesJobSpec.isCalcSpaceStats() && bHasFunctionVars) {
            double[][][] timeSeriesFormatedValuesArr = new double[valuesV.size()][][];
            valuesV.copyInto(timeSeriesFormatedValuesArr);
            return calculateStatisticsFromWhole(timeSeriesJobSpec, timeSeriesFormatedValuesArr, desiredTimeValues, spatialStatsInfo);
        } else {
            double[][][] timeSeriesFormatedValuesArr = new double[valuesV.size()][][];
            valuesV.copyInto(timeSeriesFormatedValuesArr);
            TSJobResultsNoStats tsJobResultsNoStats = new TSJobResultsNoStats(timeSeriesJobSpec.getVariableNames(), timeSeriesJobSpec.getIndices(), desiredTimeValues, timeSeriesFormatedValuesArr);
            if (!isPostProcessing && timeSeriesJobSpec.getCrossingMembraneIndices() != null && timeSeriesJobSpec.getCrossingMembraneIndices().length > 0) {
                adjustMembraneAdjacentVolumeValues(outputContext, tsJobResultsNoStats.getTimesAndValuesForVariable(timeSeriesJobSpec.getVariableNames()[0]), true, null, timeSeriesJobSpec.getIndices()[0], timeSeriesJobSpec.getCrossingMembraneIndices()[0], vcdID, timeSeriesJobSpec.getVariableNames()[0], getMesh(vcdID), timeInfo);
            }
            return tsJobResultsNoStats;
        }
    } catch (DataAccessException e) {
        lg.error(e.getMessage(), e);
        throw e;
    } catch (Throwable e) {
        lg.error(e.getMessage(), e);
        throw new DataAccessException("DataSetControllerImpl.getTimeSeriesValues: " + (e.getMessage() == null ? e.getClass().getName() : e.getMessage()));
    }
}