Examples with Precision com.amazon.randomcutforest.config.Precision used on opensource projects

Example 11 with Precision

use of com.amazon.randomcutforest.config.Precision in project random-cut-forest-by-aws by aws.

the class RandomCutForest method getApproximateDynamicScore.

/**
 * Score a point using the given scoring functions. This method will
 * short-circuit before visiting all trees if the scores that are returned from
 * a subset of trees appears to be converging to a given value. See
 * {@link OneSidedConvergingDoubleAccumulator} for more about convergence.
 *
 * @param point                   input point
 * @param precision               controls early convergence
 * @param highIsCritical          this is true for the default scoring function.
 *                                If the user wishes to use a different scoring
 *                                function where anomaly scores are low values
 *                                (for example, height in tree) then this should
 *                                be set to false.
 * @param ignoreLeafMassThreshold said threshold
 * @param seen                    scoring function when the input matches some
 *                                tuple in the leaves
 * @param unseen                  scoring function when the input is not found
 * @param damp                    dampening function for duplicates which are
 *                                same as input (applies with seen)
 * @return the dynamic score under sequential early stopping
 */
public double getApproximateDynamicScore(float[] point, double precision, boolean highIsCritical, int ignoreLeafMassThreshold, BiFunction<Double, Double, Double> seen, BiFunction<Double, Double, Double> unseen, BiFunction<Double, Double, Double> damp) {
    checkArgument(ignoreLeafMassThreshold >= 0, "ignoreLeafMassThreshold should be greater than or equal to 0");
    if (!isOutputReady()) {
        return 0.0;
    }
    VisitorFactory<Double> visitorFactory = new VisitorFactory<>((tree, y) -> new DynamicScoreVisitor(tree.projectToTree(y), tree.getMass(), ignoreLeafMassThreshold, seen, unseen, damp));
    ConvergingAccumulator<Double> accumulator = new OneSidedConvergingDoubleAccumulator(highIsCritical, precision, DEFAULT_APPROXIMATE_DYNAMIC_SCORE_MIN_VALUES_ACCEPTED, numberOfTrees);
    Function<Double, Double> finisher = x -> x / accumulator.getValuesAccepted();
    return traverseForest(transformToShingledPoint(point), visitorFactory, accumulator, finisher);
}

Example 12 with Precision

use of com.amazon.randomcutforest.config.Precision in project random-cut-forest-by-aws by aws.

the class RandomCutForest method getApproximateDynamicAttribution.

/**
 * Atrribution for dynamic sequential scoring; getL1Norm() should agree with
 * getDynamicScoringSequential
 *
 * @param point                   input
 * @param precision               parameter to stop early stopping
 * @param highIsCritical          are high values anomalous (otherwise low
 *                                values are anomalous)
 * @param ignoreLeafMassThreshold we ignore leaves with mass equal/below *
 *                                threshold
 * @param seen                    function for scoring points that have been
 *                                seen before
 * @param unseen                  function for scoring points not seen in tree
 * @param newDamp                 dampening function based on duplicates
 * @return attribution DiVector of the score
 */
public DiVector getApproximateDynamicAttribution(float[] point, double precision, boolean highIsCritical, int ignoreLeafMassThreshold, BiFunction<Double, Double, Double> seen, BiFunction<Double, Double, Double> unseen, BiFunction<Double, Double, Double> newDamp) {
    if (!isOutputReady()) {
        return new DiVector(dimensions);
    }
    VisitorFactory<DiVector> visitorFactory = new VisitorFactory<>((tree, y) -> new DynamicAttributionVisitor(y, tree.getMass(), ignoreLeafMassThreshold, seen, unseen, newDamp), (tree, x) -> x.lift(tree::liftFromTree));
    ConvergingAccumulator<DiVector> accumulator = new OneSidedConvergingDiVectorAccumulator(dimensions, highIsCritical, precision, DEFAULT_APPROXIMATE_DYNAMIC_SCORE_MIN_VALUES_ACCEPTED, numberOfTrees);
    Function<DiVector, DiVector> finisher = vector -> vector.scale(1.0 / accumulator.getValuesAccepted());
    return traverseForest(transformToShingledPoint(point), visitorFactory, accumulator, finisher);
}

Example 13 with Precision

use of com.amazon.randomcutforest.config.Precision in project random-cut-forest-by-aws by aws.

the class DynamicSampling method run.

@Override
public void run() throws Exception {
    // Create and populate a random cut forest
    int dimensions = 4;
    int numberOfTrees = 50;
    int sampleSize = 256;
    Precision precision = Precision.FLOAT_64;
    int dataSize = 4 * sampleSize;
    NormalMixtureTestData testData = new NormalMixtureTestData();
    RandomCutForest forest = RandomCutForest.builder().compact(true).dimensions(dimensions).randomSeed(0).numberOfTrees(numberOfTrees).sampleSize(sampleSize).precision(precision).build();
    RandomCutForest forest2 = RandomCutForest.builder().compact(true).dimensions(dimensions).randomSeed(0).numberOfTrees(numberOfTrees).sampleSize(sampleSize).precision(precision).build();
    int first_anomalies = 0;
    int second_anomalies = 0;
    forest2.setTimeDecay(10 * forest2.getTimeDecay());
    for (double[] point : testData.generateTestData(dataSize, dimensions)) {
        if (forest.getAnomalyScore(point) > 1.0) {
            first_anomalies++;
        }
        if (forest2.getAnomalyScore(point) > 1.0) {
            second_anomalies++;
        }
        forest.update(point);
        forest2.update(point);
    }
    System.out.println("Unusual scores: forest one " + first_anomalies + ", second one " + second_anomalies);
    // should be roughly equal
    first_anomalies = second_anomalies = 0;
    testData = new NormalMixtureTestData(-3, 40);
    for (double[] point : testData.generateTestData(dataSize, dimensions)) {
        if (forest.getAnomalyScore(point) > 1.0) {
            first_anomalies++;
        }
        if (forest2.getAnomalyScore(point) > 1.0) {
            second_anomalies++;
        }
        forest.update(point);
        forest2.update(point);
    }
    System.out.println("Unusual scores: forest one " + first_anomalies + ", second one " + second_anomalies);
    // forest2 should adapt faster
    first_anomalies = second_anomalies = 0;
    RandomCutForestMapper mapper = new RandomCutForestMapper();
    mapper.setSaveExecutorContextEnabled(true);
    RandomCutForest copyForest = mapper.toModel(mapper.toState(forest));
    copyForest.setTimeDecay(50 * forest.getTimeDecay());
    // force an adjustment to catch up
    testData = new NormalMixtureTestData(-10, -40);
    int forced_change_anomalies = 0;
    for (double[] point : testData.generateTestData(dataSize, dimensions)) {
        if (forest.getAnomalyScore(point) > 1.0) {
            first_anomalies++;
        }
        if (forest2.getAnomalyScore(point) > 1.0) {
            second_anomalies++;
        }
        if (copyForest.getAnomalyScore(point) > 1.0) {
            forced_change_anomalies++;
        }
        copyForest.update(point);
        forest.update(point);
        forest2.update(point);
    }
    // both should show the similar rate of adjustment
    System.out.println("Unusual scores: forest one " + first_anomalies + ", second one " + second_anomalies + ", forced (first) " + forced_change_anomalies);
}

Example 14 with Precision

use of com.amazon.randomcutforest.config.Precision in project random-cut-forest-by-aws by aws.

the class Thresholded1DGaussianMix method run.

@Override
public void run() throws Exception {
    // Create and populate a random cut forest
    int shingleSize = 4;
    int numberOfTrees = 50;
    int sampleSize = 256;
    Precision precision = Precision.FLOAT_32;
    int dataSize = 4 * sampleSize;
    // change this to try different number of attributes,
    // this parameter is not expected to be larger than 5 for this example
    int baseDimensions = 1;
    int count = 0;
    int dimensions = baseDimensions * shingleSize;
    ThresholdedRandomCutForest forest = new ThresholdedRandomCutForest.Builder<>().compact(true).dimensions(dimensions).randomSeed(0).numberOfTrees(numberOfTrees).shingleSize(shingleSize).sampleSize(sampleSize).precision(precision).anomalyRate(0.01).forestMode(ForestMode.TIME_AUGMENTED).build();
    long seed = new Random().nextLong();
    System.out.println("Anomalies would correspond to a run, based on a change of state.");
    System.out.println("Each change is normal <-> anomaly;  so after the second change the data is normal");
    System.out.println("seed = " + seed);
    NormalMixtureTestData normalMixtureTestData = new NormalMixtureTestData(10, 1.0, 50, 2.0, 0.01, 0.1);
    MultiDimDataWithKey dataWithKeys = normalMixtureTestData.generateTestDataWithKey(dataSize, 1, 0);
    int keyCounter = 0;
    for (double[] point : dataWithKeys.data) {
        AnomalyDescriptor result = forest.process(point, count);
        if (keyCounter < dataWithKeys.changeIndices.length && result.getInternalTimeStamp() == dataWithKeys.changeIndices[keyCounter]) {
            System.out.println("timestamp " + (result.getInputTimestamp()) + " CHANGE");
            ++keyCounter;
        }
        if (keyCounter < dataWithKeys.changeIndices.length && count == dataWithKeys.changeIndices[keyCounter]) {
            System.out.println("timestamp " + (count) + " CHANGE ");
            ++keyCounter;
        }
        if (result.getAnomalyGrade() != 0) {
            System.out.print("timestamp " + (count) + " RESULT value ");
            for (int i = 0; i < baseDimensions; i++) {
                System.out.print(result.getCurrentInput()[i] + ", ");
            }
            System.out.print("score " + result.getRCFScore() + ", grade " + result.getAnomalyGrade() + ", ");
            if (result.isExpectedValuesPresent()) {
                if (result.getRelativeIndex() != 0 && result.isStartOfAnomaly()) {
                    System.out.print(-result.getRelativeIndex() + " steps ago, instead of ");
                    for (int i = 0; i < baseDimensions; i++) {
                        System.out.print(result.getPastValues()[i] + ", ");
                    }
                    System.out.print("expected ");
                    for (int i = 0; i < baseDimensions; i++) {
                        System.out.print(result.getExpectedValuesList()[0][i] + ", ");
                        if (result.getPastValues()[i] != result.getExpectedValuesList()[0][i]) {
                            System.out.print("( " + (result.getPastValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                        }
                    }
                } else {
                    System.out.print("expected ");
                    for (int i = 0; i < baseDimensions; i++) {
                        System.out.print(result.getExpectedValuesList()[0][i] + ", ");
                        if (result.getCurrentInput()[i] != result.getExpectedValuesList()[0][i]) {
                            System.out.print("( " + (result.getCurrentInput()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                        }
                    }
                }
            }
            System.out.println();
        }
        ++count;
    }
}

Example 15 with Precision

use of com.amazon.randomcutforest.config.Precision in project random-cut-forest-by-aws by aws.

the class ThresholdedMultiDimensionalExample method run.

@Override
public void run() throws Exception {
    // Create and populate a random cut forest
    int shingleSize = 4;
    int numberOfTrees = 50;
    int sampleSize = 256;
    Precision precision = Precision.FLOAT_32;
    int dataSize = 4 * sampleSize;
    // change this to try different number of attributes,
    // this parameter is not expected to be larger than 5 for this example
    int baseDimensions = 2;
    int dimensions = baseDimensions * shingleSize;
    ThresholdedRandomCutForest forest = ThresholdedRandomCutForest.builder().compact(true).dimensions(dimensions).randomSeed(0).numberOfTrees(numberOfTrees).shingleSize(shingleSize).sampleSize(sampleSize).precision(precision).anomalyRate(0.01).forestMode(ForestMode.STANDARD).build();
    long seed = new Random().nextLong();
    System.out.println("seed = " + seed);
    // change the last argument seed for a different run
    MultiDimDataWithKey dataWithKeys = ShingledMultiDimDataWithKeys.generateShingledDataWithKey(dataSize, 50, shingleSize, baseDimensions, seed);
    int keyCounter = 0;
    int count = 0;
    for (double[] point : dataWithKeys.data) {
        AnomalyDescriptor result = forest.process(point, 0L);
        if (keyCounter < dataWithKeys.changeIndices.length && count + shingleSize - 1 == dataWithKeys.changeIndices[keyCounter]) {
            System.out.println("timestamp " + (count + shingleSize - 1) + " CHANGE " + Arrays.toString(dataWithKeys.changes[keyCounter]));
            ++keyCounter;
        }
        if (result.getAnomalyGrade() != 0) {
            System.out.print("timestamp " + (count + shingleSize - 1) + " RESULT value ");
            for (int i = (shingleSize - 1) * baseDimensions; i < shingleSize * baseDimensions; i++) {
                System.out.print(result.getCurrentInput()[i] + ", ");
            }
            System.out.print("score " + result.getRCFScore() + ", grade " + result.getAnomalyGrade() + ", ");
            if (result.isExpectedValuesPresent()) {
                if (result.getRelativeIndex() != 0 && result.isStartOfAnomaly()) {
                    System.out.print(-result.getRelativeIndex() + " steps ago, instead of ");
                    for (int i = 0; i < baseDimensions; i++) {
                        System.out.print(result.getPastValues()[i] + ", ");
                    }
                    System.out.print("expected ");
                    for (int i = 0; i < baseDimensions; i++) {
                        System.out.print(result.getExpectedValuesList()[0][i] + ", ");
                        if (result.getPastValues()[i] != result.getExpectedValuesList()[0][i]) {
                            System.out.print("( " + (result.getPastValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                        }
                    }
                } else {
                    System.out.print("expected ");
                    for (int i = 0; i < baseDimensions; i++) {
                        System.out.print(result.getExpectedValuesList()[0][i] + ", ");
                        if (result.getCurrentInput()[(shingleSize - 1) * baseDimensions + i] != result.getExpectedValuesList()[0][i]) {
                            System.out.print("( " + (result.getCurrentInput()[(shingleSize - 1) * baseDimensions + i] - result.getExpectedValuesList()[0][i]) + " ) ");
                        }
                    }
                }
            }
            System.out.println();
        }
        ++count;
    }
}