Examples with BiConsumer java.util.function.BiConsumer used on opensource projects

Example 61 with BiConsumer

use of java.util.function.BiConsumer in project elki by elki-project.

the class ComputeKNNOutlierScores method run.

@Override
public void run() {
    final Database database = inputstep.getDatabase();
    final Relation<O> relation = database.getRelation(distf.getInputTypeRestriction());
    // Ensure we don't go beyond the relation size:
    final int maxk = Math.min(this.maxk, relation.size() - 1);
    // Get a KNN query.
    final int lim = Math.min(maxk + 2, relation.size());
    KNNQuery<O> knnq = QueryUtil.getKNNQuery(relation, distf, lim);
    // Precompute kNN:
    if (!(knnq instanceof PreprocessorKNNQuery)) {
        MaterializeKNNPreprocessor<O> preproc = new MaterializeKNNPreprocessor<>(relation, distf, lim);
        preproc.initialize();
        relation.getHierarchy().add(relation, preproc);
    }
    // Test that we now get a proper index query
    knnq = QueryUtil.getKNNQuery(relation, distf, lim);
    if (!(knnq instanceof PreprocessorKNNQuery)) {
        throw new AbortException("Not using preprocessor knn query -- KNN queries using class: " + knnq.getClass());
    }
    // Warn for some known slow methods and large k:
    if (!isDisabled("LDOF") && maxk > 100) {
        LOG.verbose("Note: LODF needs O(k^2) distance computations. Use -" + Parameterizer.DISABLE_ID.getName() + " LDOF to disable.");
    }
    if (!isDisabled("FastABOD") && maxk > 100) {
        LOG.warning("Note: FastABOD needs quadratic memory. Use -" + Parameterizer.DISABLE_ID.getName() + " FastABOD to disable.");
    }
    if (!isDisabled("DWOF") && maxk > 100) {
        LOG.warning("Note: DWOF needs O(k^2) distance computations. Use -" + Parameterizer.DISABLE_ID.getName() + " DWOF to disable.");
    }
    final DBIDs ids = relation.getDBIDs();
    try (PrintStream fout = new PrintStream(outfile)) {
        // Control: print the DBIDs in case we are seeing an odd iteration
        // 
        fout.append("# Data set size: " + relation.size()).append(" data type: " + relation.getDataTypeInformation()).append(FormatUtil.NEWLINE);
        // Label outlier result (reference)
        writeResult(fout, ids, bylabel.run(database), new IdentityScaling(), "bylabel");
        final int startk = (this.startk > 0) ? this.startk : this.stepk;
        final int startkmin2 = (startk >= 2) ? startk : (startk + stepk);
        final int startkmin3 = (startk >= 3) ? startk : (startkmin2 >= 3) ? startkmin2 : (startkmin2 + stepk);
        // Output function:
        BiConsumer<String, OutlierResult> out = (kstr, result) -> writeResult(fout, ids, result, scaling, kstr);
        // KNN
        runForEachK(// 
        "KNN", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNOutlier<O>(distf, k).run(database, relation), out);
        // KNN Weight
        runForEachK(// 
        "KNNW", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNWeightOutlier<O>(distf, k).run(database, relation), out);
        // Run LOF
        runForEachK(// 
        "LOF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LOF<O>(k, distf).run(database, relation), out);
        // Run Simplified-LOF
        runForEachK(// 
        "SimplifiedLOF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new SimplifiedLOF<O>(k, distf).run(database, relation), out);
        // LoOP
        runForEachK(// 
        "LoOP", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LoOP<O>(k, k, distf, distf, 1.0).run(database, relation), out);
        // LDOF
        runForEachK(// 
        "LDOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new LDOF<O>(distf, k).run(database, relation), out);
        // Run ODIN
        runForEachK(// 
        "ODIN", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new ODIN<O>(distf, k).run(database, relation), out);
        // Run FastABOD
        runForEachK(// 
        "FastABOD", // 
        startkmin3, // 
        stepk, // 
        maxk, k -> // 
        new FastABOD<O>(new PolynomialKernelFunction(2), k).run(database, relation), out);
        // Run KDEOS with intrinsic dimensionality 2.
        runForEachK(// 
        "KDEOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> new // 
        KDEOS<O>(// 
        distf, // 
        k, // 
        k, // 
        GaussianKernelDensityFunction.KERNEL, // 
        0., 0.5 * GaussianKernelDensityFunction.KERNEL.canonicalBandwidth(), // 
        2).run(database, relation), out);
        // Run LDF
        runForEachK(// 
        "LDF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LDF<O>(k, distf, GaussianKernelDensityFunction.KERNEL, 1., .1).run(database, relation), out);
        // Run INFLO
        runForEachK(// 
        "INFLO", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new INFLO<O>(distf, 1.0, k).run(database, relation), out);
        // Run COF
        runForEachK(// 
        "COF", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new COF<O>(k, distf).run(database, relation), out);
        // Run simple Intrinsic dimensionality
        runForEachK(// 
        "Intrinsic", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new IntrinsicDimensionalityOutlier<O>(distf, k, AggregatedHillEstimator.STATIC).run(database, relation), out);
        // Run IDOS
        runForEachK(// 
        "IDOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new IDOS<O>(distf, AggregatedHillEstimator.STATIC, k, k).run(database, relation), out);
        // Run simple kernel-density LOF variant
        runForEachK(// 
        "KDLOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new SimpleKernelDensityLOF<O>(k, distf, GaussianKernelDensityFunction.KERNEL).run(database, relation), out);
        // Run DWOF (need pairwise distances, too)
        runForEachK(// 
        "DWOF", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new DWOF<O>(distf, k, 1.1).run(database, relation), out);
        // Run LIC
        runForEachK(// 
        "LIC", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new LocalIsolationCoefficient<O>(distf, k).run(database, relation), out);
        // Run VOV (requires a vector field).
        if (TypeUtil.DOUBLE_VECTOR_FIELD.isAssignableFromType(relation.getDataTypeInformation())) {
            @SuppressWarnings("unchecked") final DistanceFunction<? super DoubleVector> df = (DistanceFunction<? super DoubleVector>) distf;
            @SuppressWarnings("unchecked") final Relation<DoubleVector> rel = (Relation<DoubleVector>) (Relation<?>) relation;
            runForEachK(// 
            "VOV", // 
            startk, // 
            stepk, // 
            maxk, k -> // 
            new VarianceOfVolume<DoubleVector>(k, df).run(database, rel), out);
        }
        // Run KNN DD
        runForEachK(// 
        "KNNDD", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNDD<O>(distf, k).run(database, relation), out);
        // Run KNN SOS
        runForEachK(// 
        "KNNSOS", // 
        startk, // 
        stepk, // 
        maxk, k -> // 
        new KNNSOS<O>(distf, k).run(relation), out);
        // Run ISOS
        runForEachK(// 
        "ISOS", // 
        startkmin2, // 
        stepk, // 
        maxk, k -> // 
        new ISOS<O>(distf, k, AggregatedHillEstimator.STATIC).run(relation), out);
    } catch (FileNotFoundException e) {
        throw new AbortException("Cannot create output file.", e);
    }
}

Example 62 with BiConsumer

use of java.util.function.BiConsumer in project presto by prestodb.

the class TestPinotQueryGenerator method testUnaryAggregationHelper.

private void testUnaryAggregationHelper(BiConsumer<PlanBuilder, PlanBuilder.AggregationBuilder> aggregationFunctionBuilder, String expectedAggOutput) {
    PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare));
    PlanNode filter = buildPlan(planBuilder -> filter(planBuilder, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare), getRowExpression("fare > 3", defaultSessionHolder)));
    PlanNode anotherFilter = buildPlan(planBuilder -> filter(planBuilder, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare), getRowExpression("secondssinceepoch between 200 and 300 and regionid >= 40", defaultSessionHolder)));
    PlanNode filterWithMultiValue = buildPlan(planBuilder -> filter(planBuilder, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare, scores), getRowExpression("contains(scores, 100) OR contains(scores, 200)", defaultSessionHolder)));
    testPinotQuery(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).globalGrouping())), format("SELECT %s FROM realtimeOnly", getExpectedAggOutput(expectedAggOutput, "")));
    testPinotQuery(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(filter).globalGrouping())), format("SELECT %s FROM realtimeOnly WHERE (fare > 3)", getExpectedAggOutput(expectedAggOutput, "")));
    testPinotQuery(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(filter).singleGroupingSet(variable("regionid")))), format("SELECT %s FROM realtimeOnly WHERE (fare > 3) GROUP BY regionId %s 10000", getExpectedAggOutput(expectedAggOutput, "regionId"), getGroupByLimitKey()));
    testPinotQuery(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).singleGroupingSet(variable("regionid")))), format("SELECT %s FROM realtimeOnly GROUP BY regionId %s 10000", getExpectedAggOutput(expectedAggOutput, "regionId"), getGroupByLimitKey()));
    testPinotQuery(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(anotherFilter).singleGroupingSet(variable("regionid"), variable("city")))), format("SELECT %s FROM realtimeOnly WHERE ((secondsSinceEpoch BETWEEN 200 AND 300) AND (regionId >= 40)) GROUP BY regionId, city %s 10000", getExpectedAggOutput(expectedAggOutput, "regionId, city"), getGroupByLimitKey()));
    testPinotQuery(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(filterWithMultiValue).singleGroupingSet(variable("regionid"), variable("city")))), format("SELECT %s FROM realtimeOnly WHERE ((scores = 100) OR (scores = 200)) GROUP BY regionId, city %s 10000", getExpectedAggOutput(expectedAggOutput, "regionId, city"), getGroupByLimitKey()));
}

Example 63 with BiConsumer

use of java.util.function.BiConsumer in project presto by prestodb.

the class TestDruidQueryGenerator method testCountStar.

@Test
public void testCountStar() {
    BiConsumer<PlanBuilder, PlanBuilder.AggregationBuilder> aggregationFunctionBuilder = (planBuilder, aggregationBuilder) -> aggregationBuilder.addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder));
    PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, druidTable, regionId, secondsSinceEpoch, city, fare));
    PlanNode filter = buildPlan(planBuilder -> filter(planBuilder, tableScan(planBuilder, druidTable, regionId, secondsSinceEpoch, city, fare), getRowExpression("fare > 3", defaultSessionHolder)));
    PlanNode anotherFilter = buildPlan(planBuilder -> filter(planBuilder, tableScan(planBuilder, druidTable, regionId, secondsSinceEpoch, city, fare), getRowExpression("secondssinceepoch between 200 and 300 and \"region.id\" >= 40", defaultSessionHolder)));
    testDQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).globalGrouping())), "SELECT count(*) FROM \"realtimeOnly\"");
    testDQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(filter).globalGrouping())), "SELECT count(*) FROM \"realtimeOnly\" WHERE (\"fare\" > 3)");
    testDQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(filter).singleGroupingSet(variable("region.id")))), "SELECT \"region.Id\", count(*) FROM \"realtimeOnly\" WHERE (\"fare\" > 3) GROUP BY \"region.Id\"");
    testDQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).singleGroupingSet(variable("region.id")))), "SELECT \"region.Id\", count(*) FROM \"realtimeOnly\" GROUP BY \"region.Id\"");
    testDQL(planBuilder -> limit(planBuilder, 5L, planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).singleGroupingSet(variable("region.id"))))), "SELECT \"region.Id\", count(*) FROM \"realtimeOnly\" GROUP BY \"region.Id\" LIMIT 5");
    testDQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(anotherFilter).singleGroupingSet(variable("region.id"), variable("city")))), "SELECT \"region.Id\", \"city\", count(*) FROM \"realtimeOnly\" WHERE ((\"secondsSinceEpoch\" BETWEEN 200 AND 300) AND (\"region.Id\" >= 40)) GROUP BY \"region.Id\", \"city\"");
}

Example 64 with BiConsumer

use of java.util.function.BiConsumer in project presto by prestodb.

the class TestBlockRetainedSizeBreakdown method checkRetainedSize.

private static void checkRetainedSize(Block block, boolean getRegionCreateNewObjects) {
    AtomicLong objectSize = new AtomicLong();
    Object2LongOpenCustomHashMap<Object> trackedObjects = new Object2LongOpenCustomHashMap<>(new ObjectStrategy());
    BiConsumer<Object, Long> consumer = (object, size) -> {
        objectSize.addAndGet(size);
        trackedObjects.addTo(object, 1);
    };
    block.retainedBytesForEachPart(consumer);
    assertEquals(objectSize.get(), block.getRetainedSizeInBytes());
    Block copyBlock = block.getRegion(0, block.getPositionCount() / 2);
    copyBlock.retainedBytesForEachPart(consumer);
    assertEquals(objectSize.get(), block.getRetainedSizeInBytes() + copyBlock.getRetainedSizeInBytes());
    assertEquals(trackedObjects.getLong(block), 1);
    assertEquals(trackedObjects.getLong(copyBlock), 1);
    trackedObjects.remove(block);
    trackedObjects.remove(copyBlock);
    for (long value : trackedObjects.values()) {
        assertEquals(value, getRegionCreateNewObjects ? 1 : 2);
    }
}

Example 65 with BiConsumer

use of java.util.function.BiConsumer in project presto by prestodb.

the class TestAddExchangesPlans method assertPlanWithMergePartitionStrategy.

private void assertPlanWithMergePartitionStrategy(String sql, String partitionMergingStrategy, int remoteRepartitionExchangeCount, PlanMatchPattern pattern) {
    Session session = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(AGGREGATION_PARTITIONING_MERGING_STRATEGY, partitionMergingStrategy).setSystemProperty(TASK_CONCURRENCY, "2").build();
    BiConsumer<Plan, Integer> validateMultipleRemoteRepartitionExchange = (plan, count) -> assertEquals(searchFrom(plan.getRoot()).where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == REMOTE_STREAMING && ((ExchangeNode) node).getType() == REPARTITION).count(), count.intValue());
    assertPlanWithSession(sql, session, false, pattern, plan -> validateMultipleRemoteRepartitionExchange.accept(plan, remoteRepartitionExchangeCount));
}