Examples with HumanReadableBytes org.apache.druid.java.util.common.HumanReadableBytes used on opensource projects

Example 26 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class DruidProcessingConfig method intermediateComputeSizeBytes.

public int intermediateComputeSizeBytes() {
    HumanReadableBytes sizeBytesConfigured = intermediateComputeSizeBytesConfigured();
    if (!DEFAULT_PROCESSING_BUFFER_SIZE_BYTES.equals(sizeBytesConfigured)) {
        if (sizeBytesConfigured.getBytes() > Integer.MAX_VALUE) {
            throw new IAE("druid.processing.buffer.sizeBytes must be less than 2GiB");
        }
        return sizeBytesConfigured.getBytesInInt();
    } else if (computedBufferSizeBytes.get() != null) {
        return computedBufferSizeBytes.get();
    }
    long directSizeBytes;
    try {
        directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
        log.info("Detected max direct memory size of [%,d] bytes", directSizeBytes);
    } catch (UnsupportedOperationException e) {
        // max direct memory defaults to max heap size on recent JDK version, unless set explicitly
        directSizeBytes = computeMaxMemoryFromMaxHeapSize();
        log.info("Defaulting to at most [%,d] bytes (25%% of max heap size) of direct memory for computation buffers", directSizeBytes);
    }
    int numProcessingThreads = getNumThreads();
    int numMergeBuffers = getNumMergeBuffers();
    int totalNumBuffers = numMergeBuffers + numProcessingThreads;
    int sizePerBuffer = (int) ((double) directSizeBytes / (double) (totalNumBuffers + 1));
    final int computedSizePerBuffer = Math.min(sizePerBuffer, MAX_DEFAULT_PROCESSING_BUFFER_SIZE_BYTES);
    if (computedBufferSizeBytes.compareAndSet(null, computedSizePerBuffer)) {
        log.info("Auto sizing buffers to [%,d] bytes each for [%,d] processing and [%,d] merge buffers", computedSizePerBuffer, numProcessingThreads, numMergeBuffers);
    }
    return computedSizePerBuffer;
}

Example 27 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class ExpressionLambdaAggregatorFactoryTest method testDoubleArrayTypeFinalized.

@Test
public void testDoubleArrayTypeFinalized() {
    ExpressionLambdaAggregatorFactory agg = new ExpressionLambdaAggregatorFactory("expr_agg_name", ImmutableSet.of("some_column", "some_other_column"), null, "0.0", "ARRAY<DOUBLE>[]", null, false, false, "__acc + some_column + some_other_column", "array_set_add(__acc, expr_agg_name)", null, "array_to_string(o, ';')", new HumanReadableBytes(2048), TestExprMacroTable.INSTANCE);
    Assert.assertEquals(ColumnType.DOUBLE, agg.getIntermediateType());
    Assert.assertEquals(ColumnType.DOUBLE_ARRAY, agg.getCombiningFactory().getIntermediateType());
    Assert.assertEquals(ColumnType.STRING, agg.getResultType());
}

Example 28 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class ExpressionLambdaAggregatorFactoryTest method testStringArrayTypeFinalized.

@Test
public void testStringArrayTypeFinalized() {
    ExpressionLambdaAggregatorFactory agg = new ExpressionLambdaAggregatorFactory("expr_agg_name", ImmutableSet.of("some_column", "some_other_column"), null, "''", "ARRAY<STRING>[]", null, false, false, "concat(__acc, some_column, some_other_column)", "array_set_add(__acc, expr_agg_name)", null, "array_to_string(o, ';')", new HumanReadableBytes(2048), TestExprMacroTable.INSTANCE);
    Assert.assertEquals(ColumnType.STRING, agg.getIntermediateType());
    Assert.assertEquals(ColumnType.STRING_ARRAY, agg.getCombiningFactory().getIntermediateType());
    Assert.assertEquals(ColumnType.STRING, agg.getResultType());
}

Example 29 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class ExpressionLambdaAggregatorFactoryTest method testInitialCombineValueMustBeConstant.

@Test
public void testInitialCombineValueMustBeConstant() {
    expectedException.expect(IllegalArgumentException.class);
    expectedException.expectMessage("initial combining value must be constant");
    ExpressionLambdaAggregatorFactory agg = new ExpressionLambdaAggregatorFactory("expr_agg_name", ImmutableSet.of("some_column", "some_other_column"), null, "0.0", "x + y", true, false, false, "__acc + some_column + some_other_column", "__acc + expr_agg_name", null, null, new HumanReadableBytes(2048), TestExprMacroTable.INSTANCE);
    agg.getResultType();
}

Example 30 with HumanReadableBytes

use of org.apache.druid.java.util.common.HumanReadableBytes in project druid by druid-io.

the class ExpressionLambdaAggregatorFactoryTest method testStringArrayType.

@Test
public void testStringArrayType() {
    ExpressionLambdaAggregatorFactory agg = new ExpressionLambdaAggregatorFactory("expr_agg_name", ImmutableSet.of("some_column", "some_other_column"), null, "''", "ARRAY<STRING>[]", null, false, false, "concat(__acc, some_column, some_other_column)", "array_set_add(__acc, expr_agg_name)", null, null, new HumanReadableBytes(2048), TestExprMacroTable.INSTANCE);
    Assert.assertEquals(ColumnType.STRING, agg.getIntermediateType());
    Assert.assertEquals(ColumnType.STRING_ARRAY, agg.getCombiningFactory().getIntermediateType());
    Assert.assertEquals(ColumnType.STRING_ARRAY, agg.getResultType());
}