Examples with DataSize io.airlift.units.DataSize used on opensource projects

Example 1 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestHiveS3Config method testExplicitPropertyMappings.

@Test
public void testExplicitPropertyMappings() {
    Map<String, String> properties = new ImmutableMap.Builder<String, String>().put("hive.s3.aws-access-key", "abc123").put("hive.s3.aws-secret-key", "secret").put("hive.s3.endpoint", "endpoint.example.com").put("hive.s3.signer-type", "S3SignerType").put("hive.s3.use-instance-credentials", "false").put("hive.s3.ssl.enabled", "false").put("hive.s3.sse.enabled", "true").put("hive.s3.sse.type", "KMS").put("hive.s3.sse.kms-key-id", "KMS_KEY_ID").put("hive.s3.encryption-materials-provider", "EMP_CLASS").put("hive.s3.kms-key-id", "KEY_ID").put("hive.s3.max-client-retries", "9").put("hive.s3.max-error-retries", "8").put("hive.s3.max-backoff-time", "4m").put("hive.s3.max-retry-time", "20m").put("hive.s3.connect-timeout", "8s").put("hive.s3.socket-timeout", "4m").put("hive.s3.multipart.min-file-size", "32MB").put("hive.s3.multipart.min-part-size", "15MB").put("hive.s3.max-connections", "77").put("hive.s3.staging-directory", "/s3-staging").put("hive.s3.pin-client-to-current-region", "true").put("hive.s3.user-agent-prefix", "user-agent-prefix").build();
    HiveS3Config expected = new HiveS3Config().setS3AwsAccessKey("abc123").setS3AwsSecretKey("secret").setS3Endpoint("endpoint.example.com").setS3SignerType(PrestoS3SignerType.S3SignerType).setS3UseInstanceCredentials(false).setS3SslEnabled(false).setS3SseEnabled(true).setS3SseType(PrestoS3SseType.KMS).setS3SseKmsKeyId("KMS_KEY_ID").setS3EncryptionMaterialsProvider("EMP_CLASS").setS3KmsKeyId("KEY_ID").setS3MaxClientRetries(9).setS3MaxErrorRetries(8).setS3MaxBackoffTime(new Duration(4, TimeUnit.MINUTES)).setS3MaxRetryTime(new Duration(20, TimeUnit.MINUTES)).setS3ConnectTimeout(new Duration(8, TimeUnit.SECONDS)).setS3SocketTimeout(new Duration(4, TimeUnit.MINUTES)).setS3MultipartMinFileSize(new DataSize(32, Unit.MEGABYTE)).setS3MultipartMinPartSize(new DataSize(15, Unit.MEGABYTE)).setS3MaxConnections(77).setS3StagingDirectory(new File("/s3-staging")).setPinS3ClientToCurrentRegion(true).setS3UserAgentPrefix("user-agent-prefix");
    assertFullMapping(properties, expected);
}

Example 2 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestStageStats method assertExpectedStageStats.

public static void assertExpectedStageStats(StageStats actual) {
    assertEquals(actual.getSchedulingComplete().getMillis(), 0);
    assertEquals(actual.getGetSplitDistribution().getCount(), 1.0);
    assertEquals(actual.getScheduleTaskDistribution().getCount(), 2.0);
    assertEquals(actual.getAddSplitDistribution().getCount(), 3.0);
    assertEquals(actual.getTotalTasks(), 4);
    assertEquals(actual.getRunningTasks(), 5);
    assertEquals(actual.getCompletedTasks(), 6);
    assertEquals(actual.getTotalDrivers(), 7);
    assertEquals(actual.getQueuedDrivers(), 8);
    assertEquals(actual.getRunningDrivers(), 10);
    assertEquals(actual.getCompletedDrivers(), 11);
    assertEquals(actual.getCumulativeMemory(), 12.0);
    assertEquals(actual.getTotalMemoryReservation(), new DataSize(13, BYTE));
    assertEquals(actual.getPeakMemoryReservation(), new DataSize(14, BYTE));
    assertEquals(actual.getTotalScheduledTime(), new Duration(15, NANOSECONDS));
    assertEquals(actual.getTotalCpuTime(), new Duration(16, NANOSECONDS));
    assertEquals(actual.getTotalUserTime(), new Duration(17, NANOSECONDS));
    assertEquals(actual.getTotalBlockedTime(), new Duration(18, NANOSECONDS));
    assertEquals(actual.getRawInputDataSize(), new DataSize(19, BYTE));
    assertEquals(actual.getRawInputPositions(), 20);
    assertEquals(actual.getProcessedInputDataSize(), new DataSize(21, BYTE));
    assertEquals(actual.getProcessedInputPositions(), 22);
    assertEquals(actual.getOutputDataSize(), new DataSize(23, BYTE));
    assertEquals(actual.getOutputPositions(), 24);
}

Example 3 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestExchangeClient method testHappyPath.

@Test
public void testHappyPath() throws Exception {
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, executor), executor, deltaMemoryInBytes -> {
    });
    exchangeClient.addLocation(location);
    exchangeClient.noMoreLocations();
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
    assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
    assertEquals(exchangeClient.isClosed(), true);
    if (exchangeClient.getStatus().getBufferedPages() != 0) {
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    }
    assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 3, 3, "not scheduled");
}

Example 4 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestNodeMemoryConfig method testDefaults.

@Test
public void testDefaults() {
    // This can't use assertRecordedDefaults because the default value is dependent on the current max heap size, which varies based on the current size of the survivor space.
    for (int i = 0; i < 1_000; i++) {
        DataSize expected = new DataSize(Runtime.getRuntime().maxMemory() * 0.1, BYTE);
        NodeMemoryConfig config = new NodeMemoryConfig();
        if (expected.equals(config.getMaxQueryMemoryPerNode())) {
            return;
        }
    }
    // We can't make this 100% deterministic, since we don't know when the survivor space will change sizes, but assume that something is broken if we got the wrong answer 1000 times
    fail();
}

Example 5 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestMemoryPools method testBlocking.

@Test
public void testBlocking() throws Exception {
    Session session = testSessionBuilder().setCatalog("tpch").setSchema("tiny").setSystemProperty("task_default_concurrency", "1").build();
    LocalQueryRunner localQueryRunner = queryRunnerWithInitialTransaction(session);
    // add tpch
    localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of());
    // reserve all the memory in the pool
    MemoryPool pool = new MemoryPool(new MemoryPoolId("test"), new DataSize(10, MEGABYTE));
    QueryId fakeQueryId = new QueryId("fake");
    assertTrue(pool.tryReserve(fakeQueryId, TEN_MEGABYTES));
    MemoryPool systemPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(10, MEGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("query"), new DataSize(10, MEGABYTE), pool, systemPool, localQueryRunner.getExecutor());
    // discard all output
    OutputFactory outputFactory = new PageConsumerOutputFactory(types -> (page -> {
    }));
    TaskContext taskContext = createTaskContext(queryContext, localQueryRunner.getExecutor(), session);
    List<Driver> drivers = localQueryRunner.createDrivers("SELECT COUNT(*) FROM orders JOIN lineitem USING (orderkey)", outputFactory, taskContext);
    // run driver, until it blocks
    while (!isWaitingForMemory(drivers)) {
        for (Driver driver : drivers) {
            driver.process();
        }
    }
    // driver should be blocked waiting for memory
    for (Driver driver : drivers) {
        assertFalse(driver.isFinished());
    }
    assertTrue(pool.getFreeBytes() <= 0);
    pool.free(fakeQueryId, TEN_MEGABYTES);
    do {
        assertFalse(isWaitingForMemory(drivers));
        boolean progress = false;
        for (Driver driver : drivers) {
            ListenableFuture<?> blocked = driver.process();
            progress = progress | blocked.isDone();
        }
        // query should not block
        assertTrue(progress);
    } while (!drivers.stream().allMatch(Driver::isFinished));
}