Examples with Duration io.airlift.units.Duration used on opensource projects

Example 46 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) throws InterruptedException {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    });
    // For hard timeout, add an additional 5 seconds to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + 5000, MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}

Example 47 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class AssignmentLimiter method checkAssignFrom.

public synchronized void checkAssignFrom(String nodeIdentifier) {
    if (offlineNodes.contains(nodeIdentifier)) {
        return;
    }
    long now = ticker.read();
    long start = delayedNodes.computeIfAbsent(nodeIdentifier, key -> now);
    Duration delay = new Duration(now - start, NANOSECONDS);
    if (delay.compareTo(reassignmentDelay) < 0) {
        throw new PrestoException(RAPTOR_REASSIGNMENT_DELAY, format("Reassignment delay is in effect for node %s (elapsed: %s)", nodeIdentifier, delay.convertToMostSuccinctTimeUnit()));
    }
    if (lastOfflined.isPresent()) {
        delay = new Duration(now - lastOfflined.getAsLong(), NANOSECONDS);
        if (delay.compareTo(reassignmentInterval) < 0) {
            throw new PrestoException(RAPTOR_REASSIGNMENT_THROTTLE, format("Reassignment throttle is in effect for node %s (elapsed: %s)", nodeIdentifier, delay.convertToMostSuccinctTimeUnit()));
        }
    }
    delayedNodes.remove(nodeIdentifier);
    offlineNodes.add(nodeIdentifier);
    lastOfflined = OptionalLong.of(now);
}

Example 48 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class ShardRecoveryManager method restoreFromBackup.

@VisibleForTesting
void restoreFromBackup(UUID shardUuid, OptionalLong shardSize) {
    File storageFile = storageService.getStorageFile(shardUuid);
    if (!backupStore.get().shardExists(shardUuid)) {
        stats.incrementShardRecoveryBackupNotFound();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "No backup file found for shard: " + shardUuid);
    }
    if (storageFile.exists()) {
        if (!shardSize.isPresent() || (storageFile.length() == shardSize.getAsLong())) {
            return;
        }
        log.warn("Local shard file is corrupt. Deleting local file: %s", storageFile);
        storageFile.delete();
    }
    // create a temporary file in the staging directory
    File stagingFile = temporarySuffix(storageService.getStagingFile(shardUuid));
    storageService.createParents(stagingFile);
    // copy to temporary file
    log.info("Copying shard %s from backup...", shardUuid);
    long start = System.nanoTime();
    try {
        backupStore.get().restoreShard(shardUuid, stagingFile);
    } catch (PrestoException e) {
        stats.incrementShardRecoveryFailure();
        stagingFile.delete();
        throw e;
    }
    Duration duration = nanosSince(start);
    DataSize size = succinctBytes(stagingFile.length());
    DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize();
    stats.addShardRecoveryDataRate(rate, size, duration);
    log.info("Copied shard %s from backup in %s (%s at %s/s)", shardUuid, duration, size, rate);
    // move to final location
    storageService.createParents(storageFile);
    try {
        Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE);
    } catch (FileAlreadyExistsException e) {
    // someone else already created it (should not happen, but safe to ignore)
    } catch (IOException e) {
        stats.incrementShardRecoveryFailure();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "Failed to move shard: " + shardUuid, e);
    } finally {
        stagingFile.delete();
    }
    if (!storageFile.exists() || (shardSize.isPresent() && (storageFile.length() != shardSize.getAsLong()))) {
        stats.incrementShardRecoveryFailure();
        log.info("Files do not match after recovery. Deleting local file: " + shardUuid);
        storageFile.delete();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "File not recovered correctly: " + shardUuid);
    }
    stats.incrementShardRecoverySuccess();
}

Example 49 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class TestStorageManagerConfig method testExplicitPropertyMappings.

@Test
public void testExplicitPropertyMappings() {
    Map<String, String> properties = new ImmutableMap.Builder<String, String>().put("storage.data-directory", "/data").put("storage.min-available-space", "123GB").put("storage.orc.max-merge-distance", "16kB").put("storage.orc.max-read-size", "16kB").put("storage.orc.stream-buffer-size", "16kB").put("storage.max-deletion-threads", "999").put("storage.shard-recovery-timeout", "1m").put("storage.missing-shard-discovery-interval", "4m").put("storage.compaction-enabled", "false").put("storage.compaction-interval", "4h").put("storage.organization-enabled", "false").put("storage.organization-interval", "4h").put("storage.ejector-interval", "9h").put("storage.max-recovery-threads", "12").put("storage.max-organization-threads", "12").put("storage.max-shard-rows", "10000").put("storage.max-shard-size", "10MB").put("storage.max-buffer-size", "512MB").put("storage.one-split-per-bucket-threshold", "4").build();
    StorageManagerConfig expected = new StorageManagerConfig().setDataDirectory(new File("/data")).setMinAvailableSpace(new DataSize(123, GIGABYTE)).setOrcMaxMergeDistance(new DataSize(16, KILOBYTE)).setOrcMaxReadSize(new DataSize(16, KILOBYTE)).setOrcStreamBufferSize(new DataSize(16, KILOBYTE)).setDeletionThreads(999).setShardRecoveryTimeout(new Duration(1, MINUTES)).setMissingShardDiscoveryInterval(new Duration(4, MINUTES)).setCompactionEnabled(false).setCompactionInterval(new Duration(4, HOURS)).setOrganizationEnabled(false).setOrganizationInterval(new Duration(4, HOURS)).setShardEjectorInterval(new Duration(9, HOURS)).setRecoveryThreads(12).setOrganizationThreads(12).setMaxShardRows(10_000).setMaxShardSize(new DataSize(10, MEGABYTE)).setMaxBufferSize(new DataSize(512, MEGABYTE)).setOneSplitPerBucketThreshold(4);
    assertFullMapping(properties, expected);
}

Example 50 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class TestMetadataConfig method testExplicitPropertyMappings.

@Test
public void testExplicitPropertyMappings() {
    Map<String, String> properties = new ImmutableMap.Builder<String, String>().put("raptor.startup-grace-period", "42m").put("raptor.reassignment-delay", "6m").put("raptor.reassignment-interval", "7m").build();
    MetadataConfig expected = new MetadataConfig().setStartupGracePeriod(new Duration(42, MINUTES)).setReassignmentDelay(new Duration(6, MINUTES)).setReassignmentInterval(new Duration(7, MINUTES));
    assertFullMapping(properties, expected);
}