Examples with MILLISECONDS java.util.concurrent.TimeUnit.MILLISECONDS used on opensource projects

Example 11 with MILLISECONDS

use of java.util.concurrent.TimeUnit.MILLISECONDS in project presto by prestodb.

the class ShardCleaner method cleanBackupShards.

@VisibleForTesting
void cleanBackupShards() {
    Set<UUID> processing = newConcurrentHashSet();
    BlockingQueue<UUID> completed = new LinkedBlockingQueue<>();
    boolean fill = true;
    while (!Thread.currentThread().isInterrupted()) {
        // get a new batch if any completed and we are under the batch size
        Set<UUID> uuids = ImmutableSet.of();
        if (fill && (processing.size() < CLEANABLE_SHARDS_BATCH_SIZE)) {
            uuids = dao.getCleanableShardsBatch(maxTimestamp(backupCleanTime));
            fill = false;
        }
        if (uuids.isEmpty() && processing.isEmpty()) {
            break;
        }
        // skip any that are already processing and mark remaining as processing
        uuids = ImmutableSet.copyOf(difference(uuids, processing));
        processing.addAll(uuids);
        // execute deletes
        for (UUID uuid : uuids) {
            runAsync(() -> backupStore.get().deleteShard(uuid), backupExecutor).thenAccept(v -> completed.add(uuid)).whenComplete((v, e) -> {
                if (e != null) {
                    log.error(e, "Error cleaning backup shard: %s", uuid);
                    backupJobErrors.update(1);
                    processing.remove(uuid);
                }
            });
        }
        // get the next batch of completed deletes
        int desired = min(100, processing.size());
        Collection<UUID> done = drain(completed, desired, 100, MILLISECONDS);
        if (done.isEmpty()) {
            continue;
        }
        // remove completed deletes from database
        processing.removeAll(done);
        dao.deleteCleanedShards(done);
        backupShardsCleaned.update(done.size());
        fill = true;
    }
}

Example 12 with MILLISECONDS

use of java.util.concurrent.TimeUnit.MILLISECONDS in project presto by prestodb.

the class HiveClientModule method createStripeMetadataSourceFactory.

@Singleton
@Provides
public StripeMetadataSourceFactory createStripeMetadataSourceFactory(OrcCacheConfig orcCacheConfig, MBeanExporter exporter) {
    StripeMetadataSource stripeMetadataSource = new StorageStripeMetadataSource();
    if (orcCacheConfig.isStripeMetadataCacheEnabled()) {
        Cache<StripeId, Slice> footerCache = CacheBuilder.newBuilder().maximumWeight(orcCacheConfig.getStripeFooterCacheSize().toBytes()).weigher((id, footer) -> toIntExact(((Slice) footer).getRetainedSize())).expireAfterAccess(orcCacheConfig.getStripeFooterCacheTtlSinceLastAccess().toMillis(), MILLISECONDS).recordStats().build();
        Cache<StripeStreamId, Slice> streamCache = CacheBuilder.newBuilder().maximumWeight(orcCacheConfig.getStripeStreamCacheSize().toBytes()).weigher((id, stream) -> toIntExact(((Slice) stream).getRetainedSize())).expireAfterAccess(orcCacheConfig.getStripeStreamCacheTtlSinceLastAccess().toMillis(), MILLISECONDS).recordStats().build();
        CacheStatsMBean footerCacheStatsMBean = new CacheStatsMBean(footerCache);
        CacheStatsMBean streamCacheStatsMBean = new CacheStatsMBean(streamCache);
        exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_StripeFooter"), footerCacheStatsMBean);
        exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_StripeStream"), streamCacheStatsMBean);
        Optional<Cache<StripeStreamId, List<RowGroupIndex>>> rowGroupIndexCache = Optional.empty();
        if (orcCacheConfig.isRowGroupIndexCacheEnabled()) {
            rowGroupIndexCache = Optional.of(CacheBuilder.newBuilder().maximumWeight(orcCacheConfig.getRowGroupIndexCacheSize().toBytes()).weigher((id, rowGroupIndices) -> toIntExact(((List<RowGroupIndex>) rowGroupIndices).stream().mapToLong(RowGroupIndex::getRetainedSizeInBytes).sum())).expireAfterAccess(orcCacheConfig.getStripeStreamCacheTtlSinceLastAccess().toMillis(), MILLISECONDS).recordStats().build());
            CacheStatsMBean rowGroupIndexCacheStatsMBean = new CacheStatsMBean(rowGroupIndexCache.get());
            exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_StripeStreamRowGroupIndex"), rowGroupIndexCacheStatsMBean);
        }
        stripeMetadataSource = new CachingStripeMetadataSource(stripeMetadataSource, footerCache, streamCache, rowGroupIndexCache);
    }
    StripeMetadataSourceFactory factory = StripeMetadataSourceFactory.of(stripeMetadataSource);
    if (orcCacheConfig.isDwrfStripeCacheEnabled()) {
        factory = new DwrfAwareStripeMetadataSourceFactory(factory);
    }
    return factory;
}

Example 13 with MILLISECONDS

use of java.util.concurrent.TimeUnit.MILLISECONDS in project presto by prestodb.

the class HiveClientModule method createParquetMetadataSource.

@Singleton
@Provides
public ParquetMetadataSource createParquetMetadataSource(ParquetCacheConfig parquetCacheConfig, MBeanExporter exporter) {
    ParquetMetadataSource parquetMetadataSource = new MetadataReader();
    if (parquetCacheConfig.isMetadataCacheEnabled()) {
        Cache<ParquetDataSourceId, ParquetFileMetadata> cache = CacheBuilder.newBuilder().maximumWeight(parquetCacheConfig.getMetadataCacheSize().toBytes()).weigher((id, metadata) -> ((ParquetFileMetadata) metadata).getMetadataSize()).expireAfterAccess(parquetCacheConfig.getMetadataCacheTtlSinceLastAccess().toMillis(), MILLISECONDS).recordStats().build();
        CacheStatsMBean cacheStatsMBean = new CacheStatsMBean(cache);
        parquetMetadataSource = new CachingParquetMetadataSource(cache, parquetMetadataSource);
        exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_ParquetMetadata"), cacheStatsMBean);
    }
    return parquetMetadataSource;
}

Example 14 with MILLISECONDS

use of java.util.concurrent.TimeUnit.MILLISECONDS in project presto by prestodb.

the class HiveClientModule method createOrcFileTailSource.

@Singleton
@Provides
public OrcFileTailSource createOrcFileTailSource(OrcCacheConfig orcCacheConfig, MBeanExporter exporter) {
    int expectedFileTailSizeInBytes = toIntExact(orcCacheConfig.getExpectedFileTailSize().toBytes());
    boolean dwrfStripeCacheEnabled = orcCacheConfig.isDwrfStripeCacheEnabled();
    OrcFileTailSource orcFileTailSource = new StorageOrcFileTailSource(expectedFileTailSizeInBytes, dwrfStripeCacheEnabled);
    if (orcCacheConfig.isFileTailCacheEnabled()) {
        Cache<OrcDataSourceId, OrcFileTail> cache = CacheBuilder.newBuilder().maximumWeight(orcCacheConfig.getFileTailCacheSize().toBytes()).weigher((id, tail) -> ((OrcFileTail) tail).getFooterSize() + ((OrcFileTail) tail).getMetadataSize()).expireAfterAccess(orcCacheConfig.getFileTailCacheTtlSinceLastAccess().toMillis(), MILLISECONDS).recordStats().build();
        CacheStatsMBean cacheStatsMBean = new CacheStatsMBean(cache);
        orcFileTailSource = new CachingOrcFileTailSource(orcFileTailSource, cache);
        exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_OrcFileTail"), cacheStatsMBean);
    }
    return orcFileTailSource;
}

Example 15 with MILLISECONDS

use of java.util.concurrent.TimeUnit.MILLISECONDS in project presto by prestodb.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    }, directExecutor());
    // For hard timeout, add an additional time to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + ADDITIONAL_WAIT_TIME.toMillis(), MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}