Examples with DataSize io.airlift.units.DataSize used on opensource projects

Example 41 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TypedHistogram method addNewGroup.

private void addNewGroup(int hashPosition, int position, Block block, long count) {
    hashPositions.set(hashPosition, values.getPositionCount());
    counts.set(values.getPositionCount(), count);
    type.appendTo(block, position, values);
    // increase capacity, if necessary
    if (values.getPositionCount() >= maxFill) {
        rehash(maxFill * 2);
    }
    if (getEstimatedSize() > FOUR_MEGABYTES) {
        throw exceededLocalLimit(new DataSize(4, MEGABYTE));
    }
}

Example 42 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TypedSet method addNewElement.

private void addNewElement(int hashPosition, Block block, int position) {
    elementType.appendTo(block, position, elementBlock);
    if (elementBlock.getSizeInBytes() > FOUR_MEGABYTES) {
        throw exceededLocalLimit(new DataSize(4, MEGABYTE));
    }
    blockPositionByHash.set(hashPosition, elementBlock.getPositionCount() - 1);
    // increase capacity, if necessary
    if (elementBlock.getPositionCount() >= maxFill) {
        rehash(maxFill * 2);
    }
}

Example 43 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class ServerMainModule method setup.

@Override
protected void setup(Binder binder) {
    ServerConfig serverConfig = buildConfigObject(ServerConfig.class);
    if (serverConfig.isCoordinator()) {
        install(new CoordinatorModule());
        binder.bind(new TypeLiteral<Optional<QueryPerformanceFetcher>>() {
        }).toProvider(QueryPerformanceFetcherProvider.class).in(Scopes.SINGLETON);
    } else {
        binder.bind(new TypeLiteral<Optional<QueryPerformanceFetcher>>() {
        }).toInstance(Optional.empty());
        // Install no-op resource group manager on workers, since only coordinators manage resource groups.
        binder.bind(ResourceGroupManager.class).to(NoOpResourceGroupManager.class).in(Scopes.SINGLETON);
        // HACK: this binding is needed by SystemConnectorModule, but will only be used on the coordinator
        binder.bind(QueryManager.class).toInstance(newProxy(QueryManager.class, (proxy, method, args) -> {
            throw new UnsupportedOperationException();
        }));
    }
    configBinder(binder).bindConfig(FeaturesConfig.class);
    binder.bind(SqlParser.class).in(Scopes.SINGLETON);
    binder.bind(SqlParserOptions.class).toInstance(sqlParserOptions);
    bindFailureDetector(binder, serverConfig.isCoordinator());
    jaxrsBinder(binder).bind(ThrowableMapper.class);
    configBinder(binder).bindConfig(QueryManagerConfig.class);
    jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class);
    // session properties
    binder.bind(SessionPropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON);
    // schema properties
    binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON);
    // table properties
    binder.bind(TablePropertyManager.class).in(Scopes.SINGLETON);
    // node manager
    discoveryBinder(binder).bindSelector("presto");
    binder.bind(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    binder.bind(InternalNodeManager.class).to(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(DiscoveryNodeManager.class).withGeneratedName();
    httpClientBinder(binder).bindHttpClient("node-manager", ForNodeManager.class).withTracing().withConfigDefaults(config -> {
        config.setIdleTimeout(new Duration(30, SECONDS));
        config.setRequestTimeout(new Duration(10, SECONDS));
    });
    // node scheduler
    // TODO: remove from NodePartitioningManager and move to CoordinatorModule
    configBinder(binder).bindConfig(NodeSchedulerConfig.class);
    binder.bind(NodeScheduler.class).in(Scopes.SINGLETON);
    binder.bind(NodeSchedulerExporter.class).in(Scopes.SINGLETON);
    binder.bind(NodeTaskMap.class).in(Scopes.SINGLETON);
    newExporter(binder).export(NodeScheduler.class).withGeneratedName();
    // network topology
    // TODO: move to CoordinatorModule when NodeScheduler is moved
    install(installModuleIf(NodeSchedulerConfig.class, config -> LEGACY.equalsIgnoreCase(config.getNetworkTopology()), moduleBinder -> moduleBinder.bind(NetworkTopology.class).to(LegacyNetworkTopology.class).in(Scopes.SINGLETON)));
    install(installModuleIf(NodeSchedulerConfig.class, config -> FLAT.equalsIgnoreCase(config.getNetworkTopology()), moduleBinder -> moduleBinder.bind(NetworkTopology.class).to(FlatNetworkTopology.class).in(Scopes.SINGLETON)));
    // task execution
    jaxrsBinder(binder).bind(TaskResource.class);
    newExporter(binder).export(TaskResource.class).withGeneratedName();
    binder.bind(TaskManager.class).to(SqlTaskManager.class).in(Scopes.SINGLETON);
    // workaround for CodeCache GC issue
    if (JavaVersion.current().getMajor() == 8) {
        configBinder(binder).bindConfig(CodeCacheGcConfig.class);
        binder.bind(CodeCacheGcTrigger.class).in(Scopes.SINGLETON);
    }
    // Add monitoring for JVM pauses
    binder.bind(PauseMeter.class).in(Scopes.SINGLETON);
    newExporter(binder).export(PauseMeter.class).withGeneratedName();
    configBinder(binder).bindConfig(MemoryManagerConfig.class);
    configBinder(binder).bindConfig(NodeMemoryConfig.class);
    configBinder(binder).bindConfig(ReservedSystemMemoryConfig.class);
    binder.bind(LocalMemoryManager.class).in(Scopes.SINGLETON);
    binder.bind(LocalMemoryManagerExporter.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TaskManager.class).withGeneratedName();
    binder.bind(TaskExecutor.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TaskExecutor.class).withGeneratedName();
    binder.bind(LocalExecutionPlanner.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(CompilerConfig.class);
    binder.bind(ExpressionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExpressionCompiler.class).withGeneratedName();
    configBinder(binder).bindConfig(TaskManagerConfig.class);
    binder.bind(IndexJoinLookupStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(IndexJoinLookupStats.class).withGeneratedName();
    binder.bind(AsyncHttpExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(AsyncHttpExecutionMBean.class).withGeneratedName();
    binder.bind(JoinFilterFunctionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinFilterFunctionCompiler.class).withGeneratedName();
    binder.bind(JoinCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinCompiler.class).withGeneratedName();
    binder.bind(OrderingCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(OrderingCompiler.class).withGeneratedName();
    binder.bind(PagesIndex.Factory.class).to(PagesIndex.DefaultFactory.class);
    binder.bind(JoinProbeCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinProbeCompiler.class).withGeneratedName();
    binder.bind(LookupJoinOperators.class).in(Scopes.SINGLETON);
    jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class);
    jsonCodecBinder(binder).bindJsonCodec(StageInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class);
    jaxrsBinder(binder).bind(PagesResponseWriter.class);
    // exchange client
    binder.bind(new TypeLiteral<ExchangeClientSupplier>() {
    }).to(ExchangeClientFactory.class).in(Scopes.SINGLETON);
    httpClientBinder(binder).bindHttpClient("exchange", ForExchange.class).withTracing().withConfigDefaults(config -> {
        config.setIdleTimeout(new Duration(30, SECONDS));
        config.setRequestTimeout(new Duration(10, SECONDS));
        config.setMaxConnectionsPerServer(250);
        config.setMaxContentLength(new DataSize(32, MEGABYTE));
    });
    configBinder(binder).bindConfig(ExchangeClientConfig.class);
    binder.bind(ExchangeExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExchangeExecutionMBean.class).withGeneratedName();
    // execution
    binder.bind(LocationFactory.class).to(HttpLocationFactory.class).in(Scopes.SINGLETON);
    // memory manager
    jaxrsBinder(binder).bind(MemoryResource.class);
    jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(MemoryPoolAssignmentsRequest.class);
    // transaction manager
    configBinder(binder).bindConfig(TransactionManagerConfig.class);
    // data stream provider
    binder.bind(PageSourceManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSourceProvider.class).to(PageSourceManager.class).in(Scopes.SINGLETON);
    // page sink provider
    binder.bind(PageSinkManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSinkProvider.class).to(PageSinkManager.class).in(Scopes.SINGLETON);
    // metadata
    binder.bind(StaticCatalogStore.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(StaticCatalogStoreConfig.class);
    binder.bind(MetadataManager.class).in(Scopes.SINGLETON);
    binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON);
    // type
    binder.bind(TypeRegistry.class).in(Scopes.SINGLETON);
    binder.bind(TypeManager.class).to(TypeRegistry.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
    newSetBinder(binder, Type.class);
    // split manager
    binder.bind(SplitManager.class).in(Scopes.SINGLETON);
    // node partitioning manager
    binder.bind(NodePartitioningManager.class).in(Scopes.SINGLETON);
    // index manager
    binder.bind(IndexManager.class).in(Scopes.SINGLETON);
    // handle resolver
    binder.install(new HandleJsonModule());
    // connector
    binder.bind(ConnectorManager.class).in(Scopes.SINGLETON);
    // system connector
    binder.install(new SystemConnectorModule());
    // splits
    jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class);
    jsonCodecBinder(binder).bindJsonCodec(ConnectorSplit.class);
    jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class);
    jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class);
    jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class);
    // query monitor
    configBinder(binder).bindConfig(QueryMonitorConfig.class);
    binder.bind(QueryMonitor.class).in(Scopes.SINGLETON);
    // Determine the NodeVersion
    String prestoVersion = serverConfig.getPrestoVersion();
    if (prestoVersion == null) {
        prestoVersion = getClass().getPackage().getImplementationVersion();
    }
    checkState(prestoVersion != null, "presto.version must be provided when it cannot be automatically determined");
    NodeVersion nodeVersion = new NodeVersion(prestoVersion);
    binder.bind(NodeVersion.class).toInstance(nodeVersion);
    // presto announcement
    discoveryBinder(binder).bindHttpAnnouncement("presto").addProperty("node_version", nodeVersion.toString()).addProperty("coordinator", String.valueOf(serverConfig.isCoordinator())).addProperty("connectorIds", nullToEmpty(serverConfig.getDataSources()));
    // server info resource
    jaxrsBinder(binder).bind(ServerInfoResource.class);
    // plugin manager
    binder.bind(PluginManager.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(PluginManagerConfig.class);
    binder.bind(CatalogManager.class).in(Scopes.SINGLETON);
    // optimizers
    binder.bind(PlanOptimizers.class).in(Scopes.SINGLETON);
    // block encodings
    binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON);
    binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON);
    newSetBinder(binder, new TypeLiteral<BlockEncodingFactory<?>>() {
    });
    jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class);
    jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class);
    // thread visualizer
    jaxrsBinder(binder).bind(ThreadResource.class);
    // PageSorter
    binder.bind(PageSorter.class).to(PagesIndexPageSorter.class).in(Scopes.SINGLETON);
    // PageIndexer
    binder.bind(PageIndexerFactory.class).to(GroupByHashPageIndexerFactory.class).in(Scopes.SINGLETON);
    // Finalizer
    binder.bind(FinalizerService.class).in(Scopes.SINGLETON);
    // Spiller
    binder.bind(SpillerFactory.class).to(BinarySpillerFactory.class).in(Scopes.SINGLETON);
    newExporter(binder).export(SpillerFactory.class).withGeneratedName();
}

Example 44 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) throws InterruptedException {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    });
    // For hard timeout, add an additional 5 seconds to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + 5000, MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}

Example 45 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class ShardRecoveryManager method restoreFromBackup.

@VisibleForTesting
void restoreFromBackup(UUID shardUuid, OptionalLong shardSize) {
    File storageFile = storageService.getStorageFile(shardUuid);
    if (!backupStore.get().shardExists(shardUuid)) {
        stats.incrementShardRecoveryBackupNotFound();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "No backup file found for shard: " + shardUuid);
    }
    if (storageFile.exists()) {
        if (!shardSize.isPresent() || (storageFile.length() == shardSize.getAsLong())) {
            return;
        }
        log.warn("Local shard file is corrupt. Deleting local file: %s", storageFile);
        storageFile.delete();
    }
    // create a temporary file in the staging directory
    File stagingFile = temporarySuffix(storageService.getStagingFile(shardUuid));
    storageService.createParents(stagingFile);
    // copy to temporary file
    log.info("Copying shard %s from backup...", shardUuid);
    long start = System.nanoTime();
    try {
        backupStore.get().restoreShard(shardUuid, stagingFile);
    } catch (PrestoException e) {
        stats.incrementShardRecoveryFailure();
        stagingFile.delete();
        throw e;
    }
    Duration duration = nanosSince(start);
    DataSize size = succinctBytes(stagingFile.length());
    DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize();
    stats.addShardRecoveryDataRate(rate, size, duration);
    log.info("Copied shard %s from backup in %s (%s at %s/s)", shardUuid, duration, size, rate);
    // move to final location
    storageService.createParents(storageFile);
    try {
        Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE);
    } catch (FileAlreadyExistsException e) {
    // someone else already created it (should not happen, but safe to ignore)
    } catch (IOException e) {
        stats.incrementShardRecoveryFailure();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "Failed to move shard: " + shardUuid, e);
    } finally {
        stagingFile.delete();
    }
    if (!storageFile.exists() || (shardSize.isPresent() && (storageFile.length() != shardSize.getAsLong()))) {
        stats.incrementShardRecoveryFailure();
        log.info("Files do not match after recovery. Deleting local file: " + shardUuid);
        storageFile.delete();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "File not recovered correctly: " + shardUuid);
    }
    stats.incrementShardRecoverySuccess();
}