Search in sources :

Example 1 with SECONDS

use of java.util.concurrent.TimeUnit.SECONDS in project che by eclipse.

the class JGitConnection method commit.

@Override
public Revision commit(CommitParams params) throws GitException {
    try {
        // Check repository state
        RepositoryState repositoryState = repository.getRepositoryState();
        if (!repositoryState.canCommit()) {
            throw new GitException(format(MESSAGE_COMMIT_NOT_POSSIBLE, repositoryState.getDescription()));
        }
        if (params.isAmend() && !repositoryState.canAmend()) {
            throw new GitException(format(MESSAGE_COMMIT_AMEND_NOT_POSSIBLE, repositoryState.getDescription()));
        }
        // Check committer
        GitUser committer = getUser();
        if (committer == null) {
            throw new GitException("Committer can't be null");
        }
        String committerName = committer.getName();
        String committerEmail = committer.getEmail();
        if (committerName == null || committerEmail == null) {
            throw new GitException("Git user name and (or) email wasn't set", ErrorCodes.NO_COMMITTER_NAME_OR_EMAIL_DEFINED);
        }
        // Check commit message
        String message = params.getMessage();
        if (message == null) {
            throw new GitException("Message wasn't set");
        }
        Status status = status(StatusFormat.SHORT);
        List<String> specified = params.getFiles();
        List<String> staged = new ArrayList<>();
        staged.addAll(status.getAdded());
        staged.addAll(status.getChanged());
        staged.addAll(status.getRemoved());
        List<String> changed = new ArrayList<>(staged);
        changed.addAll(status.getModified());
        changed.addAll(status.getMissing());
        List<String> specifiedStaged = specified.stream().filter(path -> staged.stream().anyMatch(s -> s.startsWith(path))).collect(Collectors.toList());
        List<String> specifiedChanged = specified.stream().filter(path -> changed.stream().anyMatch(c -> c.startsWith(path))).collect(Collectors.toList());
        // Check that there are changes present for commit, if 'isAmend' is disabled
        if (!params.isAmend()) {
            // Check that there are staged changes present for commit, or any changes if 'isAll' is enabled
            if (status.isClean()) {
                throw new GitException("Nothing to commit, working directory clean");
            } else if (!params.isAll() && (specified.isEmpty() ? staged.isEmpty() : specifiedStaged.isEmpty())) {
                throw new GitException("No changes added to commit");
            }
        } else {
            /*
                By default Jgit doesn't allow to commit not changed specified paths. According to setAllowEmpty method documentation,
                setting this flag to true must allow such commit, but it won't because Jgit has a bug:
                https://bugs.eclipse.org/bugs/show_bug.cgi?id=510685. As a workaround, specified paths of the commit command will contain
                only changed and specified paths. If other changes are present, but the list of changed and specified paths is empty,
                throw exception to prevent committing other paths. TODO Remove this check when the bug will be fixed.
                */
            if (!specified.isEmpty() && !(params.isAll() ? changed.isEmpty() : staged.isEmpty()) && specifiedChanged.isEmpty()) {
                throw new GitException(format("Changes are present but not changed path%s specified for commit.", specified.size() > 1 ? "s were" : " was"));
            }
        }
        // TODO add 'setAllowEmpty(params.isAmend())' when https://bugs.eclipse.org/bugs/show_bug.cgi?id=510685 will be fixed
        CommitCommand commitCommand = getGit().commit().setCommitter(committerName, committerEmail).setAuthor(committerName, committerEmail).setMessage(message).setAll(params.isAll()).setAmend(params.isAmend());
        if (!params.isAll()) {
            // TODO change to 'specified.forEach(commitCommand::setOnly)' when https://bugs.eclipse.org/bugs/show_bug.cgi?id=510685 will be fixed. See description above.
            specifiedChanged.forEach(commitCommand::setOnly);
        }
        // Check if repository is configured with Gerrit Support
        String gerritSupportConfigValue = repository.getConfig().getString(ConfigConstants.CONFIG_GERRIT_SECTION, null, ConfigConstants.CONFIG_KEY_CREATECHANGEID);
        boolean isGerritSupportConfigured = gerritSupportConfigValue != null ? Boolean.valueOf(gerritSupportConfigValue) : false;
        commitCommand.setInsertChangeId(isGerritSupportConfigured);
        RevCommit result = commitCommand.call();
        GitUser gitUser = newDto(GitUser.class).withName(committerName).withEmail(committerEmail);
        return newDto(Revision.class).withBranch(getCurrentBranch()).withId(result.getId().getName()).withMessage(result.getFullMessage()).withCommitTime(MILLISECONDS.convert(result.getCommitTime(), SECONDS)).withCommitter(gitUser);
    } catch (GitAPIException exception) {
        throw new GitException(exception.getMessage(), exception);
    }
}
Also used : RebaseStatus(org.eclipse.che.api.git.shared.RebaseResponse.RebaseStatus) Status(org.eclipse.che.api.git.shared.Status) InvalidRefNameException(org.eclipse.jgit.api.errors.InvalidRefNameException) Arrays(java.util.Arrays) LsFilesParams(org.eclipse.che.api.git.params.LsFilesParams) RevObject(org.eclipse.jgit.revwalk.RevObject) RebaseResponse(org.eclipse.che.api.git.shared.RebaseResponse) RemoteConfig(org.eclipse.jgit.transport.RemoteConfig) RepositoryState(org.eclipse.jgit.lib.RepositoryState) PullParams(org.eclipse.che.api.git.params.PullParams) SshTransport(org.eclipse.jgit.transport.SshTransport) RevWalk(org.eclipse.jgit.revwalk.RevWalk) CloneParams(org.eclipse.che.api.git.params.CloneParams) RevTag(org.eclipse.jgit.revwalk.RevTag) ResetCommand(org.eclipse.jgit.api.ResetCommand) PathFilter(org.eclipse.jgit.treewalk.filter.PathFilter) Map(java.util.Map) URIish(org.eclipse.jgit.transport.URIish) PullResponse(org.eclipse.che.api.git.shared.PullResponse) TrueFileFilter(org.apache.commons.io.filefilter.TrueFileFilter) UnauthorizedException(org.eclipse.che.api.core.UnauthorizedException) RebaseStatus(org.eclipse.che.api.git.shared.RebaseResponse.RebaseStatus) GitConflictException(org.eclipse.che.api.git.exception.GitConflictException) EnumSet(java.util.EnumSet) TagCommand(org.eclipse.jgit.api.TagCommand) RepositoryCache(org.eclipse.jgit.lib.RepositoryCache) Result(org.eclipse.jgit.lib.RefUpdate.Result) GitAPIException(org.eclipse.jgit.api.errors.GitAPIException) TreeFilter(org.eclipse.jgit.treewalk.filter.TreeFilter) GitConnection(org.eclipse.che.api.git.GitConnection) RefSpec(org.eclipse.jgit.transport.RefSpec) SSLHandshakeException(javax.net.ssl.SSLHandshakeException) RebaseResult(org.eclipse.jgit.api.RebaseResult) CommitCommand(org.eclipse.jgit.api.CommitCommand) Config(org.eclipse.che.api.git.Config) RefUpdate(org.eclipse.jgit.lib.RefUpdate) OpenSshConfig(org.eclipse.jgit.transport.OpenSshConfig) Set(java.util.Set) Status(org.eclipse.che.api.git.shared.Status) Constants(org.eclipse.jgit.lib.Constants) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) TransportCommand(org.eclipse.jgit.api.TransportCommand) Nullable(org.eclipse.che.commons.annotation.Nullable) RevTree(org.eclipse.jgit.revwalk.RevTree) DirectoryFileFilter(org.apache.commons.io.filefilter.DirectoryFileFilter) RemoteUpdateParams(org.eclipse.che.api.git.params.RemoteUpdateParams) PersonIdent(org.eclipse.jgit.lib.PersonIdent) GitInvalidRefNameException(org.eclipse.che.api.git.exception.GitInvalidRefNameException) StatusFormat(org.eclipse.che.api.git.shared.StatusFormat) FileUtils(org.eclipse.jgit.util.FileUtils) Stream(java.util.stream.Stream) CredentialsLoader(org.eclipse.che.api.git.CredentialsLoader) Session(com.jcraft.jsch.Session) PushResult(org.eclipse.jgit.transport.PushResult) DirCache(org.eclipse.jgit.dircache.DirCache) GitRefNotFoundException(org.eclipse.che.api.git.exception.GitRefNotFoundException) FS(org.eclipse.jgit.util.FS) JSchException(com.jcraft.jsch.JSchException) Revision(org.eclipse.che.api.git.shared.Revision) FilenameFilter(java.io.FilenameFilter) RevCommit(org.eclipse.jgit.revwalk.RevCommit) LogCommand(org.eclipse.jgit.api.LogCommand) LogPage(org.eclipse.che.api.git.LogPage) Strings.isNullOrEmpty(com.google.common.base.Strings.isNullOrEmpty) PROVIDER_NAME(org.eclipse.che.api.git.shared.ProviderInfo.PROVIDER_NAME) ArrayList(java.util.ArrayList) LogParams(org.eclipse.che.api.git.params.LogParams) SetupUpstreamMode(org.eclipse.jgit.api.CreateBranchCommand.SetupUpstreamMode) ListMode(org.eclipse.jgit.api.ListBranchCommand.ListMode) ResetParams(org.eclipse.che.api.git.params.ResetParams) PushResponse(org.eclipse.che.api.git.shared.PushResponse) DiffCommitFile(org.eclipse.che.api.git.shared.DiffCommitFile) PushCommand(org.eclipse.jgit.api.PushCommand) RefAlreadyExistsException(org.eclipse.jgit.api.errors.RefAlreadyExistsException) PathFilterGroup(org.eclipse.jgit.treewalk.filter.PathFilterGroup) TreeWalk(org.eclipse.jgit.treewalk.TreeWalk) DiffPage(org.eclipse.che.api.git.DiffPage) ErrorCodes(org.eclipse.che.api.core.ErrorCodes) BranchListMode(org.eclipse.che.api.git.shared.BranchListMode) Tag(org.eclipse.che.api.git.shared.Tag) AddRequest(org.eclipse.che.api.git.shared.AddRequest) FileOutputStream(java.io.FileOutputStream) MergeResult(org.eclipse.che.api.git.shared.MergeResult) ProviderInfo(org.eclipse.che.api.git.shared.ProviderInfo) IOException(java.io.IOException) File(java.io.File) RefNotFoundException(org.eclipse.jgit.api.errors.RefNotFoundException) DiffParams(org.eclipse.che.api.git.params.DiffParams) DiffFormatter(org.eclipse.jgit.diff.DiffFormatter) ServerException(org.eclipse.che.api.core.ServerException) StoredConfig(org.eclipse.jgit.lib.StoredConfig) Repository(org.eclipse.jgit.lib.Repository) IOFileFilter(org.apache.commons.io.filefilter.IOFileFilter) PushParams(org.eclipse.che.api.git.params.PushParams) LineConsumerFactory(org.eclipse.che.api.core.util.LineConsumerFactory) FetchParams(org.eclipse.che.api.git.params.FetchParams) URISyntaxException(java.net.URISyntaxException) LoggerFactory(org.slf4j.LoggerFactory) GitRefAlreadyExistsException(org.eclipse.che.api.git.exception.GitRefAlreadyExistsException) AddCommand(org.eclipse.jgit.api.AddCommand) UserCredential(org.eclipse.che.api.git.UserCredential) CheckoutConflictException(org.eclipse.jgit.api.errors.CheckoutConflictException) NullOutputStream(org.eclipse.jgit.util.io.NullOutputStream) FetchResult(org.eclipse.jgit.transport.FetchResult) CommitParams(org.eclipse.che.api.git.params.CommitParams) ProxyAuthenticator(org.eclipse.che.commons.proxy.ProxyAuthenticator) ResetType(org.eclipse.jgit.api.ResetCommand.ResetType) RemoteAddParams(org.eclipse.che.api.git.params.RemoteAddParams) AUTHENTICATE_URL(org.eclipse.che.api.git.shared.ProviderInfo.AUTHENTICATE_URL) PosixFilePermission(java.nio.file.attribute.PosixFilePermission) GitException(org.eclipse.che.api.git.exception.GitException) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) DtoFactory.newDto(org.eclipse.che.dto.server.DtoFactory.newDto) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) List(java.util.List) Branch(org.eclipse.che.api.git.shared.Branch) RemoteReference(org.eclipse.che.api.git.shared.RemoteReference) Ref(org.eclipse.jgit.lib.Ref) TagCreateParams(org.eclipse.che.api.git.params.TagCreateParams) Optional(java.util.Optional) ListBranchCommand(org.eclipse.jgit.api.ListBranchCommand) Pattern(java.util.regex.Pattern) RmCommand(org.eclipse.jgit.api.RmCommand) GitUrlUtils(org.eclipse.che.api.git.GitUrlUtils) Remote(org.eclipse.che.api.git.shared.Remote) System.lineSeparator(java.lang.System.lineSeparator) JSch(com.jcraft.jsch.JSch) UsernamePasswordCredentialsProvider(org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider) CanonicalTreeParser(org.eclipse.jgit.treewalk.CanonicalTreeParser) LineConsumer(org.eclipse.che.api.core.util.LineConsumer) CloneCommand(org.eclipse.jgit.api.CloneCommand) HashMap(java.util.HashMap) RebaseCommand(org.eclipse.jgit.api.RebaseCommand) LIST_ALL(org.eclipse.che.api.git.shared.BranchListMode.LIST_ALL) LIST_LOCAL(org.eclipse.che.api.git.shared.BranchListMode.LIST_LOCAL) FetchCommand(org.eclipse.jgit.api.FetchCommand) Inject(javax.inject.Inject) HashSet(java.util.HashSet) RmParams(org.eclipse.che.api.git.params.RmParams) LIST_REMOTE(org.eclipse.che.api.git.shared.BranchListMode.LIST_REMOTE) Files(com.google.common.io.Files) SshKeyProvider(org.eclipse.che.plugin.ssh.key.script.SshKeyProvider) SshSessionFactory(org.eclipse.jgit.transport.SshSessionFactory) ResolveMerger(org.eclipse.jgit.merge.ResolveMerger) BatchingProgressMonitor(org.eclipse.jgit.lib.BatchingProgressMonitor) AndTreeFilter(org.eclipse.jgit.treewalk.filter.AndTreeFilter) EmptyTreeIterator(org.eclipse.jgit.treewalk.EmptyTreeIterator) ObjectLoader(org.eclipse.jgit.lib.ObjectLoader) AddParams(org.eclipse.che.api.git.params.AddParams) GitUserResolver(org.eclipse.che.api.git.GitUserResolver) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) CreateBranchCommand(org.eclipse.jgit.api.CreateBranchCommand) ConfigConstants(org.eclipse.jgit.lib.ConfigConstants) ShowFileContentResponse(org.eclipse.che.api.git.shared.ShowFileContentResponse) CheckoutCommand(org.eclipse.jgit.api.CheckoutCommand) ObjectId(org.eclipse.jgit.lib.ObjectId) TransportException(org.eclipse.jgit.api.errors.TransportException) OWNER_READ(java.nio.file.attribute.PosixFilePermission.OWNER_READ) RemoteRefUpdate(org.eclipse.jgit.transport.RemoteRefUpdate) LsRemoteCommand(org.eclipse.jgit.api.LsRemoteCommand) CheckoutParams(org.eclipse.che.api.git.params.CheckoutParams) TrackingRefUpdate(org.eclipse.jgit.transport.TrackingRefUpdate) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Git(org.eclipse.jgit.api.Git) DetachedHeadException(org.eclipse.jgit.api.errors.DetachedHeadException) DiffEntry(org.eclipse.jgit.diff.DiffEntry) OWNER_WRITE(java.nio.file.attribute.PosixFilePermission.OWNER_WRITE) GitUser(org.eclipse.che.api.git.shared.GitUser) Collections(java.util.Collections) JschConfigSessionFactory(org.eclipse.jgit.transport.JschConfigSessionFactory) SECONDS(java.util.concurrent.TimeUnit.SECONDS) GitException(org.eclipse.che.api.git.exception.GitException) ArrayList(java.util.ArrayList) RepositoryState(org.eclipse.jgit.lib.RepositoryState) GitUser(org.eclipse.che.api.git.shared.GitUser) GitAPIException(org.eclipse.jgit.api.errors.GitAPIException) Revision(org.eclipse.che.api.git.shared.Revision) CommitCommand(org.eclipse.jgit.api.CommitCommand) RevCommit(org.eclipse.jgit.revwalk.RevCommit)

Example 2 with SECONDS

use of java.util.concurrent.TimeUnit.SECONDS in project presto by prestodb.

the class ServerMainModule method setup.

@Override
protected void setup(Binder binder) {
    ServerConfig serverConfig = buildConfigObject(ServerConfig.class);
    if (serverConfig.isCoordinator()) {
        install(new CoordinatorModule());
        binder.bind(new TypeLiteral<Optional<QueryPerformanceFetcher>>() {
        }).toProvider(QueryPerformanceFetcherProvider.class).in(Scopes.SINGLETON);
    } else {
        binder.bind(new TypeLiteral<Optional<QueryPerformanceFetcher>>() {
        }).toInstance(Optional.empty());
        // Install no-op resource group manager on workers, since only coordinators manage resource groups.
        binder.bind(ResourceGroupManager.class).to(NoOpResourceGroupManager.class).in(Scopes.SINGLETON);
        // HACK: this binding is needed by SystemConnectorModule, but will only be used on the coordinator
        binder.bind(QueryManager.class).toInstance(newProxy(QueryManager.class, (proxy, method, args) -> {
            throw new UnsupportedOperationException();
        }));
    }
    configBinder(binder).bindConfig(FeaturesConfig.class);
    binder.bind(SqlParser.class).in(Scopes.SINGLETON);
    binder.bind(SqlParserOptions.class).toInstance(sqlParserOptions);
    bindFailureDetector(binder, serverConfig.isCoordinator());
    jaxrsBinder(binder).bind(ThrowableMapper.class);
    configBinder(binder).bindConfig(QueryManagerConfig.class);
    jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class);
    // session properties
    binder.bind(SessionPropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON);
    // schema properties
    binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON);
    // table properties
    binder.bind(TablePropertyManager.class).in(Scopes.SINGLETON);
    // node manager
    discoveryBinder(binder).bindSelector("presto");
    binder.bind(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    binder.bind(InternalNodeManager.class).to(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(DiscoveryNodeManager.class).withGeneratedName();
    httpClientBinder(binder).bindHttpClient("node-manager", ForNodeManager.class).withTracing().withConfigDefaults(config -> {
        config.setIdleTimeout(new Duration(30, SECONDS));
        config.setRequestTimeout(new Duration(10, SECONDS));
    });
    // node scheduler
    // TODO: remove from NodePartitioningManager and move to CoordinatorModule
    configBinder(binder).bindConfig(NodeSchedulerConfig.class);
    binder.bind(NodeScheduler.class).in(Scopes.SINGLETON);
    binder.bind(NodeSchedulerExporter.class).in(Scopes.SINGLETON);
    binder.bind(NodeTaskMap.class).in(Scopes.SINGLETON);
    newExporter(binder).export(NodeScheduler.class).withGeneratedName();
    // network topology
    // TODO: move to CoordinatorModule when NodeScheduler is moved
    install(installModuleIf(NodeSchedulerConfig.class, config -> LEGACY.equalsIgnoreCase(config.getNetworkTopology()), moduleBinder -> moduleBinder.bind(NetworkTopology.class).to(LegacyNetworkTopology.class).in(Scopes.SINGLETON)));
    install(installModuleIf(NodeSchedulerConfig.class, config -> FLAT.equalsIgnoreCase(config.getNetworkTopology()), moduleBinder -> moduleBinder.bind(NetworkTopology.class).to(FlatNetworkTopology.class).in(Scopes.SINGLETON)));
    // task execution
    jaxrsBinder(binder).bind(TaskResource.class);
    newExporter(binder).export(TaskResource.class).withGeneratedName();
    binder.bind(TaskManager.class).to(SqlTaskManager.class).in(Scopes.SINGLETON);
    // workaround for CodeCache GC issue
    if (JavaVersion.current().getMajor() == 8) {
        configBinder(binder).bindConfig(CodeCacheGcConfig.class);
        binder.bind(CodeCacheGcTrigger.class).in(Scopes.SINGLETON);
    }
    // Add monitoring for JVM pauses
    binder.bind(PauseMeter.class).in(Scopes.SINGLETON);
    newExporter(binder).export(PauseMeter.class).withGeneratedName();
    configBinder(binder).bindConfig(MemoryManagerConfig.class);
    configBinder(binder).bindConfig(NodeMemoryConfig.class);
    configBinder(binder).bindConfig(ReservedSystemMemoryConfig.class);
    binder.bind(LocalMemoryManager.class).in(Scopes.SINGLETON);
    binder.bind(LocalMemoryManagerExporter.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TaskManager.class).withGeneratedName();
    binder.bind(TaskExecutor.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TaskExecutor.class).withGeneratedName();
    binder.bind(LocalExecutionPlanner.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(CompilerConfig.class);
    binder.bind(ExpressionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExpressionCompiler.class).withGeneratedName();
    configBinder(binder).bindConfig(TaskManagerConfig.class);
    binder.bind(IndexJoinLookupStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(IndexJoinLookupStats.class).withGeneratedName();
    binder.bind(AsyncHttpExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(AsyncHttpExecutionMBean.class).withGeneratedName();
    binder.bind(JoinFilterFunctionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinFilterFunctionCompiler.class).withGeneratedName();
    binder.bind(JoinCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinCompiler.class).withGeneratedName();
    binder.bind(OrderingCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(OrderingCompiler.class).withGeneratedName();
    binder.bind(PagesIndex.Factory.class).to(PagesIndex.DefaultFactory.class);
    binder.bind(JoinProbeCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinProbeCompiler.class).withGeneratedName();
    binder.bind(LookupJoinOperators.class).in(Scopes.SINGLETON);
    jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class);
    jsonCodecBinder(binder).bindJsonCodec(StageInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class);
    jaxrsBinder(binder).bind(PagesResponseWriter.class);
    // exchange client
    binder.bind(new TypeLiteral<ExchangeClientSupplier>() {
    }).to(ExchangeClientFactory.class).in(Scopes.SINGLETON);
    httpClientBinder(binder).bindHttpClient("exchange", ForExchange.class).withTracing().withConfigDefaults(config -> {
        config.setIdleTimeout(new Duration(30, SECONDS));
        config.setRequestTimeout(new Duration(10, SECONDS));
        config.setMaxConnectionsPerServer(250);
        config.setMaxContentLength(new DataSize(32, MEGABYTE));
    });
    configBinder(binder).bindConfig(ExchangeClientConfig.class);
    binder.bind(ExchangeExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExchangeExecutionMBean.class).withGeneratedName();
    // execution
    binder.bind(LocationFactory.class).to(HttpLocationFactory.class).in(Scopes.SINGLETON);
    // memory manager
    jaxrsBinder(binder).bind(MemoryResource.class);
    jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(MemoryPoolAssignmentsRequest.class);
    // transaction manager
    configBinder(binder).bindConfig(TransactionManagerConfig.class);
    // data stream provider
    binder.bind(PageSourceManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSourceProvider.class).to(PageSourceManager.class).in(Scopes.SINGLETON);
    // page sink provider
    binder.bind(PageSinkManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSinkProvider.class).to(PageSinkManager.class).in(Scopes.SINGLETON);
    // metadata
    binder.bind(StaticCatalogStore.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(StaticCatalogStoreConfig.class);
    binder.bind(MetadataManager.class).in(Scopes.SINGLETON);
    binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON);
    // type
    binder.bind(TypeRegistry.class).in(Scopes.SINGLETON);
    binder.bind(TypeManager.class).to(TypeRegistry.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
    newSetBinder(binder, Type.class);
    // split manager
    binder.bind(SplitManager.class).in(Scopes.SINGLETON);
    // node partitioning manager
    binder.bind(NodePartitioningManager.class).in(Scopes.SINGLETON);
    // index manager
    binder.bind(IndexManager.class).in(Scopes.SINGLETON);
    // handle resolver
    binder.install(new HandleJsonModule());
    // connector
    binder.bind(ConnectorManager.class).in(Scopes.SINGLETON);
    // system connector
    binder.install(new SystemConnectorModule());
    // splits
    jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class);
    jsonCodecBinder(binder).bindJsonCodec(ConnectorSplit.class);
    jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class);
    jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class);
    jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class);
    // query monitor
    configBinder(binder).bindConfig(QueryMonitorConfig.class);
    binder.bind(QueryMonitor.class).in(Scopes.SINGLETON);
    // Determine the NodeVersion
    String prestoVersion = serverConfig.getPrestoVersion();
    if (prestoVersion == null) {
        prestoVersion = getClass().getPackage().getImplementationVersion();
    }
    checkState(prestoVersion != null, "presto.version must be provided when it cannot be automatically determined");
    NodeVersion nodeVersion = new NodeVersion(prestoVersion);
    binder.bind(NodeVersion.class).toInstance(nodeVersion);
    // presto announcement
    discoveryBinder(binder).bindHttpAnnouncement("presto").addProperty("node_version", nodeVersion.toString()).addProperty("coordinator", String.valueOf(serverConfig.isCoordinator())).addProperty("connectorIds", nullToEmpty(serverConfig.getDataSources()));
    // server info resource
    jaxrsBinder(binder).bind(ServerInfoResource.class);
    // plugin manager
    binder.bind(PluginManager.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(PluginManagerConfig.class);
    binder.bind(CatalogManager.class).in(Scopes.SINGLETON);
    // optimizers
    binder.bind(PlanOptimizers.class).in(Scopes.SINGLETON);
    // block encodings
    binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON);
    binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON);
    newSetBinder(binder, new TypeLiteral<BlockEncodingFactory<?>>() {
    });
    jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class);
    jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class);
    // thread visualizer
    jaxrsBinder(binder).bind(ThreadResource.class);
    // PageSorter
    binder.bind(PageSorter.class).to(PagesIndexPageSorter.class).in(Scopes.SINGLETON);
    // PageIndexer
    binder.bind(PageIndexerFactory.class).to(GroupByHashPageIndexerFactory.class).in(Scopes.SINGLETON);
    // Finalizer
    binder.bind(FinalizerService.class).in(Scopes.SINGLETON);
    // Spiller
    binder.bind(SpillerFactory.class).to(BinarySpillerFactory.class).in(Scopes.SINGLETON);
    newExporter(binder).export(SpillerFactory.class).withGeneratedName();
}
Also used : TypeManager(com.facebook.presto.spi.type.TypeManager) PageSourceProvider(com.facebook.presto.split.PageSourceProvider) TaskStatus(com.facebook.presto.execution.TaskStatus) JoinProbeCompiler(com.facebook.presto.sql.gen.JoinProbeCompiler) TypeRegistry(com.facebook.presto.type.TypeRegistry) NodeInfo(io.airlift.node.NodeInfo) ForNodeManager(com.facebook.presto.metadata.ForNodeManager) PauseMeter(io.airlift.stats.PauseMeter) QueryPerformanceFetcher(com.facebook.presto.execution.QueryPerformanceFetcher) Executors.newSingleThreadScheduledExecutor(java.util.concurrent.Executors.newSingleThreadScheduledExecutor) StageInfo(com.facebook.presto.execution.StageInfo) InternalNodeManager(com.facebook.presto.metadata.InternalNodeManager) FinalizerService(com.facebook.presto.util.FinalizerService) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) LocalMemoryManager(com.facebook.presto.memory.LocalMemoryManager) Multibinder.newSetBinder(com.google.inject.multibindings.Multibinder.newSetBinder) QueryManagerConfig(com.facebook.presto.execution.QueryManagerConfig) DiscoveryNodeManager(com.facebook.presto.metadata.DiscoveryNodeManager) JsonCodecBinder.jsonCodecBinder(io.airlift.json.JsonCodecBinder.jsonCodecBinder) PageSinkManager(com.facebook.presto.split.PageSinkManager) FlatNetworkTopology(com.facebook.presto.execution.scheduler.FlatNetworkTopology) Set(java.util.Set) ExpressionDeserializer(com.facebook.presto.sql.Serialization.ExpressionDeserializer) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) HttpClientBinder.httpClientBinder(io.airlift.http.client.HttpClientBinder.httpClientBinder) FLAT(com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.FLAT) DiscoveryBinder.discoveryBinder(io.airlift.discovery.client.DiscoveryBinder.discoveryBinder) ConfigBinder.configBinder(io.airlift.configuration.ConfigBinder.configBinder) TypeLiteral(com.google.inject.TypeLiteral) SystemSessionProperties(com.facebook.presto.SystemSessionProperties) Reflection.newProxy(com.google.common.reflect.Reflection.newProxy) Slice(io.airlift.slice.Slice) ViewDefinition(com.facebook.presto.metadata.ViewDefinition) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) MemoryResource(com.facebook.presto.memory.MemoryResource) ReservedSystemMemoryConfig(com.facebook.presto.memory.ReservedSystemMemoryConfig) LookupJoinOperators(com.facebook.presto.operator.LookupJoinOperators) StaticCatalogStore(com.facebook.presto.metadata.StaticCatalogStore) SqlTaskManager(com.facebook.presto.execution.SqlTaskManager) Binder(com.google.inject.Binder) Type(com.facebook.presto.spi.type.Type) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExchangeClientFactory(com.facebook.presto.operator.ExchangeClientFactory) PageSorter(com.facebook.presto.spi.PageSorter) NetworkTopology(com.facebook.presto.execution.scheduler.NetworkTopology) IndexJoinLookupStats(com.facebook.presto.operator.index.IndexJoinLookupStats) TransactionManager(com.facebook.presto.transaction.TransactionManager) ResourceGroupManager(com.facebook.presto.execution.resourceGroups.ResourceGroupManager) QueryMonitorConfig(com.facebook.presto.event.query.QueryMonitorConfig) BinarySpillerFactory(com.facebook.presto.spiller.BinarySpillerFactory) SystemConnectorModule(com.facebook.presto.connector.system.SystemConnectorModule) LocationFactory(com.facebook.presto.execution.LocationFactory) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) StaticCatalogStoreConfig(com.facebook.presto.metadata.StaticCatalogStoreConfig) SpillerFactory(com.facebook.presto.spiller.SpillerFactory) TaskInfo(com.facebook.presto.execution.TaskInfo) Metadata(com.facebook.presto.metadata.Metadata) LocalExecutionPlanner(com.facebook.presto.sql.planner.LocalExecutionPlanner) TablePropertyManager(com.facebook.presto.metadata.TablePropertyManager) Block(com.facebook.presto.spi.block.Block) SqlParserOptions(com.facebook.presto.sql.parser.SqlParserOptions) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) MetadataManager(com.facebook.presto.metadata.MetadataManager) NodeVersion(com.facebook.presto.client.NodeVersion) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) JsonBinder.jsonBinder(io.airlift.json.JsonBinder.jsonBinder) TaskManagerConfig(com.facebook.presto.execution.TaskManagerConfig) NoOpResourceGroupManager(com.facebook.presto.execution.resourceGroups.NoOpResourceGroupManager) Duration(io.airlift.units.Duration) PagesIndex(com.facebook.presto.operator.PagesIndex) MemoryPoolAssignmentsRequest(com.facebook.presto.memory.MemoryPoolAssignmentsRequest) ConnectorManager(com.facebook.presto.connector.ConnectorManager) ExportBinder.newExporter(org.weakref.jmx.guice.ExportBinder.newExporter) HandleJsonModule(com.facebook.presto.metadata.HandleJsonModule) PageIndexerFactory(com.facebook.presto.spi.PageIndexerFactory) QueryMonitor(com.facebook.presto.event.query.QueryMonitor) FunctionCall(com.facebook.presto.sql.tree.FunctionCall) ImmutableSet(com.google.common.collect.ImmutableSet) ConditionalModule.installModuleIf(io.airlift.configuration.ConditionalModule.installModuleIf) NodeMemoryConfig(com.facebook.presto.memory.NodeMemoryConfig) ForExchange(com.facebook.presto.operator.ForExchange) NodeSchedulerExporter(com.facebook.presto.execution.scheduler.NodeSchedulerExporter) BlockEncodingFactory(com.facebook.presto.spi.block.BlockEncodingFactory) SplitManager(com.facebook.presto.split.SplitManager) LEGACY(com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.LEGACY) SchemaPropertyManager(com.facebook.presto.metadata.SchemaPropertyManager) SqlParser(com.facebook.presto.sql.parser.SqlParser) Preconditions.checkState(com.google.common.base.Preconditions.checkState) ExchangeClientConfig(com.facebook.presto.operator.ExchangeClientConfig) DataSize(io.airlift.units.DataSize) ServiceDescriptor(io.airlift.discovery.client.ServiceDescriptor) TransactionManagerConfig(com.facebook.presto.transaction.TransactionManagerConfig) JaxrsBinder.jaxrsBinder(io.airlift.jaxrs.JaxrsBinder.jaxrsBinder) Optional(java.util.Optional) ExpressionCompiler(com.facebook.presto.sql.gen.ExpressionCompiler) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) PageSinkProvider(com.facebook.presto.split.PageSinkProvider) ForTransactionManager(com.facebook.presto.transaction.ForTransactionManager) QueryManager(com.facebook.presto.execution.QueryManager) Singleton(javax.inject.Singleton) MemoryInfo(com.facebook.presto.memory.MemoryInfo) BlockJsonSerde(com.facebook.presto.block.BlockJsonSerde) PlanOptimizers(com.facebook.presto.sql.planner.PlanOptimizers) HttpLocationFactory(com.facebook.presto.server.remotetask.HttpLocationFactory) LegacyNetworkTopology(com.facebook.presto.execution.scheduler.LegacyNetworkTopology) BlockEncodingSerde(com.facebook.presto.spi.block.BlockEncodingSerde) AbstractConfigurationAwareModule(io.airlift.configuration.AbstractConfigurationAwareModule) PageSourceManager(com.facebook.presto.split.PageSourceManager) OrderingCompiler(com.facebook.presto.sql.gen.OrderingCompiler) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) TaskManager(com.facebook.presto.execution.TaskManager) LocalMemoryManagerExporter(com.facebook.presto.memory.LocalMemoryManagerExporter) Objects.requireNonNull(java.util.Objects.requireNonNull) IndexManager(com.facebook.presto.index.IndexManager) MemoryManagerConfig(com.facebook.presto.memory.MemoryManagerConfig) SessionPropertyManager(com.facebook.presto.metadata.SessionPropertyManager) FunctionCallDeserializer(com.facebook.presto.sql.Serialization.FunctionCallDeserializer) ExecutorService(java.util.concurrent.ExecutorService) PagesIndexPageSorter(com.facebook.presto.PagesIndexPageSorter) NodePartitioningManager(com.facebook.presto.sql.planner.NodePartitioningManager) ExchangeClientSupplier(com.facebook.presto.operator.ExchangeClientSupplier) TaskExecutor(com.facebook.presto.execution.TaskExecutor) JoinFilterFunctionCompiler(com.facebook.presto.sql.gen.JoinFilterFunctionCompiler) Scopes(com.google.inject.Scopes) CatalogManager(com.facebook.presto.metadata.CatalogManager) CompilerConfig(com.facebook.presto.sql.planner.CompilerConfig) QueryPerformanceFetcherProvider(com.facebook.presto.execution.QueryPerformanceFetcherProvider) FailureDetectorModule(com.facebook.presto.failureDetector.FailureDetectorModule) TypeDeserializer(com.facebook.presto.type.TypeDeserializer) Provides(com.google.inject.Provides) Expression(com.facebook.presto.sql.tree.Expression) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) FailureDetector(com.facebook.presto.failureDetector.FailureDetector) ExpressionSerializer(com.facebook.presto.sql.Serialization.ExpressionSerializer) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) ServerInfo(com.facebook.presto.client.ServerInfo) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) SECONDS(java.util.concurrent.TimeUnit.SECONDS) BlockEncodingManager(com.facebook.presto.block.BlockEncodingManager) TypeRegistry(com.facebook.presto.type.TypeRegistry) PageSourceManager(com.facebook.presto.split.PageSourceManager) NodeVersion(com.facebook.presto.client.NodeVersion) PagesIndexPageSorter(com.facebook.presto.PagesIndexPageSorter) ExchangeClientFactory(com.facebook.presto.operator.ExchangeClientFactory) QueryPerformanceFetcher(com.facebook.presto.execution.QueryPerformanceFetcher) DataSize(io.airlift.units.DataSize) QueryMonitor(com.facebook.presto.event.query.QueryMonitor) SessionPropertyManager(com.facebook.presto.metadata.SessionPropertyManager) OrderingCompiler(com.facebook.presto.sql.gen.OrderingCompiler) ExchangeClientSupplier(com.facebook.presto.operator.ExchangeClientSupplier) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) NoOpResourceGroupManager(com.facebook.presto.execution.resourceGroups.NoOpResourceGroupManager) SqlParserOptions(com.facebook.presto.sql.parser.SqlParserOptions) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) QueryPerformanceFetcherProvider(com.facebook.presto.execution.QueryPerformanceFetcherProvider) Optional(java.util.Optional) IndexJoinLookupStats(com.facebook.presto.operator.index.IndexJoinLookupStats) StaticCatalogStore(com.facebook.presto.metadata.StaticCatalogStore) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) HttpLocationFactory(com.facebook.presto.server.remotetask.HttpLocationFactory) SqlParser(com.facebook.presto.sql.parser.SqlParser) Duration(io.airlift.units.Duration) PauseMeter(io.airlift.stats.PauseMeter) IndexManager(com.facebook.presto.index.IndexManager) SqlTaskManager(com.facebook.presto.execution.SqlTaskManager) TaskExecutor(com.facebook.presto.execution.TaskExecutor) BlockEncodingManager(com.facebook.presto.block.BlockEncodingManager) BlockJsonSerde(com.facebook.presto.block.BlockJsonSerde) Slice(io.airlift.slice.Slice) LegacyNetworkTopology(com.facebook.presto.execution.scheduler.LegacyNetworkTopology) BinarySpillerFactory(com.facebook.presto.spiller.BinarySpillerFactory) Block(com.facebook.presto.spi.block.Block) ExpressionCompiler(com.facebook.presto.sql.gen.ExpressionCompiler) PlanOptimizers(com.facebook.presto.sql.planner.PlanOptimizers) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) ExchangeClientFactory(com.facebook.presto.operator.ExchangeClientFactory) BinarySpillerFactory(com.facebook.presto.spiller.BinarySpillerFactory) LocationFactory(com.facebook.presto.execution.LocationFactory) SpillerFactory(com.facebook.presto.spiller.SpillerFactory) PageIndexerFactory(com.facebook.presto.spi.PageIndexerFactory) BlockEncodingFactory(com.facebook.presto.spi.block.BlockEncodingFactory) HttpLocationFactory(com.facebook.presto.server.remotetask.HttpLocationFactory) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) BlockEncodingFactory(com.facebook.presto.spi.block.BlockEncodingFactory) PagesIndex(com.facebook.presto.operator.PagesIndex) LocalMemoryManagerExporter(com.facebook.presto.memory.LocalMemoryManagerExporter) NodePartitioningManager(com.facebook.presto.sql.planner.NodePartitioningManager) JoinProbeCompiler(com.facebook.presto.sql.gen.JoinProbeCompiler) TypeLiteral(com.google.inject.TypeLiteral) LocalMemoryManager(com.facebook.presto.memory.LocalMemoryManager) HandleJsonModule(com.facebook.presto.metadata.HandleJsonModule) SystemConnectorModule(com.facebook.presto.connector.system.SystemConnectorModule) FunctionCall(com.facebook.presto.sql.tree.FunctionCall) SchemaPropertyManager(com.facebook.presto.metadata.SchemaPropertyManager) LookupJoinOperators(com.facebook.presto.operator.LookupJoinOperators) PageSinkManager(com.facebook.presto.split.PageSinkManager) LocalExecutionPlanner(com.facebook.presto.sql.planner.LocalExecutionPlanner) FlatNetworkTopology(com.facebook.presto.execution.scheduler.FlatNetworkTopology) JoinFilterFunctionCompiler(com.facebook.presto.sql.gen.JoinFilterFunctionCompiler) NodeSchedulerExporter(com.facebook.presto.execution.scheduler.NodeSchedulerExporter) BinarySpillerFactory(com.facebook.presto.spiller.BinarySpillerFactory) SpillerFactory(com.facebook.presto.spiller.SpillerFactory) ConnectorManager(com.facebook.presto.connector.ConnectorManager) SplitManager(com.facebook.presto.split.SplitManager) CatalogManager(com.facebook.presto.metadata.CatalogManager) Type(com.facebook.presto.spi.type.Type) SqlTaskManager(com.facebook.presto.execution.SqlTaskManager) TaskManager(com.facebook.presto.execution.TaskManager) MetadataManager(com.facebook.presto.metadata.MetadataManager) Expression(com.facebook.presto.sql.tree.Expression) FinalizerService(com.facebook.presto.util.FinalizerService) QueryManager(com.facebook.presto.execution.QueryManager) TablePropertyManager(com.facebook.presto.metadata.TablePropertyManager) DiscoveryNodeManager(com.facebook.presto.metadata.DiscoveryNodeManager) SystemSessionProperties(com.facebook.presto.SystemSessionProperties)

Example 3 with SECONDS

use of java.util.concurrent.TimeUnit.SECONDS in project presto by prestodb.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) throws InterruptedException {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    });
    // For hard timeout, add an additional 5 seconds to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + 5000, MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}
Also used : TaskStatus(com.facebook.presto.execution.TaskStatus) Status(javax.ws.rs.core.Response.Status) Page(com.facebook.presto.spi.Page) Produces(javax.ws.rs.Produces) Iterables.transform(com.google.common.collect.Iterables.transform) TaskStatus(com.facebook.presto.execution.TaskStatus) Path(javax.ws.rs.Path) AsyncResponseHandler.bindAsyncResponse(io.airlift.http.server.AsyncResponseHandler.bindAsyncResponse) TaskState(com.facebook.presto.execution.TaskState) Duration(io.airlift.units.Duration) OutputBufferId(com.facebook.presto.OutputBuffers.OutputBufferId) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) DefaultValue(javax.ws.rs.DefaultValue) HeaderParam(javax.ws.rs.HeaderParam) BufferResult(com.facebook.presto.execution.buffer.BufferResult) DELETE(javax.ws.rs.DELETE) PRESTO_PAGE_TOKEN(com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_TOKEN) Context(javax.ws.rs.core.Context) AsyncResponse(javax.ws.rs.container.AsyncResponse) GenericEntity(javax.ws.rs.core.GenericEntity) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Suspended(javax.ws.rs.container.Suspended) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) DataSize(io.airlift.units.DataSize) List(java.util.List) Response(javax.ws.rs.core.Response) CompletionCallback(javax.ws.rs.container.CompletionCallback) PRESTO_BUFFER_COMPLETE(com.facebook.presto.client.PrestoHeaders.PRESTO_BUFFER_COMPLETE) SerializedPage(com.facebook.presto.execution.buffer.SerializedPage) UriInfo(javax.ws.rs.core.UriInfo) PRESTO_CURRENT_STATE(com.facebook.presto.client.PrestoHeaders.PRESTO_CURRENT_STATE) Nested(org.weakref.jmx.Nested) PathParam(javax.ws.rs.PathParam) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) GET(javax.ws.rs.GET) TypeToken(com.google.common.reflect.TypeToken) Inject(javax.inject.Inject) PRESTO_TASK_INSTANCE_ID(com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID) ImmutableList(com.google.common.collect.ImmutableList) Managed(org.weakref.jmx.Managed) PRESTO_MAX_SIZE(com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_SIZE) TaskManager(com.facebook.presto.execution.TaskManager) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Objects.requireNonNull(java.util.Objects.requireNonNull) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) MoreFutures.addTimeout(io.airlift.concurrent.MoreFutures.addTimeout) PRESTO_PAGE_NEXT_TOKEN(com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_NEXT_TOKEN) SessionPropertyManager(com.facebook.presto.metadata.SessionPropertyManager) TimeStat(io.airlift.stats.TimeStat) Status(javax.ws.rs.core.Response.Status) POST(javax.ws.rs.POST) Executor(java.util.concurrent.Executor) Session(com.facebook.presto.Session) PRESTO_MAX_WAIT(com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT) Futures(com.google.common.util.concurrent.Futures) PRESTO_PAGES(com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES) TaskId(com.facebook.presto.execution.TaskId) TaskInfo(com.facebook.presto.execution.TaskInfo) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Duration(io.airlift.units.Duration) AsyncResponseHandler.bindAsyncResponse(io.airlift.http.server.AsyncResponseHandler.bindAsyncResponse) AsyncResponse(javax.ws.rs.container.AsyncResponse) Response(javax.ws.rs.core.Response) BufferResult(com.facebook.presto.execution.buffer.BufferResult) SerializedPage(com.facebook.presto.execution.buffer.SerializedPage) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Path(javax.ws.rs.Path) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET)

Example 4 with SECONDS

use of java.util.concurrent.TimeUnit.SECONDS in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldBeAbleToCopyStoresFromCoreToReadReplica.

@Test
public void shouldBeAbleToCopyStoresFromCoreToReadReplica() throws Exception {
    // given
    Map<String, String> params = stringMap(CausalClusteringSettings.raft_log_rotation_size.name(), "1k", CausalClusteringSettings.raft_log_pruning_frequency.name(), "500ms", CausalClusteringSettings.state_machine_flush_window_size.name(), "1", CausalClusteringSettings.raft_log_pruning_strategy.name(), "1 entries");
    Cluster cluster = clusterRule.withNumberOfReadReplicas(0).withSharedCoreParams(params).withRecordFormat(HighLimit.NAME).startCluster();
    cluster.coreTx((db, tx) -> {
        Node node = db.createNode(Label.label("L"));
        for (int i = 0; i < 10; i++) {
            node.setProperty("prop-" + i, "this is a quite long string to get to the log limit soonish");
        }
        tx.success();
    });
    long baseVersion = versionBy(cluster.awaitLeader().raftLogDirectory(), Math::max);
    CoreClusterMember coreGraphDatabase = null;
    for (int j = 0; j < 2; j++) {
        coreGraphDatabase = cluster.coreTx((db, tx) -> {
            Node node = db.createNode(Label.label("L"));
            for (int i = 0; i < 10; i++) {
                node.setProperty("prop-" + i, "this is a quite long string to get to the log limit soonish");
            }
            tx.success();
        });
    }
    File raftLogDir = coreGraphDatabase.raftLogDirectory();
    assertEventually("pruning happened", () -> versionBy(raftLogDir, Math::min), greaterThan(baseVersion), 5, SECONDS);
    // when
    cluster.addReadReplicaWithIdAndRecordFormat(4, HighLimit.NAME).start();
    // then
    for (final ReadReplica readReplica : cluster.readReplicas()) {
        assertEventually("read replica available", () -> readReplica.database().isAvailable(0), is(true), 10, SECONDS);
    }
}
Also used : ResourceIterator(org.neo4j.graphdb.ResourceIterator) AvailabilityGuard(org.neo4j.kernel.AvailabilityGuard) Log(org.neo4j.logging.Log) CausalClusteringSettings(org.neo4j.causalclustering.core.CausalClusteringSettings) ReadReplicaGraphDatabase(org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase) CoreMatchers.startsWith(org.hamcrest.CoreMatchers.startsWith) ThrowingSupplier(org.neo4j.function.ThrowingSupplier) HighLimit(org.neo4j.kernel.impl.store.format.highlimit.HighLimit) Map(java.util.Map) Is.is(org.hamcrest.core.Is.is) CoreClusterMember(org.neo4j.causalclustering.discovery.CoreClusterMember) Assert.fail(org.junit.Assert.fail) Transaction(org.neo4j.graphdb.Transaction) Path(java.nio.file.Path) Collectors.toSet(java.util.stream.Collectors.toSet) Standard(org.neo4j.kernel.impl.store.format.standard.Standard) PageCache(org.neo4j.io.pagecache.PageCache) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Collection(java.util.Collection) Set(java.util.Set) BinaryOperator(java.util.function.BinaryOperator) UnsatisfiedDependencyException(org.neo4j.kernel.impl.util.UnsatisfiedDependencyException) Cluster(org.neo4j.causalclustering.discovery.Cluster) GraphDatabaseAPI(org.neo4j.kernel.internal.GraphDatabaseAPI) CoreGraphDatabase(org.neo4j.causalclustering.core.CoreGraphDatabase) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) TransactionIdTracker(org.neo4j.kernel.api.txtracking.TransactionIdTracker) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Predicates.awaitEx(org.neo4j.function.Predicates.awaitEx) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) GraphDatabaseSettings(org.neo4j.graphdb.factory.GraphDatabaseSettings) SortedMap(java.util.SortedMap) Mockito.mock(org.mockito.Mockito.mock) Role(org.neo4j.causalclustering.core.consensus.roles.Role) ClusterMember(org.neo4j.causalclustering.discovery.ClusterMember) Label(org.neo4j.graphdb.Label) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) Iterables.count(org.neo4j.helpers.collection.Iterables.count) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Monitors(org.neo4j.kernel.monitoring.Monitors) MINUTES(java.util.concurrent.TimeUnit.MINUTES) ClusterRule(org.neo4j.test.causalclustering.ClusterRule) LabelScanStore(org.neo4j.kernel.api.labelscan.LabelScanStore) Duration.ofSeconds(java.time.Duration.ofSeconds) Function(java.util.function.Function) Node(org.neo4j.graphdb.Node) HashSet(java.util.HashSet) CatchupPollingProcess(org.neo4j.causalclustering.catchup.tx.CatchupPollingProcess) LifecycleException(org.neo4j.kernel.lifecycle.LifecycleException) HazelcastDiscoveryServiceFactory(org.neo4j.causalclustering.discovery.HazelcastDiscoveryServiceFactory) GraphDatabaseService(org.neo4j.graphdb.GraphDatabaseService) TIME(org.neo4j.kernel.impl.store.MetaDataStore.Position.TIME) BiConsumer(java.util.function.BiConsumer) TransactionFailureException(org.neo4j.kernel.api.exceptions.TransactionFailureException) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) MapUtil.stringMap(org.neo4j.helpers.collection.MapUtil.stringMap) FileNames(org.neo4j.causalclustering.core.consensus.log.segmented.FileNames) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) FileCopyMonitor(org.neo4j.causalclustering.catchup.tx.FileCopyMonitor) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) Assert.assertEventually(org.neo4j.test.assertion.Assert.assertEventually) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) SampleData.createData(org.neo4j.causalclustering.scenarios.SampleData.createData) Rule(org.junit.Rule) Paths(java.nio.file.Paths) WriteOperationsNotAllowedException(org.neo4j.graphdb.security.WriteOperationsNotAllowedException) GraphDatabaseFacade(org.neo4j.kernel.impl.factory.GraphDatabaseFacade) DbRepresentation(org.neo4j.test.DbRepresentation) Clock(java.time.Clock) ReadReplica(org.neo4j.causalclustering.discovery.ReadReplica) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Assert.assertEquals(org.junit.Assert.assertEquals) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) ReadReplica(org.neo4j.causalclustering.discovery.ReadReplica) CoreClusterMember(org.neo4j.causalclustering.discovery.CoreClusterMember) Node(org.neo4j.graphdb.Node) Cluster(org.neo4j.causalclustering.discovery.Cluster) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) File(java.io.File) Test(org.junit.Test)

Example 5 with SECONDS

use of java.util.concurrent.TimeUnit.SECONDS in project presto by prestodb.

the class TestHttpRemoteTask method testRemoteTaskMismatch.

// This timeout should never be reached because a daemon thread in test should fail the test and do proper cleanup.
@Test(timeOut = 30000)
public void testRemoteTaskMismatch() throws InterruptedException, ExecutionException {
    Duration idleTimeout = new Duration(3, SECONDS);
    Duration failTimeout = new Duration(20, SECONDS);
    JsonCodec<TaskStatus> taskStatusCodec = JsonCodec.jsonCodec(TaskStatus.class);
    JsonCodec<TaskInfo> taskInfoCodec = JsonCodec.jsonCodec(TaskInfo.class);
    TaskManagerConfig taskManagerConfig = new TaskManagerConfig();
    // Shorten status refresh wait and info update interval so that we can have a shorter test timeout
    taskManagerConfig.setStatusRefreshMaxWait(new Duration(idleTimeout.roundTo(MILLISECONDS) / 100, MILLISECONDS));
    taskManagerConfig.setInfoUpdateInterval(new Duration(idleTimeout.roundTo(MILLISECONDS) / 10, MILLISECONDS));
    AtomicLong lastActivityNanos = new AtomicLong(System.nanoTime());
    HttpProcessor httpProcessor = new HttpProcessor(taskStatusCodec, taskInfoCodec, lastActivityNanos);
    TestingHttpClient testingHttpClient = new TestingHttpClient(httpProcessor);
    HttpRemoteTaskFactory httpRemoteTaskFactory = new HttpRemoteTaskFactory(new QueryManagerConfig(), taskManagerConfig, testingHttpClient, new TestSqlTaskManager.MockLocationFactory(), taskStatusCodec, taskInfoCodec, JsonCodec.jsonCodec(TaskUpdateRequest.class), new RemoteTaskStats());
    RemoteTask remoteTask = httpRemoteTaskFactory.createRemoteTask(TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://192.0.1.2"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {
    }), true);
    httpProcessor.setInitialTaskInfo(remoteTask.getTaskInfo());
    remoteTask.start();
    CompletableFuture<Void> testComplete = new CompletableFuture<>();
    asyncRun(idleTimeout.roundTo(MILLISECONDS), failTimeout.roundTo(MILLISECONDS), lastActivityNanos, () -> testComplete.complete(null), (message, cause) -> testComplete.completeExceptionally(new AssertionError(message, cause)));
    testComplete.get();
    httpRemoteTaskFactory.stop();
    assertTrue(remoteTask.getTaskStatus().getState().isDone(), format("TaskStatus is not in a done state: %s", remoteTask.getTaskStatus()));
    assertEquals(getOnlyElement(remoteTask.getTaskStatus().getFailures()).getErrorCode(), REMOTE_TASK_MISMATCH.toErrorCode());
    assertTrue(remoteTask.getTaskInfo().getTaskStatus().getState().isDone(), format("TaskInfo is not in a done state: %s", remoteTask.getTaskInfo()));
}
Also used : OutputBuffers(com.facebook.presto.OutputBuffers) CONTENT_TYPE(javax.ws.rs.core.HttpHeaders.CONTENT_TYPE) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) TaskStatus(com.facebook.presto.execution.TaskStatus) Assert.assertEquals(org.testng.Assert.assertEquals) TaskState(com.facebook.presto.execution.TaskState) Test(org.testng.annotations.Test) CompletableFuture(java.util.concurrent.CompletableFuture) NodeVersion(com.facebook.presto.client.NodeVersion) TaskManagerConfig(com.facebook.presto.execution.TaskManagerConfig) Duration(io.airlift.units.Duration) PRESTO_TASK_INSTANCE_ID(com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) Request(io.airlift.http.client.Request) BiConsumer(java.util.function.BiConsumer) TestingResponse(io.airlift.http.client.testing.TestingResponse) QueryManagerConfig(com.facebook.presto.execution.QueryManagerConfig) URI(java.net.URI) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) REMOTE_TASK_MISMATCH(com.facebook.presto.spi.StandardErrorCode.REMOTE_TASK_MISMATCH) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) TestSqlTaskManager(com.facebook.presto.execution.TestSqlTaskManager) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) PrestoNode(com.facebook.presto.metadata.PrestoNode) StandardCharsets(java.nio.charset.StandardCharsets) PRESTO_MAX_WAIT(com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT) String.format(java.lang.String.format) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) RemoteTask(com.facebook.presto.execution.RemoteTask) TaskTestUtils(com.facebook.presto.execution.TaskTestUtils) HttpStatus(io.airlift.http.client.HttpStatus) TaskId(com.facebook.presto.execution.TaskId) HttpRemoteTaskFactory(com.facebook.presto.server.HttpRemoteTaskFactory) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Response(io.airlift.http.client.Response) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.OutputBuffers.createInitialEmptyOutputBuffers) Assert.assertTrue(org.testng.Assert.assertTrue) TaskUpdateRequest(com.facebook.presto.server.TaskUpdateRequest) TaskInfo(com.facebook.presto.execution.TaskInfo) SECONDS(java.util.concurrent.TimeUnit.SECONDS) JsonCodec(io.airlift.json.JsonCodec) TaskId(com.facebook.presto.execution.TaskId) TaskManagerConfig(com.facebook.presto.execution.TaskManagerConfig) TestSqlTaskManager(com.facebook.presto.execution.TestSqlTaskManager) PrestoNode(com.facebook.presto.metadata.PrestoNode) TaskInfo(com.facebook.presto.execution.TaskInfo) NodeVersion(com.facebook.presto.client.NodeVersion) CompletableFuture(java.util.concurrent.CompletableFuture) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) HttpRemoteTaskFactory(com.facebook.presto.server.HttpRemoteTaskFactory) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) TaskUpdateRequest(com.facebook.presto.server.TaskUpdateRequest) RemoteTask(com.facebook.presto.execution.RemoteTask) Duration(io.airlift.units.Duration) TaskStatus(com.facebook.presto.execution.TaskStatus) AtomicLong(java.util.concurrent.atomic.AtomicLong) QueryManagerConfig(com.facebook.presto.execution.QueryManagerConfig) Test(org.testng.annotations.Test)

Aggregations

SECONDS (java.util.concurrent.TimeUnit.SECONDS)6 TaskInfo (com.facebook.presto.execution.TaskInfo)3 TaskStatus (com.facebook.presto.execution.TaskStatus)3 Duration (io.airlift.units.Duration)3 NodeVersion (com.facebook.presto.client.NodeVersion)2 PRESTO_MAX_WAIT (com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT)2 PRESTO_TASK_INSTANCE_ID (com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID)2 NodeTaskMap (com.facebook.presto.execution.NodeTaskMap)2 QueryManagerConfig (com.facebook.presto.execution.QueryManagerConfig)2 TaskManager (com.facebook.presto.execution.TaskManager)2 SessionPropertyManager (com.facebook.presto.metadata.SessionPropertyManager)2 BoundedExecutor (io.airlift.concurrent.BoundedExecutor)2 DataSize (io.airlift.units.DataSize)2 File (java.io.File)2 IOException (java.io.IOException)2 Collection (java.util.Collection)2 List (java.util.List)2 Objects.requireNonNull (java.util.Objects.requireNonNull)2 Optional (java.util.Optional)2 Set (java.util.Set)2