Search in sources :

Example 11 with ImmutableListMultimap

use of com.google.common.collect.ImmutableListMultimap in project gerrit by GerritCodeReview.

the class ReceiveCommits method parseMagicBranch.

/**
 * Parse the magic branch data (refs/for/BRANCH/OPTIONALTOPIC%OPTIONS) into the magicBranch
 * member.
 *
 * <p>Assumes we are handling a magic branch here.
 */
private void parseMagicBranch(ReceiveCommand cmd) throws PermissionBackendException, IOException {
    try (TraceTimer traceTimer = newTimer("parseMagicBranch")) {
        logger.atFine().log("Found magic branch %s", cmd.getRefName());
        MagicBranchInput magicBranch = new MagicBranchInput(user, projectState, cmd, labelTypes);
        String ref;
        magicBranch.cmdLineParser = optionParserFactory.create(magicBranch);
        // Filter out plugin push options, as the parser would reject them as unknown.
        ImmutableListMultimap<String, String> pushOptionsToParse = pushOptions.entries().stream().filter(e -> !isPluginPushOption(e.getKey())).collect(toImmutableListMultimap(e -> e.getKey(), e -> e.getValue()));
        try {
            ref = magicBranch.parse(pushOptionsToParse);
        } catch (CmdLineException e) {
            if (!magicBranch.cmdLineParser.wasHelpRequestedByOption()) {
                logger.atFine().log("Invalid branch syntax");
                reject(cmd, e.getMessage());
                return;
            }
            // never happens
            ref = null;
        }
        if (magicBranch.skipValidation) {
            reject(cmd, String.format("\"--%s\" option is only supported for direct push", PUSH_OPTION_SKIP_VALIDATION));
            return;
        }
        if (magicBranch.topic != null && magicBranch.topic.length() > ChangeUtil.TOPIC_MAX_LENGTH) {
            reject(cmd, String.format("topic length exceeds the limit (%d)", ChangeUtil.TOPIC_MAX_LENGTH));
        }
        if (magicBranch.cmdLineParser.wasHelpRequestedByOption()) {
            StringWriter w = new StringWriter();
            w.write("\nHelp for refs/for/branch:\n\n");
            magicBranch.cmdLineParser.printUsage(w, null);
            String pluginPushOptionsHelp = StreamSupport.stream(pluginPushOptions.entries().spliterator(), /* parallel= */
            false).map(e -> String.format("-o %s~%s: %s", e.getPluginName(), e.get().getName(), e.get().getDescription())).sorted().collect(joining("\n"));
            if (!pluginPushOptionsHelp.isEmpty()) {
                w.write("\nPlugin push options:\n" + pluginPushOptionsHelp);
            }
            addMessage(w.toString());
            reject(cmd, "see help");
            return;
        }
        if (projectState.isAllUsers() && RefNames.REFS_USERS_SELF.equals(ref)) {
            logger.atFine().log("Handling %s", RefNames.REFS_USERS_SELF);
            ref = RefNames.refsUsers(user.getAccountId());
        }
        // configuration.
        if (receivePackRefCache.exactRef(ref) == null && !ref.equals(readHEAD(repo)) && !ref.equals(RefNames.REFS_CONFIG)) {
            logger.atFine().log("Ref %s not found", ref);
            if (ref.startsWith(Constants.R_HEADS)) {
                String n = ref.substring(Constants.R_HEADS.length());
                reject(cmd, "branch " + n + " not found");
            } else {
                reject(cmd, ref + " not found");
            }
            return;
        }
        magicBranch.dest = BranchNameKey.create(project.getNameKey(), ref);
        magicBranch.perm = permissions.ref(ref);
        Optional<AuthException> err = checkRefPermission(magicBranch.perm, RefPermission.READ).map(Optional::of).orElse(checkRefPermission(magicBranch.perm, RefPermission.CREATE_CHANGE));
        if (err.isPresent()) {
            rejectProhibited(cmd, err.get());
            return;
        }
        if (magicBranch.isPrivate && magicBranch.removePrivate) {
            reject(cmd, "the options 'private' and 'remove-private' are mutually exclusive");
            return;
        }
        boolean privateByDefault = projectCache.get(project.getNameKey()).orElseThrow(illegalState(project.getNameKey())).is(BooleanProjectConfig.PRIVATE_BY_DEFAULT);
        setChangeAsPrivate = magicBranch.isPrivate || (privateByDefault && !magicBranch.removePrivate);
        if (receiveConfig.disablePrivateChanges && setChangeAsPrivate) {
            reject(cmd, "private changes are disabled");
            return;
        }
        if (magicBranch.workInProgress && magicBranch.ready) {
            reject(cmd, "the options 'wip' and 'ready' are mutually exclusive");
            return;
        }
        if (magicBranch.publishComments && magicBranch.noPublishComments) {
            reject(cmd, "the options 'publish-comments' and 'no-publish-comments' are mutually exclusive");
            return;
        }
        if (magicBranch.submit) {
            err = checkRefPermission(magicBranch.perm, RefPermission.UPDATE_BY_SUBMIT);
            if (err.isPresent()) {
                rejectProhibited(cmd, err.get());
                return;
            }
        }
        RevWalk walk = receivePack.getRevWalk();
        RevCommit tip;
        try {
            tip = walk.parseCommit(magicBranch.cmd.getNewId());
            logger.atFine().log("Tip of push: %s", tip.name());
        } catch (IOException ex) {
            magicBranch.cmd.setResult(REJECTED_MISSING_OBJECT);
            logger.atSevere().withCause(ex).log("Invalid pack upload; one or more objects weren't sent");
            return;
        }
        String destBranch = magicBranch.dest.branch();
        try {
            if (magicBranch.merged) {
                if (magicBranch.base != null) {
                    reject(cmd, "cannot use merged with base");
                    return;
                }
                Ref refTip = receivePackRefCache.exactRef(magicBranch.dest.branch());
                if (refTip == null) {
                    reject(cmd, magicBranch.dest.branch() + " not found");
                    return;
                }
                RevCommit branchTip = receivePack.getRevWalk().parseCommit(refTip.getObjectId());
                if (!walk.isMergedInto(tip, branchTip)) {
                    reject(cmd, "not merged into branch");
                    return;
                }
            }
            // if %base or %merged was specified, ignore newChangeForAllNotInTarget.
            if (tip.getParentCount() > 1 || magicBranch.base != null || magicBranch.merged || tip.getParentCount() == 0) {
                logger.atFine().log("Forcing newChangeForAllNotInTarget = false");
                newChangeForAllNotInTarget = false;
            }
            if (magicBranch.base != null) {
                logger.atFine().log("Handling %%base: %s", magicBranch.base);
                magicBranch.baseCommit = Lists.newArrayListWithCapacity(magicBranch.base.size());
                for (ObjectId id : magicBranch.base) {
                    try {
                        magicBranch.baseCommit.add(walk.parseCommit(id));
                    } catch (IncorrectObjectTypeException notCommit) {
                        reject(cmd, "base must be a commit");
                        return;
                    } catch (MissingObjectException e) {
                        reject(cmd, "base not found");
                        return;
                    } catch (IOException e) {
                        throw new StorageException(String.format("Project %s cannot read %s", project.getName(), id.name()), e);
                    }
                }
            } else if (newChangeForAllNotInTarget) {
                Ref refTip = receivePackRefCache.exactRef(magicBranch.dest.branch());
                if (refTip != null) {
                    RevCommit branchTip = receivePack.getRevWalk().parseCommit(refTip.getObjectId());
                    magicBranch.baseCommit = Collections.singletonList(branchTip);
                    logger.atFine().log("Set baseCommit = %s", magicBranch.baseCommit.get(0).name());
                } else {
                    // repository and to review an initial project configuration.
                    if (!ref.equals(readHEAD(repo)) && !ref.equals(RefNames.REFS_CONFIG)) {
                        reject(cmd, magicBranch.dest.branch() + " not found");
                        return;
                    }
                }
            }
        } catch (IOException e) {
            throw new StorageException(String.format("Error walking to %s in project %s", destBranch, project.getName()), e);
        }
        if (validateConnected(magicBranch.cmd, magicBranch.dest, tip)) {
            this.magicBranch = magicBranch;
            this.result.magicPush(true);
        }
    }
}
Also used : NotifyInfo(com.google.gerrit.extensions.api.changes.NotifyInfo) DynamicItem(com.google.gerrit.extensions.registration.DynamicItem) MultimapBuilder(com.google.common.collect.MultimapBuilder) ChangeReportFormatter(com.google.gerrit.server.git.ChangeReportFormatter) DeadlineChecker(com.google.gerrit.server.DeadlineChecker) ReceiveCommand(org.eclipse.jgit.transport.ReceiveCommand) RevObject(org.eclipse.jgit.revwalk.RevObject) BadRequestException(com.google.gerrit.extensions.restapi.BadRequestException) HashtagsInput(com.google.gerrit.extensions.api.changes.HashtagsInput) RevWalk(org.eclipse.jgit.revwalk.RevWalk) Config(org.eclipse.jgit.lib.Config) Future(java.util.concurrent.Future) FooterLine(org.eclipse.jgit.revwalk.FooterLine) Map(java.util.Map) UrlFormatter(com.google.gerrit.server.config.UrlFormatter) Metadata(com.google.gerrit.server.logging.Metadata) MailRecipients(com.google.gerrit.server.mail.MailUtil.MailRecipients) RevSort(org.eclipse.jgit.revwalk.RevSort) Constants(org.eclipse.jgit.lib.Constants) PublishCommentsOp(com.google.gerrit.server.PublishCommentsOp) Collectors.joining(java.util.stream.Collectors.joining) RequestListener(com.google.gerrit.server.RequestListener) PersonIdent(org.eclipse.jgit.lib.PersonIdent) TagCache(com.google.gerrit.server.git.TagCache) Stream(java.util.stream.Stream) HashtagsUtil.cleanupHashtag(com.google.gerrit.server.change.HashtagsUtil.cleanupHashtag) BooleanProjectConfig(com.google.gerrit.entities.BooleanProjectConfig) Counter3(com.google.gerrit.metrics.Counter3) PluginSetContext(com.google.gerrit.server.plugincontext.PluginSetContext) ProjectConfigEntryType(com.google.gerrit.extensions.api.projects.ProjectConfigEntryType) Counter0(com.google.gerrit.metrics.Counter0) MetricMaker(com.google.gerrit.metrics.MetricMaker) PatchSetInfo(com.google.gerrit.entities.PatchSetInfo) FluentLogger(com.google.common.flogger.FluentLogger) Joiner(com.google.common.base.Joiner) ValidationError(com.google.gerrit.server.git.ValidationError) SAME_CHANGE_ID_IN_MULTIPLE_CHANGES(com.google.gerrit.server.git.receive.ReceiveConstants.SAME_CHANGE_ID_IN_MULTIPLE_CHANGES) RevCommit(org.eclipse.jgit.revwalk.RevCommit) UnprocessableEntityException(com.google.gerrit.extensions.restapi.UnprocessableEntityException) CommentsUtil(com.google.gerrit.server.CommentsUtil) RequestScopePropagator(com.google.gerrit.server.util.RequestScopePropagator) PerformanceLogger(com.google.gerrit.server.logging.PerformanceLogger) LinkedHashMap(java.util.LinkedHashMap) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) SubmissionExecutor(com.google.gerrit.server.update.SubmissionExecutor) Description(com.google.gerrit.metrics.Description) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) StreamSupport(java.util.stream.StreamSupport) MagicBranch(com.google.gerrit.server.util.MagicBranch) ProjectCache.illegalState(com.google.gerrit.server.project.ProjectCache.illegalState) NotifyResolver(com.google.gerrit.server.change.NotifyResolver) ObjectIds.abbreviateName(com.google.gerrit.git.ObjectIds.abbreviateName) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ChangeEdit(com.google.gerrit.server.edit.ChangeEdit) IdentifiedUser(com.google.gerrit.server.IdentifiedUser) Project(com.google.gerrit.entities.Project) DynamicSet(com.google.gerrit.extensions.registration.DynamicSet) NOT_ATTEMPTED(org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED) LabelVote(com.google.gerrit.server.util.LabelVote) TimeUtil(com.google.gerrit.server.util.time.TimeUtil) CommentValidator(com.google.gerrit.extensions.validators.CommentValidator) ONLY_USERS_WITH_TOGGLE_WIP_STATE_PERM_CAN_MODIFY_WIP(com.google.gerrit.server.git.receive.ReceiveConstants.ONLY_USERS_WITH_TOGGLE_WIP_STATE_PERM_CAN_MODIFY_WIP) RequestCancelledException(com.google.gerrit.server.cancellation.RequestCancelledException) RequestId(com.google.gerrit.server.logging.RequestId) REJECTED_MISSING_OBJECT(org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_MISSING_OBJECT) CommentSizeValidator(com.google.gerrit.server.git.validators.CommentSizeValidator) RefPermission(com.google.gerrit.server.permissions.RefPermission) PUSH_OPTION_SKIP_VALIDATION(com.google.gerrit.server.git.receive.ReceiveConstants.PUSH_OPTION_SKIP_VALIDATION) URLDecoder(java.net.URLDecoder) ValidationMessage(com.google.gerrit.server.git.validators.ValidationMessage) Inject(com.google.inject.Inject) MissingObjectException(org.eclipse.jgit.errors.MissingObjectException) RepoOnlyOp(com.google.gerrit.server.update.RepoOnlyOp) PatchSetInfoFactory(com.google.gerrit.server.patch.PatchSetInfoFactory) UpdateException(com.google.gerrit.server.update.UpdateException) Assisted(com.google.inject.assistedinject.Assisted) ImmutableListMultimap.toImmutableListMultimap(com.google.common.collect.ImmutableListMultimap.toImmutableListMultimap) LabelType(com.google.gerrit.entities.LabelType) AuthException(com.google.gerrit.extensions.restapi.AuthException) AutoMerger(com.google.gerrit.server.patch.AutoMerger) PerformanceLogContext(com.google.gerrit.server.logging.PerformanceLogContext) PluginConfig(com.google.gerrit.server.config.PluginConfig) NoteMap(org.eclipse.jgit.notes.NoteMap) ImmutableSetMultimap(com.google.common.collect.ImmutableSetMultimap) CommentCountValidator(com.google.gerrit.server.git.validators.CommentCountValidator) BiMap(com.google.common.collect.BiMap) ImmutableSet(com.google.common.collect.ImmutableSet) SortedSetMultimap(com.google.common.collect.SortedSetMultimap) Collection(java.util.Collection) OK(org.eclipse.jgit.transport.ReceiveCommand.Result.OK) PermissionDeniedException(com.google.gerrit.server.permissions.PermissionDeniedException) Collectors(java.util.stream.Collectors) ApprovalsUtil(com.google.gerrit.server.approval.ApprovalsUtil) Objects(java.util.Objects) CmdLineException(org.kohsuke.args4j.CmdLineException) SetPrivateOp(com.google.gerrit.server.change.SetPrivateOp) ChangeData(com.google.gerrit.server.query.change.ChangeData) Nullable(com.google.gerrit.common.Nullable) AllProjectsName(com.google.gerrit.server.config.AllProjectsName) Ref(org.eclipse.jgit.lib.Ref) InternalChangeQuery(com.google.gerrit.server.query.change.InternalChangeQuery) Providers(com.google.inject.util.Providers) Queue(java.util.Queue) RefNames.isConfigRef(com.google.gerrit.entities.RefNames.isConfigRef) RequestStateContext(com.google.gerrit.server.cancellation.RequestStateContext) CommentValidationContext(com.google.gerrit.extensions.validators.CommentValidationContext) IncorrectObjectTypeException(org.eclipse.jgit.errors.IncorrectObjectTypeException) GeneralPreferencesInfo(com.google.gerrit.extensions.client.GeneralPreferencesInfo) RepoContext(com.google.gerrit.server.update.RepoContext) MergedByPushOp(com.google.gerrit.server.git.MergedByPushOp) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) REFS_CHANGES(com.google.gerrit.entities.RefNames.REFS_CHANGES) Field(com.google.gerrit.metrics.Field) UsedAt(com.google.gerrit.common.UsedAt) Change(com.google.gerrit.entities.Change) ChangeUtil(com.google.gerrit.server.ChangeUtil) ChangeContext(com.google.gerrit.server.update.ChangeContext) SetTopicOp(com.google.gerrit.server.change.SetTopicOp) SetHashtagsOp(com.google.gerrit.server.change.SetHashtagsOp) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ProjectState(com.google.gerrit.server.project.ProjectState) Collectors.toList(java.util.stream.Collectors.toList) HashBiMap(com.google.common.collect.HashBiMap) Provider(com.google.inject.Provider) ReceivePack(org.eclipse.jgit.transport.ReceivePack) SuperprojectUpdateOnSubmission(com.google.gerrit.server.update.SuperprojectUpdateOnSubmission) Arrays(java.util.Arrays) ProjectConfig(com.google.gerrit.server.project.ProjectConfig) ListMultimap(com.google.common.collect.ListMultimap) GroupCollector(com.google.gerrit.server.git.GroupCollector) ProjectCache(com.google.gerrit.server.project.ProjectCache) PermissionBackend(com.google.gerrit.server.permissions.PermissionBackend) InvalidDeadlineException(com.google.gerrit.server.InvalidDeadlineException) NoSuchChangeException(com.google.gerrit.server.project.NoSuchChangeException) RetryHelper(com.google.gerrit.server.update.RetryHelper) GerritServerConfig(com.google.gerrit.server.config.GerritServerConfig) TraceContext(com.google.gerrit.server.logging.TraceContext) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) BranchNameKey(com.google.gerrit.entities.BranchNameKey) NotifyHandling(com.google.gerrit.extensions.api.changes.NotifyHandling) ProjectPermission(com.google.gerrit.server.permissions.ProjectPermission) NEW_PATCHSET_PATTERN(com.google.gerrit.server.git.validators.CommitValidators.NEW_PATCHSET_PATTERN) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) LazyArgs.lazy(com.google.common.flogger.LazyArgs.lazy) Singleton(com.google.inject.Singleton) Iterables(com.google.common.collect.Iterables) CommentType(com.google.gerrit.extensions.validators.CommentForValidation.CommentType) PermissionBackendException(com.google.gerrit.server.permissions.PermissionBackendException) RefOperationValidationException(com.google.gerrit.server.git.validators.RefOperationValidationException) ConfigInvalidException(org.eclipse.jgit.errors.ConfigInvalidException) CommentForValidation(com.google.gerrit.extensions.validators.CommentForValidation) TraceTimer(com.google.gerrit.server.logging.TraceContext.TraceTimer) CancellationMetrics(com.google.gerrit.server.CancellationMetrics) SubmissionListener(com.google.gerrit.server.update.SubmissionListener) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) ChangeEditUtil(com.google.gerrit.server.edit.ChangeEditUtil) PatchSet(com.google.gerrit.entities.PatchSet) RestApiException(com.google.gerrit.extensions.restapi.RestApiException) Extension(com.google.gerrit.extensions.registration.Extension) LinkedHashSet(java.util.LinkedHashSet) UNKNOWN(com.google.gerrit.server.git.MultiProgressMonitor.UNKNOWN) CommitValidationMessage(com.google.gerrit.server.git.validators.CommitValidationMessage) Sequences(com.google.gerrit.server.notedb.Sequences) CreateRefControl(com.google.gerrit.server.project.CreateRefControl) RevFilter(org.eclipse.jgit.revwalk.filter.RevFilter) AttentionSetUnchangedOp(com.google.gerrit.server.change.AttentionSetUnchangedOp) StorageException(com.google.gerrit.exceptions.StorageException) StringWriter(java.io.StringWriter) CommentValidationFailure(com.google.gerrit.extensions.validators.CommentValidationFailure) CmdLineParser(com.google.gerrit.util.cli.CmdLineParser) NoSuchProjectException(com.google.gerrit.server.project.NoSuchProjectException) ChangeNotes(com.google.gerrit.server.notedb.ChangeNotes) R_HEADS(org.eclipse.jgit.lib.Constants.R_HEADS) RefOperationValidators(com.google.gerrit.server.git.validators.RefOperationValidators) DynamicMap(com.google.gerrit.extensions.registration.DynamicMap) ObjectReader(org.eclipse.jgit.lib.ObjectReader) Repository(org.eclipse.jgit.lib.Repository) REJECTED_OTHER_REASON(org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON) MailUtil.getRecipientsFromFooters(com.google.gerrit.server.mail.MailUtil.getRecipientsFromFooters) ObjectInserter(org.eclipse.jgit.lib.ObjectInserter) HumanComment(com.google.gerrit.entities.HumanComment) SubmissionId(com.google.gerrit.entities.SubmissionId) RefNames.isRefsUsersSelf(com.google.gerrit.entities.RefNames.isRefsUsersSelf) LabelTypes(com.google.gerrit.entities.LabelTypes) ChangeIndexer(com.google.gerrit.server.index.change.ChangeIndexer) BatchUpdate(com.google.gerrit.server.update.BatchUpdate) SubmitInput(com.google.gerrit.extensions.api.changes.SubmitInput) RefNames(com.google.gerrit.entities.RefNames) COMMAND_REJECTION_MESSAGE_FOOTER(com.google.gerrit.server.git.receive.ReceiveConstants.COMMAND_REJECTION_MESSAGE_FOOTER) Splitter(com.google.common.base.Splitter) ReceivePackInitializer(com.google.gerrit.server.git.ReceivePackInitializer) LinkedListMultimap(com.google.common.collect.LinkedListMultimap) GlobalPermission(com.google.gerrit.server.permissions.GlobalPermission) Task(com.google.gerrit.server.git.MultiProgressMonitor.Task) ImmutableMap(com.google.common.collect.ImmutableMap) Account(com.google.gerrit.entities.Account) RequestInfo(com.google.gerrit.server.RequestInfo) ProjectConfigEntry(com.google.gerrit.server.config.ProjectConfigEntry) Option(org.kohsuke.args4j.Option) Streams(com.google.common.collect.Streams) Sets(com.google.common.collect.Sets) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) Optional(java.util.Optional) MoreObjects.firstNonNull(com.google.common.base.MoreObjects.firstNonNull) BatchUpdateOp(com.google.gerrit.server.update.BatchUpdateOp) UnsupportedEncodingException(java.io.UnsupportedEncodingException) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) BanCommit(com.google.gerrit.server.git.BanCommit) CommentSource(com.google.gerrit.extensions.validators.CommentForValidation.CommentSource) PublishCommentUtil(com.google.gerrit.server.PublishCommentUtil) MultiProgressMonitor(com.google.gerrit.server.git.MultiProgressMonitor) MergeOpRepoManager(com.google.gerrit.server.submit.MergeOpRepoManager) ChangePermission(com.google.gerrit.server.permissions.ChangePermission) CreateGroupPermissionSyncer(com.google.gerrit.server.CreateGroupPermissionSyncer) HashMap(java.util.HashMap) ReplyAttentionSetUpdates(com.google.gerrit.server.restapi.change.ReplyAttentionSetUpdates) MergeOp(com.google.gerrit.server.submit.MergeOp) PostUpdateContext(com.google.gerrit.server.update.PostUpdateContext) Objects.requireNonNull(java.util.Objects.requireNonNull) ChangeInserter(com.google.gerrit.server.change.ChangeInserter) AccountResolver(com.google.gerrit.server.account.AccountResolver) Iterator(java.util.Iterator) Maps(com.google.common.collect.Maps) ObjectId(org.eclipse.jgit.lib.ObjectId) RecipientType(com.google.gerrit.extensions.api.changes.RecipientType) ResourceConflictException(com.google.gerrit.extensions.restapi.ResourceConflictException) PatchSetUtil(com.google.gerrit.server.PatchSetUtil) Collections(java.util.Collections) ObjectId(org.eclipse.jgit.lib.ObjectId) AuthException(com.google.gerrit.extensions.restapi.AuthException) IncorrectObjectTypeException(org.eclipse.jgit.errors.IncorrectObjectTypeException) IOException(java.io.IOException) RevWalk(org.eclipse.jgit.revwalk.RevWalk) MissingObjectException(org.eclipse.jgit.errors.MissingObjectException) Ref(org.eclipse.jgit.lib.Ref) RefNames.isConfigRef(com.google.gerrit.entities.RefNames.isConfigRef) StringWriter(java.io.StringWriter) TraceTimer(com.google.gerrit.server.logging.TraceContext.TraceTimer) StorageException(com.google.gerrit.exceptions.StorageException) CmdLineException(org.kohsuke.args4j.CmdLineException) RevCommit(org.eclipse.jgit.revwalk.RevCommit)

Example 12 with ImmutableListMultimap

use of com.google.common.collect.ImmutableListMultimap in project gridss by PapenfussLab.

the class SingleReadEvidenceTest method assembly_scoring_should_be_symmetrical.

@Test
@Category(Hg19Tests.class)
public void assembly_scoring_should_be_symmetrical() throws FileNotFoundException {
    File sam = new File("src/test/resources/assembly_score_mismatch.sam");
    File ref = Hg19Tests.findHg19Reference();
    ProcessingContext pc = new ProcessingContext(getFSContext(), ref, new SynchronousReferenceLookupAdapter(new IndexedFastaSequenceFile(ref)), null, getConfig());
    SAMEvidenceSource ses = SES(pc);
    SamReader sr = SamReaderFactory.make().open(sam);
    List<SAMRecord> in = Lists.newArrayList(sr.iterator());
    List<SingleReadEvidence> sreList = in.stream().flatMap(record -> SingleReadEvidence.createEvidence(ses, 0, record).stream()).collect(Collectors.toList());
    ImmutableListMultimap<String, SingleReadEvidence> mm = Multimaps.index(sreList, sre -> sre.getSAMRecord().getReadName());
    for (Entry<String, Collection<SingleReadEvidence>> entry : mm.asMap().entrySet()) {
        float bequal = entry.getValue().iterator().next().getBreakendQual();
        float bpqual = ((SplitReadEvidence) entry.getValue().iterator().next()).getBreakpointQual();
        for (SingleReadEvidence sre : entry.getValue()) {
            assertEquals(bequal, sre.getBreakendQual(), 0);
            assertEquals(bpqual, ((SplitReadEvidence) sre).getBreakpointQual(), 0);
        }
    }
}
Also used : SequenceUtil(htsjdk.samtools.util.SequenceUtil) InMemoryReferenceSequenceFile(au.edu.wehi.idsv.picard.InMemoryReferenceSequenceFile) Collection(java.util.Collection) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) SynchronousReferenceLookupAdapter(au.edu.wehi.idsv.picard.SynchronousReferenceLookupAdapter) Category(org.junit.experimental.categories.Category) SamReader(htsjdk.samtools.SamReader) Collectors(java.util.stream.Collectors) SAMFileHeader(htsjdk.samtools.SAMFileHeader) File(java.io.File) FileNotFoundException(java.io.FileNotFoundException) SAMRecord(htsjdk.samtools.SAMRecord) Multimaps(com.google.common.collect.Multimaps) List(java.util.List) Lists(com.google.common.collect.Lists) IndexedFastaSequenceFile(htsjdk.samtools.reference.IndexedFastaSequenceFile) Assert.assertFalse(org.junit.Assert.assertFalse) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Entry(java.util.Map.Entry) SamReaderFactory(htsjdk.samtools.SamReaderFactory) Assert.assertEquals(org.junit.Assert.assertEquals) IndexedFastaSequenceFile(htsjdk.samtools.reference.IndexedFastaSequenceFile) SamReader(htsjdk.samtools.SamReader) SAMRecord(htsjdk.samtools.SAMRecord) Collection(java.util.Collection) InMemoryReferenceSequenceFile(au.edu.wehi.idsv.picard.InMemoryReferenceSequenceFile) File(java.io.File) IndexedFastaSequenceFile(htsjdk.samtools.reference.IndexedFastaSequenceFile) SynchronousReferenceLookupAdapter(au.edu.wehi.idsv.picard.SynchronousReferenceLookupAdapter) Category(org.junit.experimental.categories.Category) Test(org.junit.Test)

Example 13 with ImmutableListMultimap

use of com.google.common.collect.ImmutableListMultimap in project presto by prestodb.

the class OrcFileWriterFactory method createDwrfEncryption.

private Optional<DwrfWriterEncryption> createDwrfEncryption(Optional<EncryptionInformation> encryptionInformation, List<String> fileColumnNames, List<Type> types) {
    if (!encryptionInformation.isPresent()) {
        return Optional.empty();
    }
    if (!encryptionInformation.get().getDwrfEncryptionMetadata().isPresent()) {
        return Optional.empty();
    }
    DwrfEncryptionMetadata dwrfEncryptionMetadata = encryptionInformation.get().getDwrfEncryptionMetadata().get();
    List<OrcType> orcTypes = OrcType.createOrcRowType(0, fileColumnNames, types);
    Map<String, Integer> columnNamesMap = IntStream.range(0, fileColumnNames.size()).boxed().collect(toImmutableMap(fileColumnNames::get, index -> index));
    Map<Integer, Slice> keyMap = dwrfEncryptionMetadata.toKeyMap(orcTypes, columnNamesMap);
    ImmutableListMultimap.Builder<Slice, Integer> encryptionGroupsBuilder = ImmutableListMultimap.builder();
    keyMap.entrySet().stream().forEach(entry -> encryptionGroupsBuilder.put(entry.getValue(), entry.getKey()));
    ImmutableListMultimap<Slice, Integer> encryptionGroups = encryptionGroupsBuilder.build();
    List<WriterEncryptionGroup> writerEncryptionGroups = encryptionGroups.keySet().stream().map(key -> new WriterEncryptionGroup(encryptionGroups.get(key), key)).collect(toImmutableList());
    return Optional.of(new DwrfWriterEncryption(toKeyProvider(dwrfEncryptionMetadata.getEncryptionProvider()), writerEncryptionGroups));
}
Also used : HdfsOrcDataSource(com.facebook.presto.hive.orc.HdfsOrcDataSource) DateTimeZone(org.joda.time.DateTimeZone) FileSystem(org.apache.hadoop.fs.FileSystem) HiveSessionProperties.getDwrfWriterStripeCacheeMaxSize(com.facebook.presto.hive.HiveSessionProperties.getDwrfWriterStripeCacheeMaxSize) HIVE_WRITE_VALIDATION_FAILED(com.facebook.presto.hive.HiveErrorCode.HIVE_WRITE_VALIDATION_FAILED) DataSink(com.facebook.presto.common.io.DataSink) HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory) OrcConf(org.apache.orc.OrcConf) HiveSessionProperties.getOrcOptimizedWriterMinStripeSize(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMinStripeSize) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) HiveSessionProperties.getOrcOptimizedWriterValidateMode(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterValidateMode) OrcDataSource(com.facebook.presto.orc.OrcDataSource) Splitter(com.google.common.base.Splitter) ENGLISH(java.util.Locale.ENGLISH) WriterEncryptionGroup(com.facebook.presto.orc.WriterEncryptionGroup) CRYPTO_SERVICE(com.facebook.presto.orc.metadata.KeyProvider.CRYPTO_SERVICE) HIVE_UNSUPPORTED_FORMAT(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT) KeyProvider(com.facebook.presto.orc.metadata.KeyProvider) META_TABLE_COLUMN_TYPES(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) HiveSessionProperties.isDwrfWriterStripeCacheEnabled(com.facebook.presto.hive.HiveSessionProperties.isDwrfWriterStripeCacheEnabled) HiveSessionProperties.isExecutionBasedMemoryAccountingEnabled(com.facebook.presto.hive.HiveSessionProperties.isExecutionBasedMemoryAccountingEnabled) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ORC(com.facebook.presto.orc.OrcEncoding.ORC) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) DWRF(com.facebook.presto.orc.OrcEncoding.DWRF) CompressionKind(com.facebook.presto.orc.metadata.CompressionKind) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Optional(java.util.Optional) OrcOutputFormat(org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat) HiveType.toHiveTypes(com.facebook.presto.hive.HiveType.toHiveTypes) IntStream(java.util.stream.IntStream) HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize) Slice(io.airlift.slice.Slice) HiveSessionProperties.getOrcMaxMergeDistance(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxMergeDistance) Flatten(org.weakref.jmx.Flatten) Callable(java.util.concurrent.Callable) META_TABLE_COLUMNS(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS) DwrfWriterEncryption(com.facebook.presto.orc.DwrfWriterEncryption) DataSinkFactory(com.facebook.presto.hive.datasink.DataSinkFactory) PrestoException(com.facebook.presto.spi.PrestoException) Supplier(java.util.function.Supplier) UNKNOWN(com.facebook.presto.orc.metadata.KeyProvider.UNKNOWN) Inject(javax.inject.Inject) MetastoreUtil(com.facebook.presto.hive.metastore.MetastoreUtil) Managed(org.weakref.jmx.Managed) TypeManager(com.facebook.presto.common.type.TypeManager) HiveSessionProperties.getOrcMaxBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize) Objects.requireNonNull(java.util.Objects.requireNonNull) OrcType(com.facebook.presto.orc.metadata.OrcType) OrcWriterStats(com.facebook.presto.orc.OrcWriterStats) Type(com.facebook.presto.common.type.Type) DwrfEncryptionProvider(com.facebook.presto.orc.DwrfEncryptionProvider) OrcDataSourceId(com.facebook.presto.orc.OrcDataSourceId) OrcEncoding(com.facebook.presto.orc.OrcEncoding) Properties(java.util.Properties) DefaultOrcWriterFlushPolicy(com.facebook.presto.orc.DefaultOrcWriterFlushPolicy) HiveSessionProperties.getOrcStreamBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize) IOException(java.io.IOException) HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows) JobConf(org.apache.hadoop.mapred.JobConf) Collectors.toList(java.util.stream.Collectors.toList) HiveSessionProperties.getOrcStringStatisticsLimit(com.facebook.presto.hive.HiveSessionProperties.getOrcStringStatisticsLimit) HIVE_WRITER_OPEN_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR) WriterEncryptionGroup(com.facebook.presto.orc.WriterEncryptionGroup) OrcType(com.facebook.presto.orc.metadata.OrcType) Slice(io.airlift.slice.Slice) DwrfWriterEncryption(com.facebook.presto.orc.DwrfWriterEncryption) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap)

Example 14 with ImmutableListMultimap

use of com.google.common.collect.ImmutableListMultimap in project presto by prestodb.

the class TestPushProjectionThroughUnion method test.

@Test
public void test() {
    FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager());
    tester().assertThat(new PushProjectionThroughUnion()).on(p -> {
        VariableReferenceExpression a = p.variable("a");
        VariableReferenceExpression b = p.variable("b");
        VariableReferenceExpression c = p.variable("c");
        return p.project(assignment(p.variable("c_times_3"), call("c * 3", functionResolution.arithmeticFunction(MULTIPLY, BIGINT, BIGINT), BIGINT, c, constant(3L, BIGINT))), p.union(ImmutableListMultimap.<VariableReferenceExpression, VariableReferenceExpression>builder().put(c, a).put(c, b).build(), ImmutableList.of(p.values(a), p.values(b))));
    }).matches(union(project(ImmutableMap.of("a_times_3", expression("a * 3")), values(ImmutableList.of("a"))), project(ImmutableMap.of("b_times_3", expression("b * 3")), values(ImmutableList.of("b")))).withNumberOfOutputColumns(1).withAlias("a_times_3").withAlias("b_times_3"));
}
Also used : BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) PlanMatchPattern.project(com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project) ImmutableMap(com.google.common.collect.ImmutableMap) MULTIPLY(com.facebook.presto.common.function.OperatorType.MULTIPLY) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) Test(org.testng.annotations.Test) Expressions.call(com.facebook.presto.sql.relational.Expressions.call) Expressions.constant(com.facebook.presto.sql.relational.Expressions.constant) PlanMatchPattern.union(com.facebook.presto.sql.planner.assertions.PlanMatchPattern.union) BaseRuleTest(com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest) ImmutableList(com.google.common.collect.ImmutableList) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) PlanBuilder.assignment(com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) FunctionResolution(com.facebook.presto.sql.relational.FunctionResolution) PlanMatchPattern.expression(com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression) PlanMatchPattern.values(com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) FunctionResolution(com.facebook.presto.sql.relational.FunctionResolution) Test(org.testng.annotations.Test) BaseRuleTest(com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest)

Aggregations

ImmutableListMultimap (com.google.common.collect.ImmutableListMultimap)14 Map (java.util.Map)9 ImmutableList (com.google.common.collect.ImmutableList)7 ImmutableSet (com.google.common.collect.ImmutableSet)6 Collection (java.util.Collection)5 List (java.util.List)5 Optional (java.util.Optional)5 ImmutableMap (com.google.common.collect.ImmutableMap)4 Nullable (javax.annotation.Nullable)3 BuildTarget (com.facebook.buck.model.BuildTarget)2 BuildRule (com.facebook.buck.rules.BuildRule)2 BytecodeBlock (com.facebook.presto.bytecode.BytecodeBlock)2 BytecodeNode (com.facebook.presto.bytecode.BytecodeNode)2 Scope (com.facebook.presto.bytecode.Scope)2 IfStatement (com.facebook.presto.bytecode.control.IfStatement)2 LabelNode (com.facebook.presto.bytecode.instruction.LabelNode)2 Splitter (com.google.common.base.Splitter)2 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 Iterables (com.google.common.collect.Iterables)2 Lists (com.google.common.collect.Lists)2