Search in sources :

Example 6 with Triplet

use of org.javatuples.Triplet in project ldbc_snb_datagen_spark by ldbc.

the class PostGenerator method createPosts.

public Stream<Triplet<Post, Stream<Like>, Stream<Pair<Comment, Stream<Like>>>>> createPosts(RandomGeneratorFarm randomFarm, final Forum forum, final List<ForumMembership> memberships, long numPostsInForum, Iterator<Long> idIterator, long blockId) {
    Properties properties = new Properties();
    properties.setProperty("type", "post");
    return memberships.stream().flatMap(member -> {
        // generate number of posts by this member
        double numPostsPerMember = numPostsInForum / (double) memberships.size();
        if (numPostsPerMember < 1.0) {
            double prob = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_POST).nextDouble();
            if (prob < numPostsPerMember)
                numPostsPerMember = 1.0;
        } else {
            numPostsPerMember = Math.ceil(numPostsPerMember);
        }
        final int numPostsPerMemberInt = (int) numPostsPerMember;
        // 0 to 20
        int numComments = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_COMMENT).nextInt(DatagenParams.maxNumComments + 1);
        return Streams.stream(Iterators.forIterator(0, i -> i < numPostsPerMemberInt, i -> ++i, i -> {
            // create post core
            PostCore postCore = generatePostInfo(randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_POST), randomFarm.get(RandomGeneratorFarm.Aspect.TAG), randomFarm.get(RandomGeneratorFarm.Aspect.DATE), forum, member, numComments);
            if (postCore == null)
                return Iterators.ForIterator.CONTINUE();
            // create content, county, ip - sometimes randomise
            String content = this.generator.generateText(member.getPerson(), postCore.getTags(), properties);
            int country = member.getPerson().getCountry();
            IP ip = member.getPerson().getIpAddress();
            Random random = randomFarm.get(RandomGeneratorFarm.Aspect.DIFF_IP_FOR_TRAVELER);
            if (PersonBehavior.changeUsualCountry(random, postCore.getCreationDate())) {
                random = randomFarm.get(RandomGeneratorFarm.Aspect.COUNTRY);
                country = Dictionaries.places.getRandomCountryUniform(random);
                random = randomFarm.get(RandomGeneratorFarm.Aspect.IP);
                ip = Dictionaries.ips.getIP(random, country);
            }
            Post post = new Post();
            // create post with above information and from post info
            post.initialize(SN.formId(SN.composeId(idIterator.next(), postCore.getCreationDate()), blockId), postCore.getCreationDate(), postCore.getDeletionDate(), member.getPerson(), forum.getId(), content, new ArrayList<>(postCore.getTags()), country, ip, Dictionaries.browsers.getPostBrowserId(randomFarm.get(RandomGeneratorFarm.Aspect.DIFF_BROWSER), randomFarm.get(RandomGeneratorFarm.Aspect.BROWSER), member.getPerson().getBrowserId()), forum.getLanguage(), postCore.isExplicitlyDeleted());
            Stream<Like> likeStream = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ? likeGenerator.generateLikes(randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_LIKES), randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE), forum, post, Like.LikeType.POST) : Stream.empty();
            Stream<Pair<Comment, Stream<Like>>> commentStream = commentGenerator.createComments(randomFarm, forum, post, numComments, idIterator, blockId);
            return Iterators.ForIterator.RETURN(new Triplet<>(post, likeStream, commentStream));
        }));
    });
}
Also used : IP(ldbc.snb.datagen.entities.dynamic.person.IP) java.util(java.util) Comment(ldbc.snb.datagen.entities.dynamic.messages.Comment) ForumMembership(ldbc.snb.datagen.entities.dynamic.relations.ForumMembership) LikeGenerator(ldbc.snb.datagen.generator.generators.LikeGenerator) Dictionaries(ldbc.snb.datagen.generator.dictionary.Dictionaries) PersonBehavior(ldbc.snb.datagen.util.PersonBehavior) Pair(org.javatuples.Pair) CommentGenerator(ldbc.snb.datagen.generator.generators.CommentGenerator) Iterators(ldbc.snb.datagen.util.Iterators) RandomGeneratorFarm(ldbc.snb.datagen.util.RandomGeneratorFarm) Stream(java.util.stream.Stream) Triplet(org.javatuples.Triplet) DatagenParams(ldbc.snb.datagen.generator.DatagenParams) Like(ldbc.snb.datagen.entities.dynamic.relations.Like) Post(ldbc.snb.datagen.entities.dynamic.messages.Post) Forum(ldbc.snb.datagen.entities.dynamic.Forum) Streams(ldbc.snb.datagen.util.Streams) TextGenerator(ldbc.snb.datagen.generator.generators.textgenerators.TextGenerator) SN(ldbc.snb.datagen.generator.vocabulary.SN) Comment(ldbc.snb.datagen.entities.dynamic.messages.Comment) Post(ldbc.snb.datagen.entities.dynamic.messages.Post) Triplet(org.javatuples.Triplet) IP(ldbc.snb.datagen.entities.dynamic.person.IP) Stream(java.util.stream.Stream)

Example 7 with Triplet

use of org.javatuples.Triplet in project hopsworks by logicalclocks.

the class ElasticFeaturestoreBuilder method setFeatureDescriptionHighlights.

private void setFeatureDescriptionHighlights(SearchHit hitAux, ElasticFeaturestoreDTO result, DatasetAccessController.DatasetAccessCtrl accessCtrl) throws ElasticException, GenericException {
    ElasticFeaturestoreHit hit = ElasticFeaturestoreHit.instance(hitAux);
    Map<String, HighlightField> highlightFields = hitAux.getHighlightFields();
    String featureDescriptionField = FeaturestoreXAttrsConstants.getFeaturestoreElasticKey(FeaturestoreXAttrsConstants.FG_FEATURES, FeaturestoreXAttrsConstants.DESCRIPTION);
    // <highlighted text, name, description>
    Function<Triplet<String, String, String>, Boolean> matcher = (state) -> {
        // check if highlighted description equals feature description
        return removeHighlightTags(state.getValue0()).equals(state.getValue2());
    };
    BiConsumer<ElasticFeaturestoreItemDTO.Highlights, String> highlighter = ElasticFeaturestoreItemDTO.Highlights::setDescription;
    setFeatureHighlights(highlightFields.get(featureDescriptionField), hit, matcher, highlighter, result, accessCtrl);
}
Also used : HopsworksJAXBContext(io.hops.hopsworks.common.util.HopsworksJAXBContext) ProjectFacade(io.hops.hopsworks.common.dao.project.ProjectFacade) FeaturestoreXAttrsConstants(io.hops.hopsworks.common.featurestore.xattr.dto.FeaturestoreXAttrsConstants) Function(java.util.function.Function) Project(io.hops.hopsworks.persistence.entity.project.Project) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) Inject(javax.inject.Inject) TransactionAttributeType(javax.ejb.TransactionAttributeType) Triplet(org.javatuples.Triplet) Text(org.elasticsearch.common.text.Text) TransactionAttribute(javax.ejb.TransactionAttribute) Map(java.util.Map) ElasticException(io.hops.hopsworks.exceptions.ElasticException) SearchResponse(org.elasticsearch.action.search.SearchResponse) HighlightField(org.elasticsearch.search.fetch.subphase.highlight.HighlightField) BiConsumer(java.util.function.BiConsumer) EJB(javax.ejb.EJB) SearchHit(org.elasticsearch.search.SearchHit) Stateless(javax.ejb.Stateless) Set(java.util.Set) ElasticController(io.hops.hopsworks.common.elastic.ElasticController) RESTCodes(io.hops.hopsworks.restutils.RESTCodes) ServiceException(io.hops.hopsworks.exceptions.ServiceException) ElasticFeaturestoreHit(io.hops.hopsworks.common.elastic.ElasticFeaturestoreHit) GenericException(io.hops.hopsworks.exceptions.GenericException) FeaturestoreDocType(io.hops.hopsworks.common.elastic.FeaturestoreDocType) Users(io.hops.hopsworks.persistence.entity.user.Users) ElasticFeaturestoreHit(io.hops.hopsworks.common.elastic.ElasticFeaturestoreHit) Triplet(org.javatuples.Triplet) HighlightField(org.elasticsearch.search.fetch.subphase.highlight.HighlightField)

Aggregations

Triplet (org.javatuples.Triplet)7 ArrayList (java.util.ArrayList)5 Map (java.util.Map)3 Set (java.util.Set)3 Function (java.util.function.Function)3 ProjectFacade (io.hops.hopsworks.common.dao.project.ProjectFacade)2 ElasticController (io.hops.hopsworks.common.elastic.ElasticController)2 ElasticFeaturestoreHit (io.hops.hopsworks.common.elastic.ElasticFeaturestoreHit)2 FeaturestoreDocType (io.hops.hopsworks.common.elastic.FeaturestoreDocType)2 FeaturestoreXAttrsConstants (io.hops.hopsworks.common.featurestore.xattr.dto.FeaturestoreXAttrsConstants)2 HopsworksJAXBContext (io.hops.hopsworks.common.util.HopsworksJAXBContext)2 ElasticException (io.hops.hopsworks.exceptions.ElasticException)2 GenericException (io.hops.hopsworks.exceptions.GenericException)2 ServiceException (io.hops.hopsworks.exceptions.ServiceException)2 Project (io.hops.hopsworks.persistence.entity.project.Project)2 Users (io.hops.hopsworks.persistence.entity.user.Users)2 RESTCodes (io.hops.hopsworks.restutils.RESTCodes)2 IOException (java.io.IOException)2 HashSet (java.util.HashSet)2 BiConsumer (java.util.function.BiConsumer)2