Examples with Forum ldbc.snb.datagen.entities.dynamic.Forum used on opensource projects

Example 1 with Forum

use of ldbc.snb.datagen.entities.dynamic.Forum in project ldbc_snb_datagen_spark by ldbc.

the class CommentGenerator method createComments.

public Stream<Pair<Comment, Stream<Like>>> createComments(RandomGeneratorFarm randomFarm, final Forum forum, final Post post, long numComments, Iterator<Long> idIterator, long blockId) {
    List<Message> parentCandidates = new ArrayList<>();
    parentCandidates.add(post);
    Properties prop = new Properties();
    prop.setProperty("type", "comment");
    // each iteration adds a new leaf node, for the first iteration this is a child of root Post
    return Streams.stream(Iterators.forIterator(0, i -> i < numComments, i -> ++i, i -> {
        // pick from parent candidates
        int parentIndex = randomFarm.get(RandomGeneratorFarm.Aspect.REPLY_TO).nextInt(parentCandidates.size());
        Message parentMessage = parentCandidates.get(parentIndex);
        // memberships that overlap with the existence of the parent message
        List<ForumMembership> validMemberships = new ArrayList<>();
        for (ForumMembership membership : forum.getMemberships()) {
            if ((membership.getCreationDate() < parentMessage.getCreationDate() && membership.getDeletionDate() > parentMessage.getCreationDate()) || membership.getCreationDate() < parentMessage.getDeletionDate() && membership.getDeletionDate() > parentMessage.getDeletionDate()) {
                validMemberships.add(membership);
            }
        }
        if (validMemberships.size() == 0) {
            // skip if no valid membership
            return Iterators.ForIterator.BREAK();
        }
        // get random membership from valid memberships - picking who created the comment
        int membershipIndex = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX).nextInt(validMemberships.size());
        ForumMembership membership = validMemberships.get(membershipIndex);
        TreeSet<Integer> tags = new TreeSet<>();
        String content;
        boolean isShort = false;
        if (randomFarm.get(RandomGeneratorFarm.Aspect.REDUCED_TEXT).nextDouble() > 0.6666) {
            List<Integer> currentTags = new ArrayList<>();
            for (Integer tag : parentMessage.getTags()) {
                if (randomFarm.get(RandomGeneratorFarm.Aspect.TAG).nextDouble() > 0.5) {
                    tags.add(tag);
                }
                currentTags.add(tag);
            }
            for (int j = 0; j < (int) Math.ceil(parentMessage.getTags().size() / 2.0); ++j) {
                int randomTag = currentTags.get(randomFarm.get(RandomGeneratorFarm.Aspect.TAG).nextInt(currentTags.size()));
                tags.add(Dictionaries.tagMatrix.getRandomRelated(randomFarm.get(RandomGeneratorFarm.Aspect.TOPIC), randomTag));
            }
            content = this.generator.generateText(membership.getPerson(), tags, prop);
        } else {
            isShort = true;
            int index = randomFarm.get(RandomGeneratorFarm.Aspect.TEXT_SIZE).nextInt(shortComments.length);
            content = shortComments[index];
        }
        // creation date
        long minCreationDate = Math.max(parentMessage.getCreationDate(), membership.getCreationDate()) + DatagenParams.delta;
        long maxCreationDate = Collections.min(Arrays.asList(membership.getDeletionDate(), parentMessage.getDeletionDate(), Dictionaries.dates.getSimulationEnd()));
        if (maxCreationDate <= minCreationDate) {
            return Iterators.ForIterator.CONTINUE();
        }
        // powerlaw distribtion
        long creationDate = Dictionaries.dates.powerLawCommDateDay(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), minCreationDate);
        if (creationDate > maxCreationDate) {
            return Iterators.ForIterator.CONTINUE();
        }
        long deletionDate;
        boolean isExplicitlyDeleted;
        // if person is a deleter and selected for delete
        if (membership.getPerson().isMessageDeleter() && randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_COMM).nextDouble() < DatagenParams.probCommentDeleted) {
            isExplicitlyDeleted = true;
            long minDeletionDate = creationDate + DatagenParams.delta;
            long maxDeletionDate = Collections.min(Arrays.asList(parentMessage.getDeletionDate(), membership.getDeletionDate(), Dictionaries.dates.getSimulationEnd()));
            if (maxDeletionDate <= minDeletionDate) {
                return Iterators.ForIterator.CONTINUE();
            }
            deletionDate = Dictionaries.dates.powerLawDeleteDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), minDeletionDate, maxDeletionDate);
        } else {
            isExplicitlyDeleted = false;
            deletionDate = Collections.min(Arrays.asList(parentMessage.getDeletionDate(), membership.getDeletionDate()));
        }
        int country = membership.getPerson().getCountry();
        IP ip = membership.getPerson().getIpAddress();
        Random random = randomFarm.get(RandomGeneratorFarm.Aspect.DIFF_IP_FOR_TRAVELER);
        if (PersonBehavior.changeUsualCountry(random, creationDate)) {
            random = randomFarm.get(RandomGeneratorFarm.Aspect.COUNTRY);
            country = Dictionaries.places.getRandomCountryUniform(random);
            random = randomFarm.get(RandomGeneratorFarm.Aspect.IP);
            ip = Dictionaries.ips.getIP(random, country);
        }
        Comment comment = new Comment(SN.formId(SN.composeId(idIterator.next(), creationDate), blockId), creationDate, deletionDate, membership.getPerson(), forum.getId(), content, new ArrayList<>(tags), country, ip, Dictionaries.browsers.getPostBrowserId(randomFarm.get(RandomGeneratorFarm.Aspect.DIFF_BROWSER), randomFarm.get(RandomGeneratorFarm.Aspect.BROWSER), membership.getPerson().getBrowserId()), post.getMessageId(), parentMessage.getMessageId(), isExplicitlyDeleted);
        if (!isShort)
            parentCandidates.add(new Comment(comment));
        Stream<Like> likeStream = comment.getContent().length() > 10 && randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE).nextDouble() <= 0.1 ? likeGenerator.generateLikes(randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_LIKES), randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LIKE), forum, comment, Like.LikeType.COMMENT) : Stream.empty();
        return Iterators.ForIterator.RETURN(new Pair<>(comment, likeStream));
    }));
}

Example 2 with Forum

use of ldbc.snb.datagen.entities.dynamic.Forum in project ldbc_snb_datagen_spark by ldbc.

the class ForumGenerator method createWall.

/**
 * Creates a personal wall for a given Person. All friends become members
 *
 * @param randomFarm randomFarm
 * @param forumId    forumID
 * @param person     Person
 * @return Forum
 */
Forum createWall(RandomGeneratorFarm randomFarm, long forumId, Person person, long blockId) {
    int language = person.getLanguages().get(randomFarm.get(RandomGeneratorFarm.Aspect.LANGUAGE).nextInt(person.getLanguages().size()));
    // Check moderator can be added
    if (person.getDeletionDate() - person.getCreationDate() + DatagenParams.delta < 0) {
        // what to return?
        return null;
    }
    Forum forum = new Forum(SN.formId(SN.composeId(forumId, person.getCreationDate() + DatagenParams.delta), blockId), person.getCreationDate() + DatagenParams.delta, person.getDeletionDate(), new PersonSummary(person), person.getDeletionDate(), StringUtils.clampString("Wall of " + person.getFirstName() + " " + person.getLastName(), 256), person.getCityId(), language, Forum.ForumType.WALL, false);
    // wall inherits tags from person
    List<Integer> forumTags = new ArrayList<>(person.getInterests());
    forum.setTags(forumTags);
    // adds all friends as members of wall
    List<Knows> knows = person.getKnows();
    // for each friend generate hasMember edge
    for (Knows know : knows) {
        long hasMemberCreationDate = know.getCreationDate() + DatagenParams.delta;
        long hasMemberDeletionDate = Math.min(forum.getDeletionDate(), know.getDeletionDate());
        if (hasMemberDeletionDate - hasMemberCreationDate < 0) {
            continue;
        }
        forum.addMember(new ForumMembership(forum.getId(), hasMemberCreationDate, hasMemberDeletionDate, know.to(), Forum.ForumType.WALL, false));
    }
    return forum;
}

Example 3 with Forum

use of ldbc.snb.datagen.entities.dynamic.Forum in project ldbc_snb_datagen_spark by ldbc.

the class PersonActivityGenerator method generateAlbums.

/**
 * Generates the albums for a Person.
 *
 * @param person person
 */
private GenWall<Pair<Photo, Stream<Like>>> generateAlbums(Person person, long blockId) {
    // work out number of albums to generate
    int numberOfMonths = (int) Dictionaries.dates.numberOfMonths(person.getCreationDate());
    int numberOfPhotoAlbums = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_PHOTO_ALBUM).nextInt(DatagenParams.maxNumPhotoAlbumsPerMonth + 1);
    int numberOfPhotoAlbumsForMonths = numberOfPhotoAlbums == 0 ? numberOfPhotoAlbums : numberOfMonths * numberOfPhotoAlbums;
    return new GenWall<>(Streams.stream(Iterators.forIterator(0, i -> i < numberOfPhotoAlbumsForMonths, i -> ++i, i -> {
        Forum album = forumGenerator.createAlbum(randomFarm, startForumId++, person, i, blockId);
        if (album == null) {
            return Iterators.ForIterator.CONTINUE();
        }
        // number of photos to generate
        int numPhotosInAlbum = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_PHOTO).nextInt(DatagenParams.maxNumPhotoPerAlbums + 1);
        // create photos
        Stream<Pair<Photo, Stream<Like>>> photos = photoGenerator.createPhotos(randomFarm, album, numPhotosInAlbum, messageIdIterator, blockId);
        return Iterators.ForIterator.RETURN(new Triplet<>(album, album.getMemberships().stream(), photos));
    })));
}

Example 4 with Forum

use of ldbc.snb.datagen.entities.dynamic.Forum in project ldbc_snb_datagen_spark by ldbc.

the class ForumGenerator method createAlbum.

/**
 * Creates an album for a given Person.
 *
 * @param randomFarm random farm
 * @param forumId    forumId
 * @param person     Person who the album belongs it
 * @param numAlbum   number of album e.g. Album 10
 * @return Album
 */
Forum createAlbum(RandomGeneratorFarm randomFarm, long forumId, Person person, int numAlbum, long blockId) {
    long minAlbumCreationDate = person.getCreationDate() + DatagenParams.delta;
    long maxAlbumCreationDate = Math.min(person.getDeletionDate(), Dictionaries.dates.getSimulationEnd());
    long albumCreationDate = Dictionaries.dates.randomDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), minAlbumCreationDate, maxAlbumCreationDate);
    long albumDeletionDate;
    boolean isExplicitlyDeleted;
    if (randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_FORUM).nextDouble() < DatagenParams.probForumDeleted) {
        isExplicitlyDeleted = true;
        long minAlbumDeletionDate = albumCreationDate + DatagenParams.delta;
        long maxAlbumDeletionDate = Math.min(person.getDeletionDate(), Dictionaries.dates.getSimulationEnd());
        if (maxAlbumDeletionDate - minAlbumCreationDate < 0) {
            return null;
        }
        albumDeletionDate = Dictionaries.dates.randomDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), minAlbumDeletionDate, maxAlbumDeletionDate);
    } else {
        isExplicitlyDeleted = false;
        albumDeletionDate = person.getDeletionDate();
    }
    int language = person.getLanguages().get(randomFarm.get(RandomGeneratorFarm.Aspect.LANGUAGE).nextInt(person.getLanguages().size()));
    Forum forum = new Forum(SN.formId(SN.composeId(forumId, albumCreationDate), blockId), albumCreationDate, albumDeletionDate, new PersonSummary(person), person.getDeletionDate(), StringUtils.clampString("Album " + numAlbum + " of " + person.getFirstName() + " " + person.getLastName(), 256), person.getCityId(), language, Forum.ForumType.ALBUM, isExplicitlyDeleted);
    Iterator<Integer> iter = person.getInterests().iterator();
    int idx = randomFarm.get(RandomGeneratorFarm.Aspect.FORUM_INTEREST).nextInt(person.getInterests().size());
    for (int i = 0; i < idx; i++) {
        iter.next();
    }
    int interestId = iter.next();
    List<Integer> interest = new ArrayList<>();
    interest.add(interestId);
    forum.setTags(interest);
    List<Integer> countries = Dictionaries.places.getCountries();
    int randomCountry = randomFarm.get(RandomGeneratorFarm.Aspect.COUNTRY).nextInt(countries.size());
    forum.setPlaceId(countries.get(randomCountry));
    List<Knows> friends = new ArrayList<>(person.getKnows());
    for (Knows knows : friends) {
        double prob = randomFarm.get(RandomGeneratorFarm.Aspect.ALBUM_MEMBERSHIP).nextDouble();
        if (prob < 0.7) {
            long hasMemberCreationDate = Math.max(knows.to().getCreationDate(), forum.getCreationDate()) + DatagenParams.delta;
            long hasMemberDeletionDate = Collections.min(Arrays.asList(knows.to().getDeletionDate(), forum.getDeletionDate()));
            if (hasMemberDeletionDate - hasMemberCreationDate > 0) {
                forum.addMember(new ForumMembership(forum.getId(), hasMemberCreationDate, hasMemberDeletionDate, knows.to(), Forum.ForumType.ALBUM, false));
            }
        }
    }
    return forum;
}

Example 5 with Forum

use of ldbc.snb.datagen.entities.dynamic.Forum in project ldbc_snb_datagen_spark by ldbc.

the class ForumGenerator method createGroup.

/**
 * Creates a Group with the Person as the moderator. 30% membership come from friends the rest are random.
 *
 * @param randomFarm random number generator
 * @param forumId    forumID
 * @param moderator  moderator
 * @param block      person block
 * @return Group
 */
Forum createGroup(RandomGeneratorFarm randomFarm, long forumId, Person moderator, List<Person> block, long blockId) {
    // creation date
    long groupMinCreationDate = moderator.getCreationDate() + DatagenParams.delta;
    long groupMaxCreationDate = Math.min(moderator.getDeletionDate(), Dictionaries.dates.getSimulationEnd());
    long groupCreationDate = Dictionaries.dates.randomDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), groupMinCreationDate, groupMaxCreationDate);
    // deletion date
    long groupDeletionDate;
    boolean isExplicitlyDeleted;
    if (randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_FORUM).nextDouble() < DatagenParams.probForumDeleted) {
        isExplicitlyDeleted = true;
        long groupMinDeletionDate = groupCreationDate + DatagenParams.delta;
        long groupMaxDeletionDate = Dictionaries.dates.getSimulationEnd();
        groupDeletionDate = Dictionaries.dates.randomDate(randomFarm.get(RandomGeneratorFarm.Aspect.DATE), groupMinDeletionDate, groupMaxDeletionDate);
    } else {
        isExplicitlyDeleted = false;
        groupDeletionDate = Dictionaries.dates.getNetworkCollapse();
    }
    // the hasModerator edge is deleted if either the Forum (group) or the Person (moderator) is deleted
    long moderatorDeletionDate = Math.min(groupDeletionDate, moderator.getDeletionDate());
    int language = moderator.getLanguages().get(randomFarm.get(RandomGeneratorFarm.Aspect.LANGUAGE).nextInt(moderator.getLanguages().size()));
    Iterator<Integer> iter = moderator.getInterests().iterator();
    int idx = randomFarm.get(RandomGeneratorFarm.Aspect.FORUM_INTEREST).nextInt(moderator.getInterests().size());
    for (int i = 0; i < idx; i++) {
        iter.next();
    }
    int interestId = iter.next();
    List<Integer> interest = new ArrayList<>();
    interest.add(interestId);
    // Create group
    Forum forum = new Forum(SN.formId(SN.composeId(forumId, groupCreationDate), blockId), groupCreationDate, groupDeletionDate, new PersonSummary(moderator), moderatorDeletionDate, StringUtils.clampString("Group for " + Dictionaries.tags.getName(interestId).replace("\"", "\\\"") + " in " + Dictionaries.places.getPlaceName(moderator.getCityId()), 256), moderator.getCityId(), language, Forum.ForumType.GROUP, isExplicitlyDeleted);
    // Set tags of this forum
    forum.setTags(interest);
    // Add members
    TreeSet<Long> groupMembers = new TreeSet<>();
    List<Knows> moderatorKnows = new ArrayList<>(moderator.getKnows());
    int numModeratorKnows = moderatorKnows.size();
    int groupSize = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_USERS_PER_FORUM).nextInt(DatagenParams.maxGroupSize);
    int numLoop = 0;
    while ((forum.getMemberships().size() < groupSize) && (numLoop < DatagenParams.blockSize)) {
        // controls the proportion of members that are friends
        double prob = randomFarm.get(RandomGeneratorFarm.Aspect.KNOWS_LEVEL).nextDouble();
        if (prob < 0.3 && numModeratorKnows > 0) {
            // pick random knows edge from friends
            int knowsIndex = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX).nextInt(numModeratorKnows);
            Knows knows = moderatorKnows.get(knowsIndex);
            if (!groupMembers.contains(knows.to().getAccountId())) {
                // if friend not already member of group
                long minCreationDate = Math.max(forum.getCreationDate(), knows.to().getCreationDate()) + DatagenParams.delta;
                long maxCreationDate = Collections.min(Arrays.asList(forum.getDeletionDate(), knows.to().getDeletionDate(), Dictionaries.dates.getSimulationEnd()));
                if (maxCreationDate - minCreationDate > 0) {
                    Random random = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX);
                    long hasMemberCreationDate = Dictionaries.dates.randomDate(random, minCreationDate, maxCreationDate);
                    long hasMemberDeletionDate;
                    boolean isHasMemberExplicitlyDeleted;
                    if (randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_MEMB).nextDouble() < DatagenParams.probMembDeleted) {
                        isHasMemberExplicitlyDeleted = true;
                        long minDeletionDate = hasMemberCreationDate + DatagenParams.delta;
                        long maxDeletionDate = Collections.min(Arrays.asList(knows.to().getDeletionDate(), forum.getDeletionDate(), Dictionaries.dates.getSimulationEnd()));
                        if (maxDeletionDate - minDeletionDate < 0) {
                            continue;
                        }
                        hasMemberDeletionDate = Dictionaries.dates.randomDate(random, minDeletionDate, maxDeletionDate);
                    } else {
                        isHasMemberExplicitlyDeleted = false;
                        hasMemberDeletionDate = Collections.min(Arrays.asList(knows.to().getDeletionDate(), forum.getDeletionDate()));
                    }
                    ForumMembership hasMember = new ForumMembership(forum.getId(), hasMemberCreationDate, hasMemberDeletionDate, knows.to(), Forum.ForumType.GROUP, isHasMemberExplicitlyDeleted);
                    forum.addMember(hasMember);
                    groupMembers.add(knows.to().getAccountId());
                }
            }
        } else {
            // pick from the person block
            int candidateIndex = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX).nextInt(block.size());
            Person member = block.get(candidateIndex);
            prob = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP).nextDouble();
            if ((prob < 0.1) && !groupMembers.contains(member.getAccountId())) {
                long minHasMemberCreationDate = Math.max(forum.getCreationDate(), member.getCreationDate()) + DatagenParams.delta;
                long maxHasMemberCreationDate = Collections.min(Arrays.asList(forum.getDeletionDate(), member.getDeletionDate(), Dictionaries.dates.getSimulationEnd()));
                if (maxHasMemberCreationDate - minHasMemberCreationDate > 0) {
                    Random random = randomFarm.get(RandomGeneratorFarm.Aspect.MEMBERSHIP_INDEX);
                    long hasMemberCreationDate = Dictionaries.dates.randomDate(random, minHasMemberCreationDate, maxHasMemberCreationDate);
                    long hasMemberDeletionDate;
                    boolean isHasMemberExplicitlyDeleted;
                    if (randomFarm.get(RandomGeneratorFarm.Aspect.DELETION_MEMB).nextDouble() < DatagenParams.probMembDeleted) {
                        isHasMemberExplicitlyDeleted = true;
                        long minHasMemberDeletionDate = hasMemberCreationDate + DatagenParams.delta;
                        long maxHasMemberDeletionDate = Collections.min(Arrays.asList(member.getDeletionDate(), forum.getDeletionDate(), Dictionaries.dates.getSimulationEnd()));
                        if (maxHasMemberCreationDate - minHasMemberDeletionDate < 0) {
                            continue;
                        }
                        hasMemberDeletionDate = Dictionaries.dates.randomDate(random, minHasMemberDeletionDate, maxHasMemberDeletionDate);
                    } else {
                        isHasMemberExplicitlyDeleted = false;
                        hasMemberDeletionDate = Collections.min(Arrays.asList(member.getDeletionDate(), forum.getDeletionDate()));
                    }
                    forum.addMember(new ForumMembership(forum.getId(), hasMemberCreationDate, hasMemberDeletionDate, new PersonSummary(member), Forum.ForumType.GROUP, isHasMemberExplicitlyDeleted));
                    groupMembers.add(member.getAccountId());
                }
            }
        }
        numLoop++;
    }
    return forum;
}