Examples with ChunkStorage me.perrycate.groupmeutils.util.ChunkStorage used on opensource projects

Example 1 with ChunkStorage

use of me.perrycate.groupmeutils.util.ChunkStorage in project groupme-utils by TheGuyWithTheFace.

the class MessageIterator method getAllMessages.

/*
     * Get each message starting from bottom.
     *
     * @return a ChunkStorage object containing all messages belonging to the
     * given group, in chunks of GroupMessages.MAX_MESSAGES.
     *
     */
private ChunkStorage getAllMessages(GroupMe api, String groupID) {
    // We use hold messages in groups of 100 in ChunkStorage to avoid
    // attempting to store the entire group in memory while still not
    // duplicating any network calls.
    Group group = api.getGroup(groupID);
    int totalMessages = group.getMessageCount();
    String lastMessageId = group.getLastMessageId();
    ChunkStorage storage = new ChunkStorage();
    // Dump all messages to chunk storage
    GroupMessages messages;
    for (int i = 0; i < totalMessages; i += GroupMessages.MAX_MESSAGES) {
        messages = api.getMessagesBefore(groupID, lastMessageId);
        byte[] data = ByteConverter.objectToBytes(messages);
        storage.addFirst(data);
        lastMessageId = messages.getMessage(messages.getMessages().length - 1).getId();
    }
    return storage;
}

Example 2 with ChunkStorage

use of me.perrycate.groupmeutils.util.ChunkStorage in project groupme-utils by TheGuyWithTheFace.

the class Dumper method append.

// */
/**
     * Scans the groupme for any new messages ocurring after the last message in
     * inputFile, and appends new messages to the bottom.
     *
     * This method assumes that inputFile is the result of a previous
     * dumpFromTop or appendFromTop call, and thus maintains the same output
     * format.
     *
     */
public int append(File sourceFile) {
    // TODO we could in fact share more code with dump if we only
    // made note of the last message in the group, but then got messages in
    // the same order as dump() (bottom-up) and just stop when we reach the
    // message Id we made note of. The downside is that if people add to
    // the chat while append() is running, these messages won't be added.
    // On the other hand though, this would eliminate the concern of the
    // message count being innacurate for the same reason, so maybe that's
    // a good thing?
    FileReader r;
    try {
        r = new FileReader(sourceFile);
    } catch (FileNotFoundException e) {
        System.err.println("FATAL: Could not find file " + sourceFile + "!");
        return -1;
    }
    // Find the last message in the file. Ideally there is a faster way to
    // do this other than reading through it line by line from the top, but
    // I have not found it yet.
    // TODO there certainly must be, especially now that we don't have to
    // copy things into new file, we just need to get the last line in file.
    String lastLine = "";
    String tmp = "";
    try (BufferedReader reader = new BufferedReader(r)) {
        tmp = reader.readLine();
        while (tmp != null) {
            lastLine = tmp;
            tmp = reader.readLine();
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    // TODO if no id found, should tell that to the user.
    // get ID of last message in file
    String firstMessageId = readIdFromLine(lastLine);
    // Append new messages to group, storing messages in file chunks to
    // avoid keeping them all in memory.
    ChunkStorage storage = new ChunkStorage();
    GroupMessages messages = groupme.getMessagesAfter(groupId, firstMessageId);
    int nrOfMessages = messages.getMessages().length;
    while (messages.getMessages().length > 0) {
        // Have to reverse message order since getMessagesAfter's order is
        // opposite to getMessagesBefore. Inconvenient, but I'm trying to
        // keep it consistent with GroupMe's API for now, for better or
        // worse.
        List<Message> m = Arrays.asList(messages.getMessages());
        // WARNING/TODO/NOTE: fixing the "GroupMessages not being
        // defensively
        // copied bug" will require change here, since the current code is
        // working around messages being reversed.
        Collections.reverse(m);
        writeChunk(m.toArray(new Message[0]), storage);
        // Remember, we reversed the array
        firstMessageId = messages.getMessage(0).getId();
        // Different from dump(): this gets Messages AFTER messageId
        messages = groupme.getMessagesAfter(groupId, firstMessageId);
    }
    // Concatenate each chunk into a single log
    try (OutputStream o = Files.newOutputStream(sourceFile.toPath(), StandardOpenOption.APPEND, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
        BufferedOutputStream output = new BufferedOutputStream(o)) {
        while (storage.size() != 0) {
            // Different from dump(): this removes from the first, not last
            output.write(storage.removeLast());
        }
        storage.close();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return nrOfMessages;
}