Examples with DataOutput_ suite.util.DataOutput_ used on opensource projects

Example 1 with DataOutput_

use of suite.util.DataOutput_ in project suite by stupidsing.

the class JournalledFileFactory method journalled.

private static // 
JournalledPageFile journalled(// 
PageFile df, // 
PageFile jpf, // 
PageFile ppf, int pageSize) {
    Serializer<Bytes> bytesSerializer = serialize.bytes(pageSize);
    Serializer<JournalEntry> journalEntrySerializer = new Serializer<>() {

        public JournalEntry read(DataInput_ dataInput) throws IOException {
            int pointer = dataInput.readInt();
            Bytes bytes = bytesSerializer.read(dataInput);
            return new JournalEntry(pointer, bytes);
        }

        public void write(DataOutput_ dataOutput, JournalEntry journalEntry) throws IOException {
            dataOutput.writeInt(journalEntry.pointer);
            bytesSerializer.write(dataOutput, journalEntry.bytes);
        }
    };
    PageFile dataFile = df;
    SerializedPageFile<JournalEntry> journalPageFile = SerializedFileFactory.serialized(jpf, journalEntrySerializer);
    SerializedPageFile<Integer> pointerPageFile = SerializedFileFactory.serialized(ppf, serialize.int_);
    int nCommittedJournalEntries0 = pointerPageFile.load(0);
    List<JournalEntry> journalEntries = new ArrayList<>();
    for (int jp = 0; jp < nCommittedJournalEntries0; jp++) journalEntries.add(journalPageFile.load(jp));
    return new JournalledPageFile() {

        private int nCommittedJournalEntries = nCommittedJournalEntries0;

        public void close() throws IOException {
            dataFile.close();
            journalPageFile.close();
            pointerPageFile.close();
        }

        public synchronized Bytes load(int pointer) {
            IntObjPair<JournalEntry> pair = findPageInJournal(pointer);
            if (pair != null)
                return pair.t1.bytes;
            else
                return dataFile.load(pointer);
        }

        public synchronized void save(int pointer, Bytes bytes) {
            IntObjPair<JournalEntry> pair = findDirtyPageInJournal(pointer);
            int jp;
            JournalEntry journalEntry;
            if (pair != null) {
                jp = pair.t0;
                journalEntry = pair.t1;
            } else {
                jp = journalEntries.size();
                journalEntries.add(journalEntry = new JournalEntry(pointer, null));
            }
            journalEntry.bytes = bytes;
            journalPageFile.save(jp, journalEntry);
        }

        /**
         * Marks a snapshot that data can be recovered to.
         */
        public synchronized void commit() {
            while (nCommittedJournalEntries < journalEntries.size()) {
                JournalEntry journalEntry = journalEntries.get(nCommittedJournalEntries++);
                dataFile.save(journalEntry.pointer, journalEntry.bytes);
            }
            if (8 < nCommittedJournalEntries)
                saveJournal();
        }

        /**
         * Makes sure the current snapshot of data is saved and recoverable
         * on failure, upon the return of method call.
         */
        public synchronized void sync() {
            journalPageFile.sync();
            saveJournal();
            pointerPageFile.sync();
        }

        private void saveJournal() {
            pointerPageFile.save(0, nCommittedJournalEntries);
            if (128 < nCommittedJournalEntries)
                applyJournal();
        }

        /**
         * Shortens the journal by applying them to page file.
         */
        public synchronized void applyJournal() {
            // make sure all changes are written to main file
            dataFile.sync();
            // clear all committed entries
            journalEntries.subList(0, nCommittedJournalEntries).clear();
            // reset committed pointer
            pointerPageFile.save(0, nCommittedJournalEntries = 0);
            pointerPageFile.sync();
            // write back entries for next commit
            for (int jp = 0; jp < journalEntries.size(); jp++) journalPageFile.save(jp, journalEntries.get(jp));
        }

        private IntObjPair<JournalEntry> findPageInJournal(int pointer) {
            return findPageInJournal(pointer, 0);
        }

        private IntObjPair<JournalEntry> findDirtyPageInJournal(int pointer) {
            return findPageInJournal(pointer, nCommittedJournalEntries);
        }

        private IntObjPair<JournalEntry> findPageInJournal(int pointer, int start) {
            IntObjPair<JournalEntry> pair = null;
            for (int jp = start; jp < journalEntries.size(); jp++) {
                JournalEntry journalEntry = journalEntries.get(jp);
                if (journalEntry.pointer == pointer)
                    pair = IntObjPair.of(jp, journalEntry);
            }
            return pair;
        }
    };
}

Example 2 with DataOutput_

use of suite.util.DataOutput_ in project suite by stupidsing.

the class FileFactory method extentFile.

public static ExtentFile extentFile(PageFile pf) {
    Serialize serialize = Serialize.me;
    Serializer<Extent> extentSerializer = serialize.extent();
    Serializer<Bytes> bytesSerializer = serialize.variableLengthBytes;
    SerializedPageFile<Block> pageFile = SerializedFileFactory.serialized(pf, new Serializer<>() {

        public Block read(DataInput_ dataInput) throws IOException {
            Extent extent = extentSerializer.read(dataInput);
            Bytes bytes = bytesSerializer.read(dataInput);
            return new Block(extent, bytes);
        }

        public void write(DataOutput_ dataOutput, Block block) throws IOException {
            extentSerializer.write(dataOutput, block.extent);
            bytesSerializer.write(dataOutput, block.bytes);
        }
    });
    return new ExtentFile() {

        public void close() throws IOException {
            pageFile.close();
        }

        public void sync() {
            pageFile.sync();
        }

        public Bytes load(Extent extent) {
            BytesBuilder bb = new BytesBuilder();
            for (int pointer = extent.start; pointer < extent.end; pointer++) {
                Block block = pageFile.load(pointer);
                Util.assert_(Objects.equals(block.extent, extent));
                bb.append(block.bytes);
            }
            return bb.toBytes();
        }

        public void save(Extent extent, Bytes bytes) {
            for (int pointer = extent.start, p = 0; pointer < extent.end; pointer++) {
                int p1 = p + blockSize;
                pageFile.save(pointer, new Block(extent, bytes.range(p, p1)));
                p = p1;
            }
        }

        public List<Extent> scan(int start, int end) {
            List<Extent> extents = new ArrayList<>();
            int pointer = start;
            while (pointer < end) {
                Extent extent = pageFile.load(pointer).extent;
                if (start <= extent.start && extent.end <= end)
                    extents.add(extent);
                pointer = extent.end;
            }
            return extents;
        }
    };
}

Example 3 with DataOutput_

use of suite.util.DataOutput_ in project suite by stupidsing.

the class StoreCache method getOutlet.

public Outlet<Bytes> getOutlet(Bytes key, Source<Outlet<Bytes>> source) {
    return Rethrow.ex(() -> {
        long current = System.currentTimeMillis();
        Path path;
        int i = 0;
        while (Files.exists(path = path(key, i++, ""))) if (isUpToDate(path, current)) {
            InputStream is = Files.newInputStream(path);
            DataInputStream dis = new DataInputStream(is);
            if (isMatch(key, dis))
                return read(dis).closeAtEnd(is);
            dis.close();
            is.close();
        } else {
            Files.delete(path);
            break;
        }
        Pair<Boolean, Path> pair = match(key);
        if (pair.t0) {
            InputStream vis = Files.newInputStream(pair.t1);
            DataInputStream vdis = new DataInputStream(vis);
            return read(vdis).closeAtEnd(vis);
        } else {
            Outlet<Bytes> outlet = source.source();
            OutputStream vos = FileUtil.out(pair.t1);
            DataOutput_ vdo = DataOutput_.of(vos);
            return // 
            Outlet.of(() -> Rethrow.ex(() -> {
                Bytes value = outlet.next();
                if (value != null)
                    vdo.writeBytes(value);
                return value;
            })).closeAtEnd(// 
            vos).closeAtEnd(vdo);
        }
    });
}

Example 4 with DataOutput_

use of suite.util.DataOutput_ in project suite by stupidsing.

the class StoreCache method writeKey.

private void writeKey(Path path, Bytes key) throws IOException {
    try (OutputStream kos = FileUtil.out(path);
        DataOutput_ kdo = DataOutput_.of(kos)) {
        kdo.writeInt(key.size());
        kdo.writeBytes(key);
    }
}