Examples with VariableByteOutputStream org.exist.storage.io.VariableByteOutputStream used on opensource projects

Example 6 with VariableByteOutputStream

use of org.exist.storage.io.VariableByteOutputStream in project exist by eXist-db.

the class NGramIndexWorker method saveIndex.

private void saveIndex() {
    if (ngrams.isEmpty()) {
        return;
    }
    final VariableByteOutputStream buf = new VariableByteOutputStream();
    for (final Map.Entry<QNameTerm, OccurrenceList> entry : ngrams.entrySet()) {
        final QNameTerm key = entry.getKey();
        final OccurrenceList occurences = entry.getValue();
        occurences.sort();
        os.clear();
        os.writeInt(currentDoc.getDocId());
        os.writeByte(key.qname.getNameType());
        os.writeInt(occurences.getTermCount());
        // write nodeids, freq, and offsets to a `temp` buf
        try {
            NodeId previous = null;
            for (int m = 0; m < occurences.getSize(); ) {
                previous = occurences.getNode(m).write(previous, buf);
                final int freq = occurences.getOccurrences(m);
                buf.writeInt(freq);
                for (int n = 0; n < freq; n++) {
                    buf.writeInt(occurences.getOffset(m + n));
                }
                m += freq;
            }
            final byte[] bufData = buf.toByteArray();
            // clear the buf for the next iteration
            buf.clear();
            // Write length of node IDs + frequency + offsets (bytes)
            os.writeFixedInt(bufData.length);
            // Write the node IDs + frequency + offset
            os.write(bufData);
        } catch (final IOException e) {
            LOG.error("IOException while writing nGram index: {}", e.getMessage(), e);
        }
        final ByteArray data = os.data();
        if (data.size() == 0) {
            continue;
        }
        try (final ManagedLock<ReentrantLock> dbLock = lockManager.acquireBtreeWriteLock(index.db.getLockName())) {
            final NGramQNameKey value = new NGramQNameKey(currentDoc.getCollection().getId(), key.qname, index.getBrokerPool().getSymbols(), key.term);
            index.db.append(value, data);
        } catch (final LockException e) {
            LOG.warn("Failed to acquire lock for file {}", FileUtils.fileName(index.db.getFile()), e);
        } catch (final IOException e) {
            LOG.warn("IO error for file {}", FileUtils.fileName(index.db.getFile()), e);
        } catch (final ReadOnlyException e) {
            LOG.warn("Read-only error for file {}", FileUtils.fileName(index.db.getFile()), e);
        } finally {
            os.clear();
        }
    }
    ngrams.clear();
}

Example 7 with VariableByteOutputStream

use of org.exist.storage.io.VariableByteOutputStream in project exist by eXist-db.

the class SimpleACLPermissionTest method roundtrip_write_read.

@Test
public void roundtrip_write_read() throws PermissionDeniedException, IOException {
    final SecurityManager mockSecurityManager = EasyMock.createMock(SecurityManager.class);
    final Database mockDatabase = EasyMock.createMock(Database.class);
    final DBBroker mockBroker = EasyMock.createMock(DBBroker.class);
    final Subject mockCurrentSubject = EasyMock.createMock(Subject.class);
    expect(mockSecurityManager.getDatabase()).andReturn(mockDatabase).times(2);
    expect(mockDatabase.getActiveBroker()).andReturn(mockBroker).times(2);
    expect(mockBroker.getCurrentSubject()).andReturn(mockCurrentSubject).times(2);
    expect(mockCurrentSubject.hasDbaRole()).andReturn(true).times(2);
    replay(mockSecurityManager, mockDatabase, mockBroker, mockCurrentSubject);
    SimpleACLPermission permission = new SimpleACLPermission(mockSecurityManager);
    assertEquals(0, permission.getACECount());
    final int userId1 = 1;
    final int mode1 = ALL;
    permission.addUserACE(ACE_ACCESS_TYPE.ALLOWED, userId1, mode1);
    final int groupId2 = 2;
    final int mode2 = Permission.READ;
    permission.addGroupACE(ACE_ACCESS_TYPE.DENIED, groupId2, mode2);
    final VariableByteOutputStream os = new VariableByteOutputStream();
    // write the acl out
    permission.write(os);
    verify(mockSecurityManager, mockDatabase, mockBroker, mockCurrentSubject);
    assertEquals(2, permission.getACECount());
    assertEquals(ACE_ACCESS_TYPE.ALLOWED, permission.getACEAccessType(0));
    assertEquals(userId1, permission.getACEId(0));
    assertEquals(ACE_TARGET.USER, permission.getACETarget(0));
    assertEquals(mode1, permission.getACEMode(0));
    assertEquals(ACE_ACCESS_TYPE.DENIED, permission.getACEAccessType(1));
    assertEquals(groupId2, permission.getACEId(1));
    assertEquals(ACE_TARGET.GROUP, permission.getACETarget(1));
    assertEquals(mode2, permission.getACEMode(1));
    // get the written acl data
    final byte[] data = os.toByteArray();
    // create a new permission instance
    SimpleACLPermission permission2 = new SimpleACLPermission(mockSecurityManager);
    // read the acl in
    permission2.read(new VariableByteInputStream(new UnsynchronizedByteArrayInputStream(data)));
    assertEquals(2, permission2.getACECount());
    assertEquals(ACE_ACCESS_TYPE.ALLOWED, permission2.getACEAccessType(0));
    assertEquals(userId1, permission2.getACEId(0));
    assertEquals(ACE_TARGET.USER, permission2.getACETarget(0));
    assertEquals(mode1, permission2.getACEMode(0));
    assertEquals(ACE_ACCESS_TYPE.DENIED, permission2.getACEAccessType(1));
    assertEquals(groupId2, permission2.getACEId(1));
    assertEquals(ACE_TARGET.GROUP, permission2.getACETarget(1));
    assertEquals(mode2, permission2.getACEMode(1));
}

Example 8 with VariableByteOutputStream

use of org.exist.storage.io.VariableByteOutputStream in project exist by eXist-db.

the class SymbolTableTest method write_and_read_are_balanced.

@Test
public void write_and_read_are_balanced() throws IOException, BrokerPoolServiceException {
    final SymbolTable symbolTable = createSymbolTable(createTempDir());
    symbolTable.getSymbol("some-name");
    VariableByteOutputStream mockOs = createMock(VariableByteOutputStream.class);
    VariableByteInput mockIs = createMock(VariableByteInput.class);
    final Capture<Byte> byteCapture = newCapture();
    final Capture<Integer> intCapture = newCapture();
    final Capture<String> strCapture = newCapture();
    // write expectations
    mockOs.writeByte(captureByte(byteCapture));
    mockOs.writeInt(captureInt(intCapture));
    mockOs.writeUTF(capture(strCapture));
    replay(mockOs);
    symbolTable.localNameSymbols.write(mockOs);
    verify(mockOs);
    // read expectations
    expect(mockIs.available()).andReturn(1);
    expect(mockIs.readByte()).andReturn(byteCapture.getValue());
    expect(mockIs.readInt()).andReturn(intCapture.getValue());
    expect(mockIs.readUTF()).andReturn(strCapture.getValue());
    expect(mockIs.available()).andReturn(0);
    replay(mockIs);
    symbolTable.read(mockIs);
    verify(mockIs);
}

Example 9 with VariableByteOutputStream

use of org.exist.storage.io.VariableByteOutputStream in project exist by eXist-db.

the class NativeValueIndex method flush.

private <T> void flush(final PendingChanges<T> pending, final FunctionE<T, Value, EXistException> dbKeyFn) {
    final VariableByteOutputStream nodeIdOs = new VariableByteOutputStream();
    for (final Map.Entry<T, List<NodeId>> entry : pending.changes.entrySet()) {
        final T key = entry.getKey();
        final List<NodeId> gids = entry.getValue();
        final int gidsCount = gids.size();
        // Don't forget this one
        FastQSort.sort(gids, 0, gidsCount - 1);
        os.clear();
        os.writeInt(this.doc.getDocId());
        os.writeInt(gidsCount);
        // Compute the GID list
        try {
            NodeId previous = null;
            for (final NodeId nodeId : gids) {
                previous = nodeId.write(previous, nodeIdOs);
            }
            final byte[] nodeIdsData = nodeIdOs.toByteArray();
            // clear the buf for the next iteration
            nodeIdOs.clear();
            // Write length of node IDs (bytes)
            os.writeFixedInt(nodeIdsData.length);
            // write the node IDs
            os.write(nodeIdsData);
        } catch (final IOException e) {
            LOG.warn("IO error while writing range index: {}", e.getMessage(), e);
        // TODO : throw exception?
        }
        try (final ManagedLock<ReentrantLock> bfileLock = lockManager.acquireBtreeWriteLock(dbValues.getLockName())) {
            final Value v = dbKeyFn.apply(key);
            if (dbValues.append(v, os.data()) == BFile.UNKNOWN_ADDRESS) {
                LOG.warn("Could not append index data for key '{}'", key);
            // TODO : throw exception ?
            }
        } catch (final EXistException | IOException e) {
            LOG.error(e.getMessage(), e);
        } catch (final LockException e) {
            LOG.warn("Failed to acquire lock for '{}'", FileUtils.fileName(dbValues.getFile()), e);
        // TODO : return ?
        } catch (final ReadOnlyException e) {
            LOG.warn(e.getMessage(), e);
            // Return without clearing the pending entries
            return;
        } finally {
            os.clear();
        }
    }
    pending.changes.clear();
}

Example 10 with VariableByteOutputStream

use of org.exist.storage.io.VariableByteOutputStream in project exist by eXist-db.

the class SymbolTable method saveSymbols.

/**
 * Save the entire symbol table. Will only be called when initializing an
 * empty database or when upgrading an older dbx file.
 *
 * @throws EXistException in response to eXist-db error
 */
private void saveSymbols() throws EXistException {
    try (final VariableByteOutputStream os = new VariableByteOutputStream(8192);
        final OutputStream fos = new BufferedOutputStream(Files.newOutputStream(getFile()))) {
        writeAll(os);
        fos.write(os.toByteArray());
    } catch (final FileNotFoundException e) {
        throw new EXistException("File not found: " + this.getFile().toAbsolutePath().toString(), e);
    } catch (final IOException e) {
        throw new EXistException("IO error occurred while creating " + this.getFile().toAbsolutePath().toString(), e);
    }
}