Examples with InputStreamDataInput org.apache.lucene.store.InputStreamDataInput used on opensource projects

Example 1 with InputStreamDataInput

use of org.apache.lucene.store.InputStreamDataInput in project elasticsearch by elastic.

the class XAnalyzingSuggester method load.

@Override
public boolean load(InputStream input) throws IOException {
    DataInput dataIn = new InputStreamDataInput(input);
    try {
        this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
        maxAnalyzedPathsForOneInput = dataIn.readVInt();
        hasPayloads = dataIn.readByte() == 1;
    } finally {
        IOUtils.close(input);
    }
    return true;
}

Example 2 with InputStreamDataInput

use of org.apache.lucene.store.InputStreamDataInput in project jackrabbit-oak by apache.

the class OakDirectoryTest method writeFile.

private static void writeFile(Directory directory, String fileName, long size) throws Exception {
    IndexOutput o = directory.createOutput(fileName, IOContext.DEFAULT);
    o.copyBytes(new InputStreamDataInput(new NullInputStream(size)), size);
    o.close();
}

Example 3 with InputStreamDataInput

use of org.apache.lucene.store.InputStreamDataInput in project jackrabbit-oak by apache.

the class OakDirectoryTestBase method writeFile.

static void writeFile(Directory directory, String fileName, long size) throws Exception {
    IndexOutput o = directory.createOutput(fileName, IOContext.DEFAULT);
    o.copyBytes(new InputStreamDataInput(new NullInputStream(size)), size);
    o.close();
}

Example 4 with InputStreamDataInput

use of org.apache.lucene.store.InputStreamDataInput in project elasticsearch by elastic.

the class TranslogReader method open.

/**
     * Given a file channel, opens a {@link TranslogReader}, taking care of checking and validating the file header.
     *
     * @param channel the translog file channel
     * @param path the path to the translog
     * @param checkpoint the translog checkpoint
     * @param translogUUID the tranlog UUID
     * @return a new TranslogReader
     * @throws IOException if any of the file operations resulted in an I/O exception
     */
public static TranslogReader open(final FileChannel channel, final Path path, final Checkpoint checkpoint, final String translogUUID) throws IOException {
    try {
        // don't close
        InputStreamStreamInput headerStream = new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel));
        // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the
        // header, in binary this looks like:
        //
        // binary: 0011 1111 1101 0111 0110 1100 0001 0111
        // hex   :    3    f    d    7    6    c    1    7
        //
        // With version 0 of the translog, the first byte is the
        // Operation.Type, which will always be between 0-4, so we know if
        // we grab the first byte, it can be:
        // 0x3f => Lucene's magic number, so we can assume it's version 1 or later
        // 0x00 => version 0 of the translog
        //
        // otherwise the first byte of the translog is corrupted and we
        // should bail
        byte b1 = headerStream.readByte();
        if (b1 == LUCENE_CODEC_HEADER_BYTE) {
            // Read 3 more bytes, meaning a whole integer has been read
            byte b2 = headerStream.readByte();
            byte b3 = headerStream.readByte();
            byte b4 = headerStream.readByte();
            // Convert the 4 bytes that were read into an integer
            int header = ((b1 & 0xFF) << 24) + ((b2 & 0xFF) << 16) + ((b3 & 0xFF) << 8) + ((b4 & 0xFF) << 0);
            // byte separately
            if (header != CodecUtil.CODEC_MAGIC) {
                throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path);
            }
            // Confirm the rest of the header using CodecUtil, extracting
            // the translog version
            int version = CodecUtil.checkHeaderNoMagic(new InputStreamDataInput(headerStream), TranslogWriter.TRANSLOG_CODEC, 1, Integer.MAX_VALUE);
            switch(version) {
                case TranslogWriter.VERSION_CHECKSUMS:
                    throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
                case TranslogWriter.VERSION_CHECKPOINTS:
                    assert path.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX) : "new file ends with old suffix: " + path;
                    assert checkpoint.numOps >= 0 : "expected at least 0 operatin but got: " + checkpoint.numOps;
                    assert checkpoint.offset <= channel.size() : "checkpoint is inconsistent with channel length: " + channel.size() + " " + checkpoint;
                    int len = headerStream.readInt();
                    if (len > channel.size()) {
                        throw new TranslogCorruptedException("uuid length can't be larger than the translog");
                    }
                    BytesRef ref = new BytesRef(len);
                    ref.length = len;
                    headerStream.read(ref.bytes, ref.offset, ref.length);
                    BytesRef uuidBytes = new BytesRef(translogUUID);
                    if (uuidBytes.bytesEquals(ref) == false) {
                        throw new TranslogCorruptedException("expected shard UUID " + uuidBytes + " but got: " + ref + " this translog file belongs to a different translog. path:" + path);
                    }
                    final long firstOperationOffset = ref.length + CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC) + Integer.BYTES;
                    return new TranslogReader(checkpoint, channel, path, firstOperationOffset);
                default:
                    throw new TranslogCorruptedException("No known translog stream version: " + version + " path:" + path);
            }
        } else if (b1 == UNVERSIONED_TRANSLOG_HEADER_BYTE) {
            throw new IllegalStateException("pre-1.4 translog found [" + path + "]");
        } else {
            throw new TranslogCorruptedException("Invalid first byte in translog file, got: " + Long.toHexString(b1) + ", expected 0x00 or 0x3f. path:" + path);
        }
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
        throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e);
    }
}

Example 5 with InputStreamDataInput

use of org.apache.lucene.store.InputStreamDataInput in project crate by crate.

the class TranslogHeader method read.

/**
 * Read a translog header from the given path and file channel
 */
static TranslogHeader read(final String translogUUID, final Path path, final FileChannel channel) throws IOException {
    try {
        // This input is intentionally not closed because closing it will close the FileChannel.
        final BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel), channel.size()), path.toString());
        final int version;
        try {
            version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKSUMS, VERSION_PRIMARY_TERM);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
            tryReportOldVersionError(path, channel);
            throw new TranslogCorruptedException(path.toString(), "translog header corrupted", e);
        }
        if (version == VERSION_CHECKSUMS) {
            throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
        }
        // Read the translogUUID
        final int uuidLen = in.readInt();
        if (uuidLen > channel.size()) {
            throw new TranslogCorruptedException(path.toString(), "UUID length can't be larger than the translog");
        }
        if (uuidLen <= 0) {
            throw new TranslogCorruptedException(path.toString(), "UUID length must be positive");
        }
        final BytesRef uuid = new BytesRef(uuidLen);
        uuid.length = uuidLen;
        in.read(uuid.bytes, uuid.offset, uuid.length);
        // Read the primary term
        final long primaryTerm;
        if (version == VERSION_PRIMARY_TERM) {
            primaryTerm = in.readLong();
        } else {
            // This can be dropped with 5.0 as BWC with older versions is not required anymore
            assert version == VERSION_CHECKPOINTS : "Unknown header version [" + version + "]";
            primaryTerm = UNASSIGNED_PRIMARY_TERM;
        }
        // Verify the checksum (can be always verified on >= 5.0 as version must be primary term based without BWC)
        if (version >= VERSION_PRIMARY_TERM) {
            Translog.verifyChecksum(in);
        }
        assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]";
        final int headerSizeInBytes = headerSizeInBytes(version, uuid.length);
        assert channel.position() == headerSizeInBytes : "Header is not fully read; header size [" + headerSizeInBytes + "], position [" + channel.position() + "]";
        // verify UUID only after checksum, to ensure that UUID is not corrupted
        final BytesRef expectedUUID = new BytesRef(translogUUID);
        if (uuid.bytesEquals(expectedUUID) == false) {
            throw new TranslogCorruptedException(path.toString(), "expected shard UUID " + expectedUUID + " but got: " + uuid + " this translog file belongs to a different translog");
        }
        return new TranslogHeader(translogUUID, primaryTerm, headerSizeInBytes);
    } catch (EOFException e) {
        throw new TranslogCorruptedException(path.toString(), "translog header truncated", e);
    }
}