Examples with OutputStreamIndexOutput org.apache.lucene.store.OutputStreamIndexOutput used on opensource projects

Example 1 with OutputStreamIndexOutput

use of org.apache.lucene.store.OutputStreamIndexOutput in project elasticsearch by elastic.

the class ChecksumBlobStoreFormat method writeBlob.

/**
     * Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method.
     * <p>
     * The blob will be compressed and checksum will be written if required.
     *
     * @param obj           object to be serialized
     * @param blobContainer blob container
     * @param blobName          blob name
     */
protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException {
    BytesReference bytes = write(obj);
    try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, byteArrayOutputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {

                @Override
                public void close() throws IOException {
                // this is important since some of the XContentBuilders write bytes on close.
                // in order to write the footer we need to prevent closing the actual index input.
                }
            }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        BytesArray bytesArray = new BytesArray(byteArrayOutputStream.toByteArray());
        try (InputStream stream = bytesArray.streamInput()) {
            blobContainer.writeBlob(blobName, stream, bytesArray.length());
        }
    }
}

Example 2 with OutputStreamIndexOutput

use of org.apache.lucene.store.OutputStreamIndexOutput in project elasticsearch by elastic.

the class Checkpoint method write.

public static void write(ChannelFactory factory, Path checkpointFile, Checkpoint checkpoint, OpenOption... options) throws IOException {
    final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(FILE_SIZE) {

        @Override
        public synchronized byte[] toByteArray() {
            // don't clone
            return buf;
        }
    };
    final String resourceDesc = "checkpoint(path=\"" + checkpointFile + "\", gen=" + checkpoint + ")";
    try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, FILE_SIZE)) {
        CodecUtil.writeHeader(indexOutput, CHECKPOINT_CODEC, CURRENT_VERSION);
        checkpoint.write(indexOutput);
        CodecUtil.writeFooter(indexOutput);
        assert indexOutput.getFilePointer() == FILE_SIZE : "get you numbers straight; bytes written: " + indexOutput.getFilePointer() + ", buffer size: " + FILE_SIZE;
        assert indexOutput.getFilePointer() < 512 : "checkpoint files have to be smaller than 512 bytes for atomic writes; size: " + indexOutput.getFilePointer();
    }
    // now go and write to the channel, in one go.
    try (FileChannel channel = factory.open(checkpointFile, options)) {
        Channels.writeToChannel(byteOutputStream.toByteArray(), channel);
        // no need to force metadata, file size stays the same and we did the full fsync
        // when we first created the file, so the directory entry doesn't change as well
        channel.force(false);
    }
}

Example 3 with OutputStreamIndexOutput

use of org.apache.lucene.store.OutputStreamIndexOutput in project elasticsearch by elastic.

the class MetaDataStateFormat method write.

/**
     * Writes the given state to the given directories. The state is written to a
     * state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it
     * doesn't exist. The state is serialized to a temporary file in that directory and is then atomically moved to
     * it's target filename of the pattern <tt>{prefix}{version}.st</tt>.
     *
     * @param state the state object to write
     * @param locations the locations where the state should be written to.
     * @throws IOException if an IOException occurs
     */
public final void write(final T state, final Path... locations) throws IOException {
    if (locations == null) {
        throw new IllegalArgumentException("Locations must not be null");
    }
    if (locations.length <= 0) {
        throw new IllegalArgumentException("One or more locations required");
    }
    final long maxStateId = findMaxStateId(prefix, locations) + 1;
    assert maxStateId >= 0 : "maxStateId must be positive but was: [" + maxStateId + "]";
    final String fileName = prefix + maxStateId + STATE_FILE_EXTENSION;
    Path stateLocation = locations[0].resolve(STATE_DIR_NAME);
    Files.createDirectories(stateLocation);
    final Path tmpStatePath = stateLocation.resolve(fileName + ".tmp");
    final Path finalStatePath = stateLocation.resolve(fileName);
    try {
        final String resourceDesc = "MetaDataStateFormat.write(path=\"" + tmpStatePath + "\")";
        try (OutputStreamIndexOutput out = new OutputStreamIndexOutput(resourceDesc, fileName, Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) {
            CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION);
            out.writeInt(format.index());
            try (XContentBuilder builder = newXContentBuilder(format, new IndexOutputOutputStream(out) {

                @Override
                public void close() throws IOException {
                // this is important since some of the XContentBuilders write bytes on close.
                // in order to write the footer we need to prevent closing the actual index input.
                }
            })) {
                builder.startObject();
                {
                    toXContent(builder, state);
                }
                builder.endObject();
            }
            CodecUtil.writeFooter(out);
        }
        // fsync the state file
        IOUtils.fsync(tmpStatePath, false);
        Files.move(tmpStatePath, finalStatePath, StandardCopyOption.ATOMIC_MOVE);
        IOUtils.fsync(stateLocation, true);
        for (int i = 1; i < locations.length; i++) {
            stateLocation = locations[i].resolve(STATE_DIR_NAME);
            Files.createDirectories(stateLocation);
            Path tmpPath = stateLocation.resolve(fileName + ".tmp");
            Path finalPath = stateLocation.resolve(fileName);
            try {
                Files.copy(finalStatePath, tmpPath);
                // we are on the same FileSystem / Partition here we can do an atomic move
                Files.move(tmpPath, finalPath, StandardCopyOption.ATOMIC_MOVE);
                // we just fsync the dir here..
                IOUtils.fsync(stateLocation, true);
            } finally {
                Files.deleteIfExists(tmpPath);
            }
        }
    } finally {
        Files.deleteIfExists(tmpStatePath);
    }
    cleanupOldFiles(prefix, fileName, locations);
}

Example 4 with OutputStreamIndexOutput

use of org.apache.lucene.store.OutputStreamIndexOutput in project geode by apache.

the class RegionDirectory method createTempOutput.

public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
    String name = prefix + "_temp_" + UUID.randomUUID() + suffix;
    final File file = fs.createTemporaryFile(name);
    final OutputStream out = file.getOutputStream();
    return new OutputStreamIndexOutput(name, name, out, 1000);
}

Example 5 with OutputStreamIndexOutput

use of org.apache.lucene.store.OutputStreamIndexOutput in project geode by apache.

the class RegionDirectory method createOutput.

@Override
public IndexOutput createOutput(final String name, final IOContext context) throws IOException {
    ensureOpen();
    final File file = fs.createFile(name);
    final OutputStream out = file.getOutputStream();
    return new OutputStreamIndexOutput(name, name, out, 1000);
}