Examples with IOContext org.apache.lucene.store.IOContext used on opensource projects

Example 11 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class IndexWriter method addIndexes.

/**
   * Merges the provided indexes into this index.
   * 
   * <p>
   * The provided IndexReaders are not closed.
   * 
   * <p>
   * See {@link #addIndexes} for details on transactional semantics, temporary
   * free space required in the Directory, and non-CFS segments on an Exception.
   * 
   * <p>
   * <b>NOTE:</b> empty segments are dropped by this method and not added to this
   * index.
   * 
   * <p>
   * <b>NOTE:</b> this merges all given {@link LeafReader}s in one
   * merge. If you intend to merge a large number of readers, it may be better
   * to call this method multiple times, each time with a small set of readers.
   * In principle, if you use a merge policy with a {@code mergeFactor} or
   * {@code maxMergeAtOnce} parameter, you should pass that many readers in one
   * call.
   * 
   * <p>
   * <b>NOTE:</b> this method does not call or make use of the {@link MergeScheduler},
   * so any custom bandwidth throttling is at the moment ignored.
   * 
   * @return The <a href="#sequence_number">sequence number</a>
   * for this operation
   *
   * @throws CorruptIndexException
   *           if the index is corrupt
   * @throws IOException
   *           if there is a low-level IO error
   * @throws IllegalArgumentException
   *           if addIndexes would cause the index to exceed {@link #MAX_DOCS}
   */
public long addIndexes(CodecReader... readers) throws IOException {
    ensureOpen();
    // long so we can detect int overflow:
    long numDocs = 0;
    Sort indexSort = config.getIndexSort();
    long seqNo;
    try {
        if (infoStream.isEnabled("IW")) {
            infoStream.message("IW", "flush at addIndexes(CodecReader...)");
        }
        flush(false, true);
        String mergedName = newSegmentName();
        for (CodecReader leaf : readers) {
            numDocs += leaf.numDocs();
            validateMergeReader(leaf);
        }
        // Best-effort up front check:
        testReserveDocs(numDocs);
        final IOContext context = new IOContext(new MergeInfo(Math.toIntExact(numDocs), -1, false, UNBOUNDED_MAX_MERGE_SEGMENTS));
        // TODO: somehow we should fix this merge so it's
        // abortable so that IW.close(false) is able to stop it
        TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
        // We set the min version to null for now, it will be set later by SegmentMerger
        SegmentInfo info = new SegmentInfo(directoryOrig, Version.LATEST, null, mergedName, -1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), config.getIndexSort());
        SegmentMerger merger = new SegmentMerger(Arrays.asList(readers), info, infoStream, trackingDir, globalFieldNumberMap, context);
        if (!merger.shouldMerge()) {
            return docWriter.deleteQueue.getNextSequenceNumber();
        }
        // merge 'em
        merger.merge();
        SegmentCommitInfo infoPerCommit = new SegmentCommitInfo(info, 0, -1L, -1L, -1L);
        info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
        trackingDir.clearCreatedFiles();
        setDiagnostics(info, SOURCE_ADDINDEXES_READERS);
        final MergePolicy mergePolicy = config.getMergePolicy();
        boolean useCompoundFile;
        synchronized (this) {
            // Guard segmentInfos
            if (stopMerges) {
                // Safe: these files must exist
                deleteNewFiles(infoPerCommit.files());
                return docWriter.deleteQueue.getNextSequenceNumber();
            }
            ensureOpen();
            useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, infoPerCommit, this);
        }
        // Now create the compound file if needed
        if (useCompoundFile) {
            Collection<String> filesToDelete = infoPerCommit.files();
            TrackingDirectoryWrapper trackingCFSDir = new TrackingDirectoryWrapper(directory);
            // createCompoundFile tries to cleanup, but it might not always be able to...
            try {
                createCompoundFile(infoStream, trackingCFSDir, info, context);
            } finally {
                // delete new non cfs files directly: they were never
                // registered with IFD
                deleteNewFiles(filesToDelete);
            }
            info.setUseCompoundFile(true);
        }
        // Have codec write SegmentInfo.  Must do this after
        // creating CFS so that 1) .si isn't slurped into CFS,
        // and 2) .si reflects useCompoundFile=true change
        // above:
        codec.segmentInfoFormat().write(trackingDir, info, context);
        info.addFiles(trackingDir.getCreatedFiles());
        // Register the new segment
        synchronized (this) {
            if (stopMerges) {
                // Safe: these files must exist
                deleteNewFiles(infoPerCommit.files());
                return docWriter.deleteQueue.getNextSequenceNumber();
            }
            ensureOpen();
            // Now reserve the docs, just before we update SIS:
            reserveDocs(numDocs);
            segmentInfos.add(infoPerCommit);
            seqNo = docWriter.deleteQueue.getNextSequenceNumber();
            checkpoint();
        }
    } catch (VirtualMachineError tragedy) {
        tragicEvent(tragedy, "addIndexes(CodecReader...)");
        // dead code but javac disagrees:
        seqNo = -1;
    }
    maybeMerge();
    return seqNo;
}

Example 12 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class SortingTermVectorsConsumer method initTermVectorsWriter.

@Override
void initTermVectorsWriter() throws IOException {
    if (writer == null) {
        IOContext context = new IOContext(new FlushInfo(docWriter.getNumDocsInRAM(), docWriter.bytesUsed()));
        tmpDirectory = new TrackingTmpOutputDirectoryWrapper(docWriter.directory);
        writer = docWriter.codec.termVectorsFormat().vectorsWriter(tmpDirectory, docWriter.getSegmentInfo(), context);
        lastDocID = 0;
    }
}

Example 13 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class HdfsDirectoryTest method assertInputsEquals.

private void assertInputsEquals(String name, Directory fsDir, HdfsDirectory hdfs) throws IOException {
    int reads = random.nextInt(MAX_NUMBER_OF_READS);
    IndexInput fsInput = fsDir.openInput(name, new IOContext());
    IndexInput hdfsInput = hdfs.openInput(name, new IOContext());
    assertEquals(fsInput.length(), hdfsInput.length());
    int fileLength = (int) fsInput.length();
    for (int i = 0; i < reads; i++) {
        int nextInt = Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength);
        byte[] fsBuf = new byte[random.nextInt(nextInt > 0 ? nextInt : 1) + MIN_BUFFER_SIZE];
        byte[] hdfsBuf = new byte[fsBuf.length];
        int offset = random.nextInt(fsBuf.length);
        nextInt = fsBuf.length - offset;
        int length = random.nextInt(nextInt > 0 ? nextInt : 1);
        nextInt = fileLength - length;
        int pos = random.nextInt(nextInt > 0 ? nextInt : 1);
        fsInput.seek(pos);
        fsInput.readBytes(fsBuf, offset, length);
        hdfsInput.seek(pos);
        hdfsInput.readBytes(hdfsBuf, offset, length);
        for (int f = offset; f < length; f++) {
            if (fsBuf[f] != hdfsBuf[f]) {
                fail();
            }
        }
    }
    fsInput.close();
    hdfsInput.close();
}

Example 14 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class HdfsDirectoryTest method testEof.

private void testEof(String name, Directory directory, long length) throws IOException {
    IndexInput input = directory.openInput(name, new IOContext());
    input.seek(length);
    try {
        input.readByte();
        fail("should throw eof");
    } catch (IOException e) {
    }
}

Example 15 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class HdfsDirectoryTest method testRename.

public void testRename() throws IOException {
    String[] listAll = directory.listAll();
    for (String file : listAll) {
        directory.deleteFile(file);
    }
    IndexOutput output = directory.createOutput("testing.test", new IOContext());
    output.writeInt(12345);
    output.close();
    directory.rename("testing.test", "testing.test.renamed");
    assertFalse(slowFileExists(directory, "testing.test"));
    assertTrue(slowFileExists(directory, "testing.test.renamed"));
    IndexInput input = directory.openInput("testing.test.renamed", new IOContext());
    assertEquals(12345, input.readInt());
    assertEquals(input.getFilePointer(), input.length());
    input.close();
    directory.deleteFile("testing.test.renamed");
    assertFalse(slowFileExists(directory, "testing.test.renamed"));
}