Examples with IOContext org.apache.lucene.store.IOContext used on opensource projects

Example 21 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnInput2.

/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (prefix.equals("unsorted")) {
                    return new CorruptingIndexOutput(dir0, 22, out) {

                        @Override
                        protected void corruptFile() throws IOException {
                            String newTempName;
                            try (IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                                IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                                newTempName = tmpOut.getName();
                                // Replace length at the end with a too-long value:
                                short v = in.readShort();
                                assertEquals(256, v);
                                tmpOut.writeShort(Short.MAX_VALUE);
                                tmpOut.copyBytes(in, in.length() - Short.BYTES);
                            }
                            // Delete original and copy corrupt version back:
                            dir0.deleteFile(out.getName());
                            dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
                            dir0.deleteFile(newTempName);
                        }
                    };
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed(5 * 1024));
        // This corruption made OfflineSorter fail with its own exception, but we verify it also went and added (as suppressed) that the
        // checksum was wrong:
        EOFException e = expectThrows(EOFException.class, () -> {
            new OfflineSorter(dir, "foo").sort(unsorted.getName());
        });
        assertEquals(1, e.getSuppressed().length);
        assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
        assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 22 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class RandomPostingsTester method buildIndex.

// maxAllowed = the "highest" we can index, but we will still
// randomly index at lower IndexOption
public FieldsProducer buildIndex(Codec codec, Directory dir, IndexOptions maxAllowed, boolean allowPayloads, boolean alwaysTestMax) throws IOException {
    SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "_0", maxDoc, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
    int maxIndexOption = Arrays.asList(IndexOptions.values()).indexOf(maxAllowed);
    if (LuceneTestCase.VERBOSE) {
        System.out.println("\nTEST: now build index");
    }
    // TODO use allowPayloads
    FieldInfo[] newFieldInfoArray = new FieldInfo[fields.size()];
    for (int fieldUpto = 0; fieldUpto < fields.size(); fieldUpto++) {
        FieldInfo oldFieldInfo = fieldInfos.fieldInfo(fieldUpto);
        // Randomly picked the IndexOptions to index this
        // field with:
        IndexOptions indexOptions = IndexOptions.values()[alwaysTestMax ? maxIndexOption : TestUtil.nextInt(random, 1, maxIndexOption)];
        boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
        newFieldInfoArray[fieldUpto] = new FieldInfo(oldFieldInfo.name, fieldUpto, false, false, doPayloads, indexOptions, DocValuesType.NONE, -1, new HashMap<>(), 0, 0);
    }
    FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);
    // Estimate that flushed segment size will be 25% of
    // what we use in RAM:
    long bytes = totalPostings * 8 + totalPayloadBytes;
    SegmentWriteState writeState = new SegmentWriteState(null, dir, segmentInfo, newFieldInfos, null, new IOContext(new FlushInfo(maxDoc, bytes)));
    Fields seedFields = new SeedFields(fields, newFieldInfos, maxAllowed, allowPayloads);
    FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState);
    boolean success = false;
    try {
        consumer.write(seedFields);
        success = true;
    } finally {
        if (success) {
            IOUtils.close(consumer);
        } else {
            IOUtils.closeWhileHandlingException(consumer);
        }
    }
    if (LuceneTestCase.VERBOSE) {
        System.out.println("TEST: after indexing: files=");
        for (String file : dir.listAll()) {
            System.out.println("  " + file + ": " + dir.fileLength(file) + " bytes");
        }
    }
    currentFieldInfos = newFieldInfos;
    SegmentReadState readState = new SegmentReadState(dir, segmentInfo, newFieldInfos, IOContext.READ);
    return codec.postingsFormat().fieldsProducer(readState);
}

Example 23 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class TestBKD method testBitFlippedOnPartition2.

/** Make sure corruption on a recursed partition is caught, when BKDWriter does get angry */
public void testBitFlippedOnPartition2() throws Exception {
    // Generate fixed data set:
    int numDocs = atLeast(10000);
    int numBytesPerDim = 4;
    int numDims = 3;
    byte[][][] docValues = new byte[numDocs][][];
    byte counter = 0;
    for (int docID = 0; docID < numDocs; docID++) {
        byte[][] values = new byte[numDims][];
        for (int dim = 0; dim < numDims; dim++) {
            values[dim] = new byte[numBytesPerDim];
            for (int i = 0; i < values[dim].length; i++) {
                values[dim][i] = counter;
                counter++;
            }
        }
        docValues[docID] = values;
    }
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                //System.out.println("prefix=" + prefix + " suffix=" + suffix);
                if (corrupted == false && suffix.equals("bkd_left1")) {
                    //System.out.println("now corrupt byte=" + x + " prefix=" + prefix + " suffix=" + suffix);
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 22072, out);
                } else {
                    return out;
                }
            }
        };
        Throwable t = expectThrows(CorruptIndexException.class, () -> {
            verify(dir, docValues, null, numDims, numBytesPerDim, 50, 0.1);
        });
        assertCorruptionDetected(t);
    }
}

Example 24 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class LuceneTestCase method newIOContext.

/** TODO: javadoc */
public static IOContext newIOContext(Random random, IOContext oldContext) {
    final int randomNumDocs = random.nextInt(4192);
    final int size = random.nextInt(512) * randomNumDocs;
    if (oldContext.flushInfo != null) {
        // the incoming IOContext:
        return new IOContext(new FlushInfo(randomNumDocs, Math.max(oldContext.flushInfo.estimatedSegmentSize, size)));
    } else if (oldContext.mergeInfo != null) {
        // the incoming IOContext:
        return new IOContext(new MergeInfo(randomNumDocs, Math.max(oldContext.mergeInfo.estimatedMergeBytes, size), random.nextBoolean(), TestUtil.nextInt(random, 1, 100)));
    } else {
        // Make a totally random IOContext:
        final IOContext context;
        switch(random.nextInt(5)) {
            case 0:
                context = IOContext.DEFAULT;
                break;
            case 1:
                context = IOContext.READ;
                break;
            case 2:
                context = IOContext.READONCE;
                break;
            case 3:
                context = new IOContext(new MergeInfo(randomNumDocs, size, true, -1));
                break;
            case 4:
                context = new IOContext(new FlushInfo(randomNumDocs, size));
                break;
            default:
                context = IOContext.DEFAULT;
        }
        return context;
    }
}

Example 25 with IOContext

use of org.apache.lucene.store.IOContext in project lucene-solr by apache.

the class FSTTester method doTest.

FST<T> doTest(int prune1, int prune2, boolean allowRandomSuffixSharing) throws IOException {
    if (LuceneTestCase.VERBOSE) {
        System.out.println("\nTEST: prune1=" + prune1 + " prune2=" + prune2);
    }
    final Builder<T> builder = new Builder<>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, prune1, prune2, prune1 == 0 && prune2 == 0, allowRandomSuffixSharing ? random.nextBoolean() : true, allowRandomSuffixSharing ? TestUtil.nextInt(random, 1, 10) : Integer.MAX_VALUE, outputs, true, 15);
    for (InputOutput<T> pair : pairs) {
        if (pair.output instanceof List) {
            @SuppressWarnings("unchecked") List<Long> longValues = (List<Long>) pair.output;
            @SuppressWarnings("unchecked") final Builder<Object> builderObject = (Builder<Object>) builder;
            for (Long value : longValues) {
                builderObject.add(pair.input, value);
            }
        } else {
            builder.add(pair.input, pair.output);
        }
    }
    FST<T> fst = builder.finish();
    if (random.nextBoolean() && fst != null) {
        IOContext context = LuceneTestCase.newIOContext(random);
        IndexOutput out = dir.createOutput("fst.bin", context);
        fst.save(out);
        out.close();
        IndexInput in = dir.openInput("fst.bin", context);
        try {
            fst = new FST<>(in, outputs);
        } finally {
            in.close();
            dir.deleteFile("fst.bin");
        }
    }
    if (LuceneTestCase.VERBOSE && pairs.size() <= 20 && fst != null) {
        System.out.println("Printing FST as dot file to stdout:");
        final Writer w = new OutputStreamWriter(System.out, Charset.defaultCharset());
        Util.toDot(fst, w, false, false);
        w.flush();
        System.out.println("END dot file");
    }
    if (LuceneTestCase.VERBOSE) {
        if (fst == null) {
            System.out.println("  fst has 0 nodes (fully pruned)");
        } else {
            System.out.println("  fst has " + builder.getNodeCount() + " nodes and " + builder.getArcCount() + " arcs");
        }
    }
    if (prune1 == 0 && prune2 == 0) {
        verifyUnPruned(inputMode, fst);
    } else {
        verifyPruned(inputMode, fst, prune1, prune2);
    }
    nodeCount = builder.getNodeCount();
    arcCount = builder.getArcCount();
    return fst;
}