Examples with Codec org.apache.lucene.codecs.Codec used on opensource projects

Example 41 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseFieldInfoFormatTestCase method testExceptionOnCloseOutput.

/** 
   * Test field infos write that hits exception on close.
   * make sure we get our exception back, no file handle leaks, etc. 
   */
public void testExceptionOnCloseOutput() throws Exception {
    Failure fail = new Failure() {

        @Override
        public void eval(MockDirectoryWrapper dir) throws IOException {
            for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
                if (doFail && "close".equals(e.getMethodName())) {
                    throw new FakeIOException();
                }
            }
        }
    };
    MockDirectoryWrapper dir = newMockDirectory();
    dir.failOn(fail);
    Codec codec = getCodec();
    SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
    FieldInfos.Builder builder = new FieldInfos.Builder();
    FieldInfo fi = builder.getOrAdd("field");
    fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
    addAttributes(fi);
    FieldInfos infos = builder.finish();
    fail.setDoFail();
    expectThrows(FakeIOException.class, () -> {
        codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
    });
    fail.clearDoFail();
    dir.close();
}

Example 42 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseFieldInfoFormatTestCase method testExceptionOnCreateOutput.

/** 
   * Test field infos write that hits exception immediately on open.
   * make sure we get our exception back, no file handle leaks, etc. 
   */
public void testExceptionOnCreateOutput() throws Exception {
    Failure fail = new Failure() {

        @Override
        public void eval(MockDirectoryWrapper dir) throws IOException {
            for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
                if (doFail && "createOutput".equals(e.getMethodName())) {
                    throw new FakeIOException();
                }
            }
        }
    };
    MockDirectoryWrapper dir = newMockDirectory();
    dir.failOn(fail);
    Codec codec = getCodec();
    SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
    FieldInfos.Builder builder = new FieldInfos.Builder();
    FieldInfo fi = builder.getOrAdd("field");
    fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
    addAttributes(fi);
    FieldInfos infos = builder.finish();
    fail.setDoFail();
    expectThrows(FakeIOException.class, () -> {
        codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
    });
    fail.clearDoFail();
    dir.close();
}

Example 43 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseFieldInfoFormatTestCase method testRandom.

// TODO: more tests
/** Test field infos read/write with random fields, with different values. */
public void testRandom() throws Exception {
    Directory dir = newDirectory();
    Codec codec = getCodec();
    SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
    // generate a bunch of fields
    int numFields = atLeast(2000);
    Set<String> fieldNames = new HashSet<>();
    for (int i = 0; i < numFields; i++) {
        fieldNames.add(TestUtil.randomUnicodeString(random()));
    }
    FieldInfos.Builder builder = new FieldInfos.Builder();
    for (String field : fieldNames) {
        IndexableFieldType fieldType = randomFieldType(random());
        FieldInfo fi = builder.getOrAdd(field);
        IndexOptions indexOptions = fieldType.indexOptions();
        if (indexOptions != IndexOptions.NONE) {
            fi.setIndexOptions(indexOptions);
            if (fieldType.omitNorms()) {
                fi.setOmitsNorms();
            }
        }
        fi.setDocValuesType(fieldType.docValuesType());
        if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
            if (random().nextBoolean()) {
                fi.setStorePayloads();
            }
        }
        addAttributes(fi);
    }
    FieldInfos infos = builder.finish();
    codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
    FieldInfos infos2 = codec.fieldInfosFormat().read(dir, segmentInfo, "", IOContext.DEFAULT);
    assertEquals(infos, infos2);
    dir.close();
}

Example 44 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseIndexFileFormatTestCase method testMultiClose.

/** Calls close multiple times on closeable codec apis */
public void testMultiClose() throws IOException {
    // first make a one doc index
    Directory oneDocIndex = applyCreatedVersionMajor(newDirectory());
    IndexWriter iw = new IndexWriter(oneDocIndex, new IndexWriterConfig(new MockAnalyzer(random())));
    Document oneDoc = new Document();
    FieldType customType = new FieldType(TextField.TYPE_STORED);
    customType.setStoreTermVectors(true);
    Field customField = new Field("field", "contents", customType);
    oneDoc.add(customField);
    oneDoc.add(new NumericDocValuesField("field", 5));
    iw.addDocument(oneDoc);
    LeafReader oneDocReader = getOnlyLeafReader(DirectoryReader.open(iw));
    iw.close();
    // now feed to codec apis manually
    // we use FSDir, things like ramdir are not guaranteed to cause fails if you write to them after close(), etc
    Directory dir = newFSDirectory(createTempDir("justSoYouGetSomeChannelErrors"));
    Codec codec = getCodec();
    SegmentInfo segmentInfo = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "_0", 1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
    FieldInfo proto = oneDocReader.getFieldInfos().fieldInfo("field");
    FieldInfo field = new FieldInfo(proto.name, proto.number, proto.hasVectors(), proto.omitsNorms(), proto.hasPayloads(), proto.getIndexOptions(), proto.getDocValuesType(), proto.getDocValuesGen(), new HashMap<>(), proto.getPointDimensionCount(), proto.getPointNumBytes());
    FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { field });
    SegmentWriteState writeState = new SegmentWriteState(null, dir, segmentInfo, fieldInfos, null, new IOContext(new FlushInfo(1, 20)));
    SegmentReadState readState = new SegmentReadState(dir, segmentInfo, fieldInfos, IOContext.READ);
    // PostingsFormat
    try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState)) {
        consumer.write(oneDocReader.fields());
        IOUtils.close(consumer);
        IOUtils.close(consumer);
    }
    try (FieldsProducer producer = codec.postingsFormat().fieldsProducer(readState)) {
        IOUtils.close(producer);
        IOUtils.close(producer);
    }
    // DocValuesFormat
    try (DocValuesConsumer consumer = codec.docValuesFormat().fieldsConsumer(writeState)) {
        consumer.addNumericField(field, new EmptyDocValuesProducer() {

            @Override
            public NumericDocValues getNumeric(FieldInfo field) {
                return new NumericDocValues() {

                    int docID = -1;

                    @Override
                    public int docID() {
                        return docID;
                    }

                    @Override
                    public int nextDoc() {
                        docID++;
                        if (docID == 1) {
                            docID = NO_MORE_DOCS;
                        }
                        return docID;
                    }

                    @Override
                    public int advance(int target) {
                        if (docID <= 0 && target == 0) {
                            docID = 0;
                        } else {
                            docID = NO_MORE_DOCS;
                        }
                        return docID;
                    }

                    @Override
                    public boolean advanceExact(int target) throws IOException {
                        docID = target;
                        return target == 0;
                    }

                    @Override
                    public long cost() {
                        return 1;
                    }

                    @Override
                    public long longValue() {
                        return 5;
                    }
                };
            }
        });
        IOUtils.close(consumer);
        IOUtils.close(consumer);
    }
    try (DocValuesProducer producer = codec.docValuesFormat().fieldsProducer(readState)) {
        IOUtils.close(producer);
        IOUtils.close(producer);
    }
    // NormsFormat
    try (NormsConsumer consumer = codec.normsFormat().normsConsumer(writeState)) {
        consumer.addNormsField(field, new NormsProducer() {

            @Override
            public NumericDocValues getNorms(FieldInfo field) {
                return new NumericDocValues() {

                    int docID = -1;

                    @Override
                    public int docID() {
                        return docID;
                    }

                    @Override
                    public int nextDoc() {
                        docID++;
                        if (docID == 1) {
                            docID = NO_MORE_DOCS;
                        }
                        return docID;
                    }

                    @Override
                    public int advance(int target) {
                        if (docID <= 0 && target == 0) {
                            docID = 0;
                        } else {
                            docID = NO_MORE_DOCS;
                        }
                        return docID;
                    }

                    @Override
                    public boolean advanceExact(int target) throws IOException {
                        docID = target;
                        return target == 0;
                    }

                    @Override
                    public long cost() {
                        return 1;
                    }

                    @Override
                    public long longValue() {
                        return 5;
                    }
                };
            }

            @Override
            public void checkIntegrity() {
            }

            @Override
            public void close() {
            }

            @Override
            public long ramBytesUsed() {
                return 0;
            }
        });
        IOUtils.close(consumer);
        IOUtils.close(consumer);
    }
    try (NormsProducer producer = codec.normsFormat().normsProducer(readState)) {
        IOUtils.close(producer);
        IOUtils.close(producer);
    }
    // TermVectorsFormat
    try (TermVectorsWriter consumer = codec.termVectorsFormat().vectorsWriter(dir, segmentInfo, writeState.context)) {
        consumer.startDocument(1);
        consumer.startField(field, 1, false, false, false);
        consumer.startTerm(new BytesRef("testing"), 2);
        consumer.finishTerm();
        consumer.finishField();
        consumer.finishDocument();
        consumer.finish(fieldInfos, 1);
        IOUtils.close(consumer);
        IOUtils.close(consumer);
    }
    try (TermVectorsReader producer = codec.termVectorsFormat().vectorsReader(dir, segmentInfo, fieldInfos, readState.context)) {
        IOUtils.close(producer);
        IOUtils.close(producer);
    }
    // StoredFieldsFormat
    try (StoredFieldsWriter consumer = codec.storedFieldsFormat().fieldsWriter(dir, segmentInfo, writeState.context)) {
        consumer.startDocument();
        consumer.writeField(field, customField);
        consumer.finishDocument();
        consumer.finish(fieldInfos, 1);
        IOUtils.close(consumer);
        IOUtils.close(consumer);
    }
    try (StoredFieldsReader producer = codec.storedFieldsFormat().fieldsReader(dir, segmentInfo, fieldInfos, readState.context)) {
        IOUtils.close(producer);
        IOUtils.close(producer);
    }
    IOUtils.close(oneDocReader, oneDocIndex, dir);
}

Example 45 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseFieldInfoFormatTestCase method testExceptionOnOpenInput.

/** 
   * Test field infos read that hits exception immediately on open.
   * make sure we get our exception back, no file handle leaks, etc. 
   */
public void testExceptionOnOpenInput() throws Exception {
    Failure fail = new Failure() {

        @Override
        public void eval(MockDirectoryWrapper dir) throws IOException {
            for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
                if (doFail && "openInput".equals(e.getMethodName())) {
                    throw new FakeIOException();
                }
            }
        }
    };
    MockDirectoryWrapper dir = newMockDirectory();
    dir.failOn(fail);
    Codec codec = getCodec();
    SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
    FieldInfos.Builder builder = new FieldInfos.Builder();
    FieldInfo fi = builder.getOrAdd("field");
    fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
    addAttributes(fi);
    FieldInfos infos = builder.finish();
    codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
    fail.setDoFail();
    expectThrows(FakeIOException.class, () -> {
        codec.fieldInfosFormat().read(dir, segmentInfo, "", IOContext.DEFAULT);
    });
    fail.clearDoFail();
    dir.close();
}