Examples with Codec org.apache.lucene.codecs.Codec used on opensource projects

Example 1 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class TestSuggestField method iwcWithSuggestField.

static IndexWriterConfig iwcWithSuggestField(Analyzer analyzer, final Set<String> suggestFields) {
    IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
    iwc.setMergePolicy(newLogMergePolicy());
    Codec filterCodec = new Lucene70Codec() {

        PostingsFormat postingsFormat = new Completion50PostingsFormat();

        @Override
        public PostingsFormat getPostingsFormatForField(String field) {
            if (suggestFields.contains(field)) {
                return postingsFormat;
            }
            return super.getPostingsFormatForField(field);
        }
    };
    iwc.setCodec(filterCodec);
    return iwc;
}

Example 2 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class TestNamedSPILoader method testLookup.

public void testLookup() {
    String currentName = TestUtil.getDefaultCodec().getName();
    Codec codec = Codec.forName(currentName);
    assertEquals(currentName, codec.getName());
}

Example 3 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseSegmentInfoFormatTestCase method testExceptionOnOpenInput.

/** 
   * Test segment infos read that hits exception immediately on open.
   * make sure we get our exception back, no file handle leaks, etc. 
   */
public void testExceptionOnOpenInput() throws Exception {
    Failure fail = new Failure() {

        @Override
        public void eval(MockDirectoryWrapper dir) throws IOException {
            for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
                if (doFail && "openInput".equals(e.getMethodName())) {
                    throw new FakeIOException();
                }
            }
        }
    };
    MockDirectoryWrapper dir = newMockDirectory();
    dir.failOn(fail);
    Codec codec = getCodec();
    byte[] id = StringHelper.randomId();
    SegmentInfo info = new SegmentInfo(dir, getVersions()[0], getVersions()[0], "_123", 1, false, codec, Collections.<String, String>emptyMap(), id, new HashMap<>(), null);
    info.setFiles(Collections.<String>emptySet());
    codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
    fail.setDoFail();
    expectThrows(FakeIOException.class, () -> {
        codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
    });
    fail.clearDoFail();
    dir.close();
}

Example 4 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseStoredFieldsFormatTestCase method testWriteReadMerge.

public void testWriteReadMerge() throws IOException {
    // get another codec, other than the default: so we are merging segments across different codecs
    final Codec otherCodec;
    if ("SimpleText".equals(Codec.getDefault().getName())) {
        otherCodec = TestUtil.getDefaultCodec();
    } else {
        otherCodec = new SimpleTextCodec();
    }
    Directory dir = newDirectory();
    IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
    final int docCount = atLeast(200);
    final byte[][][] data = new byte[docCount][][];
    for (int i = 0; i < docCount; ++i) {
        final int fieldCount = rarely() ? RandomNumbers.randomIntBetween(random(), 1, 500) : RandomNumbers.randomIntBetween(random(), 1, 5);
        data[i] = new byte[fieldCount][];
        for (int j = 0; j < fieldCount; ++j) {
            final int length = rarely() ? random().nextInt(1000) : random().nextInt(10);
            final int max = rarely() ? 256 : 2;
            data[i][j] = randomByteArray(length, max);
        }
    }
    final FieldType type = new FieldType(StringField.TYPE_STORED);
    type.setIndexOptions(IndexOptions.NONE);
    type.freeze();
    IntPoint id = new IntPoint("id", 0);
    StoredField idStored = new StoredField("id", 0);
    for (int i = 0; i < data.length; ++i) {
        Document doc = new Document();
        doc.add(id);
        doc.add(idStored);
        id.setIntValue(i);
        idStored.setIntValue(i);
        for (int j = 0; j < data[i].length; ++j) {
            Field f = new Field("bytes" + j, data[i][j], type);
            doc.add(f);
        }
        iw.w.addDocument(doc);
        if (random().nextBoolean() && (i % (data.length / 10) == 0)) {
            iw.w.close();
            IndexWriterConfig iwConfNew = newIndexWriterConfig(new MockAnalyzer(random()));
            // test merging against a non-compressing codec
            if (iwConf.getCodec() == otherCodec) {
                iwConfNew.setCodec(Codec.getDefault());
            } else {
                iwConfNew.setCodec(otherCodec);
            }
            iwConf = iwConfNew;
            iw = new RandomIndexWriter(random(), dir, iwConf);
        }
    }
    for (int i = 0; i < 10; ++i) {
        final int min = random().nextInt(data.length);
        final int max = min + random().nextInt(20);
        iw.deleteDocuments(IntPoint.newRangeQuery("id", min, max - 1));
    }
    // force merges with deletions
    iw.forceMerge(2);
    iw.commit();
    final DirectoryReader ir = DirectoryReader.open(dir);
    assertTrue(ir.numDocs() > 0);
    int numDocs = 0;
    for (int i = 0; i < ir.maxDoc(); ++i) {
        final Document doc = ir.document(i);
        if (doc == null) {
            continue;
        }
        ++numDocs;
        final int docId = doc.getField("id").numericValue().intValue();
        assertEquals(data[docId].length + 1, doc.getFields().size());
        for (int j = 0; j < data[docId].length; ++j) {
            final byte[] arr = data[docId][j];
            final BytesRef arr2Ref = doc.getBinaryValue("bytes" + j);
            final byte[] arr2 = Arrays.copyOfRange(arr2Ref.bytes, arr2Ref.offset, arr2Ref.offset + arr2Ref.length);
            assertArrayEquals(arr, arr2);
        }
    }
    assertTrue(ir.numDocs() <= numDocs);
    ir.close();
    iw.deleteAll();
    iw.commit();
    iw.forceMerge(1);
    iw.close();
    dir.close();
}

Example 5 with Codec

use of org.apache.lucene.codecs.Codec in project lucene-solr by apache.

the class BaseSegmentInfoFormatTestCase method testExceptionOnCloseInput.

/** 
   * Test segment infos read that hits exception on close
   * make sure we get our exception back, no file handle leaks, etc. 
   */
public void testExceptionOnCloseInput() throws Exception {
    Failure fail = new Failure() {

        @Override
        public void eval(MockDirectoryWrapper dir) throws IOException {
            for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
                if (doFail && "close".equals(e.getMethodName())) {
                    throw new FakeIOException();
                }
            }
        }
    };
    MockDirectoryWrapper dir = newMockDirectory();
    dir.failOn(fail);
    Codec codec = getCodec();
    byte[] id = StringHelper.randomId();
    SegmentInfo info = new SegmentInfo(dir, getVersions()[0], getVersions()[0], "_123", 1, false, codec, Collections.<String, String>emptyMap(), id, new HashMap<>(), null);
    info.setFiles(Collections.<String>emptySet());
    codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
    fail.setDoFail();
    expectThrows(FakeIOException.class, () -> {
        codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
    });
    fail.clearDoFail();
    dir.close();
}