Examples with Lucene70Codec org.apache.lucene.codecs.lucene70.Lucene70Codec used on opensource projects

Example 1 with Lucene70Codec

use of org.apache.lucene.codecs.lucene70.Lucene70Codec in project lucene-solr by apache.

the class TestSuggestField method iwcWithSuggestField.

static IndexWriterConfig iwcWithSuggestField(Analyzer analyzer, final Set<String> suggestFields) {
    IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
    iwc.setMergePolicy(newLogMergePolicy());
    Codec filterCodec = new Lucene70Codec() {

        PostingsFormat postingsFormat = new Completion50PostingsFormat();

        @Override
        public PostingsFormat getPostingsFormatForField(String field) {
            if (suggestFields.contains(field)) {
                return postingsFormat;
            }
            return super.getPostingsFormatForField(field);
        }
    };
    iwc.setCodec(filterCodec);
    return iwc;
}

Example 2 with Lucene70Codec

use of org.apache.lucene.codecs.lucene70.Lucene70Codec in project lucene-solr by apache.

the class SchemaCodecFactory method init.

@Override
public void init(NamedList args) {
    super.init(args);
    assert codec == null;
    String compressionModeStr = (String) args.get(COMPRESSION_MODE);
    Mode compressionMode;
    if (compressionModeStr != null) {
        try {
            compressionMode = Mode.valueOf(compressionModeStr.toUpperCase(Locale.ROOT));
        } catch (IllegalArgumentException e) {
            throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid compressionMode: '" + compressionModeStr + "'. Value must be one of " + Arrays.toString(Mode.values()));
        }
        log.debug("Using compressionMode: " + compressionMode);
    } else {
        compressionMode = SOLR_DEFAULT_COMPRESSION_MODE;
        log.debug("Using default compressionMode: " + compressionMode);
    }
    codec = new Lucene70Codec(compressionMode) {

        @Override
        public PostingsFormat getPostingsFormatForField(String field) {
            final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field);
            if (schemaField != null) {
                String postingsFormatName = schemaField.getType().getPostingsFormat();
                if (postingsFormatName != null) {
                    return PostingsFormat.forName(postingsFormatName);
                }
            }
            return super.getPostingsFormatForField(field);
        }

        @Override
        public DocValuesFormat getDocValuesFormatForField(String field) {
            final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field);
            if (schemaField != null) {
                String docValuesFormatName = schemaField.getType().getDocValuesFormat();
                if (docValuesFormatName != null) {
                    return DocValuesFormat.forName(docValuesFormatName);
                }
            }
            return super.getDocValuesFormatForField(field);
        }
    };
}

Example 3 with Lucene70Codec

use of org.apache.lucene.codecs.lucene70.Lucene70Codec in project lucene-solr by apache.

the class TestLucene50StoredFieldsFormatHighCompression method testMixedCompressions.

/**
   * Change compression params (leaving it the same for old segments)
   * and tests that nothing breaks.
   */
public void testMixedCompressions() throws Exception {
    Directory dir = newDirectory();
    for (int i = 0; i < 10; i++) {
        IndexWriterConfig iwc = newIndexWriterConfig();
        iwc.setCodec(new Lucene70Codec(RandomPicks.randomFrom(random(), Mode.values())));
        IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
        Document doc = new Document();
        doc.add(new StoredField("field1", "value1"));
        doc.add(new StoredField("field2", "value2"));
        iw.addDocument(doc);
        if (random().nextInt(4) == 0) {
            iw.forceMerge(1);
        }
        iw.commit();
        iw.close();
    }
    DirectoryReader ir = DirectoryReader.open(dir);
    assertEquals(10, ir.numDocs());
    for (int i = 0; i < 10; i++) {
        Document doc = ir.document(i);
        assertEquals("value1", doc.get("field1"));
        assertEquals("value2", doc.get("field2"));
    }
    ir.close();
    // checkindex
    dir.close();
}

Example 4 with Lucene70Codec

use of org.apache.lucene.codecs.lucene70.Lucene70Codec in project lucene-solr by apache.

the class CreateIndexTask method createWriterConfig.

public static IndexWriterConfig createWriterConfig(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) {
    @SuppressWarnings("deprecation") IndexWriterConfig iwConf = new IndexWriterConfig(runData.getAnalyzer());
    iwConf.setOpenMode(mode);
    IndexDeletionPolicy indexDeletionPolicy = getIndexDeletionPolicy(config);
    iwConf.setIndexDeletionPolicy(indexDeletionPolicy);
    if (commit != null) {
        iwConf.setIndexCommit(commit);
    }
    final String mergeScheduler = config.get("merge.scheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
    if (mergeScheduler.equals(NoMergeScheduler.class.getName())) {
        iwConf.setMergeScheduler(NoMergeScheduler.INSTANCE);
    } else {
        try {
            iwConf.setMergeScheduler(Class.forName(mergeScheduler).asSubclass(MergeScheduler.class).newInstance());
        } catch (Exception e) {
            throw new RuntimeException("unable to instantiate class '" + mergeScheduler + "' as merge scheduler", e);
        }
        if (mergeScheduler.equals("org.apache.lucene.index.ConcurrentMergeScheduler")) {
            ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) iwConf.getMergeScheduler();
            int maxThreadCount = config.get("concurrent.merge.scheduler.max.thread.count", ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS);
            int maxMergeCount = config.get("concurrent.merge.scheduler.max.merge.count", ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS);
            cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
        }
    }
    final String defaultCodec = config.get("default.codec", null);
    if (defaultCodec != null) {
        try {
            Class<? extends Codec> clazz = Class.forName(defaultCodec).asSubclass(Codec.class);
            iwConf.setCodec(clazz.newInstance());
        } catch (Exception e) {
            throw new RuntimeException("Couldn't instantiate Codec: " + defaultCodec, e);
        }
    }
    final String postingsFormat = config.get("codec.postingsFormat", null);
    if (defaultCodec == null && postingsFormat != null) {
        try {
            final PostingsFormat postingsFormatChosen = PostingsFormat.forName(postingsFormat);
            iwConf.setCodec(new Lucene70Codec() {

                @Override
                public PostingsFormat getPostingsFormatForField(String field) {
                    return postingsFormatChosen;
                }
            });
        } catch (Exception e) {
            throw new RuntimeException("Couldn't instantiate Postings Format: " + postingsFormat, e);
        }
    }
    final String mergePolicy = config.get("merge.policy", "org.apache.lucene.index.LogByteSizeMergePolicy");
    boolean isCompound = config.get("compound", true);
    iwConf.setUseCompoundFile(isCompound);
    if (mergePolicy.equals(NoMergePolicy.class.getName())) {
        iwConf.setMergePolicy(NoMergePolicy.INSTANCE);
    } else {
        try {
            iwConf.setMergePolicy(Class.forName(mergePolicy).asSubclass(MergePolicy.class).newInstance());
        } catch (Exception e) {
            throw new RuntimeException("unable to instantiate class '" + mergePolicy + "' as merge policy", e);
        }
        iwConf.getMergePolicy().setNoCFSRatio(isCompound ? 1.0 : 0.0);
        if (iwConf.getMergePolicy() instanceof LogMergePolicy) {
            LogMergePolicy logMergePolicy = (LogMergePolicy) iwConf.getMergePolicy();
            logMergePolicy.setMergeFactor(config.get("merge.factor", OpenIndexTask.DEFAULT_MERGE_PFACTOR));
        }
    }
    final double ramBuffer = config.get("ram.flush.mb", OpenIndexTask.DEFAULT_RAM_FLUSH_MB);
    final int maxBuffered = config.get("max.buffered", OpenIndexTask.DEFAULT_MAX_BUFFERED);
    if (maxBuffered == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
        iwConf.setRAMBufferSizeMB(ramBuffer);
        iwConf.setMaxBufferedDocs(maxBuffered);
    } else {
        iwConf.setMaxBufferedDocs(maxBuffered);
        iwConf.setRAMBufferSizeMB(ramBuffer);
    }
    return iwConf;
}

Example 5 with Lucene70Codec

use of org.apache.lucene.codecs.lucene70.Lucene70Codec in project lucene-solr by apache.

the class TestRuleSetupAndRestoreClassEnv method before.

@Override
protected void before() throws Exception {
    // we do this in beforeClass, because some tests currently disable it
    if (System.getProperty("solr.directoryFactory") == null) {
        System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory");
    }
    // if verbose: print some debugging stuff about which codecs are loaded.
    if (VERBOSE) {
        System.out.println("Loaded codecs: " + Codec.availableCodecs());
        System.out.println("Loaded postingsFormats: " + PostingsFormat.availablePostingsFormats());
    }
    savedInfoStream = InfoStream.getDefault();
    final Random random = RandomizedContext.current().getRandom();
    final boolean v = random.nextBoolean();
    if (INFOSTREAM) {
        InfoStream.setDefault(new ThreadNameFixingPrintStreamInfoStream(System.out));
    } else if (v) {
        InfoStream.setDefault(new NullInfoStream());
    }
    Class<?> targetClass = RandomizedContext.current().getTargetClass();
    avoidCodecs = new HashSet<>();
    if (targetClass.isAnnotationPresent(SuppressCodecs.class)) {
        SuppressCodecs a = targetClass.getAnnotation(SuppressCodecs.class);
        avoidCodecs.addAll(Arrays.asList(a.value()));
    }
    savedCodec = Codec.getDefault();
    int randomVal = random.nextInt(11);
    if ("default".equals(TEST_CODEC)) {
        // just use the default, don't randomize
        codec = savedCodec;
    } else if (("random".equals(TEST_POSTINGSFORMAT) == false) || ("random".equals(TEST_DOCVALUESFORMAT) == false)) {
        // the user wired postings or DV: this is messy
        // refactor into RandomCodec....
        final PostingsFormat format;
        if ("random".equals(TEST_POSTINGSFORMAT)) {
            format = new AssertingPostingsFormat();
        } else if ("MockRandom".equals(TEST_POSTINGSFORMAT)) {
            format = new MockRandomPostingsFormat(new Random(random.nextLong()));
        } else {
            format = PostingsFormat.forName(TEST_POSTINGSFORMAT);
        }
        final DocValuesFormat dvFormat;
        if ("random".equals(TEST_DOCVALUESFORMAT)) {
            dvFormat = new AssertingDocValuesFormat();
        } else {
            dvFormat = DocValuesFormat.forName(TEST_DOCVALUESFORMAT);
        }
        codec = new AssertingCodec() {

            @Override
            public PostingsFormat getPostingsFormatForField(String field) {
                return format;
            }

            @Override
            public DocValuesFormat getDocValuesFormatForField(String field) {
                return dvFormat;
            }

            @Override
            public String toString() {
                return super.toString() + ": " + format.toString() + ", " + dvFormat.toString();
            }
        };
    } else if ("SimpleText".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 9 && LuceneTestCase.rarely(random) && !shouldAvoidCodec("SimpleText"))) {
        codec = new SimpleTextCodec();
    } else if ("CheapBastard".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 8 && !shouldAvoidCodec("CheapBastard") && !shouldAvoidCodec("Lucene41"))) {
        // we also avoid this codec if Lucene41 is avoided, since thats the postings format it uses.
        codec = new CheapBastardCodec();
    } else if ("Asserting".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 7 && !shouldAvoidCodec("Asserting"))) {
        codec = new AssertingCodec();
    } else if ("Compressing".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) {
        codec = CompressingCodec.randomInstance(random);
    } else if ("Lucene70".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene70"))) {
        codec = new Lucene70Codec(RandomPicks.randomFrom(random, Lucene50StoredFieldsFormat.Mode.values()));
    } else if (!"random".equals(TEST_CODEC)) {
        codec = Codec.forName(TEST_CODEC);
    } else if ("random".equals(TEST_POSTINGSFORMAT)) {
        codec = new RandomCodec(random, avoidCodecs);
    } else {
        assert false;
    }
    Codec.setDefault(codec);
    // Initialize locale/ timezone.
    String testLocale = System.getProperty("tests.locale", "random");
    String testTimeZone = System.getProperty("tests.timezone", "random");
    // Always pick a random one for consistency (whether tests.locale was specified or not).
    savedLocale = Locale.getDefault();
    Locale randomLocale = randomLocale(random);
    locale = testLocale.equals("random") ? randomLocale : localeForLanguageTag(testLocale);
    Locale.setDefault(locale);
    savedTimeZone = TimeZone.getDefault();
    TimeZone randomTimeZone = randomTimeZone(random());
    timeZone = testTimeZone.equals("random") ? randomTimeZone : TimeZone.getTimeZone(testTimeZone);
    TimeZone.setDefault(timeZone);
    similarity = new RandomSimilarity(random());
    // Check codec restrictions once at class level.
    try {
        checkCodecRestrictions(codec);
    } catch (AssumptionViolatedException e) {
        System.err.println("NOTE: " + e.getMessage() + " Suppressed codecs: " + Arrays.toString(avoidCodecs.toArray()));
        throw e;
    }
    // We have "stickiness" so that sometimes all we do is vary the RAM buffer size, other times just the doc count to flush by, else both.
    // This way the assertMemory in DocumentsWriterFlushControl sometimes runs (when we always flush by RAM).
    LiveIWCFlushMode flushMode;
    switch(random().nextInt(3)) {
        case 0:
            flushMode = LiveIWCFlushMode.BY_RAM;
            break;
        case 1:
            flushMode = LiveIWCFlushMode.BY_DOCS;
            break;
        case 2:
            flushMode = LiveIWCFlushMode.EITHER;
            break;
        default:
            throw new AssertionError();
    }
    LuceneTestCase.setLiveIWCFlushMode(flushMode);
    initialized = true;
}