Examples with AnalyzerSettings io.zulia.message.ZuliaIndex.AnalyzerSettings used on opensource projects

Example 1 with AnalyzerSettings

use of io.zulia.message.ZuliaIndex.AnalyzerSettings in project zuliasearch by zuliaio.

the class ServerIndexConfig method configure.

public void configure(IndexSettings indexSettings) {
    this.indexSettings = indexSettings;
    this.analyzerMap = new ConcurrentHashMap<>();
    analyzerMap.put(DefaultAnalyzers.STANDARD, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.STOPWORDS).build());
    analyzerMap.put(DefaultAnalyzers.KEYWORD, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.KEYWORD).setTokenizer(Tokenizer.KEYWORD).build());
    analyzerMap.put(DefaultAnalyzers.LC_KEYWORD, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.LC_KEYWORD).setTokenizer(Tokenizer.KEYWORD).addFilter(Filter.LOWERCASE).build());
    analyzerMap.put(DefaultAnalyzers.MIN_STEM, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.MIN_STEM).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.STOPWORDS).addFilter(Filter.ENGLISH_MIN_STEM).build());
    analyzerMap.put(DefaultAnalyzers.TWO_TWO_SHINGLE, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.TWO_TWO_SHINGLE).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.TWO_TWO_SHINGLE).build());
    analyzerMap.put(DefaultAnalyzers.THREE_THREE_SHINGLE, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.THREE_THREE_SHINGLE).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.THREE_THREE_SHINGLE).build());
    analyzerMap.put(DefaultAnalyzers.LC_CONCAT_ALL, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.LC_CONCAT_ALL).setTokenizer(Tokenizer.KEYWORD).addFilter(Filter.LOWERCASE).addFilter(Filter.CONCAT_ALL).build());
    analyzerMap.put(DefaultAnalyzers.KSTEMMED, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.KSTEMMED).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.STOPWORDS).addFilter(Filter.KSTEM).build());
    analyzerMap.put(DefaultAnalyzers.LSH, AnalyzerSettings.newBuilder().setName(DefaultAnalyzers.LSH).setTokenizer(Tokenizer.STANDARD).addFilter(Filter.LOWERCASE).addFilter(Filter.ASCII_FOLDING).addFilter(Filter.KSTEM).addFilter(Filter.STOPWORDS).addFilter(Filter.FIVE_FIVE_SHINGLE).addFilter(Filter.MINHASH).build());
    for (AnalyzerSettings analyzerSettings : indexSettings.getAnalyzerSettingsList()) {
        analyzerMap.put(analyzerSettings.getName(), analyzerSettings);
    }
    this.fieldConfigMap = new ConcurrentHashMap<>();
    for (FieldConfig fc : indexSettings.getFieldConfigList()) {
        fieldConfigMap.put(fc.getStoredFieldName(), fc);
    }
    this.indexAsMap = new ConcurrentHashMap<>();
    this.indexToStoredMap = new ConcurrentHashMap<>();
    for (String storedFieldName : fieldConfigMap.keySet()) {
        FieldConfig fc = fieldConfigMap.get(storedFieldName);
        for (IndexAs indexAs : fc.getIndexAsList()) {
            indexAsMap.put(indexAs.getIndexFieldName(), indexAs);
            indexToStoredMap.put(indexAs.getIndexFieldName(), storedFieldName);
        }
    }
    this.facetAsMap = new ConcurrentHashMap<>();
    for (String storedFieldName : fieldConfigMap.keySet()) {
        FieldConfig fc = fieldConfigMap.get(storedFieldName);
        for (FacetAs facetAs : fc.getFacetAsList()) {
            facetAsMap.put(facetAs.getFacetName(), facetAs);
        }
    }
    this.indexFieldType = new ConcurrentHashMap<>();
    for (String storedFieldName : fieldConfigMap.keySet()) {
        FieldConfig fc = fieldConfigMap.get(storedFieldName);
        for (IndexAs indexAs : fc.getIndexAsList()) {
            indexFieldType.put(indexAs.getIndexFieldName(), fc.getFieldType());
        }
    }
    this.sortFieldType = new ConcurrentHashMap<>();
    for (String storedFieldName : fieldConfigMap.keySet()) {
        FieldConfig fc = fieldConfigMap.get(storedFieldName);
        for (SortAs sortAs : fc.getSortAsList()) {
            sortFieldType.put(sortAs.getSortFieldName(), fc.getFieldType());
        }
    }
    this.superbitConfigMap = new ConcurrentHashMap<>();
    this.superbitMap = new ConcurrentHashMap<>();
    for (String storedFieldName : fieldConfigMap.keySet()) {
        FieldConfig fc = fieldConfigMap.get(storedFieldName);
        for (ProjectAs projectAs : fc.getProjectAsList()) {
            String field = projectAs.getField();
            if (projectAs.hasSuperbit()) {
                Superbit superbit = projectAs.getSuperbit();
                superbitConfigMap.put(field, superbit);
                SuperBit superBit = new SuperBit(superbit.getInputDim(), superbit.getInputDim(), superbit.getBatches(), superbit.getSeed());
                superbitMap.put(field, superBit);
            }
        }
    }
}

Example 2 with AnalyzerSettings

use of io.zulia.message.ZuliaIndex.AnalyzerSettings in project zuliasearch by zuliaio.

the class ClientIndexConfig method configure.

public void configure(IndexSettings indexSettings) {
    this.indexName = indexSettings.getIndexName();
    this.numberOfShards = indexSettings.getNumberOfShards();
    this.numberOfReplicas = indexSettings.getNumberOfReplicas();
    this.defaultSearchFields = indexSettings.getDefaultSearchFieldList();
    this.analyzerSettingsMap = new TreeMap<>();
    for (AnalyzerSettings analyzerSettings : indexSettings.getAnalyzerSettingsList()) {
        analyzerSettingsMap.put(analyzerSettings.getName(), analyzerSettings);
    }
    this.fieldMap = new TreeMap<>();
    for (FieldConfig fc : indexSettings.getFieldConfigList()) {
        fieldMap.put(fc.getStoredFieldName(), fc);
    }
    this.requestFactor = indexSettings.getRequestFactor();
    this.minShardRequest = indexSettings.getMinShardRequest();
    this.shardTolerance = indexSettings.getShardTolerance();
    this.shardQueryCacheSize = indexSettings.getShardQueryCacheSize();
    this.shardQueryCacheMaxAmount = indexSettings.getShardQueryCacheMaxAmount();
    this.idleTimeWithoutCommit = indexSettings.getIdleTimeWithoutCommit();
    this.shardCommitInterval = indexSettings.getShardCommitInterval();
    this.indexWeight = indexSettings.getIndexWeight();
    this.ramBufferMB = indexSettings.getRamBufferMB();
}