Examples with Indexer org.opennms.newts.api.search.Indexer used on opensource projects

Example 1 with Indexer

use of org.opennms.newts.api.search.Indexer in project newts by OpenNMS.

the class CassandraIndexerITCase method testDelete.

@Test
public void testDelete() {
    ResourceMetadataCache cache = mock(ResourceMetadataCache.class);
    when(cache.get(any(Context.class), any(Resource.class))).thenReturn(Optional.<ResourceMetadata>absent());
    MetricRegistry registry = new MetricRegistry();
    ContextConfigurations contextConfigurations = new ContextConfigurations();
    CassandraSession session = newtsInstance.getCassandraSession();
    CassandraIndexingOptions options = new CassandraIndexingOptions.Builder().withHierarchicalIndexing(false).build();
    Indexer indexer = new CassandraIndexer(session, 86400, cache, registry, options, new SimpleResourceIdSplitter(), contextConfigurations);
    CassandraSearcher searcher = new CassandraSearcher(session, registry, contextConfigurations);
    Map<String, String> base = map("meat", "people", "bread", "beer");
    List<Sample> samples = Lists.newArrayList();
    samples.add(sampleFor(new Resource("aaa", Optional.of(base)), "m0"));
    samples.add(sampleFor(new Resource("aab", Optional.of(map(base, "music", "metal", "beverage", "beer"))), "m0"));
    samples.add(sampleFor(new Resource("aac:aaa", Optional.of(map(base, "music", "country"))), "m0"));
    indexer.update(samples);
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("aaa")).size(), equalTo(2));
    indexer.delete(Context.DEFAULT_CONTEXT, new Resource("aaa", Optional.of(base)));
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("aaa")).size(), equalTo(1));
    indexer.delete(Context.DEFAULT_CONTEXT, new Resource("aaa", Optional.of(base)));
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("aaa")).size(), equalTo(1));
    indexer.delete(Context.DEFAULT_CONTEXT, new Resource("aac:aaa", Optional.of(base)));
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("aaa")).size(), equalTo(0));
}

Example 2 with Indexer

use of org.opennms.newts.api.search.Indexer in project newts by OpenNMS.

the class CassandraIndexerITCase method testCacheExpiring.

@Test
public void testCacheExpiring() throws Exception {
    final MetricRegistry registry = new MetricRegistry();
    final ResourceMetadataCache cache = new GuavaResourceMetadataCache(100, registry);
    final ContextConfigurations contextConfigurations = new ContextConfigurations();
    final CassandraSession session = newtsInstance.getCassandraSession();
    final CassandraIndexingOptions options = new CassandraIndexingOptions.Builder().withHierarchicalIndexing(false).build();
    final Indexer indexer = new CassandraIndexer(session, 10, cache, registry, options, new SimpleResourceIdSplitter(), contextConfigurations);
    final Resource resource = new Resource("aaa", Optional.of(map("beverage", "beer")));
    indexer.update(Collections.singletonList(sampleFor(resource, "m0")));
    assertThat(session.execute("SELECT count(*) as count FROM terms").one().getLong("count"), is(3L));
    // Let the TTL expire
    Thread.sleep(15_000);
    assertThat(session.execute("SELECT count(*) as count FROM terms").one().getLong("count"), is(0L));
    // Reindex
    indexer.update(Collections.singletonList(sampleFor(resource, "m0")));
    assertThat(session.execute("SELECT count(*) as count FROM terms").one().getLong("count"), is(3L));
    // Reindex before TTL * 3 / 4
    Thread.sleep(8_000);
    indexer.update(Collections.singletonList(sampleFor(resource, "m0")));
    // Let initial TTL pass - 12s after reindex
    Thread.sleep(4_000);
    assertThat(session.execute("SELECT count(*) as count FROM terms").one().getLong("count"), is(3L));
    // Let remaining TTL pass - 20s after reindex, 12s after last update
    Thread.sleep(8_000);
    assertThat(session.execute("SELECT count(*) as count FROM terms").one().getLong("count"), is(0L));
}

Example 3 with Indexer

use of org.opennms.newts.api.search.Indexer in project newts by OpenNMS.

the class CassandraGuiceModule method configure.

@Override
protected void configure() {
    bind(String.class).annotatedWith(named("cassandra.keyspace")).toInstance(m_newtsConf.getCassandraKeyspace());
    bind(String.class).annotatedWith(named("cassandra.hostname")).toInstance(m_newtsConf.getCassandraHost());
    bind(Integer.class).annotatedWith(named("cassandra.port")).toInstance(m_newtsConf.getCassandraPort());
    bind(String.class).annotatedWith(named("cassandra.compression")).toInstance(m_newtsConf.getCassandraCompression());
    bind(String.class).annotatedWith(named("cassandra.username")).toInstance(m_newtsConf.getCassandraUsername());
    bind(String.class).annotatedWith(named("cassandra.password")).toInstance(m_newtsConf.getCassandraPassword());
    bind(Boolean.class).annotatedWith(named("cassandra.ssl")).toInstance(m_newtsConf.getCassandraSsl());
    bind(Integer.class).annotatedWith(named("cassandra.pool.core-connections-per-host")).toInstance(m_newtsConf.getCoreConnectionsPerHost());
    bind(Integer.class).annotatedWith(named("cassandra.pool.max-connections-per-host")).toInstance(m_newtsConf.getMaxConnectionsPerHost());
    bind(Integer.class).annotatedWith(named("cassandra.pool.max-requests-per-connection")).toInstance(m_newtsConf.getMaxRequestsPerConnection());
    bind(Integer.class).annotatedWith(named("samples.cassandra.time-to-live")).toInstance(m_newtsConf.getCassandraColumnTTL());
    bind(Integer.class).annotatedWith(named("search.cassandra.time-to-live")).toInstance(m_newtsConf.getCassandraColumnTTL());
    bind(Integer.class).annotatedWith(named("sampleProcessor.maxThreads")).toInstance(m_newtsConf.getMaxSampleProcessorThreads());
    bind(Long.class).annotatedWith(named("search.resourceMetadata.maxCacheEntries")).toInstance(m_newtsConf.getSearchConfig().getMaxCacheEntries());
    bind(Boolean.class).annotatedWith(named("search.hierarical-indexing")).toInstance(m_newtsConf.getSearchConfig().isHierarchicalIndexingEnabled());
    bind(CassandraSession.class).to(CassandraSessionImpl.class);
    bind(ResourceMetadataCache.class).to(GuavaResourceMetadataCache.class);
    bind(Searcher.class).to(CassandraSearcher.class);
    bind(SampleRepository.class).to(CassandraSampleRepository.class);
    bind(Indexer.class).to(CassandraIndexer.class);
    Multibinder<SampleProcessor> processors = Multibinder.newSetBinder(binder(), SampleProcessor.class);
    if (m_newtsConf.getSearchConfig().isSeparatorEscapingEnabled()) {
        bind(ResourceIdSplitter.class).to(EscapableResourceIdSplitter.class);
    } else {
        bind(ResourceIdSplitter.class).to(SimpleResourceIdSplitter.class);
    }
    // Only add the search indexer if search is enabled
    if (m_newtsConf.getSearchConfig().isEnabled()) {
        processors.addBinding().to(CassandraIndexerSampleProcessor.class);
    }
    // Pull in context specific attributes
    ContextConfigurations contextConfigurations = new ContextConfigurations();
    for (ContextConfig contextConfig : m_newtsConf.getContextConfigs().values()) {
        contextConfigurations.addContextConfig(contextConfig.getContext(), contextConfig.getResourceShard(), contextConfig.getReadConsistency(), contextConfig.getWriteConsistency());
    }
    bind(ContextConfigurations.class).toInstance(contextConfigurations);
}

Example 4 with Indexer

use of org.opennms.newts.api.search.Indexer in project newts by OpenNMS.

the class NewtsService method run.

@Override
public void run(final NewtsConfig config, Environment environment) throws Exception {
    // Filters
    configureCors(environment);
    configureUIRedirect(environment);
    configureAuthentication(environment, config);
    final Injector injector = Guice.createInjector(new NewtsGuiceModule(), new CassandraGuiceModule(config), new GraphiteGuiceModule(config));
    MetricRegistry metricRegistry = injector.getInstance(MetricRegistry.class);
    // Create/start a JMX reporter for our MetricRegistry
    final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry).inDomain("newts").build();
    // Create (and start if so configured), a Graphite line-protocol listener
    final GraphiteListenerThread listener = new GraphiteListenerThread(injector.getInstance(GraphiteListener.class));
    environment.lifecycle().manage(new Managed() {

        @Override
        public void stop() throws Exception {
            reporter.stop();
        }

        @Override
        public void start() throws Exception {
            reporter.start();
            if (config.getGraphiteConfig().isEnabled()) {
                listener.start();
            }
        }
    });
    SampleRepository repository = injector.getInstance(SampleRepository.class);
    Indexer indexer = injector.getInstance(Indexer.class);
    // Rest resources
    environment.jersey().register(new MeasurementsResource(repository, config.getReports()));
    environment.jersey().register(new SamplesResource(repository, indexer));
    // Add search resource only if search is enabled
    if (config.getSearchConfig().isEnabled()) {
        environment.jersey().register(new SearchResource(injector.getInstance(Searcher.class)));
    }
    // Health checks
    environment.healthChecks().register("repository", new RepositoryHealthCheck(repository));
    // Mapped exceptions
    environment.jersey().register(IllegalArgumentExceptionMapper.class);
}

Example 5 with Indexer

use of org.opennms.newts.api.search.Indexer in project newts by OpenNMS.

the class CassandraIndexerITCase method test.

@Test
public void test() {
    Map<String, String> base = map("meat", "people", "bread", "beer");
    List<Sample> samples = Lists.newArrayList();
    samples.add(sampleFor(new Resource("aaa", Optional.of(base)), "m0"));
    samples.add(sampleFor(new Resource("aab", Optional.of(map(base, "music", "metal", "beverage", "beer"))), "m0"));
    samples.add(sampleFor(new Resource("aac:aaa", Optional.of(map(base, "music", "country"))), "m0"));
    CassandraSession session = newtsInstance.getCassandraSession();
    ResourceMetadataCache mockCache = mock(ResourceMetadataCache.class);
    when(mockCache.get(any(Context.class), any(Resource.class))).thenReturn(Optional.<ResourceMetadata>absent());
    MetricRegistry registry = new MetricRegistry();
    ContextConfigurations contextConfigurations = new ContextConfigurations();
    CassandraIndexingOptions options = new CassandraIndexingOptions.Builder().withHierarchicalIndexing(false).build();
    Indexer indexer = new CassandraIndexer(session, 86400, mockCache, registry, options, new SimpleResourceIdSplitter(), contextConfigurations);
    indexer.update(samples);
    CassandraSearcher searcher = new CassandraSearcher(session, registry, contextConfigurations);
    // Match path components
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("aaa")).size(), equalTo(2));
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("aac")).size(), equalTo(1));
    // Match attribute values
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("people")).size(), equalTo(3));
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("metal")).size(), equalTo(1));
    // Match attribute key + value pairs
    BooleanQuery query = new BooleanQuery();
    query.add(new TermQuery(new Term("beverage", "beer")), Operator.OR);
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, query).size(), equalTo(1));
    // Or'd terms
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("metal", "country")).size(), equalTo(2));
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("beer", "wine")).size(), equalTo(3));
    // And'd terms
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAllValues("metal", "country")).size(), equalTo(0));
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAllValues("aaa", "aac")).size(), equalTo(1));
    // Groups queries
    // (beer AND metal) OR (aaa AND country)
    BooleanQuery subquery1 = new BooleanQuery();
    subquery1.add(new TermQuery(new Term("beer")), Operator.OR);
    subquery1.add(new TermQuery(new Term("metal")), Operator.AND);
    BooleanQuery subquery2 = new BooleanQuery();
    subquery2.add(new TermQuery(new Term("aaa")), Operator.OR);
    subquery2.add(new TermQuery(new Term("country")), Operator.AND);
    query = new BooleanQuery();
    query.add(subquery1, Operator.OR);
    query.add(subquery2, Operator.OR);
    assertThat(searcher.search(Context.DEFAULT_CONTEXT, query).size(), equalTo(2));
    // Attributes are retrieved
    Result r = searcher.search(Context.DEFAULT_CONTEXT, QueryBuilder.matchAnyValue("metal")).iterator().next();
    assertThat(r.getResource().getId(), is(equalTo("aab")));
    assertThat(r.getResource().getAttributes().isPresent(), is(true));
    assertThat(r.getResource().getAttributes().get(), equalTo(map(base, "music", "metal", "beverage", "beer")));
    // Metrics too
    assertThat(r.getMetrics().size(), equalTo(1));
    assertThat(r.getMetrics().iterator().next(), equalTo("m0"));
}