Examples with SolrYard org.apache.stanbol.entityhub.yard.solr.impl.SolrYard used on opensource projects

Example 1 with SolrYard

use of org.apache.stanbol.entityhub.yard.solr.impl.SolrYard in project stanbol by apache.

the class SolrYardIndexingDestinationTest method validateSolrDestination.

/**
     * Checks if the SolrYardIndexingDestination returned by the 
     * {@link IndexingConfig} is valid and functional
     * @param config the configuration
     * @throws YardException indicates problems while working with the {@link SolrYard}
     * returned by {@link IndexingDestination#getYard()}
     * @throws IOException indicates problems while validating the SolrArchives
     * created by the {@link IndexingDestination#finalise()} method
     */
private void validateSolrDestination(IndexingConfig config) throws YardException, IOException {
    //get the destination
    IndexingDestination destination = config.getIndexingDestination();
    assertNotNull(destination);
    assertEquals(destination.getClass(), SolrYardIndexingDestination.class);
    //initialise
    assertTrue(destination.needsInitialisation());
    destination.initialise();
    //test that the returned Yard instance is functional
    Yard yard = destination.getYard();
    assertNotNull(yard);
    assertEquals(yard.getClass(), SolrYard.class);
    Representation rep = yard.getValueFactory().createRepresentation("http://www.example.com/entity#123");
    rep.add(NamespaceEnum.rdfs + "label", "test");
    rep.add(NamespaceEnum.rdfs + "description", "Representation to test storage while indexing");
    rep.add(RdfResourceEnum.entityRank.getUri(), Float.valueOf(0.8f));
    yard.store(rep);
    //finalise
    destination.finalise();
    //test the archives
    File expectedSolrArchiveFile = new File(config.getDistributionFolder(), config.getName() + ".solrindex.zip");
    assertTrue(expectedSolrArchiveFile.isFile());
    // validate the archive
    ZipFile archive = new ZipFile(expectedSolrArchiveFile);
    Set<String> expected = new HashSet<String>(EXPECTED_INDEX_ARCHIVE_FILE_NAMES);
    for (Enumeration<? extends ZipEntry> entries = archive.entries(); entries.hasMoreElements(); ) {
        ZipEntry entry = entries.nextElement();
        //the name of the index MUST be the root folder within the Archive!
        assertTrue(entry.getName().startsWith(config.getName()));
        String name = FilenameUtils.getName(entry.getName());
        if (expected.remove(name)) {
            log.info("found expected Entry '{}'", entry.getName());
        }
        Assert.assertFalse("found unexpected Entry '" + entry.getName() + "' in " + "SolrIndexArchive", UNEXPECTED_INDEX_ARCHIVE_FILE_NAMES.contains(name));
    }
    assertTrue("missing Files in index archive: " + expected, expected.isEmpty());
//TODO: reimplement to validate the created bundle!
//        //check for the solrArchive reference file and validate required properties
//        File expectedSolrArchiveReferenceFile = 
//            new File(,config.getName()+".solrindex.ref");
//        assertTrue(expectedSolrArchiveReferenceFile.isFile());
//        Properties solrRefProperties = new Properties();
//        solrRefProperties.load(new FileInputStream(expectedSolrArchiveReferenceFile));
//        assertTrue(solrRefProperties.getProperty("Index-Archive").equals(expectedSolrArchiveFile.getName()));
//        assertTrue(solrRefProperties.getProperty("Name") != null);
}

Example 2 with SolrYard

use of org.apache.stanbol.entityhub.yard.solr.impl.SolrYard in project stanbol by apache.

the class LDPathTestBase method setup.

@BeforeClass
public static void setup() throws Exception {
    // get the working directory
    // use property substitution to test this feature!
    String prefix = System.getProperty("basedir") == null ? "." : "${basedir}";
    String solrServerDir = prefix + TEST_INDEX_REL_PATH;
    log.info("Test Solr Server Directory: {}", solrServerDir);
    System.setProperty(ManagedSolrServer.MANAGED_SOLR_DIR_PROPERTY, solrServerDir);
    SolrYardConfig config = new SolrYardConfig(TEST_YARD_ID, TEST_SOLR_CORE_NAME);
    config.setAllowInitialisation(false);
    //the dbpedia default data
    config.setIndexConfigurationName(TEST_SOLR_CORE_CONFIGURATION);
    //init from datafile provider
    config.setAllowInitialisation(true);
    config.setName("DBpedia.org default data");
    config.setDescription("Data used for the LDPath setup");
    // create the Yard used for the tests
    IndexReference solrIndexRef = IndexReference.parse(config.getSolrServerLocation());
    SolrServer server = StandaloneEmbeddedSolrServerProvider.getInstance().getSolrServer(solrIndexRef, config.getIndexConfigurationName());
    Assert.assertNotNull("Unable to initialise SolrServer for testing", server);
    yard = new SolrYard(server, config, null);
    backend = new YardBackend(yard);
}

Example 3 with SolrYard

use of org.apache.stanbol.entityhub.yard.solr.impl.SolrYard in project stanbol by apache.

the class FstLinkingEngineTest method setup.

@BeforeClass
public static void setup() throws Exception {
    // get the working directory
    // use property substitution to test this feature!
    String prefix = System.getProperty("basedir") == null ? "." : "${basedir}";
    String solrServerDir = prefix + TEST_INDEX_REL_PATH;
    log.info("Test Solr Server Directory: {}", solrServerDir);
    System.setProperty(ManagedSolrServer.MANAGED_SOLR_DIR_PROPERTY, solrServerDir);
    SolrYardConfig config = new SolrYardConfig(TEST_YARD_ID, TEST_SOLR_CORE_NAME);
    config.setAllowInitialisation(false);
    //the dbpedia default data
    config.setIndexConfigurationName(TEST_SOLR_CORE_CONFIGURATION);
    //init from datafile provider
    config.setAllowInitialisation(true);
    config.setName("DBpedia.org default data");
    config.setDescription("Data used for FstLinkingEngie tests");
    // create the Yard used for the tests
    IndexReference solrIndexRef = IndexReference.parse(config.getSolrServerLocation());
    SolrServer server = StandaloneEmbeddedSolrServerProvider.getInstance().getSolrServer(solrIndexRef, config.getIndexConfigurationName());
    Assert.assertNotNull("Unable to initialise SolrServer for testing", server);
    core = ((EmbeddedSolrServer) server).getCoreContainer().getCore(solrIndexRef.getIndex());
    Assert.assertNotNull("Unable to get SolrCore '" + config.getIndexConfigurationName() + "' from SolrServer " + server, core);
    yard = new SolrYard(server, config, null);
    //setup the index configuration
    LanguageConfiguration langConf = new LanguageConfiguration("not.used", new String[] { "en;field=dbpedia-ont:surfaceForm;generate=true" });
    fstConfig = new IndexConfiguration(langConf, core, FieldEncodingEnum.SolrYard, "");
    fstConfig.setExecutorService(Executors.newFixedThreadPool(1));
    fstConfig.setTypeField("rdf:type");
    fstConfig.setRankingField("entityhub:entityRank");
    //fstConfig.setEntityCacheManager(new FastLRUCacheManager(2048));
    fstConfig.setOrigin(new PlainLiteralImpl(TEST_ORIGIN));
    //activate the FST config
    //activate this configuration
    fstConfig.activate();
    //validate that the index contains the expected entities
    validateTestIndex();
    //now create the FST models
    List<Future<?>> creationTasks = new ArrayList<Future<?>>();
    for (CorpusInfo corpus : fstConfig.getCorpora()) {
        Assert.assertTrue("Failure in UnitTest - all FST models need to be generate=true", corpus.allowCreation);
        if (!corpus.isFstFile()) {
            //create a task on the FST corpus creation service
            creationTasks.add(fstConfig.getExecutorService().submit(new CorpusCreationTask(fstConfig, corpus)));
        }
    }
    //typical hardware
    for (Future<?> future : creationTasks) {
        try {
            future.get(FST_CREATION_WAIT_TIME, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
        // we assert on future.isDone instead
        }
        Assert.assertTrue("FST Model creation not finished after " + FST_CREATION_WAIT_TIME + "seconds", future.isDone());
    }
}

Example 4 with SolrYard

use of org.apache.stanbol.entityhub.yard.solr.impl.SolrYard in project stanbol by apache.

the class SolrYardTest method initYard.

@BeforeClass
public static final void initYard() throws YardException, IOException {
    // get the working directory
    // use property substitution to test this feature!
    String prefix = System.getProperty("basedir") == null ? "." : "${basedir}";
    String solrServerDir = prefix + TEST_INDEX_REL_PATH;
    log.info("Test Solr Server Directory: " + solrServerDir);
    SolrYardConfig config = new SolrYardConfig(TEST_YARD_ID, TEST_SOLR_CORE_NAME);
    config.setName("Solr Yard Test");
    config.setDescription("The Solr Yard instance used to execute the Unit Tests defined for the Yard Interface");
    config.setAllowInitialisation(true);
    //for unit testing we want immidiate commits (required after STANBOL-1092
    // as the default changed to false)
    config.setImmediateCommit(true);
    //init the ManagedSolrServer used for the UnitTest
    System.setProperty(ManagedSolrServer.MANAGED_SOLR_DIR_PROPERTY, solrServerDir);
    IndexReference solrServerRef = IndexReference.parse(config.getSolrServerLocation());
    solrServerProvider = StandaloneEmbeddedSolrServerProvider.getInstance();
    SolrServer server = solrServerProvider.getSolrServer(solrServerRef, config.isAllowInitialisation() ? config.getIndexConfigurationName() : null);
    //Optional support for the nsPrefix service
    final NamespacePrefixService nsPrefixService;
    ServiceLoader<NamespacePrefixService> spsl = ServiceLoader.load(NamespacePrefixService.class);
    Iterator<NamespacePrefixService> it = spsl.iterator();
    if (it.hasNext()) {
        nsPrefixService = it.next();
    } else {
        nsPrefixService = null;
    }
    yard = new SolrYard(server, config, nsPrefixService);
}

Example 5 with SolrYard

use of org.apache.stanbol.entityhub.yard.solr.impl.SolrYard in project stanbol by apache.

the class SolrYardIndexingDestination method initialise.

@Override
public void initialise() {
    log.info("initialise {}", getClass().getSimpleName());
    //The constructors and the setConfiguration(..) only validate the parsed
    //parameters and initialise the member variables. This method performs 
    //the the actual initialisation of the SolrYard!
    //copy a custom configuration (if present)
    EmbeddedSolrServer server;
    IndexReference solrServerRef = IndexReference.parse(solrYardConfig.getSolrServerLocation());
    if (solrIndexConfig != null) {
        //copy the configuration
        try {
            log.info(" ... copy Solr Configuration form {} to {}", solrIndexConfig, solrIndexLocation);
            FileUtils.copyDirectory(solrIndexConfig, solrIndexLocation);
        } catch (IOException e) {
            throw new IllegalStateException(String.format("Unable to copy the Solr index configuration from %s to %s!", solrIndexConfig, solrIndexLocation), e);
        }
        solrYardConfig.setAllowInitialisation(Boolean.FALSE);
        server = StandaloneEmbeddedSolrServerProvider.getInstance().getSolrServer(solrServerRef, solrServerRef.getIndex());
        this.core = server.getCoreContainer().getCore(solrServerRef.getIndex());
    } else {
        //allow the default initialisation
        solrYardConfig.setAllowInitialisation(Boolean.TRUE);
        StandaloneEmbeddedSolrServerProvider.getInstance();
        server = StandaloneEmbeddedSolrServerProvider.getInstance().getSolrServer(solrServerRef, solrYardConfig.getIndexConfigurationName());
        if (server != null) {
            log.info("   ... initialised SolrCore with default configuration");
            this.core = server.getCoreContainer().getCore(solrServerRef.getIndex());
        } else if (solrServerRef.isPath() && new File(solrServerRef.getIndex()).isAbsolute()) {
            //the parsed absolute path is not within the managed SolrServer
            //so we need to create some CoreContainer and init/register
            //the core at the parsed location
            StandaloneManagedSolrServer s;
            if (solrServerRef.getServer() == null) {
                s = StandaloneManagedSolrServer.getManagedServer();
            } else {
                s = StandaloneManagedSolrServer.getManagedServer(solrServerRef.getServer());
            }
            CoreContainer cc = s.getCoreContainer();
            CoreDescriptor cd = new CoreDescriptor(cc, "dummy", solrServerRef.getIndex());
            this.core = cc.create(cd);
            cc.register(core, false);
            server = new EmbeddedSolrServer(cc, "dummy");
            log.info("   ... initialised existing SolrCore at {}", solrServerRef.getIndex());
        } else {
            throw new IllegalStateException("Unable to initialise SolrCore " + solrServerRef);
        }
    }
    log.info("   ... create SolrYard");
    this.solrYard = new SolrYard(server, solrYardConfig, namespacePrefixService);
}