Examples with DIHConfiguration org.apache.solr.handler.dataimport.config.DIHConfiguration used on opensource projects

Example 6 with DIHConfiguration

use of org.apache.solr.handler.dataimport.config.DIHConfiguration in project lucene-solr by apache.

the class TestDocBuilder method templateXPath.

@Test
public void templateXPath() {
    DataImporter di = new DataImporter();
    di.loadAndInit(dc_variableXpath);
    DIHConfiguration cfg = di.getConfig();
    RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
    List<Map<String, Object>> l = new ArrayList<>();
    l.add(createMap("id", 1, "name", "iphone", "manufacturer", "Apple"));
    l.add(createMap("id", 2, "name", "ipad", "manufacturer", "Apple"));
    l.add(createMap("id", 3, "name", "pixel", "manufacturer", "Google"));
    MockDataSource.setIterator("select * from x", l.iterator());
    List<Map<String, Object>> nestedData = new ArrayList<>();
    nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
    nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
    nestedData.add(createMap("founded", "Menlo Park, California, U.S", "year", "1998", "year2", "1998"));
    MockStringDataSource.setData("companies.xml", xml_attrVariableXpath);
    MockStringDataSource.setData("companies2.xml", xml_variableXpath);
    MockStringDataSource.setData("companies3.xml", xml_variableForEach);
    SolrWriterImpl swi = new SolrWriterImpl();
    di.runCmd(rp, swi);
    assertEquals(Boolean.TRUE, swi.deleteAllCalled);
    assertEquals(Boolean.TRUE, swi.commitCalled);
    assertEquals(Boolean.TRUE, swi.finishCalled);
    assertEquals(3, swi.docs.size());
    for (int i = 0; i < l.size(); i++) {
        SolrInputDocument doc = swi.docs.get(i);
        Map<String, Object> map = l.get(i);
        for (Map.Entry<String, Object> entry : map.entrySet()) {
            assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
        }
        map = nestedData.get(i);
        for (Map.Entry<String, Object> entry : map.entrySet()) {
            assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
        }
    }
    assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
    assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
}

Example 7 with DIHConfiguration

use of org.apache.solr.handler.dataimport.config.DIHConfiguration in project lucene-solr by apache.

the class TestDocBuilder method testImportCommand.

@Test
public void testImportCommand() {
    DataImporter di = new DataImporter();
    di.loadAndInit(dc_singleEntity);
    DIHConfiguration cfg = di.getConfig();
    Entity ent = cfg.getEntities().get(0);
    List<Map<String, Object>> l = new ArrayList<>();
    l.add(createMap("id", 1, "desc", "one"));
    MockDataSource.setIterator("select * from x", l.iterator());
    RequestInfo rp = new RequestInfo(null, createMap("command", "import"), null);
    SolrWriterImpl swi = new SolrWriterImpl();
    di.runCmd(rp, swi);
    assertEquals(Boolean.FALSE, swi.deleteAllCalled);
    assertEquals(Boolean.TRUE, swi.commitCalled);
    assertEquals(Boolean.TRUE, swi.finishCalled);
    assertEquals(1, swi.docs.size());
    assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
    assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
    assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
    for (int i = 0; i < l.size(); i++) {
        Map<String, Object> map = (Map<String, Object>) l.get(i);
        SolrInputDocument doc = swi.docs.get(i);
        for (Map.Entry<String, Object> entry : map.entrySet()) {
            assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
        }
    }
}

Example 8 with DIHConfiguration

use of org.apache.solr.handler.dataimport.config.DIHConfiguration in project lucene-solr by apache.

the class TestDocBuilder method testDeltaImportNoRows_MustNotCommit.

@Test
public void testDeltaImportNoRows_MustNotCommit() {
    DataImporter di = new DataImporter();
    di.loadAndInit(dc_deltaConfig);
    redirectTempProperties(di);
    DIHConfiguration cfg = di.getConfig();
    Entity ent = cfg.getEntities().get(0);
    MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
    MockDataSource.setIterator("select id from x", new ArrayList<Map<String, Object>>().iterator());
    RequestInfo rp = new RequestInfo(null, createMap("command", "delta-import"), null);
    SolrWriterImpl swi = new SolrWriterImpl();
    di.runCmd(rp, swi);
    assertEquals(Boolean.FALSE, swi.deleteAllCalled);
    assertEquals(Boolean.FALSE, swi.commitCalled);
    assertEquals(Boolean.TRUE, swi.finishCalled);
    assertEquals(0, swi.docs.size());
    assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
    assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
    assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
}

Example 9 with DIHConfiguration

use of org.apache.solr.handler.dataimport.config.DIHConfiguration in project lucene-solr by apache.

the class TestDocBuilder method singleEntityOneRow.

@Test
public void singleEntityOneRow() {
    DataImporter di = new DataImporter();
    di.loadAndInit(dc_singleEntity);
    DIHConfiguration cfg = di.getConfig();
    Entity ent = cfg.getEntities().get(0);
    List<Map<String, Object>> l = new ArrayList<>();
    l.add(createMap("id", 1, "desc", "one"));
    MockDataSource.setIterator("select * from x", l.iterator());
    RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
    SolrWriterImpl swi = new SolrWriterImpl();
    di.runCmd(rp, swi);
    assertEquals(Boolean.TRUE, swi.deleteAllCalled);
    assertEquals(Boolean.TRUE, swi.commitCalled);
    assertEquals(Boolean.TRUE, swi.finishCalled);
    assertEquals(1, swi.docs.size());
    assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
    assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
    assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
    for (int i = 0; i < l.size(); i++) {
        Map<String, Object> map = l.get(i);
        SolrInputDocument doc = swi.docs.get(i);
        for (Map.Entry<String, Object> entry : map.entrySet()) {
            assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
        }
    }
}

Example 10 with DIHConfiguration

use of org.apache.solr.handler.dataimport.config.DIHConfiguration in project lucene-solr by apache.

the class DataImporter method loadDataConfig.

public DIHConfiguration loadDataConfig(InputSource configFile) {
    DIHConfiguration dihcfg = null;
    try {
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        // only enable xinclude, if a a SolrCore and SystemId is present (makes no sense otherwise)
        if (core != null && configFile.getSystemId() != null) {
            try {
                dbf.setXIncludeAware(true);
                dbf.setNamespaceAware(true);
            } catch (UnsupportedOperationException e) {
                LOG.warn("XML parser doesn't support XInclude option");
            }
        }
        DocumentBuilder builder = dbf.newDocumentBuilder();
        if (core != null)
            builder.setEntityResolver(new SystemIdResolver(core.getResourceLoader()));
        builder.setErrorHandler(XMLLOG);
        Document document;
        try {
            document = builder.parse(configFile);
        } finally {
            // some XML parsers are broken and don't close the byte stream (but they should according to spec)
            IOUtils.closeQuietly(configFile.getByteStream());
        }
        dihcfg = readFromXml(document);
        LOG.info("Data Configuration loaded successfully");
    } catch (Exception e) {
        throw new DataImportHandlerException(SEVERE, "Data Config problem: " + e.getMessage(), e);
    }
    for (Entity e : dihcfg.getEntities()) {
        if (e.getAllAttributes().containsKey(SqlEntityProcessor.DELTA_QUERY)) {
            isDeltaImportSupported = true;
            break;
        }
    }
    return dihcfg;
}