Examples with AccumuloRyaQueryEngine org.apache.rya.accumulo.query.AccumuloRyaQueryEngine used on opensource projects

Example 11 with AccumuloRyaQueryEngine

use of org.apache.rya.accumulo.query.AccumuloRyaQueryEngine in project incubator-rya by apache.

the class CreateFluoPcj method importHistoricResultsIntoFluo.

private void importHistoricResultsIntoFluo(FluoClient fluo, FluoQuery fluoQuery, Connector accumulo, String ryaInstance) throws RyaDAOException {
    // Reuse the same set object while performing batch inserts.
    final Set<RyaStatement> queryBatch = new HashSet<>();
    // historic matches into Fluo.
    for (final StatementPatternMetadata patternMetadata : fluoQuery.getStatementPatternMetadata()) {
        // Get an iterator over all of the binding sets that match the
        // statement pattern.
        final StatementPattern pattern = FluoStringConverter.toStatementPattern(patternMetadata.getStatementPattern());
        queryBatch.add(spToRyaStatement(pattern));
    }
    // Create AccumuloRyaQueryEngine to query for historic results
    final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration();
    conf.setTablePrefix(ryaInstance);
    conf.setAuths(getAuths(accumulo));
    try (final AccumuloRyaQueryEngine queryEngine = new AccumuloRyaQueryEngine(accumulo, conf);
        CloseableIterable<RyaStatement> queryIterable = queryEngine.query(new BatchRyaQuery(queryBatch))) {
        final Set<RyaStatement> triplesBatch = new HashSet<>();
        // Insert batches of the binding sets into Fluo.
        for (final RyaStatement ryaStatement : queryIterable) {
            if (triplesBatch.size() == spInsertBatchSize) {
                writeBatch(fluo, triplesBatch);
                triplesBatch.clear();
            }
            triplesBatch.add(ryaStatement);
        }
        if (!triplesBatch.isEmpty()) {
            writeBatch(fluo, triplesBatch);
            triplesBatch.clear();
        }
    } catch (final IOException e) {
        log.warn("Ignoring IOException thrown while closing the AccumuloRyaQueryEngine used by CreatePCJ.", e);
    }
}

Example 12 with AccumuloRyaQueryEngine

use of org.apache.rya.accumulo.query.AccumuloRyaQueryEngine in project incubator-rya by apache.

the class AccumuloRyaDAO method init.

@Override
public void init() throws RyaDAOException {
    if (isInitialized.get()) {
        return;
    }
    try {
        checkNotNull(conf);
        checkNotNull(connector);
        if (batchWriterConfig == null) {
            batchWriterConfig = new BatchWriterConfig();
            batchWriterConfig.setMaxMemory(MAX_MEMORY);
            batchWriterConfig.setTimeout(MAX_TIME, TimeUnit.MILLISECONDS);
            batchWriterConfig.setMaxWriteThreads(NUM_THREADS);
        }
        tableLayoutStrategy = conf.getTableLayoutStrategy();
        ryaContext = RyaTripleContext.getInstance(conf);
        ryaTableMutationsFactory = new RyaTableMutationsFactory(ryaContext);
        secondaryIndexers = conf.getAdditionalIndexers();
        flushEachUpdate.set(conf.flushEachUpdate());
        final TableOperations tableOperations = connector.tableOperations();
        AccumuloRdfUtils.createTableIfNotExist(tableOperations, tableLayoutStrategy.getSpo());
        AccumuloRdfUtils.createTableIfNotExist(tableOperations, tableLayoutStrategy.getPo());
        AccumuloRdfUtils.createTableIfNotExist(tableOperations, tableLayoutStrategy.getOsp());
        AccumuloRdfUtils.createTableIfNotExist(tableOperations, tableLayoutStrategy.getNs());
        for (final AccumuloIndexer index : secondaryIndexers) {
            index.setConf(conf);
        }
        mt_bw = connector.createMultiTableBatchWriter(batchWriterConfig);
        // get the batch writers for tables
        bw_spo = mt_bw.getBatchWriter(tableLayoutStrategy.getSpo());
        bw_po = mt_bw.getBatchWriter(tableLayoutStrategy.getPo());
        bw_osp = mt_bw.getBatchWriter(tableLayoutStrategy.getOsp());
        bw_ns = mt_bw.getBatchWriter(tableLayoutStrategy.getNs());
        for (final AccumuloIndexer index : secondaryIndexers) {
            index.setConnector(connector);
            index.setMultiTableBatchWriter(mt_bw);
            index.init();
        }
        queryEngine = new AccumuloRyaQueryEngine(connector, conf);
        checkVersion();
        isInitialized.set(true);
    } catch (final Exception e) {
        throw new RyaDAOException(e);
    }
}

Example 13 with AccumuloRyaQueryEngine

use of org.apache.rya.accumulo.query.AccumuloRyaQueryEngine in project incubator-rya by apache.

the class AccumuloRyaDAOTest method testSameLiteralStringTypes.

@Test
public void testSameLiteralStringTypes() throws Exception {
    RyaURI cpu = new RyaURI(litdupsNS + "cpu");
    RyaURI loadPerc = new RyaURI(litdupsNS + "loadPerc");
    RyaType longLit = new RyaType(XMLSchema.LONG, "10");
    RyaType strLit = new RyaType(XMLSchema.STRING, new String(RyaContext.getInstance().serializeType(longLit)[0]));
    RyaStatement expected = new RyaStatement(cpu, loadPerc, longLit);
    dao.add(expected);
    dao.add(new RyaStatement(cpu, loadPerc, strLit));
    AccumuloRyaQueryEngine queryEngine = dao.getQueryEngine();
    CloseableIteration<RyaStatement, RyaDAOException> query = queryEngine.query(new RyaStatement(cpu, loadPerc, longLit), conf);
    assertTrue(query.hasNext());
    RyaStatement next = query.next();
    assertEquals(new Long(longLit.getData()), new Long(next.getObject().getData()));
    assertEquals(longLit.getDataType(), next.getObject().getDataType());
    assertFalse(query.hasNext());
    query.close();
}

Example 14 with AccumuloRyaQueryEngine

use of org.apache.rya.accumulo.query.AccumuloRyaQueryEngine in project incubator-rya by apache.

the class AccumuloRyaDAOTest method testQueryDates.

@Test
public void testQueryDates() throws Exception {
    RyaURI cpu = new RyaURI(litdupsNS + "cpu");
    RyaURI loadPerc = new RyaURI(litdupsNS + "loadPerc");
    // How handles local time
    RyaType uri0 = new RyaType(XMLSchema.DATETIME, "1960-01-01");
    // See Magadan Time
    RyaType uri1 = new RyaType(XMLSchema.DATETIME, "1992-01-01T+10:00");
    // How it handles UTC.
    RyaType uri2 = new RyaType(XMLSchema.DATETIME, "2000-01-01TZ");
    RyaType uri3 = new RyaType(XMLSchema.DATETIME, "2000-01-01T00:00:01.111Z");
    // duplicate
    RyaType uri4 = new RyaType(XMLSchema.DATETIME, "2000-01-01T00:00:01.111Z");
    RyaType uri5 = new RyaType(XMLSchema.DATETIME, "2000-01-01T00:00:01-00:00");
    // duplicate
    RyaType uri6 = new RyaType(XMLSchema.DATETIME, "2000-01-01T00:00:01Z");
    RyaType uri7 = new RyaType(XMLSchema.DATETIME, "-2000-01-01T00:00:01Z");
    RyaType uri8 = new RyaType(XMLSchema.DATETIME, "111-01-01T00:00:01Z");
    RyaType uri9 = new RyaType(XMLSchema.DATETIME, "12345-01-01T00:00:01Z");
    dao.add(new RyaStatement(cpu, loadPerc, uri0));
    dao.add(new RyaStatement(cpu, loadPerc, uri1));
    dao.add(new RyaStatement(cpu, loadPerc, uri2));
    dao.add(new RyaStatement(cpu, loadPerc, uri3));
    dao.add(new RyaStatement(cpu, loadPerc, uri4));
    dao.add(new RyaStatement(cpu, loadPerc, uri5));
    dao.add(new RyaStatement(cpu, loadPerc, uri6));
    dao.add(new RyaStatement(cpu, loadPerc, uri7));
    dao.add(new RyaStatement(cpu, loadPerc, uri8));
    dao.add(new RyaStatement(cpu, loadPerc, uri9));
    AccumuloRyaQueryEngine queryEngine = dao.getQueryEngine();
    Collection<RyaStatement> coll = new ArrayList();
    coll.add(new RyaStatement(null, loadPerc, uri0));
    coll.add(new RyaStatement(null, loadPerc, uri1));
    coll.add(new RyaStatement(null, loadPerc, uri2));
    CloseableIteration<RyaStatement, RyaDAOException> iter = queryEngine.batchQuery(coll, conf);
    int count = 0;
    while (iter.hasNext()) {
        count++;
        iter.next();
    }
    iter.close();
    assertEquals("Three time zones should be normalized when stored, then normalized same when queried.", 3, count);
    // now use batchscanner
    AccumuloRdfConfiguration queryConf = new AccumuloRdfConfiguration(conf);
    queryConf.setMaxRangesForScanner(2);
    coll = new ArrayList();
    coll.add(new RyaStatement(null, loadPerc, uri0));
    coll.add(new RyaStatement(null, loadPerc, uri1));
    coll.add(new RyaStatement(null, loadPerc, uri2));
    coll.add(new RyaStatement(null, loadPerc, uri3));
    coll.add(new RyaStatement(null, loadPerc, uri4));
    coll.add(new RyaStatement(null, loadPerc, uri5));
    coll.add(new RyaStatement(null, loadPerc, uri6));
    coll.add(new RyaStatement(null, loadPerc, uri7));
    coll.add(new RyaStatement(null, loadPerc, uri8));
    coll.add(new RyaStatement(null, loadPerc, uri9));
    iter = queryEngine.batchQuery(coll, queryConf);
    count = 0;
    while (iter.hasNext()) {
        count++;
        iter.next();
    }
    iter.close();
    assertEquals("Variety of time specs, including BC, pre-1970, duplicate pair ovewrite,future, 3 digit year.", 8, count);
}

Example 15 with AccumuloRyaQueryEngine

use of org.apache.rya.accumulo.query.AccumuloRyaQueryEngine in project incubator-rya by apache.

the class AccumuloRyaDAOTest method testAddValue.

@Test
public void testAddValue() throws Exception {
    RyaURI cpu = new RyaURI(litdupsNS + "cpu");
    RyaURI loadPerc = new RyaURI(litdupsNS + "loadPerc");
    RyaURI uri1 = new RyaURI(litdupsNS + "uri1");
    String myval = "myval";
    byte[] columnVis = null;
    dao.add(new RyaStatement(cpu, loadPerc, uri1, null, null, columnVis, myval.getBytes()));
    AccumuloRyaQueryEngine queryEngine = dao.getQueryEngine();
    CloseableIteration<RyaStatement, RyaDAOException> iter = queryEngine.query(new RyaStatement(cpu, loadPerc, null), conf);
    assertTrue(iter.hasNext());
    assertEquals(myval, new String(iter.next().getValue()));
    iter.close();
}