Examples with AccumuloRdfConfiguration org.apache.rya.accumulo.AccumuloRdfConfiguration used on opensource projects

Example 76 with AccumuloRdfConfiguration

use of org.apache.rya.accumulo.AccumuloRdfConfiguration in project incubator-rya by apache.

the class CreateFluoPcj method importHistoricResultsIntoFluo.

private void importHistoricResultsIntoFluo(FluoClient fluo, FluoQuery fluoQuery, Connector accumulo, String ryaInstance) throws RyaDAOException {
    // Reuse the same set object while performing batch inserts.
    final Set<RyaStatement> queryBatch = new HashSet<>();
    // historic matches into Fluo.
    for (final StatementPatternMetadata patternMetadata : fluoQuery.getStatementPatternMetadata()) {
        // Get an iterator over all of the binding sets that match the
        // statement pattern.
        final StatementPattern pattern = FluoStringConverter.toStatementPattern(patternMetadata.getStatementPattern());
        queryBatch.add(spToRyaStatement(pattern));
    }
    // Create AccumuloRyaQueryEngine to query for historic results
    final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration();
    conf.setTablePrefix(ryaInstance);
    conf.setAuths(getAuths(accumulo));
    try (final AccumuloRyaQueryEngine queryEngine = new AccumuloRyaQueryEngine(accumulo, conf);
        CloseableIterable<RyaStatement> queryIterable = queryEngine.query(new BatchRyaQuery(queryBatch))) {
        final Set<RyaStatement> triplesBatch = new HashSet<>();
        // Insert batches of the binding sets into Fluo.
        for (final RyaStatement ryaStatement : queryIterable) {
            if (triplesBatch.size() == spInsertBatchSize) {
                writeBatch(fluo, triplesBatch);
                triplesBatch.clear();
            }
            triplesBatch.add(ryaStatement);
        }
        if (!triplesBatch.isEmpty()) {
            writeBatch(fluo, triplesBatch);
            triplesBatch.clear();
        }
    } catch (final IOException e) {
        log.warn("Ignoring IOException thrown while closing the AccumuloRyaQueryEngine used by CreatePCJ.", e);
    }
}

Example 77 with AccumuloRdfConfiguration

use of org.apache.rya.accumulo.AccumuloRdfConfiguration in project incubator-rya by apache.

the class PcjTablesWithMockTest method setupRya.

private static RyaSailRepository setupRya(Connector accumuloConn) throws AccumuloException, AccumuloSecurityException, RepositoryException {
    // Setup the Rya Repository that will be used to create Repository
    // Connections.
    final RdfCloudTripleStore ryaStore = new RdfCloudTripleStore();
    final AccumuloRyaDAO crdfdao = new AccumuloRyaDAO();
    crdfdao.setConnector(accumuloConn);
    // Setup Rya configuration values.
    final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration();
    conf.setTablePrefix("demo_");
    conf.setDisplayQueryPlan(false);
    conf.setBoolean(USE_MOCK_INSTANCE, true);
    conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, RYA_TABLE_PREFIX);
    conf.set(CLOUDBASE_USER, "root");
    conf.set(CLOUDBASE_PASSWORD, "");
    conf.set(CLOUDBASE_INSTANCE, "instance");
    crdfdao.setConf(conf);
    ryaStore.setRyaDAO(crdfdao);
    final RyaSailRepository ryaRepo = new RyaSailRepository(ryaStore);
    ryaRepo.initialize();
    return ryaRepo;
}

Example 78 with AccumuloRdfConfiguration

use of org.apache.rya.accumulo.AccumuloRdfConfiguration in project incubator-rya by apache.

the class MergeTool method importChildFilesToTempParentTable.

/**
 * Imports the child files that hold the table data into the parent instance as a temporary table.
 * @param childTableName the name of the child table to import into a temporary parent table.
 * @throws Exception
 */
public void importChildFilesToTempParentTable(final String childTableName) throws Exception {
    // Create a temporary table in the parent instance to import the child files to.  Then run the merge process on the parent table and temp child table.
    final String tempChildTable = childTableName + TEMP_SUFFIX;
    createTempTableIfNeeded(tempChildTable);
    final AccumuloRdfConfiguration parentAccumuloRdfConfiguration = new AccumuloRdfConfiguration(conf);
    parentAccumuloRdfConfiguration.setTablePrefix(childTablePrefix);
    final Connector parentConnector = AccumuloRyaUtils.setupConnector(parentAccumuloRdfConfiguration);
    final TableOperations parentTableOperations = parentConnector.tableOperations();
    final Path localWorkDir = CopyTool.getPath(localMergeFileImportDir, childTableName);
    final Path hdfsBaseWorkDir = CopyTool.getPath(baseImportDir, childTableName);
    CopyTool.copyLocalToHdfs(localWorkDir, hdfsBaseWorkDir, conf);
    final Path files = CopyTool.getPath(hdfsBaseWorkDir.toString(), "files");
    final Path failures = CopyTool.getPath(hdfsBaseWorkDir.toString(), "failures");
    final FileSystem fs = FileSystem.get(conf);
    // With HDFS permissions on, we need to make sure the Accumulo user can read/move the files
    fs.setPermission(hdfsBaseWorkDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
    if (fs.exists(failures)) {
        fs.delete(failures, true);
    }
    fs.mkdirs(failures);
    parentTableOperations.importDirectory(tempChildTable, files.toString(), failures.toString(), false);
    AccumuloRyaUtils.printTablePretty(tempChildTable, conf);
}

Example 79 with AccumuloRdfConfiguration

use of org.apache.rya.accumulo.AccumuloRdfConfiguration in project incubator-rya by apache.

the class BaseCopyToolMapper method setup.

@Override
protected void setup(final Context context) throws IOException, InterruptedException {
    super.setup(context);
    log.info("Setting up mapper");
    parentConfig = context.getConfiguration();
    childConfig = MergeToolMapper.getChildConfig(parentConfig);
    startTimeString = parentConfig.get(MergeTool.START_TIME_PROP, null);
    if (startTimeString != null) {
        startTime = MergeTool.convertStartTimeStringToDate(startTimeString);
    }
    final String runTimeString = parentConfig.get(CopyTool.COPY_RUN_TIME_PROP, null);
    if (runTimeString != null) {
        runTime = MergeTool.convertStartTimeStringToDate(runTimeString);
    }
    final String offsetString = parentConfig.get(CopyTool.PARENT_TIME_OFFSET_PROP, null);
    if (offsetString != null) {
        timeOffset = Long.valueOf(offsetString);
    }
    useCopyFileOutput = parentConfig.getBoolean(CopyTool.USE_COPY_FILE_OUTPUT, false);
    parentTableName = parentConfig.get(MergeTool.TABLE_NAME_PROP, null);
    parentTablePrefix = parentConfig.get(MRUtils.TABLE_PREFIX_PROPERTY, null);
    childTablePrefix = childConfig.get(MRUtils.TABLE_PREFIX_PROPERTY, null);
    childTableName = parentTableName.replaceFirst(parentTablePrefix, childTablePrefix);
    childTableNameText = new Text(childTableName);
    log.info("Copying data from parent table, \"" + parentTableName + "\", to child table, \"" + childTableName + "\"");
    parentUser = parentConfig.get(MRUtils.AC_USERNAME_PROP, null);
    childUser = childConfig.get(MRUtils.AC_USERNAME_PROP, null);
    parentAccumuloRdfConfiguration = new AccumuloRdfConfiguration(parentConfig);
    parentAccumuloRdfConfiguration.setTablePrefix(parentTablePrefix);
    parentConnector = AccumuloRyaUtils.setupConnector(parentAccumuloRdfConfiguration);
    childAccumuloRdfConfiguration = new AccumuloRdfConfiguration(childConfig);
    childAccumuloRdfConfiguration.setTablePrefix(childTablePrefix);
    childRyaContext = RyaTripleContext.getInstance(childAccumuloRdfConfiguration);
    if (useCopyFileOutput) {
        fixSplitsInCachedLocalFiles();
    } else {
        childConnector = AccumuloRyaUtils.setupConnector(childAccumuloRdfConfiguration);
        childDao = AccumuloRyaUtils.setupDao(childConnector, childAccumuloRdfConfiguration);
        createTableIfNeeded();
        copyAuthorizations();
    }
    // Add the run time and split time to the table
    addMetadataKeys(context);
    log.info("Finished setting up mapper");
}

Example 80 with AccumuloRdfConfiguration

use of org.apache.rya.accumulo.AccumuloRdfConfiguration in project incubator-rya by apache.

the class TestVertexFormat method testRyaInput.

/*
    Write a simple parent-child directed graph to Cloudbase.
    Run a job which reads the values
    into subclasses that extend CloudbaseVertex I/O formats.
    Check the output after the job.
    */
@Test
public void testRyaInput() throws Exception {
    AccumuloRdfConfiguration conf = getConf();
    AccumuloRyaDAO ryaDAO = RyaSailFactory.getAccumuloDAO(conf);
    ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), new RyaURI("urn:test#pred1"), new RyaURI("urn:test#obj1")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), new RyaURI("urn:test#pred2"), new RyaURI("urn:test#obj2")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), new RyaURI("urn:test#pred3"), new RyaURI("urn:test#obj3")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:test#1234"), new RyaURI("urn:test#pred4"), new RyaURI("urn:test#obj4")));
    ryaDAO.flush();
    GiraphJob job = new GiraphJob(conf, getCallingMethodName());
    setupConfiguration(job);
    GiraphConfiguration giraphConf = job.getConfiguration();
    giraphConf.setComputationClass(EdgeNotification.class);
    giraphConf.setVertexInputFormatClass(RyaVertexInputFormat.class);
    giraphConf.setVertexOutputFormatClass(TestTextOutputFormat.class);
    if (log.isInfoEnabled())
        log.info("Running edge notification job using Rya Vertex input");
}