Examples with RDFFormat org.eclipse.rdf4j.rio.RDFFormat used on opensource projects

Example 11 with RDFFormat

use of org.eclipse.rdf4j.rio.RDFFormat in project nanopub-server by tkuhn.

the class LoadFiles method checkFilesToLoad.

private void checkFilesToLoad() {
    logger.info("Check whether there are files to load...");
    for (File f : loadDir.listFiles()) {
        stillAlive();
        if (f.isDirectory())
            continue;
        logger.info("Try to load file: " + f);
        try {
            final File processingFile = new File(processingDir, f.getName());
            f.renameTo(processingFile);
            RDFFormat format = Rio.getParserFormatForFileName(processingFile.getName()).orElse(null);
            MultiNanopubRdfHandler.process(format, processingFile, new NanopubHandler() {

                @Override
                public void handleNanopub(Nanopub np) {
                    if (!ourPattern.matchesUri(np.getUri().toString()))
                        return;
                    try {
                        db.loadNanopub(np);
                    } catch (Exception ex) {
                        logger.error("Failed to load nanopublication", ex);
                    }
                    stillAlive();
                }
            });
            processingFile.renameTo(new File(doneDir, f.getName()));
            logger.info("File loaded: " + processingFile);
        } catch (Exception ex) {
            logger.error("Failed to load file: " + f, ex);
        }
    }
}

Example 12 with RDFFormat

use of org.eclipse.rdf4j.rio.RDFFormat in project incubator-rya by apache.

the class ConformanceTest method run.

@Override
public int run(final String[] args) throws Exception {
    // Validate command
    if (args.length < 1 || args.length > 2) {
        System.out.println("Usage:\n");
        System.out.println("\tConformanceTest [configuration options] " + "<test-file> <temp-dir>\n");
        System.out.println("to load test data from an RDF file " + "(configuration property " + MRUtils.FORMAT_PROP + " specifies the format, default RDF/XML); or\n");
        System.out.println("\tConformanceTest [configuration options] <temp-dir>\n");
        System.out.println("to load test data from a Rya instance (specified " + "using standard configuration properties).\n");
        System.out.println("For each test given, run the reasoner over the " + "premise ontology using a temporary Mini Accumulo instance " + "at <temp-dir>, then report conformance results.");
        System.exit(1);
    }
    final Set<Value> conformanceTestURIs = new HashSet<>();
    Collection<OwlTest> conformanceTests = new LinkedList<>();
    final List<OwlTest> successes = new LinkedList<>();
    final List<OwlTest> failures = new LinkedList<>();
    final Configuration conf = getConf();
    Repository repo;
    File workingDir;
    // If tests are in a file, stick them in a repository for querying
    if (args.length == 2) {
        workingDir = new File(PathUtils.clean(args[1]));
        RDFFormat inputFormat = RDFFormat.RDFXML;
        final String formatString = conf.get(MRUtils.FORMAT_PROP);
        if (formatString != null) {
            inputFormat = RdfFormatUtils.getRdfFormatFromName(formatString);
        }
        repo = new SailRepository(new MemoryStore());
        repo.initialize();
        final RepositoryConnection conn = repo.getConnection();
        FileInputStream fileInput = new FileInputStream(PathUtils.clean(args[0]));
        conn.add(fileInput, "", inputFormat);
        fileInput.close();
        conn.close();
    } else // Otherwise, get a Rya repository
    {
        workingDir = new File(PathUtils.clean(args[0]));
        repo = MRReasoningUtils.getRepository(conf);
        repo.initialize();
    }
    // Query for the tests we're interested in
    final RepositoryConnection conn = repo.getConnection();
    conformanceTestURIs.addAll(getTestURIs(conn, TEST_INCONSISTENCY));
    conformanceTestURIs.addAll(getTestURIs(conn, TEST_CONSISTENCY));
    conformanceTestURIs.addAll(getTestURIs(conn, TEST_ENTAILMENT));
    conformanceTestURIs.addAll(getTestURIs(conn, TEST_NONENTAILMENT));
    conformanceTests = getTests(conn, conformanceTestURIs);
    conn.close();
    repo.shutDown();
    // Set up a MiniAccumulo cluster and set up conf to connect to it
    final String username = "root";
    final String password = "root";
    final MiniAccumuloCluster mini = new MiniAccumuloCluster(workingDir, password);
    mini.start();
    conf.set(MRUtils.AC_INSTANCE_PROP, mini.getInstanceName());
    conf.set(MRUtils.AC_ZK_PROP, mini.getZooKeepers());
    conf.set(MRUtils.AC_USERNAME_PROP, username);
    conf.set(MRUtils.AC_PWD_PROP, password);
    conf.setBoolean(MRUtils.AC_MOCK_PROP, false);
    conf.set(MRUtils.TABLE_PREFIX_PROPERTY, "temp_");
    // Run the conformance tests
    int result;
    for (final OwlTest test : conformanceTests) {
        System.out.println(test.iri);
        result = runTest(conf, args, test);
        if (result != 0) {
            return result;
        }
        if (test.success) {
            successes.add(test);
            System.out.println("(SUCCESS)");
        } else {
            failures.add(test);
            System.out.println("(FAIL)");
        }
    }
    mini.stop();
    System.out.println("\n" + successes.size() + " successful tests:");
    for (final OwlTest test : successes) {
        System.out.println("\t[SUCCESS] " + test.type() + " " + test.name);
    }
    System.out.println("\n" + failures.size() + " failed tests:");
    for (final OwlTest test : failures) {
        System.out.println("\t[FAIL] " + test.type() + " " + test.name);
        System.out.println("\t\t(" + test.description + ")");
        for (final Statement triple : test.error) {
            if (test.types.contains(TEST_ENTAILMENT)) {
                System.out.println("\t\tExpected: " + triple);
            } else if (test.types.contains(TEST_NONENTAILMENT)) {
                System.out.println("\t\tUnexpected: " + triple);
            }
        }
    }
    return 0;
}

Example 13 with RDFFormat

use of org.eclipse.rdf4j.rio.RDFFormat in project incubator-rya by apache.

the class KafkaLoadStatements method fromFile.

@Override
public void fromFile(final Path statementsPath, final String visibilities) throws RyaStreamsException {
    requireNonNull(statementsPath);
    requireNonNull(visibilities);
    if (!statementsPath.toFile().exists()) {
        throw new RyaStreamsException("Could not load statements at path '" + statementsPath + "' because that " + "does not exist. Make sure you've entered the correct path.");
    }
    // Create an RDF Parser whose format is derived from the statementPath's file extension.
    final String filename = statementsPath.getFileName().toString();
    final RDFFormat format = RdfFormatUtils.forFileName(filename);
    if (format == null) {
        throw new UnsupportedRDFormatException("Unknown RDF format for the file: " + filename);
    }
    final RDFParser parser = Rio.createParser(format);
    // Set a handler that writes the statements to the specified kafka topic.
    parser.setRDFHandler(new AbstractRDFHandler() {

        @Override
        public void startRDF() throws RDFHandlerException {
            log.trace("Starting loading statements.");
        }

        @Override
        public void handleStatement(final Statement stmnt) throws RDFHandlerException {
            final VisibilityStatement visiStatement = new VisibilityStatement(stmnt, visibilities);
            producer.send(new ProducerRecord<>(topic, visiStatement));
        }

        @Override
        public void endRDF() throws RDFHandlerException {
            producer.flush();
            log.trace("Done.");
        }
    });
    // Do the parse and load.
    try {
        parser.parse(Files.newInputStream(statementsPath), "");
    } catch (RDFParseException | RDFHandlerException | IOException e) {
        throw new RyaStreamsException("Could not load the RDF file's Statements into Rya Streams.", e);
    }
}

Example 14 with RDFFormat

use of org.eclipse.rdf4j.rio.RDFFormat in project incubator-rya by apache.

the class AccumuloRyaSailFactoryLoadFilesIT method testFileLoading.

@Test
public void testFileLoading() throws Exception {
    log.info("Starting file loading test...");
    final String query = "SELECT * WHERE { ?s ?p ?o . FILTER(?p != <" + RdfCloudTripleStoreConstants.RTS_VERSION_PREDICATE + ">) }";
    final String deleteQuery = "DELETE WHERE { ?s ?p ?o . }";
    for (final TestFile testFile : TestFileUtils.TEST_FILES) {
        final String testFilePath = testFile.getPath();
        final RDFFormat rdfFormat = RdfFormatUtils.forFileName(testFilePath, null);
        log.info("Loading file \"" + testFilePath + "\" with RDFFormat: " + rdfFormat.getName());
        try (final InputStream rdfContent = getClass().getResourceAsStream(testFilePath)) {
            addTriples(ryaRepository, rdfContent, rdfFormat);
        }
        SailRepositoryConnection queryConn = null;
        try {
            log.info("Querying for triples in the repository from the " + rdfFormat.getName() + " file.");
            queryConn = ryaRepository.getConnection();
            final int count = performTupleQuery(query, queryConn);
            assertEquals("Expected number of triples not found in: " + testFilePath, testFile.getExpectedCount(), count);
        } finally {
            closeQuietly(queryConn);
        }
        SailRepositoryConnection deleteConn = null;
        try {
            log.info("Deleting triples in the repository from the " + rdfFormat.getName() + " file.");
            deleteConn = ryaRepository.getConnection();
            final Update update = deleteConn.prepareUpdate(QueryLanguage.SPARQL, deleteQuery);
            update.execute();
        } finally {
            closeQuietly(deleteConn);
        }
    }
    log.info("File loading test finished.");
}

Example 15 with RDFFormat

use of org.eclipse.rdf4j.rio.RDFFormat in project incubator-rya by apache.

the class WriteStatementsCommand method execute.

@Override
public void execute(final String[] args) throws ArgumentsException, ExecutionException {
    requireNonNull(args);
    // Parse the command line arguments.
    final WriteParameters params = new WriteParameters();
    try {
        new JCommander(params, args);
    } catch (final ParameterException e) {
        throw new ArgumentsException("Could not stream the query's results because of invalid command line parameters.", e);
    }
    // Verify the configured statements file path.
    final Path statementsPath = Paths.get(params.statementsFile);
    if (!statementsPath.toFile().exists()) {
        throw new ArgumentsException("Could not load statements at path '" + statementsPath + "' because that " + "file does not exist. Make sure you've entered the correct path.");
    }
    // Create an RDF Parser whose format is derived from the statementPath's file extension.
    final String filename = statementsPath.getFileName().toString();
    final RDFFormat format = RdfFormatUtils.forFileName(filename);
    if (format == null) {
        throw new UnsupportedRDFormatException("Unknown RDF format for the file: " + filename);
    }
    final RDFParser parser = Rio.createParser(format);
    // Set up the producer.
    try (Producer<String, Set<Statement>> producer = makeProducer(params)) {
        // Set a handler that writes the statements to the specified kafka topic. It writes batches of 5 Statements.
        parser.setRDFHandler(new AbstractRDFHandler() {

            private Set<Statement> batch = new HashSet<>(5);

            @Override
            public void startRDF() throws RDFHandlerException {
                log.trace("Starting loading statements.");
            }

            @Override
            public void handleStatement(final Statement stmnt) throws RDFHandlerException {
                log.trace("Adding statement.");
                batch.add(stmnt);
                if (batch.size() == 5) {
                    flushBatch();
                }
            }

            @Override
            public void endRDF() throws RDFHandlerException {
                if (!batch.isEmpty()) {
                    flushBatch();
                }
                log.trace("Done.");
            }

            private void flushBatch() {
                log.trace("Flushing batch of size " + batch.size());
                producer.send(new ProducerRecord<>(params.topic, null, batch));
                batch = new HashSet<>(5);
                producer.flush();
            }
        });
        // Do the parse and load.
        try {
            parser.parse(Files.newInputStream(statementsPath), "");
        } catch (RDFParseException | RDFHandlerException | IOException e) {
            throw new ExecutionException("Could not load the RDF file's Statements into the Kafka topic.", e);
        }
    }
}