Examples with BatchInserterImpl org.neo4j.kernel.impl.batchinsert.BatchInserterImpl used on opensource projects

Example 1 with BatchInserterImpl

use of org.neo4j.kernel.impl.batchinsert.BatchInserterImpl in project graphdb by neo4j-attic.

the class TestBatchInsert method newBatchInserter.

private BatchInserter newBatchInserter() {
    String storePath = AbstractNeo4jTestCase.getStorePath("neo-batch");
    AbstractNeo4jTestCase.deleteFileOrDirectory(new File(storePath));
    return new BatchInserterImpl(storePath);
}

Example 2 with BatchInserterImpl

use of org.neo4j.kernel.impl.batchinsert.BatchInserterImpl in project graphdb by neo4j-attic.

the class TestProveFiveBillion method proveIt.

@Test
public void proveIt() throws Exception {
    deleteFileOrDirectory(new File(PATH));
    BatchInserter inserter = new BatchInserterImpl(PATH);
    // Create one giant chain of nodes n1->n2->n3 where each node will have
    // an int property and each rel a long string property. This will yield
    // 5b nodes/relationships, 10b property records and 5b dynamic records.
    // Start off by creating the first 4 billion (or so) entities with the
    // batch inserter just to speed things up a little
    long first = inserter.getReferenceNode();
    int max = (int) pow(2, 32) - 1000;
    Map<String, Object> nodeProperties = map("number", 123);
    Map<String, Object> relationshipProperties = map("string", "A long string, which is longer than shortstring boundaries");
    long i = 0;
    for (; i < max; i++) {
        long second = inserter.createNode(nodeProperties);
        inserter.createRelationship(first, second, TYPE, relationshipProperties);
        if (i > 0 && i % 1000000 == 0)
            System.out.println((i / 1000000) + "M");
        first = second;
    }
    inserter.shutdown();
    System.out.println("Switch to embedded");
    // Then create the rest with embedded graph db.
    GraphDatabaseService db = new EmbeddedGraphDatabase(PATH);
    Node firstNode = db.getNodeById(first);
    Transaction tx = db.beginTx();
    for (; i < 5000000000L; i++) {
        Node secondNode = db.createNode();
        firstNode.createRelationshipTo(secondNode, TYPE);
        firstNode = secondNode;
        if (i % 100000 == 0) {
            tx.success();
            tx.finish();
            System.out.println((i / 1000000) + "M");
            tx = db.beginTx();
        }
    }
    // Here we have a huge db. Loop through it and count chain length.
    /*        long count = 0;
        Node node = db.getReferenceNode();
        while ( true )
        {
            Relationship relationship = node.getSingleRelationship( TYPE, Direction.OUTGOING );
            if ( relationship == null )
            {
                break;
            }
        }
        System.out.println( count );
        assertTrue( count > 4900000000L );*/
    db.shutdown();
}

Example 3 with BatchInserterImpl

use of org.neo4j.kernel.impl.batchinsert.BatchInserterImpl in project graphdb by neo4j-attic.

the class TestLuceneBatchInsert method testNumericValues.

@Test
public void testNumericValues() {
    String path = new File(PATH, "7").getAbsolutePath();
    BatchInserter inserter = new BatchInserterImpl(path);
    BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(inserter);
    BatchInserterIndex index = provider.nodeIndex("mine", EXACT_CONFIG);
    long node1 = inserter.createNode(null);
    index.add(node1, map("number", numeric(45)));
    long node2 = inserter.createNode(null);
    index.add(node2, map("number", numeric(21)));
    assertContains(index.query("number", newIntRange("number", 21, 50, true, true)), node1, node2);
    provider.shutdown();
    inserter.shutdown();
    GraphDatabaseService db = new EmbeddedGraphDatabase(path);
    Node n1 = db.getNodeById(node1);
    Node n2 = db.getNodeById(node2);
    Index<Node> idx = db.index().forNodes("mine");
    assertContains(idx.query("number", newIntRange("number", 21, 45, false, true)), n1);
    db.shutdown();
}

Example 4 with BatchInserterImpl

use of org.neo4j.kernel.impl.batchinsert.BatchInserterImpl in project graphdb by neo4j-attic.

the class TestLuceneBatchInsert method indexNumbers.

@Test
public void indexNumbers() throws Exception {
    BatchInserter inserter = new BatchInserterImpl(new File(PATH, "8").getAbsolutePath());
    BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(inserter);
    BatchInserterIndex index = provider.nodeIndex("mine", EXACT_CONFIG);
    long id = inserter.createNode(null);
    Map<String, Object> props = new HashMap<String, Object>();
    props.put("key", 123L);
    index.add(id, props);
    index.flush();
    assertEquals(1, index.get("key", 123L).size());
    assertEquals(1, index.get("key", "123").size());
    provider.shutdown();
    inserter.shutdown();
}

Example 5 with BatchInserterImpl

use of org.neo4j.kernel.impl.batchinsert.BatchInserterImpl in project graphdb by neo4j-attic.

the class TestLuceneBatchInsert method testInsertionSpeed.

@Ignore
@Test
public void testInsertionSpeed() {
    BatchInserter inserter = new BatchInserterImpl(new File(PATH, "3").getAbsolutePath());
    BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(inserter);
    BatchInserterIndex index = provider.nodeIndex("yeah", EXACT_CONFIG);
    index.setCacheCapacity("key", 1000000);
    long t = currentTimeMillis();
    for (int i = 0; i < 1000000; i++) {
        Map<String, Object> properties = map("key", "value" + i);
        long id = inserter.createNode(properties);
        index.add(id, properties);
    }
    System.out.println("insert:" + (currentTimeMillis() - t));
    index.flush();
    t = currentTimeMillis();
    for (int i = 0; i < 1000000; i++) {
        count((Iterator<Long>) index.get("key", "value" + i));
    }
    System.out.println("get:" + (currentTimeMillis() - t));
}