Examples with InputRelationship org.neo4j.unsafe.impl.batchimport.input.InputRelationship used on opensource projects

Example 1 with InputRelationship

use of org.neo4j.unsafe.impl.batchimport.input.InputRelationship in project neo4j by neo4j.

the class ParallelBatchImporterTest method relationships.

private InputIterable<InputRelationship> relationships(final long randomSeed, final long count, final InputIdGenerator idGenerator, final IdGroupDistribution groups) {
    return new InputIterable<InputRelationship>() {

        private int calls;

        @Override
        public InputIterator<InputRelationship> iterator() {
            calls++;
            assertTrue("Unexpected use of input iterator " + multiPassIterators + ", " + calls, multiPassIterators || (!multiPassIterators && calls == 1));
            // since we use it to compare the imported data against after the import has been completed.
            return new SimpleInputIterator<InputRelationship>("test relationships") {

                private final Random random = new Random(randomSeed);

                private final Randoms randoms = new Randoms(random, Randoms.DEFAULT);

                private int cursor;

                private final MutableLong nodeIndex = new MutableLong(-1);

                @Override
                protected InputRelationship fetchNextOrNull() {
                    if (cursor < count) {
                        Object[] properties = randomProperties(randoms, "Name " + cursor);
                        try {
                            Object startNode = idGenerator.randomExisting(random, nodeIndex);
                            Group startNodeGroup = groups.groupOf(nodeIndex.longValue());
                            Object endNode = idGenerator.randomExisting(random, nodeIndex);
                            Group endNodeGroup = groups.groupOf(nodeIndex.longValue());
                            // miss some
                            startNode = idGenerator.miss(random, startNode, 0.001f);
                            endNode = idGenerator.miss(random, endNode, 0.001f);
                            String type = idGenerator.randomType(random);
                            if (random.nextFloat() < 0.00005) {
                                // Let there be a small chance of introducing a one-off relationship
                                // with a type that no, or at least very few, other relationships have.
                                type += "_odd";
                            }
                            return new InputRelationship(sourceDescription, itemNumber, itemNumber, properties, null, startNodeGroup, startNode, endNodeGroup, endNode, type, null);
                        } finally {
                            cursor++;
                        }
                    }
                    return null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return multiPassIterators;
        }
    };
}

Example 2 with InputRelationship

use of org.neo4j.unsafe.impl.batchimport.input.InputRelationship in project neo4j by neo4j.

the class ParallelBatchImporterTest method verifyData.

protected void verifyData(int nodeCount, int relationshipCount, GraphDatabaseService db, IdGroupDistribution groups, long nodeRandomSeed, long relationshipRandomSeed) {
    // Read all nodes, relationships and properties ad verify against the input data.
    try (InputIterator<InputNode> nodes = nodes(nodeRandomSeed, nodeCount, inputIdGenerator, groups).iterator();
        InputIterator<InputRelationship> relationships = relationships(relationshipRandomSeed, relationshipCount, inputIdGenerator, groups).iterator()) {
        // Nodes
        Map<String, Node> nodeByInputId = new HashMap<>(nodeCount);
        Iterator<Node> dbNodes = db.getAllNodes().iterator();
        int verifiedNodes = 0;
        while (nodes.hasNext()) {
            InputNode input = nodes.next();
            Node node = dbNodes.next();
            assertNodeEquals(input, node);
            String inputId = uniqueId(input.group(), node);
            assertNull(nodeByInputId.put(inputId, node));
            verifiedNodes++;
            assertDegrees(node);
        }
        assertEquals(nodeCount, verifiedNodes);
        // Relationships
        Map<String, Relationship> relationshipByName = new HashMap<>();
        for (Relationship relationship : db.getAllRelationships()) {
            relationshipByName.put((String) relationship.getProperty("id"), relationship);
        }
        int verifiedRelationships = 0;
        while (relationships.hasNext()) {
            InputRelationship input = relationships.next();
            if (!inputIdGenerator.isMiss(input.startNode()) && !inputIdGenerator.isMiss(input.endNode())) {
                // A relationship referring to missing nodes. The InputIdGenerator is expected to generate
                // some (very few) of those. Skip it.
                String name = (String) propertyOf(input, "id");
                Relationship relationship = relationshipByName.get(name);
                assertNotNull("Expected there to be a relationship with name '" + name + "'", relationship);
                assertEquals(nodeByInputId.get(uniqueId(input.startNodeGroup(), input.startNode())), relationship.getStartNode());
                assertEquals(nodeByInputId.get(uniqueId(input.endNodeGroup(), input.endNode())), relationship.getEndNode());
                assertRelationshipEquals(input, relationship);
            }
            verifiedRelationships++;
        }
        assertEquals(relationshipCount, verifiedRelationships);
    }
}

Example 3 with InputRelationship

use of org.neo4j.unsafe.impl.batchimport.input.InputRelationship in project neo4j by neo4j.

the class CalculateDenseNodesStepTest method shouldCollectBadRelationships.

@Test
public void shouldCollectBadRelationships() throws Exception {
    // GIVEN
    NodeRelationshipCache cache = mock(NodeRelationshipCache.class);
    Collector collector = mock(Collector.class);
    try (CalculateDenseNodesStep step = new CalculateDenseNodesStep(mock(StageControl.class), DEFAULT, cache, collector)) {
        step.processors(4);
        step.start(0);
        // WHEN
        Batch<InputRelationship, RelationshipRecord> batch = batch(relationship(1, 5), relationship(3, 10), // <-- bad relationship with missing start node
        relationship("a", 2, -1, 2), // <-- bad relationship with missing end node
        relationship(2, "b", 2, -1), // <-- bad relationship with missing start and end node
        relationship("c", "d", -1, -1));
        step.receive(0, batch);
        step.endOfUpstream();
        while (!step.isCompleted()) {
        //wait
        }
        // THEN
        verify(collector, times(1)).collectBadRelationship(any(InputRelationship.class), eq("a"));
        verify(collector, times(1)).collectBadRelationship(any(InputRelationship.class), eq("b"));
        verify(collector, times(1)).collectBadRelationship(any(InputRelationship.class), eq("c"));
        verify(collector, times(1)).collectBadRelationship(any(InputRelationship.class), eq("d"));
    }
}

Example 4 with InputRelationship

use of org.neo4j.unsafe.impl.batchimport.input.InputRelationship in project neo4j by neo4j.

the class CalculateDenseNodesStepTest method shouldNotProcessLoopsTwice.

@Test
public void shouldNotProcessLoopsTwice() throws Exception {
    // GIVEN
    NodeRelationshipCache cache = mock(NodeRelationshipCache.class);
    try (CalculateDenseNodesStep step = new CalculateDenseNodesStep(mock(StageControl.class), DEFAULT, cache, mock(Collector.class))) {
        step.processors(4);
        step.start(0);
        // WHEN
        Batch<InputRelationship, RelationshipRecord> batch = batch(relationship(1, 5), relationship(3, 10), // <-- the loop
        relationship(2, 2), relationship(4, 1));
        step.receive(0, batch);
        step.endOfUpstream();
        while (!step.isCompleted()) {
        // wait
        }
        // THEN
        verify(cache, times(2)).incrementCount(eq(1L));
        verify(cache, times(1)).incrementCount(eq(2L));
        verify(cache, times(1)).incrementCount(eq(3L));
        verify(cache, times(1)).incrementCount(eq(4L));
        verify(cache, times(1)).incrementCount(eq(5L));
        verify(cache, times(1)).incrementCount(eq(10L));
    }
}

Example 5 with InputRelationship

use of org.neo4j.unsafe.impl.batchimport.input.InputRelationship in project neo4j by neo4j.

the class CsvInputBatchImportIT method buildUpExpectedData.

private void buildUpExpectedData(List<InputNode> nodeData, List<InputRelationship> relationshipData, Map<String, InputNode> expectedNodes, Map<String, String[]> expectedNodeNames, Map<String, Map<String, Map<String, AtomicInteger>>> expectedRelationships, Map<String, AtomicLong> nodeCounts, Map<String, Map<String, Map<String, AtomicLong>>> relationshipCounts) {
    for (InputNode node : nodeData) {
        expectedNodes.put((String) node.id(), node);
        expectedNodeNames.put(nameOf(node), node.labels());
        countNodeLabels(nodeCounts, node.labels());
    }
    for (InputRelationship relationship : relationshipData) {
        // Expected relationship counts per node, type and direction
        InputNode startNode = expectedNodes.get(relationship.startNode());
        InputNode endNode = expectedNodes.get(relationship.endNode());
        {
            expectedRelationships.get(nameOf(startNode)).get(nameOf(endNode)).get(relationship.type()).incrementAndGet();
        }
        // Expected counts per start/end node label ids
        // Let's do what CountsState#addRelationship does, roughly
        relationshipCounts.get(null).get(null).get(null).incrementAndGet();
        relationshipCounts.get(null).get(relationship.type()).get(null).incrementAndGet();
        for (String startNodeLabelName : asSet(startNode.labels())) {
            Map<String, Map<String, AtomicLong>> startLabelCounts = relationshipCounts.get(startNodeLabelName);
            startLabelCounts.get(null).get(null).incrementAndGet();
            Map<String, AtomicLong> typeCounts = startLabelCounts.get(relationship.type());
            typeCounts.get(null).incrementAndGet();
            if (COMPUTE_DOUBLE_SIDED_RELATIONSHIP_COUNTS) {
                for (String endNodeLabelName : asSet(endNode.labels())) {
                    startLabelCounts.get(null).get(endNodeLabelName).incrementAndGet();
                    typeCounts.get(endNodeLabelName).incrementAndGet();
                }
            }
        }
        for (String endNodeLabelName : asSet(endNode.labels())) {
            relationshipCounts.get(null).get(null).get(endNodeLabelName).incrementAndGet();
            relationshipCounts.get(null).get(relationship.type()).get(endNodeLabelName).incrementAndGet();
        }
    }
}