Examples with Group org.neo4j.unsafe.impl.batchimport.input.Group used on opensource projects

Example 1 with Group

use of org.neo4j.unsafe.impl.batchimport.input.Group in project neo4j by neo4j.

the class ParallelBatchImporterTest method relationships.

private InputIterable<InputRelationship> relationships(final long randomSeed, final long count, final InputIdGenerator idGenerator, final IdGroupDistribution groups) {
    return new InputIterable<InputRelationship>() {

        private int calls;

        @Override
        public InputIterator<InputRelationship> iterator() {
            calls++;
            assertTrue("Unexpected use of input iterator " + multiPassIterators + ", " + calls, multiPassIterators || (!multiPassIterators && calls == 1));
            // since we use it to compare the imported data against after the import has been completed.
            return new SimpleInputIterator<InputRelationship>("test relationships") {

                private final Random random = new Random(randomSeed);

                private final Randoms randoms = new Randoms(random, Randoms.DEFAULT);

                private int cursor;

                private final MutableLong nodeIndex = new MutableLong(-1);

                @Override
                protected InputRelationship fetchNextOrNull() {
                    if (cursor < count) {
                        Object[] properties = randomProperties(randoms, "Name " + cursor);
                        try {
                            Object startNode = idGenerator.randomExisting(random, nodeIndex);
                            Group startNodeGroup = groups.groupOf(nodeIndex.longValue());
                            Object endNode = idGenerator.randomExisting(random, nodeIndex);
                            Group endNodeGroup = groups.groupOf(nodeIndex.longValue());
                            // miss some
                            startNode = idGenerator.miss(random, startNode, 0.001f);
                            endNode = idGenerator.miss(random, endNode, 0.001f);
                            String type = idGenerator.randomType(random);
                            if (random.nextFloat() < 0.00005) {
                                // Let there be a small chance of introducing a one-off relationship
                                // with a type that no, or at least very few, other relationships have.
                                type += "_odd";
                            }
                            return new InputRelationship(sourceDescription, itemNumber, itemNumber, properties, null, startNodeGroup, startNode, endNodeGroup, endNode, type, null);
                        } finally {
                            cursor++;
                        }
                    }
                    return null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return multiPassIterators;
        }
    };
}

Example 2 with Group

use of org.neo4j.unsafe.impl.batchimport.input.Group in project neo4j by neo4j.

the class ParallelBatchImporterTest method nodes.

private InputIterable<InputNode> nodes(final long randomSeed, final long count, final InputIdGenerator inputIdGenerator, final IdGroupDistribution groups) {
    return new InputIterable<InputNode>() {

        private int calls;

        @Override
        public InputIterator<InputNode> iterator() {
            calls++;
            assertTrue("Unexpected use of input iterator " + multiPassIterators + ", " + calls, multiPassIterators || (!multiPassIterators && calls == 1));
            return new SimpleInputIterator<InputNode>("test nodes") {

                private final Random random = new Random(randomSeed);

                private final Randoms randoms = new Randoms(random, Randoms.DEFAULT);

                private int cursor;

                @Override
                protected InputNode fetchNextOrNull() {
                    if (cursor < count) {
                        Object nodeId = inputIdGenerator.nextNodeId(random);
                        Object[] properties = randomProperties(randoms, nodeId);
                        String[] labels = randoms.selection(TOKENS, 0, TOKENS.length, true);
                        try {
                            Group group = groups.groupOf(cursor);
                            return new InputNode(sourceDescription, itemNumber, itemNumber, group, nodeId, properties, null, labels, null);
                        } finally {
                            cursor++;
                        }
                    }
                    return null;
                }
            };
        }

        @Override
        public boolean supportsMultiplePasses() {
            return multiPassIterators;
        }
    };
}

Example 3 with Group

use of org.neo4j.unsafe.impl.batchimport.input.Group in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups.

@Test
public void shouldBeAbleToHaveDuplicateInputIdButInDifferentGroups() throws Exception {
    // GIVEN
    Monitor monitor = mock(Monitor.class);
    IdMapper mapper = mapper(new StringEncoder(), Radix.STRING, monitor);
    InputIterable<Object> ids = wrap("source", Arrays.<Object>asList("10", "9", "10"));
    Groups groups = new Groups();
    Group firstGroup = groups.getOrCreate("first"), secondGroup = groups.getOrCreate("second");
    try (ResourceIterator<Object> iterator = ids.iterator()) {
        int id = 0;
        // group 0
        mapper.put(iterator.next(), id++, firstGroup);
        mapper.put(iterator.next(), id++, firstGroup);
        // group 1
        mapper.put(iterator.next(), id++, secondGroup);
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // WHEN/THEN
    verifyNoMoreInteractions(collector);
    verify(monitor).numberOfCollisions(0);
    assertEquals(0L, mapper.get("10", firstGroup));
    assertEquals(1L, mapper.get("9", firstGroup));
    assertEquals(2L, mapper.get("10", secondGroup));
}

Example 4 with Group

use of org.neo4j.unsafe.impl.batchimport.input.Group in project neo4j by neo4j.

the class EncodingIdMapperTest method shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups.

@Test
public void shouldDetectCorrectDuplicateInputIdsWhereManyAccidentalInManyGroups() throws Exception {
    // GIVEN
    final ControlledEncoder encoder = new ControlledEncoder(new LongEncoder());
    IdMapper mapper = mapper(encoder, Radix.LONG, NO_MONITOR);
    final int idsPerGroup = 20, groups = 5;
    final AtomicReference<Group> group = new AtomicReference<>();
    InputIterable<Object> ids = SimpleInputIteratorWrapper.wrap("source", new Iterable<Object>() {

        @Override
        public Iterator<Object> iterator() {
            return new PrefetchingIterator<Object>() {

                private int i;

                @Override
                protected Object fetchNextOrNull() {
                    // Change group every <idsPerGroup> id
                    if (i % idsPerGroup == 0) {
                        int groupId = i / idsPerGroup;
                        if (groupId == groups) {
                            return null;
                        }
                        group.set(new Group.Adapter(groupId, "Group " + groupId));
                    }
                    try {
                        // i.e. all first 10% in each group collides with all other first 10% in each group
                        if (i % idsPerGroup < 2) {
                            // Let these colliding values encode into the same eId as well,
                            // so that they are definitely marked as collisions
                            encoder.useThisIdToEncodeNoMatterWhatComesIn(Long.valueOf(1234567));
                            return Long.valueOf(i % idsPerGroup);
                        }
                        // The other 90% will be accidental collisions for something else
                        encoder.useThisIdToEncodeNoMatterWhatComesIn(Long.valueOf(123456 - group.get().id()));
                        return Long.valueOf(i);
                    } finally {
                        i++;
                    }
                }
            };
        }
    });
    // WHEN
    long actualId = 0;
    for (Object id : ids) {
        mapper.put(id, actualId++, group.get());
    }
    Collector collector = mock(Collector.class);
    mapper.prepare(ids, collector, NONE);
    // THEN
    verifyNoMoreInteractions(collector);
    actualId = 0;
    for (Object id : ids) {
        assertEquals(actualId++, mapper.get(id, group.get()));
    }
}

Example 5 with Group

use of org.neo4j.unsafe.impl.batchimport.input.Group in project neo4j by neo4j.

the class CsvInputTest method shouldHaveNodesBelongToGroupSpecifiedInHeader.

@Test
public void shouldHaveNodesBelongToGroupSpecifiedInHeader() throws Exception {
    // GIVEN
    IdType idType = IdType.ACTUAL;
    Iterable<DataFactory<InputNode>> data = dataIterable(data("123,one\n" + "456,two"));
    Groups groups = new Groups();
    Group group = groups.getOrCreate("MyGroup");
    Input input = new CsvInput(data, header(entry(null, Type.ID, group.name(), idType.extractor(extractors)), entry("name", Type.PROPERTY, extractors.string())), null, null, idType, config(COMMAS), silentBadCollector(0), getRuntime().availableProcessors());
    // WHEN/THEN
    try (InputIterator<InputNode> nodes = input.nodes().iterator()) {
        assertNode(nodes.next(), group, 123L, properties("name", "one"), labels());
        assertNode(nodes.next(), group, 456L, properties("name", "two"), labels());
        assertFalse(nodes.hasNext());
    }
}