Examples with InputEntity org.neo4j.internal.batchimport.input.InputEntity used on opensource projects

Example 6 with InputEntity

use of org.neo4j.internal.batchimport.input.InputEntity in project neo4j by neo4j.

the class CsvInputBatchImportIT method buildUpExpectedData.

private static void buildUpExpectedData(List<InputEntity> nodeData, List<InputEntity> relationshipData, Map<String, InputEntity> expectedNodes, Map<String, String[]> expectedNodeNames, Map<String, Map<String, Consumer<Object>>> expectedNodePropertyVerifiers, Map<String, Map<String, Map<String, AtomicInteger>>> expectedRelationships, Map<String, AtomicLong> nodeCounts, Map<String, Map<String, Map<String, AtomicLong>>> relationshipCounts) {
    for (InputEntity node : nodeData) {
        expectedNodes.put((String) node.id(), node);
        expectedNodeNames.put(nameOf(node), node.labels());
        // Build default verifiers for all the properties that compares the property value using equals
        Assertions.assertFalse(node.hasIntPropertyKeyIds);
        Map<String, Consumer<Object>> propertyVerifiers = new TreeMap<>();
        for (int i = 0; i < node.propertyCount(); i++) {
            final Object expectedValue = node.propertyValue(i);
            Consumer verify;
            if (expectedValue instanceof TemporalAmount) {
                // Since there is no straightforward comparison for TemporalAmount we add it to a reference
                // point in time and compare the result
                verify = actualValue -> {
                    LocalDateTime referenceTemporal = LocalDateTime.of(0, 1, 1, 0, 0);
                    LocalDateTime expected = referenceTemporal.plus((TemporalAmount) expectedValue);
                    LocalDateTime actual = referenceTemporal.plus((TemporalAmount) actualValue);
                    assertEquals(expected, actual);
                };
            } else if (expectedValue instanceof Temporal) {
                final LocalDate expectedDate = ((Temporal) expectedValue).query(TemporalQueries.localDate());
                final LocalTime expectedTime = ((Temporal) expectedValue).query(TemporalQueries.localTime());
                final ZoneId expectedZoneId = ((Temporal) expectedValue).query(TemporalQueries.zone());
                verify = actualValue -> {
                    LocalDate actualDate = ((Temporal) actualValue).query(TemporalQueries.localDate());
                    LocalTime actualTime = ((Temporal) actualValue).query(TemporalQueries.localTime());
                    ZoneId actualZoneId = ((Temporal) actualValue).query(TemporalQueries.zone());
                    assertEquals(expectedDate, actualDate);
                    assertEquals(expectedTime, actualTime);
                    if (expectedZoneId == null) {
                        if (actualZoneId != null) {
                            // If the actual value is zoned it should have the default zone
                            assertEquals(testDefaultTimeZone.get(), actualZoneId);
                        }
                    } else {
                        assertEquals(expectedZoneId, actualZoneId);
                    }
                };
            } else if (expectedValue instanceof float[]) {
                verify = actualValue -> assertArrayEquals((float[]) expectedValue, (float[]) actualValue);
            } else if (expectedValue.getClass().isArray()) {
                verify = actualValue -> assertArrayEquals((Object[]) expectedValue, (Object[]) actualValue);
            } else {
                verify = actualValue -> assertEquals(expectedValue, actualValue);
            }
            propertyVerifiers.put((String) node.propertyKey(i), verify);
        }
        // Special verifier for pointA property
        Consumer verifyPointA = actualValue -> {
            // The y-coordinate should match the node number modulo 90 (so we don't break wgs boundaries)
            PointValue v = (PointValue) actualValue;
            double actualY = v.getCoordinates().get(0).getCoordinate().get(1);
            double expectedY = indexOf(node) % 90;
            String message = actualValue + " does not have y=" + expectedY;
            assertEquals(expectedY, actualY, 0.1, message);
            message = actualValue + " does not have crs=wgs-84";
            assertEquals(CoordinateReferenceSystem.WGS84.getName(), v.getCoordinateReferenceSystem().getName(), message);
        };
        propertyVerifiers.put("pointA", verifyPointA);
        // Special verifier for pointB property
        Consumer verifyPointB = actualValue -> {
            // The y-coordinate should match the node number
            PointValue v = (PointValue) actualValue;
            double actualY = v.getCoordinates().get(0).getCoordinate().get(1);
            double expectedY = indexOf(node);
            String message = actualValue + " does not have y=" + expectedY;
            assertEquals(expectedY, actualY, 0.1, message);
            message = actualValue + " does not have crs=cartesian";
            assertEquals(CoordinateReferenceSystem.Cartesian.getName(), v.getCoordinateReferenceSystem().getName(), message);
        };
        propertyVerifiers.put("pointB", verifyPointB);
        // Special verifier for pointArray property
        Consumer verifyPointArray = actualValue -> verifyPointB.accept(((PointValue[]) actualValue)[0]);
        propertyVerifiers.put("pointArray", verifyPointArray);
        expectedNodePropertyVerifiers.put(nameOf(node), propertyVerifiers);
        countNodeLabels(nodeCounts, node.labels());
    }
    for (InputEntity relationship : relationshipData) {
        // Expected relationship counts per node, type and direction
        InputEntity startNode = expectedNodes.get(relationship.startId());
        InputEntity endNode = expectedNodes.get(relationship.endId());
        {
            expectedRelationships.get(nameOf(startNode)).get(nameOf(endNode)).get(relationship.stringType).incrementAndGet();
        }
        // Expected counts per start/end node label ids
        // Let's do what CountsState#addRelationship does, roughly
        relationshipCounts.get(null).get(null).get(null).incrementAndGet();
        relationshipCounts.get(null).get(relationship.stringType).get(null).incrementAndGet();
        for (String startNodeLabelName : asSet(startNode.labels())) {
            Map<String, Map<String, AtomicLong>> startLabelCounts = relationshipCounts.get(startNodeLabelName);
            startLabelCounts.get(null).get(null).incrementAndGet();
            Map<String, AtomicLong> typeCounts = startLabelCounts.get(relationship.stringType);
            typeCounts.get(null).incrementAndGet();
            if (COMPUTE_DOUBLE_SIDED_RELATIONSHIP_COUNTS) {
                for (String endNodeLabelName : asSet(endNode.labels())) {
                    startLabelCounts.get(null).get(endNodeLabelName).incrementAndGet();
                    typeCounts.get(endNodeLabelName).incrementAndGet();
                }
            }
        }
        for (String endNodeLabelName : asSet(endNode.labels())) {
            relationshipCounts.get(null).get(null).get(endNodeLabelName).incrementAndGet();
            relationshipCounts.get(null).get(relationship.stringType).get(endNodeLabelName).incrementAndGet();
        }
    }
}

Example 7 with InputEntity

use of org.neo4j.internal.batchimport.input.InputEntity in project neo4j by neo4j.

the class CsvInputBatchImportIT method shouldImportDataComingFromCsvFiles.

@Test
void shouldImportDataComingFromCsvFiles() throws Exception {
    // GIVEN
    Config dbConfig = Config.newBuilder().set(db_timezone, LogTimeZone.SYSTEM).set(dense_node_threshold, 5).build();
    try (JobScheduler scheduler = new ThreadPoolJobScheduler()) {
        BatchImporter importer = new ParallelBatchImporter(databaseLayout, fileSystem, PageCacheTracer.NULL, smallBatchSizeConfig(), NullLogService.getInstance(), ExecutionMonitor.INVISIBLE, EMPTY, dbConfig, defaultFormat(), ImportLogic.NO_MONITOR, scheduler, Collector.EMPTY, TransactionLogInitializer.getLogFilesInitializer(), new IndexImporterFactoryImpl(dbConfig), INSTANCE);
        List<InputEntity> nodeData = randomNodeData();
        List<InputEntity> relationshipData = randomRelationshipData(nodeData);
        // WHEN
        importer.doImport(csv(nodeDataAsFile(nodeData), relationshipDataAsFile(relationshipData), IdType.STRING, lowBufferSize(COMMAS)));
        // THEN
        verifyImportedData(nodeData, relationshipData);
    }
}

Example 8 with InputEntity

use of org.neo4j.internal.batchimport.input.InputEntity in project neo4j by neo4j.

the class CsvInputBatchImportIT method randomRelationshipData.

private List<InputEntity> randomRelationshipData(List<InputEntity> nodeData) {
    List<InputEntity> relationships = new ArrayList<>();
    for (int i = 0; i < 1000; i++) {
        InputEntity relationship = new InputEntity();
        relationship.startId(nodeData.get(random.nextInt(nodeData.size())).id(), Group.GLOBAL);
        relationship.endId(nodeData.get(random.nextInt(nodeData.size())).id(), Group.GLOBAL);
        relationship.type("TYPE_" + random.nextInt(3));
        relationships.add(relationship);
    }
    return relationships;
}

Example 9 with InputEntity

use of org.neo4j.internal.batchimport.input.InputEntity in project neo4j by neo4j.

the class CsvInputBatchImportIT method verifyImportedData.

// ======================================================
// Below is code for verifying the imported data
// ======================================================
private void verifyImportedData(List<InputEntity> nodeData, List<InputEntity> relationshipData) {
    // Build up expected data for the verification below
    Map<String, InputEntity> /*id*/
    expectedNodes = new HashMap<>();
    Map<String, String[]> expectedNodeNames = new HashMap<>();
    Map<String, Map<String, Consumer<Object>>> expectedNodePropertyVerifiers = new HashMap<>();
    Map<String, Map<String, Map<String, AtomicInteger>>> /*end node name*/
    expectedRelationships = new AutoCreatingHashMap<>(nested(nested(values(AtomicInteger.class))));
    Map<String, AtomicLong> expectedNodeCounts = new AutoCreatingHashMap<>(values(AtomicLong.class));
    Map<String, Map<String, Map<String, AtomicLong>>> expectedRelationshipCounts = new AutoCreatingHashMap<>(nested(nested(values(AtomicLong.class))));
    buildUpExpectedData(nodeData, relationshipData, expectedNodes, expectedNodeNames, expectedNodePropertyVerifiers, expectedRelationships, expectedNodeCounts, expectedRelationshipCounts);
    // Do the verification
    DatabaseManagementService managementService = new TestDatabaseManagementServiceBuilder(testDirectory.homePath()).build();
    GraphDatabaseService db = managementService.database(DEFAULT_DATABASE_NAME);
    try (Transaction tx = db.beginTx()) {
        // Verify nodes
        for (Node node : tx.getAllNodes()) {
            String name = (String) node.getProperty("name");
            String[] labels = expectedNodeNames.remove(name);
            assertEquals(asSet(labels), names(node.getLabels()));
            // Verify node properties
            Map<String, Consumer<Object>> expectedPropertyVerifiers = expectedNodePropertyVerifiers.remove(name);
            Map<String, Object> actualProperties = node.getAllProperties();
            // The id does not exist in expected properties
            actualProperties.remove("id");
            for (Map.Entry actualProperty : actualProperties.entrySet()) {
                Consumer v = expectedPropertyVerifiers.get(actualProperty.getKey());
                if (v != null) {
                    v.accept(actualProperty.getValue());
                }
            }
        }
        assertEquals(0, expectedNodeNames.size());
        // Verify relationships
        for (Relationship relationship : tx.getAllRelationships()) {
            String startNodeName = (String) relationship.getStartNode().getProperty("name");
            Map<String, Map<String, AtomicInteger>> inner = expectedRelationships.get(startNodeName);
            String endNodeName = (String) relationship.getEndNode().getProperty("name");
            Map<String, AtomicInteger> innerInner = inner.get(endNodeName);
            String type = relationship.getType().name();
            int countAfterwards = innerInner.get(type).decrementAndGet();
            assertThat(countAfterwards).isGreaterThanOrEqualTo(0);
            if (countAfterwards == 0) {
                innerInner.remove(type);
                if (innerInner.isEmpty()) {
                    inner.remove(endNodeName);
                    if (inner.isEmpty()) {
                        expectedRelationships.remove(startNodeName);
                    }
                }
            }
        }
        assertEquals(0, expectedRelationships.size());
        RecordStorageEngine storageEngine = ((GraphDatabaseAPI) db).getDependencyResolver().resolveDependency(RecordStorageEngine.class);
        NeoStores neoStores = storageEngine.testAccessNeoStores();
        CountsAccessor counts = storageEngine.countsAccessor();
        Function<String, Integer> labelTranslationTable = translationTable(neoStores.getLabelTokenStore(), ANY_LABEL);
        for (Pair<Integer, Long> count : allNodeCounts(labelTranslationTable, expectedNodeCounts)) {
            assertEquals(count.other().longValue(), counts.nodeCount(count.first(), NULL), "Label count mismatch for label " + count.first());
        }
        Function<String, Integer> relationshipTypeTranslationTable = translationTable(neoStores.getRelationshipTypeTokenStore(), ANY_RELATIONSHIP_TYPE);
        for (Pair<RelationshipCountKey, Long> count : allRelationshipCounts(labelTranslationTable, relationshipTypeTranslationTable, expectedRelationshipCounts)) {
            RelationshipCountKey key = count.first();
            assertEquals(count.other().longValue(), counts.relationshipCount(key.startLabel, key.type, key.endLabel, NULL), "Label count mismatch for label " + key);
        }
        tx.commit();
    } finally {
        managementService.shutdown();
    }
}

Example 10 with InputEntity

use of org.neo4j.internal.batchimport.input.InputEntity in project neo4j by neo4j.

the class CsvInputBatchImportIT method randomNodeData.

// ======================================================
// Below is code for generating import data
// ======================================================
private List<InputEntity> randomNodeData() {
    List<InputEntity> nodes = new ArrayList<>();
    for (int i = 0; i < 300; i++) {
        InputEntity node = new InputEntity();
        node.id(UUID.randomUUID().toString(), Group.GLOBAL);
        node.property("name", "Node " + i);
        node.property("pointA", "\"   { x : -4.2, y : " + i % 90 + ", crs: WGS-84 } \"");
        node.property("pointB", "\" { x : -8, y : " + i + " } \"");
        node.property("date", LocalDate.of(2018, i % 12 + 1, i % 28 + 1));
        node.property("time", OffsetTime.of(1, i % 60, 0, 0, ZoneOffset.ofHours(9)));
        node.property("dateTime", ZonedDateTime.of(2011, 9, 11, 8, i % 60, 0, 0, ZoneId.of("Europe/Stockholm")));
        node.property("dateTime2", // No zone specified
        LocalDateTime.of(2011, 9, 11, 8, i % 60, 0, 0));
        node.property("localTime", LocalTime.of(1, i % 60, 0));
        node.property("localDateTime", LocalDateTime.of(2011, 9, 11, 8, i % 60));
        node.property("duration", Period.of(2, -3, i % 30));
        node.property("floatArray", new float[] { 1.0f, 2.0f, 3.0f });
        node.property("dateArray", new LocalDate[] { LocalDate.of(2018, i % 12 + 1, i % 28 + 1) });
        node.property("pointArray", "\" { x : -8, y : " + i + " } \"");
        node.labels(randomLabels(random.random()));
        nodes.add(node);
    }
    return nodes;
}