Examples with ConsistencyCheckService org.neo4j.consistency.ConsistencyCheckService used on opensource projects

Example 1 with ConsistencyCheckService

use of org.neo4j.consistency.ConsistencyCheckService in project neo4j by neo4j.

the class DynamicIndexStoreViewIT method populateDbWithConcurrentUpdates.

@Test
public void populateDbWithConcurrentUpdates() throws Exception {
    GraphDatabaseService database = new TestGraphDatabaseFactory().newEmbeddedDatabase(testDirectory.graphDbDir());
    try {
        int counter = 1;
        for (int j = 0; j < 100; j++) {
            try (Transaction transaction = database.beginTx()) {
                for (int i = 0; i < 5; i++) {
                    Node node = database.createNode(Label.label("label" + counter));
                    node.setProperty("property", ThreadLocalRandom.current().nextInt());
                }
                transaction.success();
            }
            counter++;
        }
        int populatorCount = 5;
        ExecutorService executor = Executors.newFixedThreadPool(populatorCount);
        CountDownLatch startSignal = new CountDownLatch(1);
        AtomicBoolean endSignal = new AtomicBoolean();
        for (int i = 0; i < populatorCount; i++) {
            executor.submit(new Populator(database, counter, startSignal, endSignal));
        }
        try {
            try (Transaction transaction = database.beginTx()) {
                database.schema().indexFor(Label.label("label10")).on("property").create();
                transaction.success();
            }
            startSignal.countDown();
            try (Transaction transaction = database.beginTx()) {
                database.schema().awaitIndexesOnline(populatorCount, TimeUnit.MINUTES);
                transaction.success();
            }
        } finally {
            endSignal.set(true);
            executor.shutdown();
        // Basically we don't care to await their completion because they've done their job
        }
    } finally {
        database.shutdown();
        ConsistencyCheckService consistencyCheckService = new ConsistencyCheckService();
        Config config = Config.defaults();
        config = config.with(stringMap(GraphDatabaseSettings.pagecache_memory.name(), "8m"));
        consistencyCheckService.runFullConsistencyCheck(testDirectory.graphDbDir(), config, ProgressMonitorFactory.NONE, FormattedLogProvider.toOutputStream(System.out), false);
    }
}

Example 2 with ConsistencyCheckService

use of org.neo4j.consistency.ConsistencyCheckService in project neo4j by neo4j.

the class DatabaseRebuildTool method console.

private ConsoleInput console(final File fromPath, final GraphDatabaseBuilder dbBuilder, InputStream in, Listener<PrintStream> prompt, LifeSupport life) throws Exception {
    // We must have this indirection here since in order to perform CC (one of the commands) we must shut down
    // the database and let CC instantiate its own to run on. After that completes the db
    // should be restored. The commands has references to providers of things to accommodate for this.
    final AtomicReference<Store> store = new AtomicReference<>(new Store(dbBuilder));
    final Supplier<StoreAccess> storeAccess = () -> store.get().access;
    final Supplier<GraphDatabaseAPI> dbAccess = () -> store.get().db;
    ConsoleInput consoleInput = life.add(new ConsoleInput(in, out, prompt));
    consoleInput.add("apply", new ApplyTransactionsCommand(fromPath, dbAccess));
    consoleInput.add(DumpRecordsCommand.NAME, new DumpRecordsCommand(storeAccess));
    consoleInput.add("cc", new ArgsCommand() {

        @Override
        public void run(Args action, PrintStream out) throws Exception {
            File storeDir = store.get().storeDir;
            store.get().shutdown();
            try {
                Result result = new ConsistencyCheckService().runFullConsistencyCheck(storeDir, Config.defaults(), ProgressMonitorFactory.textual(out), FormattedLogProvider.toOutputStream(System.out), false);
                out.println(result.isSuccessful() ? "consistent" : "INCONSISTENT");
            } finally {
                store.set(new Store(dbBuilder));
            }
        }

        @Override
        public String toString() {
            return "Runs consistency check on the database for data that has been applied up to this point";
        }
    });
    life.add(new LifecycleAdapter() {

        @Override
        public void shutdown() {
            store.get().shutdown();
        }
    });
    return consoleInput;
}

Example 3 with ConsistencyCheckService

use of org.neo4j.consistency.ConsistencyCheckService in project neo4j by neo4j.

the class RestartIT method readReplicaTest.

@Test
public void readReplicaTest() throws Exception {
    // given
    Cluster cluster = clusterRule.withNumberOfCoreMembers(2).withNumberOfReadReplicas(1).startCluster();
    // when
    final GraphDatabaseService coreDB = cluster.awaitLeader(5, TimeUnit.SECONDS).database();
    try (Transaction tx = coreDB.beginTx()) {
        Node node = coreDB.createNode(label("boo"));
        node.setProperty("foobar", "baz_bat");
        tx.success();
    }
    cluster.addCoreMemberWithId(2).start();
    cluster.shutdown();
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction()) {
        for (CoreClusterMember core : cluster.coreMembers()) {
            ConsistencyCheckService.Result result = new ConsistencyCheckService().runFullConsistencyCheck(core.storeDir(), Config.embeddedDefaults(), ProgressMonitorFactory.NONE, NullLogProvider.getInstance(), fileSystem, false, new CheckConsistencyConfig(true, true, true, false));
            assertTrue("Inconsistent: " + core, result.isSuccessful());
        }
        for (ReadReplica readReplica : cluster.readReplicas()) {
            ConsistencyCheckService.Result result = new ConsistencyCheckService().runFullConsistencyCheck(readReplica.storeDir(), Config.embeddedDefaults(), ProgressMonitorFactory.NONE, NullLogProvider.getInstance(), fileSystem, false, new CheckConsistencyConfig(true, true, true, false));
            assertTrue("Inconsistent: " + readReplica, result.isSuccessful());
        }
    }
}

Example 4 with ConsistencyCheckService

use of org.neo4j.consistency.ConsistencyCheckService in project neo4j by neo4j.

the class StoreMigrationIT method shouldMigrate.

@Test
public void shouldMigrate() throws Exception {
    File db = testDir.directory(baseDirName(to, from));
    FileSystemAbstraction fs = fileSystemRule.get();
    fs.deleteRecursively(db);
    GraphDatabaseService database = getGraphDatabaseService(db, from.storeVersion());
    database.execute("CREATE INDEX ON :Person(name)");
    database.execute("CREATE INDEX ON :Person(born)");
    database.execute("CREATE CONSTRAINT ON (person:Person) ASSERT exists(person.name)");
    database.execute(CREATE_QUERY);
    long beforeNodes;
    long beforeLabels;
    long beforeKeys;
    long beforeRels;
    long beforeRelTypes;
    long beforeIndexes;
    long beforeConstraints;
    try (Transaction ignore = database.beginTx()) {
        beforeNodes = database.getAllNodes().stream().count();
        beforeLabels = database.getAllLabels().stream().count();
        beforeKeys = database.getAllPropertyKeys().stream().count();
        beforeRels = database.getAllRelationships().stream().count();
        beforeRelTypes = database.getAllRelationshipTypes().stream().count();
        beforeIndexes = stream(database.schema().getIndexes()).count();
        beforeConstraints = stream(database.schema().getConstraints()).count();
    }
    database.shutdown();
    database = getGraphDatabaseService(db, to.storeVersion());
    long afterNodes;
    long afterLabels;
    long afterKeys;
    long afterRels;
    long afterRelTypes;
    long afterIndexes;
    long afterConstraints;
    try (Transaction ignore = database.beginTx()) {
        afterNodes = database.getAllNodes().stream().count();
        afterLabels = database.getAllLabels().stream().count();
        afterKeys = database.getAllPropertyKeys().stream().count();
        afterRels = database.getAllRelationships().stream().count();
        afterRelTypes = database.getAllRelationshipTypes().stream().count();
        afterIndexes = stream(database.schema().getIndexes()).count();
        afterConstraints = stream(database.schema().getConstraints()).count();
    }
    database.shutdown();
    //171
    assertEquals(beforeNodes, afterNodes);
    //2
    assertEquals(beforeLabels, afterLabels);
    //8
    assertEquals(beforeKeys, afterKeys);
    //253
    assertEquals(beforeRels, afterRels);
    //6
    assertEquals(beforeRelTypes, afterRelTypes);
    //2
    assertEquals(beforeIndexes, afterIndexes);
    //1
    assertEquals(beforeConstraints, afterConstraints);
    ConsistencyCheckService consistencyCheckService = new ConsistencyCheckService();
    ConsistencyCheckService.Result result = runConsistencyChecker(db, fs, consistencyCheckService, to.storeVersion());
    if (!result.isSuccessful()) {
        fail("Database is inconsistent after migration.");
    }
}

Example 5 with ConsistencyCheckService

use of org.neo4j.consistency.ConsistencyCheckService in project neo4j by neo4j.

the class StoreMigratorFrom21IT method mustMendDuplicatePropertiesWhenUpgradingFromVersion21.

@Test
public void mustMendDuplicatePropertiesWhenUpgradingFromVersion21() throws Exception {
    // The rules:
    // If an index is present, all duplicates should be removed and the property set to the value in the index
    // If an index is not present, the property should be set to the value of the last duplicate in the property
    // chain, all duplicates except the first should be removed
    // If an index is not present, the first property in the duplicate chain should be kept for the users
    // benefit, moved to a special property value, `__DUPLICATE_<propkey>`
    //
    // This is the broken store that we are upgrading:
    //
    //   (#0:Label { keyA: "actual", keyA: "phony!", keyA: "phony!" })
    //   (#1 { keyA: "actual", keyA: "actual", keyA: "actual" })
    //   (#2:Label { keyA: "real1", keyA: "phony", keyA: "phony", keyD: "real2", keyD: "phony", keyD: "phony" })
    //   (#3 { keyA: "real1", keyA: "phony", keyA: "phony", keyD: "real2", keyD: "phony", keyD: "phony" })
    //   (#4 { keyA: "actual", keyB: "actual", keyC: "actual" })
    //   (#0)-[#0:REL { keyA: "actual", keyA: "actual", keyA: "actual" }]->(#1)
    //   (#0)-[#1:REL { keyA: "real1", keyA: "phony", keyA: "phony",
    //                  keyD: "real2", keyE: "phony", keyF: "phony" }]->(#1)
    //   (#2)-[#2:REL { keyA: "actual", keyB: "actual", keyC: "actual" }]->(#0)
    //
    // And this is what we want to end up with, after upgrading:
    //
    //   (#0:Label { keyA: "actual" })
    //   (#1 { keyA: "actual", __DUPLICATE_keyA: "actual" })
    //   (#2:Label { keyA: "real1", keyD: "real2" })
    //   (#3 { keyA: "real1", __DUPLICATE_keyA_1: "real1", __DUPLICATE_keyA_2: "real1",
    //         keyD: "real2", __DUPLICATE_keyD_1: "real2", __DUPLICATE_keyD_2: "real2" })
    //   (#4 { keyA: "actual", keyB: "actual", keyC: "actual" })
    //   (#0)-[#0:REL { keyA: "actual", __DUPLICATE_keyA: "actual" }]->(#1)
    //   (#0)-[#1:REL { keyA: "real1", __DUPLICATE_keyA_1: "real1", __DUPLICATE_keyA_2: "real1",
    //                  keyD: "real2", __DUPLICATE_keyD_1: "real2", __DUPLICATE_keyD_2: "real2" }]->(#1)
    //   (#2)-[#2:REL { keyA: "actual", keyB: "actual", keyC: "actual" }]->(#0)
    File dir = MigrationTestUtils.find21FormatStoreDirectoryWithDuplicateProperties(storeDir.directory());
    TestGraphDatabaseFactory factory = new TestGraphDatabaseFactory();
    GraphDatabaseBuilder builder = factory.newEmbeddedDatabaseBuilder(dir).setConfig(GraphDatabaseSettings.allow_store_upgrade, "true");
    GraphDatabaseService database = builder.newGraphDatabase();
    database.shutdown();
    ConsistencyCheckService service = new ConsistencyCheckService();
    ConsistencyCheckService.Result result = service.runFullConsistencyCheck(dir.getAbsoluteFile(), Config.empty(), ProgressMonitorFactory.NONE, NullLogProvider.getInstance(), false);
    assertTrue(result.isSuccessful());
    database = builder.newGraphDatabase();
    // Upgrade is now completed. Verify the contents:
    DependencyResolver dependencyResolver = ((GraphDatabaseAPI) database).getDependencyResolver();
    // Verify that the properties appear correct to the outside world:
    try (Transaction ignore = database.beginTx()) {
        verifyProperties(database.getNodeById(0), Pair.of("keyA", new Object[] { "actual", "phony!", "phony!" }));
        verifyProperties(database.getNodeById(1), Pair.of("keyA", new Object[] { "actual", "actual", "actual" }));
        verifyProperties(database.getNodeById(2), Pair.of("keyA", new Object[] { "real1", "phony", "phony" }), Pair.of("keyD", new Object[] { "real2", "phony", "phony" }));
        verifyProperties(database.getNodeById(3), Pair.of("keyA", new Object[] { "real1", "real1", "real1" }), Pair.of("keyD", new Object[] { "real2", "real2", "real2" }));
        verifyProperties(database.getNodeById(4), Pair.of("keyA", new Object[] { "actual" }), Pair.of("keyB", new Object[] { "actual" }), Pair.of("keyC", new Object[] { "actual" }));
        verifyProperties(database.getRelationshipById(0), Pair.of("keyA", new Object[] { "actual", "actual", "actual" }));
        verifyProperties(database.getRelationshipById(1), Pair.of("keyA", new Object[] { "real1", "real1", "real1" }), Pair.of("keyD", new Object[] { "real2", "real2", "real2" }));
        verifyProperties(database.getRelationshipById(2), Pair.of("keyA", new Object[] { "actual" }), Pair.of("keyB", new Object[] { "actual" }), Pair.of("keyC", new Object[] { "actual" }));
    }
    // Verify that there are no two properties on the entities, that have the same key:
    // (This is important because the verification above cannot tell if we have two keys with the same value)
    KernelAPI kernel = dependencyResolver.resolveDependency(KernelAPI.class);
    try (KernelTransaction tx = kernel.newTransaction(KernelTransaction.Type.implicit, AnonymousContext.read());
        Statement statement = tx.acquireStatement()) {
        Iterators.asUniqueSet(statement.readOperations().nodeGetPropertyKeys(0));
        Iterators.asUniqueSet(statement.readOperations().nodeGetPropertyKeys(1));
        Iterators.asUniqueSet(statement.readOperations().nodeGetPropertyKeys(2));
        Iterators.asUniqueSet(statement.readOperations().relationshipGetPropertyKeys(0));
        Iterators.asUniqueSet(statement.readOperations().relationshipGetPropertyKeys(1));
    }
    database.shutdown();
}