Examples with IterationStatistic com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic used on opensource projects

Example 1 with IterationStatistic

use of com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic in project conquery by bakdata.

the class SerializingStoreDumpTest method testCorruptKeyDump.

/**
 * Tests if entries with corrupted keys are dumped.
 */
@Test
public void testCorruptKeyDump() throws IOException {
    // Set dump directory to this tests temp-dir
    config.setUnreadableDataDumpDirectory(tmpDir);
    {
        // Open a store and insert a valid key-value pair (UserId & User)
        SerializingStore<UserId, User> store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID);
        store.add(new UserId("testU1"), user);
    }
    {
        // Open that store again, with a different config to insert a corrupt entry
        // (String & ManagedQuery)
        SerializingStore<String, QueryDescription> store = createSerializedStore(config, env, Validators.newValidator(), new StoreInfo<>(USER_STORE_ID.getName(), String.class, QueryDescription.class));
        store.add("not a valid conquery Id", cQuery);
    }
    {
        // Reopen the store with the initial value and try to iterate over all entries
        // (this triggers the dump or removal of invalid entries)
        SerializingStore<UserId, User> store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID);
        IterationStatistic expectedResult = new IterationStatistic();
        expectedResult.setTotalProcessed(2);
        expectedResult.setFailedKeys(1);
        expectedResult.setFailedValues(0);
        // Iterate (do nothing with the entries themselves)
        IterationStatistic result = store.forEach((k, v, s) -> {
        });
        assertThat(result).isEqualTo(expectedResult);
    }
    // Test if the correct number of dumpfiles was generated
    Condition<File> dumpFileCond = new Condition<>(f -> f.getName().endsWith(SerializingStore.DUMP_FILE_EXTENTION), "dump file");
    assertThat(tmpDir.listFiles()).areExactly(1, dumpFileCond);
    // Test if the dump is correct
    File dumpFile = getDumpFile(dumpFileCond);
    assertThat((QueryDescription) Jackson.MAPPER.readerFor(QueryDescription.class).readValue(dumpFile)).isEqualTo(cQuery);
}

Example 2 with IterationStatistic

use of com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic in project conquery by bakdata.

the class SerializingStoreDumpTest method testCorruptValueDump.

/**
 * Tests if entries with corrupted values are dumped.
 */
@Test
public void testCorruptValueDump() throws IOException {
    // Set dump directory to this tests temp-dir
    config.setUnreadableDataDumpDirectory(tmpDir);
    {
        // Open a store and insert a valid key-value pair (UserId & User)
        SerializingStore<UserId, User> store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID);
        store.add(user.getId(), user);
    }
    {
        // Open that store again, with a different config to insert a corrupt entry
        // (UserId & ManagedQuery)
        SerializingStore<UserId, QueryDescription> store = createSerializedStore(config, env, Validators.newValidator(), new StoreInfo<>(USER_STORE_ID.getName(), UserId.class, QueryDescription.class));
        store.add(new UserId("testU2"), cQuery);
    }
    {
        // Reopen the store with the initial value and try to iterate over all entries
        // (this triggers the dump or removal of invalid entries)
        SerializingStore<UserId, User> store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID);
        IterationStatistic expectedResult = new IterationStatistic();
        expectedResult.setTotalProcessed(2);
        expectedResult.setFailedKeys(0);
        expectedResult.setFailedValues(1);
        // Iterate (do nothing with the entries themselves)
        IterationStatistic result = store.forEach((k, v, s) -> {
        });
        assertThat(result).isEqualTo(expectedResult);
    }
    // Test if the correct number of dumpfiles was generated
    Condition<File> dumpFileCond = new Condition<>(f -> f.getName().endsWith(SerializingStore.DUMP_FILE_EXTENTION), "dump file");
    assertThat(tmpDir.listFiles()).areExactly(1, dumpFileCond);
    // Test if the dump is correct
    File dumpFile = getDumpFile(dumpFileCond);
    assertThat((QueryDescription) Jackson.MAPPER.readerFor(QueryDescription.class).readValue(dumpFile)).isEqualTo(cQuery);
}

Example 3 with IterationStatistic

use of com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic in project conquery by bakdata.

the class SerializingStoreDumpTest method testCorruptionRemoval.

/**
 * Tests if entries with corrupted are removed from the store if configured so.
 * The dump itself is not testet.
 */
@Test
public void testCorruptionRemoval() {
    log.info("This test will throw some warnings from the SerializingStore.");
    // Set config to remove corrupt entries
    config.setRemoveUnreadableFromStore(true);
    {
        // Open a store and insert a valid key-value pair (UserId & User)
        SerializingStore<UserId, User> store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID);
        store.add(new UserId("testU1"), user);
    }
    {
        // Insert two corrupt entries. One with a corrupt key and the other one with a
        // corrupt value
        {
            SerializingStore<String, QueryDescription> store = createSerializedStore(config, env, Validators.newValidator(), new StoreInfo<>(USER_STORE_ID.getName(), String.class, QueryDescription.class));
            store.add("not a valid conquery Id", cQuery);
        }
        {
            SerializingStore<UserId, QueryDescription> store = createSerializedStore(config, env, Validators.newValidator(), new StoreInfo<>(USER_STORE_ID.getName(), UserId.class, QueryDescription.class));
            store.add(new UserId("testU2"), cQuery);
        }
    }
    {
        // Reopen the store with correct configuration and try to iterate over all
        // entries (this triggers the dump or removal of invalid entries)
        SerializingStore<UserId, User> store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID);
        IterationStatistic expectedResult = new IterationStatistic();
        expectedResult.setTotalProcessed(3);
        expectedResult.setFailedKeys(1);
        expectedResult.setFailedValues(1);
        // Iterate (do nothing with the entries themselves)
        IterationStatistic result = store.forEach((k, v, s) -> {
        });
        assertThat(result).isEqualTo(expectedResult);
    }
    {
        // Reopen again to check that the corrupted values have been removed previously
        SerializingStore<UserId, User> store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID);
        IterationStatistic expectedResult = new IterationStatistic();
        expectedResult.setTotalProcessed(1);
        expectedResult.setFailedKeys(0);
        expectedResult.setFailedValues(0);
        // Iterate (do nothing with the entries themselves)
        IterationStatistic result = store.forEach((k, v, s) -> {
        });
        assertThat(result).isEqualTo(expectedResult);
    }
}