Examples with StringStringIsCleanHandler nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler used on opensource projects

Example 1 with StringStringIsCleanHandler

use of nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler in project timbuctoo by HuygensING.

the class BdbTripleStoreTest method makeCollection.

@Before
public void makeCollection() throws Exception {
    databaseCreator = new BdbNonPersistentEnvironmentCreator();
    tripleStore = new BdbTripleStore(databaseCreator.getDatabase("userId", "dataSetId", "rdfData", true, TupleBinding.getPrimitiveBinding(String.class), TupleBinding.getPrimitiveBinding(String.class), new StringStringIsCleanHandler()));
    // to make the test work on slow systems
    Thread.sleep(2000);
}

Example 2 with StringStringIsCleanHandler

use of nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler in project timbuctoo by HuygensING.

the class BdbRmlDataSourceStoreTest method itWorks.

@Test
public void itWorks() throws Exception {
    BdbNonPersistentEnvironmentCreator dbCreator = new BdbNonPersistentEnvironmentCreator();
    DataSetMetaData dataSetMetadata = new BasicDataSetMetaData("userid", "datasetid", "http://timbuctoo.huygens.knaw.nl/v5/userid/datasetid", "http://example.org/prefix/", false, false);
    final RmlDataSourceStore rmlDataSourceStore = new BdbRmlDataSourceStore(dbCreator.getDatabase("userid", "datasetid", "rmlSource", true, TupleBinding.getPrimitiveBinding(String.class), TupleBinding.getPrimitiveBinding(String.class), new StringStringIsCleanHandler()), new ImportStatus(new LogList()));
    RdfSerializer rdfSerializer = new RmlDataSourceRdfSerializer(rmlDataSourceStore);
    RawUploadRdfSaver rawUploadRdfSaver = new RawUploadRdfSaver(dataSetMetadata, "fileName", APPLICATION_OCTET_STREAM_TYPE, rdfSerializer, "origFileName", Clock.systemUTC());
    final String inputCol1 = rawUploadRdfSaver.addCollection("collection1");
    ImportPropertyDescriptions importPropertyDescriptions = new ImportPropertyDescriptions();
    importPropertyDescriptions.getOrCreate(1).setPropertyName("propName1");
    importPropertyDescriptions.getOrCreate(2).setPropertyName("propName2");
    rawUploadRdfSaver.addPropertyDescriptions(inputCol1, importPropertyDescriptions);
    rawUploadRdfSaver.addEntity(inputCol1, ImmutableMap.of("propName1", "value1", "propName2", "val2"));
    rawUploadRdfSaver.addEntity(inputCol1, ImmutableMap.of("propName1", "entVal1", "propName2", "entVal2"));
    final String inputCol2 = rawUploadRdfSaver.addCollection("collection2");
    ImportPropertyDescriptions importPropertyDescriptions1 = new ImportPropertyDescriptions();
    importPropertyDescriptions1.getOrCreate(1).setPropertyName("prop3");
    importPropertyDescriptions1.getOrCreate(2).setPropertyName("prop4");
    rawUploadRdfSaver.addPropertyDescriptions(inputCol2, importPropertyDescriptions1);
    rawUploadRdfSaver.addEntity(inputCol2, ImmutableMap.of("prop3", "value1", "prop4", "val2"));
    rawUploadRdfSaver.addEntity(inputCol2, ImmutableMap.of("prop3", "entVal1", "prop4", "entVal2"));
    rdfSerializer.close();
    RdfDataSource rdfDataSource = new RdfDataSource(rmlDataSourceStore, inputCol1, new JexlRowFactory(ImmutableMap.of(), new HashMapBasedJoinHandler()));
    RdfDataSource rdfDataSource2 = new RdfDataSource(rmlDataSourceStore, inputCol2, new JexlRowFactory(ImmutableMap.of(), new HashMapBasedJoinHandler()));
    final List<String> collection1;
    final List<String> collection2;
    try (Stream<Row> stream = rdfDataSource.getRows(new ThrowingErrorHandler())) {
        collection1 = stream.map(x -> x.getRawValue("propName1") + ":" + x.getRawValue("propName2")).collect(toList());
    }
    try (Stream<Row> stream = rdfDataSource2.getRows(new ThrowingErrorHandler())) {
        collection2 = stream.map(x -> x.getRawValue("prop3") + ":" + x.getRawValue("prop4")).collect(toList());
    }
    assertThat(collection1, contains("value1:val2", "entVal1:entVal2"));
    assertThat(collection2, contains("value1:val2", "entVal1:entVal2"));
    dbCreator.close();
}

Example 3 with StringStringIsCleanHandler

use of nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler in project timbuctoo by HuygensING.

the class BdbDumpTask method execute.

@Override
public void execute(ImmutableMultimap<String, String> parameters, PrintWriter output) throws Exception {
    if (!parameters.keySet().containsAll(REQUIRED_PARAMS)) {
        output.write("Make sure you provide the following parameters: " + REQUIRED_PARAMS);
        return;
    }
    BdbWrapper<String, String> database = environmentCreator.getDatabase(getParam(parameters, OWNER), getParam(parameters, DATA_SET), getParam(parameters, DATABASE), true, STRING_BINDER, STRING_BINDER, new StringStringIsCleanHandler());
    String prefix = parameters.containsKey("prefix") ? getParam(parameters, "prefix") : "";
    int start = parameters.containsKey("start") ? Integer.parseInt(getParam(parameters, "start")) : 0;
    int count = parameters.containsKey("count") ? Integer.parseInt(getParam(parameters, "count")) : 10;
    output.write("uncommitted data: " + database.dump(prefix, start, count, LockMode.READ_UNCOMMITTED));
    output.write("committed data: " + database.dump(prefix, start, count, LockMode.READ_COMMITTED));
}

Example 4 with StringStringIsCleanHandler

use of nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler in project timbuctoo by HuygensING.

the class DataSet method dataSet.

public static DataSet dataSet(DataSetMetaData metadata, ExecutorService executorService, String rdfPrefix, BdbEnvironmentCreator dataStoreFactory, Runnable onUpdated, DataSetStorage dataSetStorage) throws IOException, DataStoreCreationException {
    String userId = metadata.getOwnerId();
    String dataSetId = metadata.getDataSetId();
    File descriptionFile = dataSetStorage.getResourceSyncDescriptionFile();
    FileStorage fileStorage = dataSetStorage.getFileStorage();
    ImportManager importManager = new ImportManager(dataSetStorage.getLogList(), fileStorage, fileStorage, dataSetStorage.getLogStorage(), executorService, dataSetStorage.getRdfIo(), onUpdated);
    try {
        importManager.subscribeToRdf(new RdfDescriptionSaver(descriptionFile, metadata.getBaseUri(), importManager.getImportStatus()));
    } catch (ParserConfigurationException | SAXException e) {
        LOG.error("Could not construct import manager of data set", e);
    }
    final TupleBinding<String> stringBinding = TupleBinding.getPrimitiveBinding(String.class);
    try {
        StringStringIsCleanHandler stringStringIsCleanHandler = new StringStringIsCleanHandler();
        BdbTripleStore quadStore = new BdbTripleStore(dataStoreFactory.getDatabase(userId, dataSetId, "rdfData", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final BdbTypeNameStore typeNameStore = new BdbTypeNameStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "typenames", false, stringBinding, stringBinding, stringStringIsCleanHandler)), rdfPrefix);
        final BdbSchemaStore schema = new BdbSchemaStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "schema", false, stringBinding, stringBinding, stringStringIsCleanHandler)), importManager.getImportStatus());
        final BdbTruePatchStore truePatchStore = new BdbTruePatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "truePatch", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final TupleBinding<Integer> integerBinding = TupleBinding.getPrimitiveBinding(Integer.class);
        final UpdatedPerPatchStore updatedPerPatchStore = new UpdatedPerPatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "updatedPerPatch", true, integerBinding, stringBinding, new IsCleanHandler<Integer, String>() {

            @Override
            public Integer getKey() {
                return Integer.MAX_VALUE;
            }

            @Override
            public String getValue() {
                return "isClean";
            }
        }));
        final BdbRmlDataSourceStore rmlDataSourceStore = new BdbRmlDataSourceStore(dataStoreFactory.getDatabase(userId, dataSetId, "rmlSource", true, stringBinding, stringBinding, stringStringIsCleanHandler), importManager.getImportStatus());
        VersionStore versionStore = new VersionStore(dataStoreFactory.getDatabase(userId, dataSetId, "versions", false, stringBinding, integerBinding, new IsCleanHandler<String, Integer>() {

            @Override
            public String getKey() {
                return "isClean";
            }

            @Override
            public Integer getValue() {
                return Integer.MAX_VALUE;
            }
        }));
        final StoreUpdater storeUpdater = new StoreUpdater(dataStoreFactory, quadStore, typeNameStore, truePatchStore, updatedPerPatchStore, Lists.newArrayList(schema, rmlDataSourceStore), versionStore, importManager.getImportStatus());
        importManager.subscribeToRdf(storeUpdater);
        ImmutableDataSet dataSet = ImmutableDataSet.builder().ownerId(userId).dataSetName(dataSetId).bdbEnvironmentCreator(dataStoreFactory).metadata(metadata).quadStore(quadStore).typeNameStore(typeNameStore).schemaStore(schema).dataSource(new RdfDataSourceFactory(rmlDataSourceStore)).schemaStore(schema).importManager(importManager).dataSetStorage(dataSetStorage).build();
        importManager.init(dataSet);
        if (!quadStore.isClean() || !typeNameStore.isClean() || !schema.isClean() || !truePatchStore.isClean() || !updatedPerPatchStore.isClean() || !rmlDataSourceStore.isClean() || !versionStore.isClean()) {
            LOG.error("Data set '{}__{}' data is corrupted, starting to reimport.", userId, dataSetId);
            quadStore.empty();
            typeNameStore.empty();
            schema.empty();
            truePatchStore.empty();
            updatedPerPatchStore.empty();
            rmlDataSourceStore.empty();
            versionStore.empty();
            importManager.reprocessLogs();
        } else {
            // process unprocessed logs
            importManager.processLogs();
        }
        return dataSet;
    } catch (BdbDbCreationException e) {
        throw new DataStoreCreationException(e.getCause());
    }
}

Example 5 with StringStringIsCleanHandler

use of nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler in project timbuctoo by HuygensING.

the class ChangeFetcherImplTest method showsAdditions.

@Test
public void showsAdditions() throws Exception {
    final BdbNonPersistentEnvironmentCreator databaseCreator = new BdbNonPersistentEnvironmentCreator();
    final BdbTripleStore bdbTripleStore = new BdbTripleStore(databaseCreator.getDatabase("a", "b", "rdfData", true, TupleBinding.getPrimitiveBinding(String.class), TupleBinding.getPrimitiveBinding(String.class), new StringStringIsCleanHandler()));
    final BdbTruePatchStore truePatchStore = new BdbTruePatchStore(databaseCreator.getDatabase("a", "b", "truePatch", true, TupleBinding.getPrimitiveBinding(String.class), TupleBinding.getPrimitiveBinding(String.class), new StringStringIsCleanHandler()));
    bdbTripleStore.putQuad("subj", "pred", OUT, "obj", null, null);
    truePatchStore.put("subj", 0, "pred", OUT, true, "obj", null, null);
    ChangeFetcherImpl changeFetcher = new ChangeFetcherImpl(truePatchStore, bdbTripleStore, 0);
    try (Stream<CursorQuad> predicates = changeFetcher.getPredicates("subj", "pred", OUT, false, false, true)) {
        assertThat(predicates.count(), is(1L));
    }
    try (Stream<CursorQuad> predicates = changeFetcher.getPredicates("subj", "pred", OUT, true, true, true)) {
        assertThat(predicates.count(), is(1L));
    }
    try (Stream<CursorQuad> predicates = changeFetcher.getPredicates("subj", "pred", OUT, true, false, true)) {
        assertThat(predicates.count(), is(1L));
    }
    bdbTripleStore.putQuad("subj", "pred", OUT, "obj2", null, null);
    truePatchStore.put("subj", 1, "pred", OUT, true, "obj2", null, null);
    changeFetcher = new ChangeFetcherImpl(truePatchStore, bdbTripleStore, 1);
    try (Stream<CursorQuad> predicates = changeFetcher.getPredicates("subj", "pred", OUT, false, false, true)) {
        assertThat(predicates.count(), is(1L));
    }
    try (Stream<CursorQuad> predicates = changeFetcher.getPredicates("subj", "pred", OUT, true, true, true)) {
        assertThat(predicates.count(), is(2L));
    }
    try (Stream<CursorQuad> predicates = changeFetcher.getPredicates("subj", "pred", OUT, true, false, true)) {
        assertThat(predicates.count(), is(1L));
    }
}