Examples with DatasetPredicate org.apache.gobblin.metastore.predicates.DatasetPredicate used on opensource projects

Example 1 with DatasetPredicate

use of org.apache.gobblin.metastore.predicates.DatasetPredicate in project incubator-gobblin by apache.

the class DatasetStoreDatasetFinder method buildPredicate.

private StateStorePredicate buildPredicate() {
    StateStorePredicate predicate = null;
    String storeName = null;
    String datasetUrn;
    if (ConfigUtils.hasNonEmptyPath(this.config, STORE_NAME_FILTER)) {
        storeName = this.config.getString(STORE_NAME_FILTER);
        predicate = new StoreNamePredicate(storeName, x -> true);
    }
    if (ConfigUtils.hasNonEmptyPath(this.config, DATASET_URN_FILTER)) {
        if (storeName == null) {
            throw new IllegalArgumentException(DATASET_URN_FILTER + " requires " + STORE_NAME_FILTER + " to also be defined.");
        }
        datasetUrn = this.config.getString(DATASET_URN_FILTER);
        predicate = new DatasetPredicate(storeName, datasetUrn, x -> true);
    }
    return predicate == null ? new StateStorePredicate(x -> true) : predicate;
}

Example 2 with DatasetPredicate

use of org.apache.gobblin.metastore.predicates.DatasetPredicate in project incubator-gobblin by apache.

the class FsDatasetStateStoreTest method testGetMetadataForTables.

@Test
public void testGetMetadataForTables() throws Exception {
    File tmpDir = Files.createTempDir();
    tmpDir.deleteOnExit();
    FsDatasetStateStore store = new FsDatasetStateStore(FileSystem.getLocal(new Configuration()), tmpDir.getAbsolutePath());
    JobState.DatasetState dataset2State = new JobState.DatasetState("job1", "job1_id2");
    dataset2State.setDatasetUrn("dataset2");
    dataset2State.setId("dataset2");
    TaskState taskState = new TaskState();
    taskState.setJobId("job1_id2");
    taskState.setTaskId("task123");
    taskState.setProp("key", "value");
    dataset2State.addTaskState(taskState);
    store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id1"));
    store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id2"));
    store.persistDatasetState("dataset2", dataset2State);
    store.persistDatasetState("dataset1", new JobState.DatasetState("job2", "job2_id1"));
    store.persistDatasetState("", new JobState.DatasetState("job3", "job3_id1"));
    List<FsDatasetStateStoreEntryManager> metadataList = store.getMetadataForTables(new StateStorePredicate(x -> true));
    // 5 explicitly stored states, plus 4 current links, one per job-dataset
    Assert.assertEquals(metadataList.size(), 9);
    metadataList = store.getMetadataForTables(new StoreNamePredicate("job1", x -> true));
    // 3 explicitly stored states, plus 2 current links, one per dataset
    Assert.assertEquals(metadataList.size(), 5);
    metadataList = store.getMetadataForTables(new DatasetPredicate("job1", "dataset1", x -> true));
    Assert.assertEquals(metadataList.size(), 3);
    metadataList = store.getMetadataForTables(new DatasetPredicate("job1", "dataset2", meta -> ((DatasetStateStoreEntryManager) meta).getStateId().equals(DatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX)));
    Assert.assertEquals(metadataList.size(), 1);
    DatasetStateStoreEntryManager metadata = metadataList.get(0);
    Assert.assertEquals(metadata.getStoreName(), "job1");
    Assert.assertEquals(metadata.getSanitizedDatasetUrn(), "dataset2");
    Assert.assertEquals(metadata.getStateId(), DatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX);
    Assert.assertEquals(metadata.getDatasetStateStore(), store);
    JobState.DatasetState readState = (JobState.DatasetState) metadata.readState();
    TaskState readTaskState = readState.getTaskStates().get(0);
    Assert.assertEquals(readTaskState.getProp("key"), "value");
    metadata.delete();
    // verify it got deleted
    metadataList = store.getMetadataForTables(new DatasetPredicate("job1", "dataset2", meta -> ((DatasetStateStoreEntryManager) meta).getStateId().equals(DatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX)));
    Assert.assertTrue(metadataList.isEmpty());
}