Examples with DatasetStoreDataset org.apache.gobblin.metastore.DatasetStoreDataset used on opensource projects

Example 1 with DatasetStoreDataset

use of org.apache.gobblin.metastore.DatasetStoreDataset in project incubator-gobblin by apache.

the class CleanableDatasetStoreDatasetTest method testCleanStateStore.

@Test
public void testCleanStateStore() throws IOException {
    File tmpDir = Files.createTempDir();
    tmpDir.deleteOnExit();
    FileSystem fs = FileSystem.getLocal(new Configuration());
    FsDatasetStateStore store = new FsDatasetStateStore(fs, tmpDir.getAbsolutePath());
    store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id1"));
    store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id2"));
    store.persistDatasetState("dataset1", new JobState.DatasetState("job2", "job2_id1"));
    store.persistDatasetState("", new JobState.DatasetState("job3", "job3_id1"));
    Properties props = new Properties();
    props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, tmpDir.getAbsolutePath());
    props.setProperty("selection.timeBased.lookbackTime", "0m");
    TimeBasedDatasetStoreDatasetFinder datasetFinder = new TimeBasedDatasetStoreDatasetFinder(fs, props);
    List<DatasetStoreDataset> datasets = datasetFinder.findDatasets();
    for (DatasetStoreDataset dataset : datasets) {
        ((CleanableDataset) dataset).clean();
        File jobDir = new File(tmpDir.getAbsolutePath(), dataset.getKey().getStoreName());
        Assert.assertEquals(jobDir.list().length, 1);
    }
}

Example 2 with DatasetStoreDataset

use of org.apache.gobblin.metastore.DatasetStoreDataset in project incubator-gobblin by apache.

the class TimestampedDatasetStateStoreVersionFinder method findDatasetVersions.

@Override
public Collection<TimestampedDatasetStateStoreVersion> findDatasetVersions(Dataset dataset) throws IOException {
    DatasetStoreDataset storeDataset = ((DatasetStoreDataset) dataset);
    List<TimestampedDatasetStateStoreVersion> versions = Lists.newArrayList();
    for (DatasetStateStoreEntryManager entry : storeDataset.getDatasetStateStoreMetadataEntries()) {
        versions.add(new TimestampedDatasetStateStoreVersion(entry));
    }
    return versions;
}