Examples with FsDatasetStateStore org.apache.gobblin.runtime.FsDatasetStateStore used on opensource projects

Example 1 with FsDatasetStateStore

use of org.apache.gobblin.runtime.FsDatasetStateStore in project incubator-gobblin by apache.

the class CleanableDatasetStoreDatasetTest method testCleanStateStore.

@Test
public void testCleanStateStore() throws IOException {
    File tmpDir = Files.createTempDir();
    tmpDir.deleteOnExit();
    FileSystem fs = FileSystem.getLocal(new Configuration());
    FsDatasetStateStore store = new FsDatasetStateStore(fs, tmpDir.getAbsolutePath());
    store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id1"));
    store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id2"));
    store.persistDatasetState("dataset1", new JobState.DatasetState("job2", "job2_id1"));
    store.persistDatasetState("", new JobState.DatasetState("job3", "job3_id1"));
    Properties props = new Properties();
    props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, tmpDir.getAbsolutePath());
    props.setProperty("selection.timeBased.lookbackTime", "0m");
    TimeBasedDatasetStoreDatasetFinder datasetFinder = new TimeBasedDatasetStoreDatasetFinder(fs, props);
    List<DatasetStoreDataset> datasets = datasetFinder.findDatasets();
    for (DatasetStoreDataset dataset : datasets) {
        ((CleanableDataset) dataset).clean();
        File jobDir = new File(tmpDir.getAbsolutePath(), dataset.getKey().getStoreName());
        Assert.assertEquals(jobDir.list().length, 1);
    }
}