Examples with State org.apache.gobblin.configuration.State used on opensource projects

Example 71 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class HiveCopyEntityHelperTest method testAddTableDeregisterSteps.

@Test
public void testAddTableDeregisterSteps() throws Exception {
    HiveDataset dataset = Mockito.mock(HiveDataset.class);
    Mockito.when(dataset.getProperties()).thenReturn(new Properties());
    HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class);
    Mockito.when(helper.getDeleteMethod()).thenReturn(DeregisterFileDeleteMethod.NO_DELETE);
    Mockito.when(helper.getTargetURI()).thenReturn(Optional.of("/targetURI"));
    Mockito.when(helper.getHiveRegProps()).thenReturn(new HiveRegProps(new State()));
    Mockito.when(helper.getDataset()).thenReturn(dataset);
    Mockito.when(helper.addTableDeregisterSteps(Mockito.any(List.class), Mockito.any(String.class), Mockito.anyInt(), Mockito.any(org.apache.hadoop.hive.ql.metadata.Table.class))).thenCallRealMethod();
    org.apache.hadoop.hive.ql.metadata.Table meta_table = Mockito.mock(org.apache.hadoop.hive.ql.metadata.Table.class);
    org.apache.hadoop.hive.metastore.api.Table api_table = Mockito.mock(org.apache.hadoop.hive.metastore.api.Table.class);
    Mockito.when(api_table.getDbName()).thenReturn("TestDB");
    Mockito.when(api_table.getTableName()).thenReturn("TestTable");
    Mockito.when(meta_table.getTTable()).thenReturn(api_table);
    List<CopyEntity> copyEntities = new ArrayList<CopyEntity>();
    String fileSet = "testFileSet";
    int initialPriority = 0;
    int priority = helper.addTableDeregisterSteps(copyEntities, fileSet, initialPriority, meta_table);
    Assert.assertTrue(priority == 1);
    Assert.assertTrue(copyEntities.size() == 1);
    Assert.assertTrue(copyEntities.get(0) instanceof PostPublishStep);
    PostPublishStep p = (PostPublishStep) (copyEntities.get(0));
    Assert.assertTrue(p.getStep().toString().contains("Deregister table TestDB.TestTable on Hive metastore /targetURI"));
}

Example 72 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class CopyDataPublisherTest method testPublishOverlappingDatasets.

@Test
@SuppressWarnings("unchecked")
public void testPublishOverlappingDatasets() throws Exception {
    State state = getTestState("testPublishOverlappingDatasets");
    state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/");
    Path testMethodTempPath = new Path(testClassTempPath, "testPublishOverlappingDatasets");
    CopyDataPublisher copyDataPublisher = closer.register(new CopyDataPublisher(state));
    TestDatasetManager dataset1Manager = new TestDatasetManager(testMethodTempPath, state, "datasetTargetPath", ImmutableList.of("a/b"));
    dataset1Manager.createDatasetFiles();
    TestDatasetManager dataset2Manager = new TestDatasetManager(testMethodTempPath, state, "datasetTargetPath/subDir", ImmutableList.of("a/c", "d/e"));
    dataset2Manager.createDatasetFiles();
    dataset1Manager.verifyDoesntExist();
    dataset2Manager.verifyDoesntExist();
    copyDataPublisher.publishData(combine(dataset1Manager.getWorkUnitStates(), dataset2Manager.getWorkUnitStates()));
    dataset1Manager.verifyExists();
    dataset2Manager.verifyExists();
}

Example 73 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class CopyDataPublisherTest method testPublishMultipleDatasets.

@Test
@SuppressWarnings("unchecked")
public void testPublishMultipleDatasets() throws Exception {
    State state = getTestState("testPublishMultipleDatasets");
    state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/");
    Path testMethodTempPath = new Path(testClassTempPath, "testPublishMultipleDatasets");
    CopyDataPublisher copyDataPublisher = closer.register(new CopyDataPublisher(state));
    TestDatasetManager dataset1Manager = new TestDatasetManager(testMethodTempPath, state, "dataset1TargetPath", ImmutableList.of("a/b", "a/c", "d/e"));
    dataset1Manager.createDatasetFiles();
    TestDatasetManager dataset2Manager = new TestDatasetManager(testMethodTempPath, state, "dataset2TargetPath", ImmutableList.of("a/b", "a/c", "d/e"));
    dataset2Manager.createDatasetFiles();
    dataset1Manager.verifyDoesntExist();
    dataset2Manager.verifyDoesntExist();
    copyDataPublisher.publishData(combine(dataset1Manager.getWorkUnitStates(), dataset2Manager.getWorkUnitStates()));
    dataset1Manager.verifyExists();
    dataset2Manager.verifyExists();
}

Example 74 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class DeletingCopyDataPublisherTest method testDeleteOnSource.

@Test
public void testDeleteOnSource() throws Exception {
    State state = getTestState("testDeleteOnSource");
    Path testMethodTempPath = new Path(testClassTempPath, "testDeleteOnSource");
    DeletingCopyDataPublisher copyDataPublisher = closer.register(new DeletingCopyDataPublisher(state));
    File outputDir = new File(testMethodTempPath.toString(), "task-output/jobid/1f042f494d1fe2198e0e71a17faa233f33b5099b");
    outputDir.mkdirs();
    outputDir.deleteOnExit();
    WorkUnitState wus = new WorkUnitState();
    CopyableDataset copyableDataset = new TestCopyableDataset(new Path("origin"));
    CopyableDatasetMetadata metadata = new CopyableDatasetMetadata(copyableDataset);
    CopyEntity cf = CopyableFileUtils.createTestCopyableFile(new Path(testMethodTempPath, "test.txt").toString());
    CopySource.serializeCopyableDataset(wus, metadata);
    CopySource.serializeCopyEntity(wus, cf);
    Assert.assertTrue(fs.exists(new Path(testMethodTempPath, "test.txt")));
    wus.setWorkingState(WorkingState.SUCCESSFUL);
    copyDataPublisher.publishData(ImmutableList.of(wus));
    Assert.assertFalse(fs.exists(new Path(testMethodTempPath, "test.txt")));
}

Example 75 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class RecoveryHelperTest method testGetPersistDir.

@Test
public void testGetPersistDir() throws Exception {
    State state = new State();
    Assert.assertFalse(RecoveryHelper.getPersistDir(state).isPresent());
    state.setProp(RecoveryHelper.PERSIST_DIR_KEY, this.tmpDir.getAbsolutePath());
    Assert.assertTrue(RecoveryHelper.getPersistDir(state).isPresent());
    Assert.assertTrue(RecoveryHelper.getPersistDir(state).get().toUri().getPath().startsWith(this.tmpDir.getAbsolutePath()));
}