Examples with LoadFileDesc org.apache.hadoop.hive.ql.plan.LoadFileDesc used on opensource projects

Example 11 with LoadFileDesc

use of org.apache.hadoop.hive.ql.plan.LoadFileDesc in project hive by apache.

the class TestGenMapRedUtilsCreateConditionalTask method testMergePathValidMoveWorkReturnsNewMoveWork.

@Test
public void testMergePathValidMoveWorkReturnsNewMoveWork() {
    final Path condInputPath = new Path("s3a://bucket/scratch/-ext-10000");
    final Path condOutputPath = new Path("s3a://bucket/scratch/-ext-10002");
    final Path targetMoveWorkPath = new Path("s3a://bucket/scratch/-ext-10003");
    final MoveWork mockWork = mock(MoveWork.class);
    final LineageState lineageState = new LineageState();
    MoveWork newWork;
    // test using loadFileWork
    when(mockWork.getLoadFileWork()).thenReturn(new LoadFileDesc(condOutputPath, targetMoveWorkPath, false, "", "", false));
    newWork = GenMapRedUtils.mergeMovePaths(condInputPath, mockWork, lineageState);
    assertNotNull(newWork);
    assertNotEquals(newWork, mockWork);
    assertEquals(condInputPath, newWork.getLoadFileWork().getSourcePath());
    assertEquals(targetMoveWorkPath, newWork.getLoadFileWork().getTargetDir());
    // test using loadTableWork
    TableDesc tableDesc = new TableDesc();
    reset(mockWork);
    when(mockWork.getLoadTableWork()).thenReturn(new LoadTableDesc(condOutputPath, tableDesc, null));
    newWork = GenMapRedUtils.mergeMovePaths(condInputPath, mockWork, lineageState);
    assertNotNull(newWork);
    assertNotEquals(newWork, mockWork);
    assertEquals(condInputPath, newWork.getLoadTableWork().getSourcePath());
    assertTrue(newWork.getLoadTableWork().getTable().equals(tableDesc));
}

Example 12 with LoadFileDesc

use of org.apache.hadoop.hive.ql.plan.LoadFileDesc in project hive by apache.

the class TestGenMapRedUtilsCreateConditionalTask method createMoveTask.

private Task<MoveWork> createMoveTask(Path source, Path destination) {
    Task<MoveWork> moveTask = mock(MoveTask.class);
    MoveWork moveWork = new MoveWork();
    moveWork.setLoadFileWork(new LoadFileDesc(source, destination, true, null, null, false));
    when(moveTask.getWork()).thenReturn(moveWork);
    return moveTask;
}