Examples with Context org.apache.hadoop.hive.ql.Context used on opensource projects

Example 71 with Context

use of org.apache.hadoop.hive.ql.Context in project hive by apache.

the class TestDbTxnManager2 method setUp.

@Before
public void setUp() throws Exception {
    SessionState.start(conf);
    ctx = new Context(conf);
    driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null);
    TxnDbUtil.cleanDb(conf);
    TxnDbUtil.prepDb(conf);
    SessionState ss = SessionState.get();
    ss.initTxnMgr(conf);
    txnMgr = ss.getTxnMgr();
    Assert.assertTrue(txnMgr instanceof DbTxnManager);
    txnHandler = TxnUtils.getTxnStore(conf);
}

Example 72 with Context

use of org.apache.hadoop.hive.ql.Context in project hive by apache.

the class GenSparkUtils method createMoveTask.

/**
 * Create and add any dependent move tasks.
 *
 * This is forked from {@link GenMapRedUtils}. The difference is that it doesn't check
 * 'isLinkedFileSink' and does not set parent dir for the linked file sinks.
 */
public static Path createMoveTask(Task<?> currTask, boolean chDir, FileSinkOperator fsOp, ParseContext parseCtx, List<Task<MoveWork>> mvTasks, HiveConf hconf, DependencyCollectionTask dependencyTask) {
    Path dest = null;
    FileSinkDesc fileSinkDesc = fsOp.getConf();
    if (chDir) {
        dest = fsOp.getConf().getFinalDirName();
        // generate the temporary file
        // it must be on the same file system as the current destination
        Context baseCtx = parseCtx.getContext();
        Path tmpDir = null;
        // In such case we shouldn't request an external tmp dir as it will end up inside the mr temp dir
        if (baseCtx.isMRTmpFileURI(dest.toUri().getPath())) {
            tmpDir = baseCtx.getMRTmpPath();
        } else {
            tmpDir = baseCtx.getExternalTmpPath(dest);
        }
        // Change all the linked file sink descriptors
        if (fileSinkDesc.getLinkedFileSinkDesc() != null) {
            for (FileSinkDesc fsConf : fileSinkDesc.getLinkedFileSinkDesc()) {
                fsConf.setDirName(tmpDir);
            }
        } else {
            fileSinkDesc.setDirName(tmpDir);
        }
    }
    Task<MoveWork> mvTask = null;
    if (!chDir) {
        mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(mvTasks, fileSinkDesc.getFinalDirName(), false, fileSinkDesc.isDirectInsert(), fileSinkDesc.getMoveTaskId(), fileSinkDesc.getAcidOperation());
    }
    // Set the move task to be dependent on the current task
    if (mvTask != null) {
        GenMapRedUtils.addDependentMoveTasks(mvTask, hconf, currTask, dependencyTask);
    }
    return dest;
}

Example 73 with Context

use of org.apache.hadoop.hive.ql.Context in project hive by apache.

the class LineageInfo method main.

public static void main(String[] args) throws Exception {
    String query = args[0];
    LineageInfo lep = new LineageInfo();
    Context ctx = new Context(new HiveConf());
    lep.getLineageInfo(query, ctx);
    for (String tab : lep.getInputTableList()) {
        System.out.println("InputTable=" + tab);
    }
    for (String tab : lep.getOutputTableList()) {
        System.out.println("OutputTable=" + tab);
    }
}

Example 74 with Context

use of org.apache.hadoop.hive.ql.Context in project hive by apache.

the class TestUtilities method testGetInputPathsWithMultipleThreadsAndEmptyPartitions.

/**
 * Check that calling {@link Utilities#getInputPaths(JobConf, MapWork, Path, Context, boolean)}
 * can process two different tables that both have empty partitions when using multiple threads.
 * Some extra logic is placed at the end of the test to validate no race conditions put the
 * {@link MapWork} object in an invalid state.
 */
@Test
public void testGetInputPathsWithMultipleThreadsAndEmptyPartitions() throws Exception {
    int numPartitions = 15;
    JobConf jobConf = new JobConf();
    jobConf.setInt(HiveConf.ConfVars.HIVE_EXEC_INPUT_LISTING_MAX_THREADS.varname, Runtime.getRuntime().availableProcessors() * 2);
    MapWork mapWork = new MapWork();
    Path testTablePath = new Path("testTable");
    Path[] testPartitionsPaths = new Path[numPartitions];
    PartitionDesc mockPartitionDesc = mock(PartitionDesc.class);
    TableDesc mockTableDesc = mock(TableDesc.class);
    when(mockTableDesc.isNonNative()).thenReturn(false);
    when(mockTableDesc.getProperties()).thenReturn(new Properties());
    when(mockPartitionDesc.getProperties()).thenReturn(new Properties());
    when(mockPartitionDesc.getTableDesc()).thenReturn(mockTableDesc);
    doReturn(HiveSequenceFileOutputFormat.class).when(mockPartitionDesc).getOutputFileFormatClass();
    for (int i = 0; i < numPartitions; i++) {
        String testPartitionName = "p=" + i;
        testPartitionsPaths[i] = new Path(testTablePath, "p=" + i);
        mapWork.getPathToAliases().put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName));
        mapWork.getAliasToWork().put(testPartitionName, mock(Operator.class));
        mapWork.getPathToPartitionInfo().put(testPartitionsPaths[i], mockPartitionDesc);
    }
    FileSystem fs = FileSystem.getLocal(jobConf);
    try {
        fs.mkdirs(testTablePath);
        List<Path> inputPaths = Utilities.getInputPaths(jobConf, mapWork, new Path(HiveConf.getVar(jobConf, HiveConf.ConfVars.LOCALSCRATCHDIR)), mock(Context.class), false);
        assertEquals(inputPaths.size(), numPartitions);
        for (int i = 0; i < numPartitions; i++) {
            assertNotEquals(inputPaths.get(i), testPartitionsPaths[i]);
        }
        assertEquals(mapWork.getPathToAliases().size(), numPartitions);
        assertEquals(mapWork.getPathToPartitionInfo().size(), numPartitions);
        assertEquals(mapWork.getAliasToWork().size(), numPartitions);
        for (Map.Entry<Path, List<String>> entry : mapWork.getPathToAliases().entrySet()) {
            assertNotNull(entry.getKey());
            assertNotNull(entry.getValue());
            assertEquals(entry.getValue().size(), 1);
            assertTrue(entry.getKey().getFileSystem(new Configuration()).exists(entry.getKey()));
        }
    } finally {
        if (fs.exists(testTablePath)) {
            fs.delete(testTablePath, true);
        }
    }
}

Example 75 with Context

use of org.apache.hadoop.hive.ql.Context in project hive by apache.

the class TestMapRedTask method mrTaskSumbitViaChildWithImpersonation.

@Test
public void mrTaskSumbitViaChildWithImpersonation() throws IOException, LoginException {
    Utils.getUGI().setAuthenticationMethod(PROXY);
    Context ctx = Mockito.mock(Context.class);
    when(ctx.getLocalTmpPath()).thenReturn(new Path(System.getProperty("java.io.tmpdir")));
    TaskQueue taskQueue = new TaskQueue(ctx);
    QueryState queryState = new QueryState.Builder().build();
    HiveConf conf = queryState.getConf();
    conf.setBoolVar(HiveConf.ConfVars.SUBMITVIACHILD, true);
    MapredWork mrWork = new MapredWork();
    mrWork.setMapWork(Mockito.mock(MapWork.class));
    MapRedTask mrTask = Mockito.spy(new MapRedTask());
    mrTask.setWork(mrWork);
    mrTask.initialize(queryState, null, taskQueue, ctx);
    mrTask.jobExecHelper = Mockito.mock(HadoopJobExecHelper.class);
    when(mrTask.jobExecHelper.progressLocal(Mockito.any(Process.class), Mockito.anyString())).thenReturn(0);
    mrTask.execute();
    ArgumentCaptor<String[]> captor = ArgumentCaptor.forClass(String[].class);
    verify(mrTask).spawn(Mockito.anyString(), Mockito.anyString(), captor.capture());
    String expected = "HADOOP_PROXY_USER=" + Utils.getUGI().getUserName();
    Assert.assertTrue(Arrays.asList(captor.getValue()).contains(expected));
}