Examples with FileSinkDesc org.apache.hadoop.hive.ql.plan.FileSinkDesc used on opensource projects

Example 21 with FileSinkDesc

use of org.apache.hadoop.hive.ql.plan.FileSinkDesc in project hive by apache.

the class TestUtilities method runRemoveTempOrDuplicateFilesTestCase.

private List<Path> runRemoveTempOrDuplicateFilesTestCase(String executionEngine, boolean dPEnabled) throws Exception {
    Configuration hconf = new HiveConf(this.getClass());
    // do this to verify that Utilities.removeTempOrDuplicateFiles does not revert to default scheme information
    hconf.set("fs.defaultFS", "hdfs://should-not-be-used/");
    hconf.set(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname, executionEngine);
    FileSystem localFs = FileSystem.getLocal(hconf);
    DynamicPartitionCtx dpCtx = getDynamicPartitionCtx(dPEnabled);
    Path tempDirPath = setupTempDirWithSingleOutputFile(hconf);
    FileSinkDesc conf = getFileSinkDesc(tempDirPath);
    List<Path> paths = Utilities.removeTempOrDuplicateFiles(localFs, tempDirPath, dpCtx, conf, hconf);
    String expectedScheme = tempDirPath.toUri().getScheme();
    String expectedAuthority = tempDirPath.toUri().getAuthority();
    assertPathsMatchSchemeAndAuthority(expectedScheme, expectedAuthority, paths);
    return paths;
}

Example 22 with FileSinkDesc

use of org.apache.hadoop.hive.ql.plan.FileSinkDesc in project SQLWindowing by hbutani.

the class MRExecutor method createReduceSideTree.

/**
	 * The reduce-side plan always looks like 
	 * ExtratOperator->PTFOperator->FileSinkOperator.
	 * Use the data structures initialized in MRUtils to 
	 * create the operators here.
	 * @param qdef
	 * @param mrUtils
	 * @param mr
	 * @throws WindowingException
	 */
@SuppressWarnings("unchecked")
private void createReduceSideTree(QueryDef qdef, MRUtils mrUtils, MapredWork mr) throws WindowingException {
    // reduce side work
    Operator<FileSinkDesc> op4 = OperatorFactory.get(new FileSinkDesc(mrUtils.getOutputPath(), createOutputTableDesc(qdef), false));
    Operator<PTFDesc> op3 = RuntimeUtils.createPTFOperator(new PTFDesc(SerializationUtils.serializeQueryDef(qdef)), op4);
    Operator<ExtractDesc> op2 = OperatorFactory.get(new ExtractDesc(TranslateUtils.getStringColumn(Utilities.ReduceField.VALUE.toString())), op3);
    mr.setReducer(op2);
}