Examples with ThriftJDBCBinarySerDe org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe used on opensource projects

Example 1 with ThriftJDBCBinarySerDe

use of org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe in project hive by apache.

the class FileSinkOperator method closeOp.

@Override
public void closeOp(boolean abort) throws HiveException {
    row_count.set(numRows);
    LOG.info(toString() + ": records written - " + numRows);
    if (!bDynParts && !filesCreated) {
        boolean skipFiles = "tez".equalsIgnoreCase(HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE));
        if (skipFiles) {
            Class<?> clazz = conf.getTableInfo().getOutputFileFormatClass();
            skipFiles = !StreamingOutputFormat.class.isAssignableFrom(clazz);
        }
        if (!skipFiles) {
            createBucketFiles(fsp);
        }
    }
    lastProgressReport = System.currentTimeMillis();
    if (!abort) {
        // (the size of buffer is kept track of in the ThriftJDBCBinarySerDe).
        if (conf.isUsingThriftJDBCBinarySerDe()) {
            try {
                recordValue = serializer.serialize(null, inputObjInspectors[0]);
                if (null != fpaths) {
                    rowOutWriters = fpaths.outWriters;
                    rowOutWriters[0].write(recordValue);
                }
            } catch (SerDeException | IOException e) {
                throw new HiveException(e);
            }
        }
        for (FSPaths fsp : valToPaths.values()) {
            fsp.closeWriters(abort);
            // accumulated statistics which will be aggregated in case of spray writers
            if (conf.isGatherStats() && isCollectRWStats) {
                if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
                    for (int idx = 0; idx < fsp.outWriters.length; idx++) {
                        RecordWriter outWriter = fsp.outWriters[idx];
                        if (outWriter != null) {
                            SerDeStats stats = ((StatsProvidingRecordWriter) outWriter).getStats();
                            if (stats != null) {
                                fsp.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
                                fsp.stat.addToStat(StatsSetupConst.ROW_COUNT, stats.getRowCount());
                            }
                        }
                    }
                } else {
                    for (int i = 0; i < fsp.updaters.length; i++) {
                        if (fsp.updaters[i] != null) {
                            SerDeStats stats = fsp.updaters[i].getStats();
                            if (stats != null) {
                                fsp.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
                                fsp.stat.addToStat(StatsSetupConst.ROW_COUNT, stats.getRowCount());
                            }
                        }
                    }
                }
            }
            if (isNativeTable) {
                fsp.commit(fs);
            }
        }
        // Only publish stats if this operator's flag was set to gather stats
        if (conf.isGatherStats()) {
            publishStats();
        }
    } else {
        // reduce().
        for (FSPaths fsp : valToPaths.values()) {
            fsp.abortWriters(fs, abort, !autoDelete && isNativeTable);
        }
    }
    fsp = prevFsp = null;
    super.closeOp(abort);
}

Example 2 with ThriftJDBCBinarySerDe

use of org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe in project hive by apache.

the class ThriftJDBCBinarySerDe method initialize.

@Override
public void initialize(Configuration conf, Properties tbl) throws SerDeException {
    // Get column names
    MAX_BUFFERED_ROWS = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE);
    LOG.info("ThriftJDBCBinarySerDe max number of buffered columns: " + MAX_BUFFERED_ROWS);
    String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
    String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
    final String columnNameDelimiter = tbl.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tbl.getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
    if (columnNameProperty.length() == 0) {
        columnNames = new ArrayList<String>();
    } else {
        columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
    }
    if (columnTypeProperty.length() == 0) {
        columnTypes = new ArrayList<TypeInfo>();
    } else {
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    }
    rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    rowObjectInspector = (StructObjectInspector) TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(rowTypeInfo);
    initializeRowAndColumns();
    try {
        thriftFormatter.initialize(conf, tbl);
    } catch (Exception e) {
        new SerDeException(e);
    }
}